CI: Two Machines (#402)

This patch starts using two machines to test testgres.

Key things
- Option '--init' is used in a command docker run to prevent a problem with lots of zomby processes on altlinux-11, alpine-pg10 and alpine-pg11 targets.

Co-authored-by: Mark G. mark@google.com

Author: dmitry-lipetsk

.github/workflows/ci.yml

@@ -74,82 +74,115 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - platform: "std2-all"
+          - platform: "alpine"
             python: "3.7"
             postgres: "17"
-          - platform: "std2-all"
+            case_suffix: "py3_07_xx-pg17_xx"
+          - platform: "alpine"
             python: "3.8.0"
             postgres: "17"
-          - platform: "std2-all"
+            case_suffix: "py3_08_88-pg17_xx"
+          - platform: "alpine"
             python: "3.8"
             postgres: "17"
-          - platform: "std2-all"
+            case_suffix: "py3_08_xx-pg17_xx"
+          - platform: "alpine"
             python: "3.9"
             postgres: "17"
-          - platform: "std2-all"
+            case_suffix: "py3_09_xx-pg17_xx"
+          - platform: "alpine"
             python: "3.10"
             postgres: "17"
-          - platform: "std2-all"
+            case_suffix: "py3_10_xx-pg17_xx"
+          - platform: "alpine"
             python: "3.11"
             postgres: "17"
-          - platform: "std2-all"
+            case_suffix: "py3_11_xx-pg17_xx"
+          - platform: "alpine"
             python: "3.12"
             postgres: "17"
-          - platform: "std2-all"
+            case_suffix: "py3_12_xx-pg17_xx"
+          - platform: "alpine"
             python: "3.13"
             postgres: "17"
-          - platform: "std2-all"
+            case_suffix: "py3_13_xx-pg17_xx"
+          - platform: "alpine"
             python: "3.14"
             postgres: "17"
-          - platform: "std"
-            python: "3"
+            case_suffix: "py3_14_xx-pg17_xx"
+          - platform: "alpine"
+            python: "3.12"
             postgres: "10"
-          - platform: "std"
-            python: "3"
+            case_suffix: "py3_12_xx-pg10_xx"
+          - platform: "alpine"
+            python: "3.12"
             postgres: "11"
-          - platform: "std"
-            python: "3"
+            case_suffix: "py3_12_xx-pg11_xx"
+          - platform: "alpine"
+            python: "3.12"
             postgres: "12"
-          - platform: "std"
-            python: "3"
+            case_suffix: "py3_12_xx-pg12_xx"
+          - platform: "alpine"
+            python: "3.12"
             postgres: "13"
-          - platform: "std"
-            python: "3"
+            case_suffix: "py3_12_xx-pg13_xx"
+          - platform: "alpine"
+            python: "3.12"
             postgres: "14"
-          - platform: "std"
-            python: "3"
+            case_suffix: "py3_12_xx-pg14_xx"
+          - platform: "alpine"
+            python: "3.12"
             postgres: "15"
-          - platform: "std"
-            python: "3"
+            case_suffix: "py3_12_xx-pg15_xx"
+          - platform: "alpine"
+            python: "3.12"
             postgres: "16"
-          - platform: "std-all"
-            python: "3"
-            postgres: "17"
-          - platform: "std-all"
-            python: "3"
+            case_suffix: "py3_12_xx-pg16_xx"
+          - platform: "alpine"
+            python: "3.12"
             postgres: "18"
+            case_suffix: "py3_12_xx-pg18_xx"
           - platform: "ubuntu_24_04"
             python: "3"
             postgres: "17"
+            case_suffix: "py3_xx_xx-pg17_xx"
           - platform: "altlinux_10"
             python: "3"
             postgres: "17"
+            case_suffix: "py3_xx_xx-pg17_xx"
           - platform: "altlinux_11"
             python: "3"
             postgres: "17"
+            case_suffix: "py3_xx_xx-pg17_xx"
           - platform: "astralinux_1_7"
             python: "3"
             postgres: "17"
+            case_suffix: "py3_xx_xx-pg17_xx"
+
+    name: "test: ${{ matrix.platform }} | ${{ matrix.case_suffix }}"
 
     env:
-      BASE_SIGN: "${{ matrix.platform }}-py${{ matrix.python }}-pg${{ matrix.postgres }}"
+      BASE_SIGN: "${{ matrix.platform }}-${{ matrix.case_suffix }}"
+
+      DOCKER_HIGHLOAD_FLAGS: >-
+        --init
+        --sysctl net.core.somaxconn=4096
+        --sysctl net.ipv4.tcp_max_syn_backlog=4096
+        --ulimit nofile=524288:524288
+        --ulimit nproc=65535:65535
+        --sysctl net.ipv4.tcp_tw_reuse=1
+        --sysctl net.ipv4.ip_local_port_range="1024 65535"
 
     steps:
     - name: Prepare variables
       run: |
         echo "RUN_CFG__NOW=$(date +'%Y%m%d_%H%M%S')" >> $GITHUB_ENV
         echo "RUN_CFG__LOGS_DIR=logs-${{ env.BASE_SIGN }}" >> $GITHUB_ENV
         echo "RUN_CFG__DOCKER_IMAGE_NAME=tests-${{ env.BASE_SIGN }}" >> $GITHUB_ENV
+        echo "RUN_CFG__NETWORK_NAME=network-${{ env.BASE_SIGN }}" >> $GITHUB_ENV
+        echo "RUN_CFG__MACHINE1_NAME=machine1-${{ env.BASE_SIGN }}" >> $GITHUB_ENV
+        echo "RUN_CFG__MACHINE2_NAME=machine2-${{ env.BASE_SIGN }}" >> $GITHUB_ENV
+        echo "RUN_CFG__SSH_KEY_NAME=id_ed25519-${{ env.BASE_SIGN }}" >> $GITHUB_ENV
         echo "---------- [$GITHUB_ENV]"
         cat $GITHUB_ENV
     - name: Checkout
@@ -158,10 +191,83 @@ jobs:
       run: mkdir -p "${{ env.RUN_CFG__LOGS_DIR }}"
     - name: Adjust logs folder permission
       run: chmod -R 777 "${{ env.RUN_CFG__LOGS_DIR }}"
-    - name: Build local image ${{ matrix.alpine }}
+    - name: Build local image
       run: docker build --build-arg PG_VERSION="${{ matrix.postgres }}" --build-arg PYTHON_VERSION="${{ matrix.python }}" -t "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" -f Dockerfile--${{ matrix.platform }}.tmpl .
-    - name: Run
-      run: docker run $(bash <(curl -s https://codecov.io/env)) -t -v ${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}:/home/test/testgres/logs "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}"
+    - name: Generate temporary SSH Key pair for CI
+      run: |
+        # Generate a key without a password directly on the GitHub Actions host
+        ssh-keygen -q -t ed25519 -N "" -f ${{ github.workspace }}/${{ env.RUN_CFG__SSH_KEY_NAME }}
+
+    - name: Setup network
+      run: |
+        docker network create ${{ env.RUN_CFG__NETWORK_NAME }}
+
+    - name: Run machine2 (for remote operations)
+      run: |
+        #
+        # "- sleep infinity" is a right command. Docker ignores "-" and runs "sleep infinity".
+        #
+        docker run -d -t \
+          ${{ env.DOCKER_HIGHLOAD_FLAGS }} \
+          --sysctl net.ipv4.tcp_tw_reuse=1 \
+          --sysctl net.ipv4.ip_local_port_range="1024 65535" \
+          --network ${{ env.RUN_CFG__NETWORK_NAME }} \
+          --name ${{ env.RUN_CFG__MACHINE2_NAME }} \
+          "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" \
+          sleep infinity
+
+        echo "Waiting for container ${{ env.RUN_CFG__MACHINE2_NAME }} to start ..."
+        
+        # Set the timeout using the attempt counter
+        MAX_ATTEMPTS=50
+        ATTEMPT=0
+        
+        until [ "$(docker container inspect --format '{{.State.Running}}' "${{ env.RUN_CFG__MACHINE2_NAME }}")" = "true" ]; do
+          ATTEMPT=$((ATTEMPT + 1))
+          if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then
+            echo "Error: Container did not start within the allotted time (timeout)!"
+            exit 1
+          fi
+          sleep 0.2
+        done
+        
+        echo "Container successfully launched in $((ATTEMPT * 2 / 10)) second(s)!"
+
+        #
+        # Setup authorized_keys
+        #
+        cat ${{ github.workspace }}/${{ env.RUN_CFG__SSH_KEY_NAME }}.pub | \
+        docker exec -i -u test ${{ env.RUN_CFG__MACHINE2_NAME }} sh -c " \
+          cat >> /home/test/.ssh/authorized_keys && \
+          chmod 600 /home/test/.ssh/authorized_keys \
+        "
+
+        #
+        # Log status
+        #
+        docker exec -u root ${{ env.RUN_CFG__MACHINE2_NAME }} sh -c "
+          echo '--- INSIDE CONTAINER OS ---' && \
+          cat /etc/os-release && \
+          echo '--- INSIDE CONTAINER .SSH ---' && \
+          ls -la /home/test/.ssh/ \
+        "
+
+    - name: Run machine1 (main test container)
+      run: |
+        # Launch the main container, passing the private key to the .ssh folder of the test user.
+        # Pass environment variables so Python tests know where to start.
+        # Added the --rm flag so that the container is automatically deleted after the tests are completed
+        docker run --rm -t \
+          ${{ env.DOCKER_HIGHLOAD_FLAGS }} \
+          --network ${{ env.RUN_CFG__NETWORK_NAME }} \
+          --name ${{ env.RUN_CFG__MACHINE1_NAME }} \
+          -v ${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}:/home/test/testgres/logs \
+          -v ${{ github.workspace }}/${{ env.RUN_CFG__SSH_KEY_NAME }}:/home/test/testgres/id_ed25519_test:ro \
+          -e TEST_CFG__REMOTE_HOST="${{ env.RUN_CFG__MACHINE2_NAME }}" \
+          -e TEST_CFG__REMOTE_PORT="22" \
+          -e TEST_CFG__REMOTE_USERNAME="test" \
+          -e TEST_CFG__REMOTE_SSH_KEY="/home/test/testgres/id_ed25519_test" \
+          "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}"
     - name: Upload Logs
       uses: actions/upload-artifact@v7
       if: always() # IT IS IMPORTANT!

Dockerfile--alpine.tmpl

@@ -0,0 +1,148 @@
+ARG PG_VERSION=17
+ARG PYTHON_VERSION=3.12
+
+# --------------------------------------------- base1
+FROM postgres:${PG_VERSION}-alpine AS base1
+
+RUN apk add --no-cache \
+    coreutils \
+    bash \
+    mc \
+    procps \
+    openssh \
+    sshpass \
+    sudo \
+    git
+
+RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \
+    sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \
+    sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config
+
+# --------------------------------------------- base2_with_python-3
+FROM base1 AS base2_with_python-3
+
+RUN apk add --no-cache \
+    curl \
+    python3 \
+    python3-dev \
+    build-base \
+    musl-dev \
+    linux-headers \
+    # For pyenv \
+    patch \
+    xz-dev \
+    zip \
+    zlib-dev \
+    libffi-dev \
+    readline-dev \
+    openssl openssl-dev \
+    sqlite-dev \
+    bzip2-dev
+
+ENV PYTHON_BINARY=python3
+
+# --------------------------------------------- base3_with_python-3.7
+FROM base2_with_python-3 AS base3_with_python-3.7
+ENV PYTHON_VERSION=3.7
+
+# --------------------------------------------- base3_with_python-3.8.0
+FROM base2_with_python-3 AS base3_with_python-3.8.0
+ENV PYTHON_VERSION=3.8.0
+
+# --------------------------------------------- base3_with_python-3.8
+FROM base2_with_python-3 AS base3_with_python-3.8
+ENV PYTHON_VERSION=3.8
+
+# --------------------------------------------- base3_with_python-3.9
+FROM base2_with_python-3 AS base3_with_python-3.9
+ENV PYTHON_VERSION=3.9
+
+# --------------------------------------------- base3_with_python-3.10
+FROM base2_with_python-3 AS base3_with_python-3.10
+ENV PYTHON_VERSION=3.10
+
+# --------------------------------------------- base3_with_python-3.11
+FROM base2_with_python-3 AS base3_with_python-3.11
+ENV PYTHON_VERSION=3.11
+
+# --------------------------------------------- base3_with_python-3.12
+FROM base2_with_python-3 AS base3_with_python-3.12
+ENV PYTHON_VERSION=3.12
+
+# --------------------------------------------- base3_with_python-3.13
+FROM base2_with_python-3 AS base3_with_python-3.13
+ENV PYTHON_VERSION=3.13
+
+# --------------------------------------------- base3_with_python-3.14
+FROM base2_with_python-3 AS base3_with_python-3.14
+ENV PYTHON_VERSION=3.14
+
+# --------------------------------------------- final
+FROM base3_with_python-${PYTHON_VERSION} AS final
+
+EXPOSE 22
+RUN ssh-keygen -A
+
+RUN adduser -D test && addgroup -S sudo && adduser test sudo
+
+# It allows to use sudo without password
+RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD
+RUN echo "test:*" | chpasswd -e
+
+USER test
+RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash
+RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION}
+USER root
+
+COPY --chown=test:test . /home/test/testgres
+WORKDIR /home/test/testgres
+
+ENV LANG=C.UTF-8
+
+RUN chmod 700 /home/test/ && \
+    mkdir -p /home/test/.ssh && \
+    chown -R test:test /home/test/.ssh && \
+    mkdir /home/test/testgres/logs && \
+    chown -R test:test /home/test/testgres/logs
+
+#
+# \"$@\"
+#   - quote is important!
+#   - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ.
+#
+ENTRYPOINT ["sh", "-c", " \
+    set -eux; \
+    echo 'SYSTEM START: PREPARING SSH'; \
+    /usr/sbin/sshd; \
+    ls -la /home/test/.ssh/; \
+    \"$@\" \
+", "DUMMY-DUMMY-DUMMY"]
+
+# Run tests by default (master machine role)
+CMD ["bash", "-c",  " \
+set -eux; \
+echo \"HOME DIR IS [`realpath ~/`]\"; \
+echo \"WORK DIR IS [$(pwd)]\"; \
+if [ ! -f /home/test/.ssh/id_rsa ]; then \
+    su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \
+    su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \
+    chmod 600 /home/test/.ssh/authorized_keys; \
+fi; \
+ls -la /home/test/.ssh/; \
+su test -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\"; \
+su test -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\"; \
+if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \
+    su test -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\"; \
+fi; \
+if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \
+    cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \
+    export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \
+    chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \
+    chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \
+    ls -la /home/test/.ssh/; \
+fi; \
+ls -la ./; \
+su test -c \"TEST_FILTER='' PATH=\"/home/test/.pyenv/bin:$PATH\" bash ./run_tests2.sh\"; \
+"]