Exp/comp speedups

.ci/scripts/checkout_complement.sh

@@ -15,11 +15,11 @@ mkdir -p complement
 # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
 #    (GITHUB_BASE_REF for pull requests).
 # 3. Use the default complement branch ("HEAD").
-for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "HEAD"; do
+# for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "HEAD"; do
   # Skip empty branch names and merge commits.
-  if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
-    continue
-  fi
+#  if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+#    continue
+#  fi
 
-  (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
-done
+wget -O - "https://github.com/realtyem/complement/archive/refactor-ssapi-tests.tar.gz" | tar -xz --strip-components=1 -C complement
+#done

.github/workflows/docker.yml

@@ -47,8 +47,7 @@ jobs:
             latest=${{ github.ref == 'refs/heads/unraid_develop' }}
           tags: |
             type=raw,value=${{ github.ref_name }},enable=${{ github.ref != 'refs/heads/unraid_develop' }}
-            type=raw,value=unified,enable=${{ github.ref == 'refs/heads/unraid_develop' }}
-            type=semver,pattern={{version}}
+            type=pep440,pattern={{raw}}
 
       - name: Build and push to repo
 

.github/workflows/latest_deps.yml

@@ -208,7 +208,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@77399b6110ef82b94c1c9f9f615acf9e604f7f56 # v2.5.0, 2020-12-06
+      - uses: JasonEtco/create-an-issue@3a8ba796516b57db8cb2ee6dfc65bc76cd39d56d # v2.8.2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

.github/workflows/tests.yml

@@ -58,7 +58,7 @@ jobs:
       - run: scripts-dev/check_schema_delta.py --force-colors
 
   lint:
-    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v2"
     with:
       typechecking-extras: "all"
 
@@ -161,33 +161,37 @@ jobs:
 
       - run: cargo fmt --check
 
-  prewarm-cache:
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Build and push
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          push: false
-          file: "docker/Dockerfile-unified"
-          tags: "realtyem/synapse:latest"
-          outputs: type=docker,dest=/tmp/synapse-prewarm.tar
-          cache-from: type=gha
+  # prewarm-cache:
+  #   runs-on: ubuntu-22.04
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+
+      # - name: Run Docker on tmpfs
+      #   uses: JonasAlfredsson/docker-on-tmpfs@v1
+      #   with:
+      #     tmpfs_size: 5
+
+  #     - name: Set up Docker Buildx
+  #       id: buildx
+  #       uses: docker/setup-buildx-action@v2
+
+  #     - name: Build and push
+  #       uses: docker/build-push-action@v3
+  #       with:
+  #         context: .
+  #         push: false
+  #         file: "docker/Dockerfile-unified"
+  #         tags: "realtyem/synapse:latest"
+  #         outputs: type=docker,dest=/tmp/synapse-prewarm.tar
+  #         cache-from: type=gha
           #cache-to: type=gha,mode=max
 
-      - name: Upload artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: synapse-prewarm
-          path: /tmp/synapse-prewarm.tar
-
+  #     - name: Upload artifact
+  #       uses: actions/upload-artifact@v3
+  #       with:
+  #         name: synapse-prewarm
+  #         path: /tmp/synapse-prewarm.tar
 
   # Dummy step to gate other tests on without repeating the whole list
   linting-done:
@@ -201,7 +205,7 @@ jobs:
       - check-schema-delta
       - lint-clippy
       - lint-rustfmt
-      - prewarm-cache
+      # - prewarm-cache
     runs-on: ubuntu-22.04
     steps:
       - run: "true"
@@ -528,7 +532,78 @@ jobs:
             ported.sql
             schema_diff
 
-  complement:
+  complement-csapi:
+    if: "${{ !failure() && !cancelled() }}"
+    needs: linting-done
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arrangement: monolith
+            database: SQLite
+
+          - arrangement: monolith
+            database: Postgres
+
+          - arrangement: workers
+            database: Postgres
+
+    steps:
+      - name: Increase Swap Memory space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 13
+
+      - name: Run actions/checkout@v3 for synapse
+        uses: actions/checkout@v3
+        with:
+          path: synapse
+
+      # - name: Run Docker on tmpfs
+      #   uses: JonasAlfredsson/docker-on-tmpfs@v1
+      #   with:
+      #     tmpfs_size: 5
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Install Rust
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
+        with:
+            toolchain: 1.58.1
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Prepare Complement's Prerequisites
+        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
+
+      - name: Check Disk and Memory Free Space
+        run: |
+          df -h
+          swapon --show
+          free -h
+
+      - run: |
+          set -o pipefail
+          ASYNCIO_REACTOR=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement-csapi.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+        shell: bash
+        env:
+          PASS_SYNAPSE_POSTGRES_ENABLE_SYNCHRONOUS_COMMIT: false
+          SYNAPSE_TEST_LOG_LEVEL: DEBUG
+        name: Run Complement Tests
+
+      - name: Check Disk and Memory Free Space
+        run: |
+          df -h
+          swapon --show
+          free -h
+
+  complement-fed:
     if: "${{ !failure() && !cancelled() }}"
     needs: linting-done
     runs-on: ubuntu-latest
@@ -547,11 +622,21 @@ jobs:
             database: Postgres
 
     steps:
+      - name: Increase Swap Memory space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 13
+
       - name: Run actions/checkout@v3 for synapse
         uses: actions/checkout@v3
         with:
           path: synapse
 
+      # - name: Run Docker on tmpfs
+      #   uses: JonasAlfredsson/docker-on-tmpfs@v1
+      #   with:
+      #     tmpfs_size: 5
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v2
@@ -568,44 +653,106 @@ jobs:
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
-      - name: Download artifact
-        uses: actions/download-artifact@v3
+      - name: Check Disk and Memory Free Space
+        run: |
+          df -h
+          swapon --show
+          free -h
+
+      - run: |
+          set -o pipefail
+          ASYNCIO_REACTOR=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement-fed.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+        shell: bash
+        env:
+          SYNAPSE_TEST_LOG_LEVEL: DEBUG
+          PASS_SYNAPSE_POSTGRES_ENABLE_SYNCHRONOUS_COMMIT: false
+        name: Run Complement Tests
+
+      - name: Check Disk and Memory Free Space
+        run: |
+          df -h
+          swapon --show
+          free -h
+
+  complement:
+    if: "${{ !failure() && !cancelled() }}"
+    needs: linting-done
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arrangement: monolith
+            database: SQLite
+
+          - arrangement: monolith
+            database: Postgres
+
+          - arrangement: workers
+            database: Postgres
+
+    steps:
+      - name: Increase Swap Memory space
+        uses: pierotofy/set-swap-space@master
         with:
-          name: synapse-prewarm
-          path: /tmp
+          swap-size-gb: 13
 
-      - name: Load image
+      - name: Adjust Memory Values
         run: |
-          docker load --input /tmp/synapse-prewarm.tar
-          docker image ls -a
-
-      # - name: Build base
-        # uses: docker/build-push-action@v3
-        # with:
-          # context: .
-          # push: false
-          # file: "docker/Dockerfile-unified"
-          # tags: "synapse:latest"
-          # cache-from: type=gha
-          # cache-to: type=gha,mode=max
-
-      - name: Build Complement
-        uses: docker/build-push-action@v3
-        with:
-          context: ./synapse/docker/complement
-          push: false
-          load: true
-          file: "synapse/docker/complement/Dockerfile"
-          tags: "complement-synapse:latest"
-          # cache-from: type=gha
-          # cache-to: type=gha,mode=max
+          sudo cat /proc/sys/vm/swappiness
+          sudo sysctl vm.swappiness=10
+          sudo cat /proc/sys/vm/swappiness
+          echo 0 | sudo tee /sys/class/block/sda/queue/rotational
+          echo 1024 | sudo tee /sys/class/block/sda/queue/read_ahead_kb
+
+      - name: Run actions/checkout@v3 for synapse
+        uses: actions/checkout@v3
+        with:
+          path: synapse
+
+      # - name: Run Docker on tmpfs
+      #   uses: JonasAlfredsson/docker-on-tmpfs@v1
+      #   with:
+      #     tmpfs_size: 5
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Install Rust
+        # There don't seem to be versioned releases of this action per se: for each rust
+        # version there is a branch which gets constantly rebased on top of master.
+        # We pin to a specific commit for paranoia's sake.
+        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
+        with:
+            toolchain: 1.58.1
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Prepare Complement's Prerequisites
+        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
+
+      - name: Check Disk and Memory Free Space
+        run: |
+          df -h
+          swapon --show
+          free -h
 
       - run: |
           set -o pipefail
-          POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -f -json 2>&1 | synapse/.ci/scripts/gotestfmt
+          ASYNCIO_REACTOR=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
         shell: bash
+        env:
+          SYNAPSE_TEST_LOG_LEVEL: DEBUG
+          PASS_SYNAPSE_POSTGRES_ENABLE_SYNCHRONOUS_COMMIT: false
         name: Run Complement Tests
 
+      - name: Check Disk and Memory Free Space
+        run: |
+          df -h
+          swapon --show
+          free -h
+
   cargo-test:
     if: ${{ needs.changes.outputs.rust == 'true' }}
     runs-on: ubuntu-22.04
@@ -636,6 +783,8 @@ jobs:
       - sytest
       - export-data
       - portdb
+      - complement-fed
+      - complement-csapi
       - complement
       - cargo-test
     runs-on: ubuntu-22.04

.github/workflows/twisted_trunk.yml

@@ -174,7 +174,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@77399b6110ef82b94c1c9f9f615acf9e604f7f56 # v2.5.0, 2020-12-06
+      - uses: JasonEtco/create-an-issue@3a8ba796516b57db8cb2ee6dfc65bc76cd39d56d # v2.8.2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with: