pgvector

.github/workflows/release-image.yml

@@ -6,18 +6,19 @@ on:
       image_tag:
         description: 'The tag name of image'
         required: true
-        default: ''
+        default: '14.7.2-pgvector-v0.8.0'
       pg_version:
         description: 'The version of postgres'
         required: true
+        default: '14.7.2'
 
   release:
     types:
       - published
 
 env:
   RELEASE_VERSION: ${{ github.ref_name }}
-  IMAGE_NAME: "apecloud/spilo"
+  IMAGE_NAME: "wallyxjh/spilo"
   DOCKER_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
   DOCKER_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
   ALIYUN_USER: ${{ secrets.ALIYUN_REGISTRY_USER }}
@@ -33,103 +34,84 @@ jobs:
     outputs:
       tag-name: ${{ steps.get_tag_name.outputs.tag_name }}
     steps:
-      - name: Get Tag Name
+      - name: Determine Image Tag
         id: get_tag_name
         run: |
           TAG_NAME=${{ inputs.image_tag }}
-          #if [ -z "$TAG_NAME" ]; then
-          #    TAG_NAME=${{ env.RELEASE_VERSION }}
-          #fi
-          #TAG_NAME="${TAG_NAME/v/}"
-          echo tag_name=$TAG_NAME >> $GITHUB_OUTPUT
+          echo "tag-name=$TAG_NAME" >> $GITHUB_OUTPUT
 
   release-image:
     needs: image-tag
-    name: Push ${{ matrix.registry }}
+    name: Push to ${{ matrix.registry }}
     strategy:
       fail-fast: false
       matrix:
-        registry: [ docker.io, apecloud-registry.cn-zhangjiakou.cr.aliyuncs.com, infracreate-registry.cn-zhangjiakou.cr.aliyuncs.com ]
+        registry: [docker.io]
     runs-on: ubuntu-latest
     permissions:
-      contents: 'read'
-      packages: 'write'
+      contents: read
+      packages: write
     defaults:
       run:
         shell: bash
     steps:
-      - name: Checkout
+      - name: Checkout Repository
         uses: actions/checkout@v3
 
-      - name: Set up Python
+      - name: Set up Python and dependencies
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
 
-      - name: Install flake8 and docker-compose
-        run: python -m pip install flake8 docker-compose==1.17.1
+      - name: Install system dependencies
+        run: sudo apt update && sudo apt install -y build-essential python3-dev libyaml-dev python3-pip python3-setuptools python3-wheel python3-venv
 
-      - name: Derive spilo image name
+      - name: Force upgrade pip, setuptools, and wheel
+        run: python -m pip install --upgrade --force-reinstall pip setuptools wheel
+
+      - name: Install PyYAML from source
+        run: pip install --no-cache-dir --no-binary pyyaml pyyaml
+
+      - name: Define Spilo Image Name
         id: image
-        working-directory: postgres-appliance
         run: |
-          IMAGE="${{ matrix.registry }}/${{ env.IMAGE_NAME }}:${{ needs.image-tag.outputs.tag-name }}"
+          IMAGE="${{ matrix.registry }}/${{ env.IMAGE_NAME }}:${{ inputs.image_tag }}"
           echo "NAME=$IMAGE" >> $GITHUB_OUTPUT
 
-      - name: Set up QEMU
+      - name: Set up QEMU and Docker Buildx
         uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - if: ${{ contains(matrix.registry, 'docker') }}
-        name: Login to ${{ matrix.registry }}
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ matrix.registry }}
-          username: ${{ env.DOCKER_USER }}
-          password: ${{ env.DOCKER_PASSWORD }}
-
-      - if: ${{ contains(matrix.registry, 'aliyuncs') && ! contains(matrix.registry, 'infracreate') }}
-        name: Login to ${{ matrix.registry }}
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ matrix.registry }}
-          username: ${{ env.ALIYUN_USER }}
-          password: ${{ env.ALIYUN_PASSWORD }}
-
-      - if: ${{ contains(matrix.registry, 'aliyuncs') && contains(matrix.registry, 'infracreate') }}
-        name: Login to ${{ matrix.registry }}
+      - name: Authenticate to ${{ matrix.registry }}
         uses: docker/login-action@v2
         with:
           registry: ${{ matrix.registry }}
-          username: ${{ env.ALIYUN_USER_NEW }}
-          password: ${{ env.ALIYUN_PASSWORD_NEW }}
+          username: ${{ contains(matrix.registry, 'docker') && env.DOCKER_USER || (contains(matrix.registry, 'infracreate') && env.ALIYUN_USER_NEW || env.ALIYUN_USER) }}
+          password: ${{ contains(matrix.registry, 'docker') && env.DOCKER_PASSWORD || (contains(matrix.registry, 'infracreate') && env.ALIYUN_PASSWORD_NEW || env.ALIYUN_PASSWORD) }}
 
-      - name: Build and export to local docker for testing
+      - name: Build and Load Image for Testing
         uses: docker/build-push-action@v3
         with:
           context: "postgres-appliance/"
           load: true
           tags: ${{ steps.image.outputs.NAME }}
-          build-args: |
-            PGVERSION=${{ inputs.pg_version }}
-            PGOLDVERSIONS=${{ inputs.pg_old_versions }}
+          build-args: PGVERSION=${{ inputs.pg_version }}
 
-      - name: Test spilo docker image
+      - name: Test Spilo Docker Image
         continue-on-error: true
         env:
           SPILO_TEST_IMAGE: "${{ steps.image.outputs.NAME }}"
         run: |
+          set -e
           bash postgres-appliance/tests/test_spilo.sh 
 
-      - name: Build arm64 additionally and push multi-arch image to ${{ matrix.registry }}
+      - name: Build and Push Multi-Arch Image
         uses: docker/build-push-action@v3
         with:
           context: "postgres-appliance/"
           push: true
           tags: "${{ steps.image.outputs.NAME }}"
           platforms: linux/amd64,linux/arm64
-          build-args: |
-            PGVERSION=${{ inputs.pg_version }}
-            PGOLDVERSIONS=
+          build-args: PGVERSION=${{ inputs.pg_version }}

.github/workflows/tests.yaml

@@ -17,7 +17,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: 3.7
+        python-version: 3.9
     - name: Install flake8 and docker-compose
       run: python -m pip install flake8 docker-compose==1.17.1
     - name: Run shellcheck

docs/prerequisites.md

@@ -6,6 +6,7 @@ see the [Stups Installation Guide](http://docs.stups.io/en/latest/installation/)
 
 To be able to deploy and troubleshoot Spilo, you need the stups tooling installed locally:
 ```bash
+$ sudo pip3 install --upgrade pip
 $ sudo pip3 install --upgrade stups
 ```
 

postgres-appliance/Dockerfile

@@ -1,6 +1,6 @@
 ARG BASE_IMAGE=ubuntu:22.04
-ARG PGVERSION=16
-ARG PGVECTOR="v0.7.4"
+ARG PGVERSION=14.7.2
+ARG PGVECTOR="v0.8.0"
 ARG TIMESCALEDB="2.11.2 2.14.2"
 ARG DEMO=false
 ARG COMPRESS=false

postgres-appliance/build_scripts/base.sh

@@ -79,47 +79,47 @@ sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/create
 for version in $DEB_PG_SUPPORTED_VERSIONS; do
     sed -i "s/ main.*$/ main $version/g" /etc/apt/sources.list.d/pgdg.list
     apt-get update
-
+    major_version=$(echo "$version" | awk -F. '{print $1}')
     if [ "$DEMO" != "true" ]; then
-        EXTRAS=("postgresql-pltcl-${version}"
-                "postgresql-${version}-dirtyread"
-                "postgresql-${version}-extra-window-functions"
-                "postgresql-${version}-first-last-agg"
-                "postgresql-${version}-hll"
-                "postgresql-${version}-hypopg"
-                "postgresql-${version}-plproxy"
-                "postgresql-${version}-partman"
-                "postgresql-${version}-pgaudit"
-                "postgresql-${version}-pldebugger"
-                "postgresql-${version}-pglogical"
-                "postgresql-${version}-pglogical-ticker"
-                "postgresql-${version}-plpgsql-check"
-                "postgresql-${version}-pg-checksums"
-                "postgresql-${version}-pgl-ddl-deploy"
-                "postgresql-${version}-pgq-node"
-                "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}"
-                "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}-scripts"
-                "postgresql-${version}-repack"
-                "postgresql-${version}-wal2json"
-                "postgresql-${version}-decoderbufs"
-                "postgresql-${version}-pllua"
-                "postgresql-${version}-pgvector")
+        EXTRAS=("postgresql-pltcl-${major_version}"
+                "postgresql-${major_version}-dirtyread"
+                "postgresql-${major_version}-extra-window-functions"
+                "postgresql-${major_version}-first-last-agg"
+                "postgresql-${major_version}-hll"
+                "postgresql-${major_version}-hypopg"
+                "postgresql-${major_version}-plproxy"
+                "postgresql-${major_version}-partman"
+                "postgresql-${major_version}-pgaudit"
+                "postgresql-${major_version}-pldebugger"
+                "postgresql-${major_version}-pglogical"
+                "postgresql-${major_version}-pglogical-ticker"
+                "postgresql-${major_version}-plpgsql-check"
+                "postgresql-${major_version}-pg-checksums"
+                "postgresql-${major_version}-pgl-ddl-deploy"
+                "postgresql-${major_version}-pgq-node"
+                "postgresql-${major_version}-postgis-${POSTGIS_VERSION%.*}"
+                "postgresql-${major_version}-postgis-${POSTGIS_VERSION%.*}-scripts"
+                "postgresql-${major_version}-repack"
+                "postgresql-${major_version}-wal2json"
+                "postgresql-${major_version}-decoderbufs"
+                "postgresql-${major_version}-pllua"
+                "postgresql-${major_version}-pgvector")
 
         if [ "$WITH_PERL" = "true" ]; then
-            EXTRAS+=("postgresql-plperl-${version}")
+            EXTRAS+=("postgresql-plperl-${major_version}")
         fi
 
     fi
-
+    major_version=$(echo "$version" | awk -F. '{print $1}')
     # Install PostgreSQL binaries, contrib, plproxy and multiple pl's
     apt-get install --allow-downgrades -y \
-        "postgresql-${version}-cron" \
-        "postgresql-contrib-${version}" \
-        "postgresql-${version}-pgextwlist" \
-        "postgresql-plpython3-${version}" \
-        "postgresql-server-dev-${version}" \
-        "postgresql-${version}-pgq3" \
-        "postgresql-${version}-pg-stat-kcache" \
+        "postgresql-${major_version}-cron" \
+        "postgresql-contrib-${major_version}" \
+        "postgresql-${major_version}-pgextwlist" \
+        "postgresql-plpython3-${major_version}" \
+        "postgresql-server-dev-${major_version}" \
+        "postgresql-${major_version}-pgq3" \
+        "postgresql-${major_version}-pg-stat-kcache" \
         "${EXTRAS[@]}"
 
     # Install 3rd party stuff
@@ -131,10 +131,10 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
             git checkout "$v"
             sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt
             if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \
-                    -DTAP_CHECKS=OFF -DPG_CONFIG="/usr/lib/postgresql/$version/bin/pg_config" \
+                    -DTAP_CHECKS=OFF -DPG_CONFIG="/usr/lib/postgresql/$major_version/bin/pg_config" \
                     -DAPACHE_ONLY="$TIMESCALEDB_APACHE_ONLY" -DSEND_TELEMETRY_DEFAULT=NO; then
                 make -C build install
-                strip /usr/lib/postgresql/"$version"/lib/timescaledb*.so
+                strip /usr/lib/postgresql/"$major_version"/lib/timescaledb*.so
             fi
             git reset --hard
             git clean -f -d
@@ -146,7 +146,7 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
         cd pgvector
         for v in $PGVECTOR; do
             git checkout "$v"
-            export PG_CONFIG="/usr/lib/postgresql/$version/bin/pg_config"
+            export PG_CONFIG="/usr/lib/postgresql/$major_version/bin/pg_config"
             # fix Illegal instruction, https://github.com/pgvector/pgvector/issues/54#issuecomment-1562071614
             # overwrite OPTFLAGS to remove -march=native
             make OPTFLAGS="" && make install
@@ -161,10 +161,10 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
         curl -L https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor > /usr/share/keyrings/timescale_E7391C94080429FF.gpg
 
         apt-get update
-        if [ "$(apt-cache search --names-only "^timescaledb-toolkit-postgresql-${version}$" | wc -l)" -eq 1 ]; then
-            apt-get install "timescaledb-toolkit-postgresql-$version"
+        if [ "$(apt-cache search --names-only "^timescaledb-toolkit-postgresql-${major_version}$" | wc -l)" -eq 1 ]; then
+            apt-get install "timescaledb-toolkit-postgresql-$major_version"
         else
-            echo "Skipping timescaledb-toolkit-postgresql-$version as it's not found in the repository"
+            echo "Skipping timescaledb-toolkit-postgresql-$major_version as it's not found in the repository"
         fi
 
         rm /etc/apt/sources.list.d/timescaledb.list
@@ -192,13 +192,18 @@ sed -i "s/ main.*$/ main/g" /etc/apt/sources.list.d/pgdg.list
 apt-get update
 apt-get install -y postgresql postgresql-server-dev-all postgresql-all libpq-dev
 for version in $DEB_PG_SUPPORTED_VERSIONS; do
-    apt-get install -y "postgresql-server-dev-${version}"
+    major_version=$(echo "$version" | awk -F. '{print $1}')
+    apt-get install -y "postgresql-server-dev-${major_version}"
 done
 
 if [ "$DEMO" != "true" ]; then
     for version in $DEB_PG_SUPPORTED_VERSIONS; do
         # create postgis symlinks to make it possible to perform update
-        ln -s "postgis-${POSTGIS_VERSION%.*}.so" "/usr/lib/postgresql/${version}/lib/postgis-2.5.so"
+        major_version=$(echo "$version" | awk -F. '{print $1}')
+        if [ -e "/usr/lib/postgresql/${major_version}/lib/postgis-2.5.so" ]; then
+            rm "/usr/lib/postgresql/${major_version}/lib/postgis-2.5.so"
+        fi
+        ln -s "postgis-${POSTGIS_VERSION%.*}.so" "/usr/lib/postgresql/${major_version}/lib/postgis-2.5.so"
     done
 fi
 
@@ -229,7 +234,8 @@ dpkg -l | grep '^rc' | awk '{print $2}' | xargs apt-get purge -y
 
 # Try to minimize size by creating symlinks instead of duplicate files
 if [ "$DEMO" != "true" ]; then
-    cd "/usr/lib/postgresql/$PGVERSION/bin"
+    major_version=$(echo "$PGVERSION" | awk -F. '{print $1}')
+    cd "/usr/lib/postgresql/$major_version/bin"
     for u in clusterdb \
             pg_archivecleanup \
             pg_basebackup \
@@ -241,17 +247,22 @@ if [ "$DEMO" != "true" ]; then
             reindexdb \
             vacuumlo *.py; do
         for v in /usr/lib/postgresql/*; do
-            if [ "$v" != "/usr/lib/postgresql/$PGVERSION" ] && [ -f "$v/bin/$u" ]; then
+            if [ "$v" != "/usr/lib/postgresql/$major_version" ] && [ -f "$v/bin/$u" ]; then
                 rm "$v/bin/$u"
-                ln -s "../../$PGVERSION/bin/$u" "$v/bin/$u"
+                ln -s "../../$major_version/bin/$u" "$v/bin/$u"
             fi
         done
     done
 
     set +x
 
     for v1 in $(find /usr/share/postgresql -type d -mindepth 1 -maxdepth 1 | sort -Vr); do
+        major_version=$(echo "$PGVERSION" | awk -F. '{print $1}')
         # relink files with the same content
+        if [[ "$v1" != "/usr/share/postgresql/$major_version" ]]; then
+                continue
+        fi
+
         cd "$v1/extension"
         while IFS= read -r -d '' orig
         do
@@ -274,14 +285,21 @@ if [ "$DEMO" != "true" ]; then
                 done
             fi
         done
-
         # relink files with the same name and content across different major versions
         started=0
         for v2 in $(find /usr/share/postgresql -type d -mindepth 1 -maxdepth 1 | sort -Vr); do
+            major_version=$(echo "$PGVERSION" | awk -F. '{print $1}')
+            # relink files with the same content
+            if [[ "$v2" != "/usr/share/postgresql/$major_version" ]]; then
+                continue
+            fi
             if [ "$v1" = "$v2" ]; then
                 started=1
             elif [ $started = 1 ]; then
                 for d1 in extension contrib contrib/postgis-$POSTGIS_VERSION; do
+                    if [[ "$d1" != "/usr/share/postgresql/$major_version" ]]; then
+                      continue
+                    fi
                     cd "$v1/$d1"
                     d2="$d1"
                     d1="../../${v1##*/}/$d1"