Merge branch 'main' into qg/safety

.github/workflows/_comps-workflow.yml

@@ -0,0 +1,123 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Comps jobs
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      node:
+        required: true
+        type: string
+      service:
+        required: true
+        type: string
+      tag:
+        default: "comps"
+        required: false
+        type: string
+      build:
+        default: true
+        required: false
+        type: boolean
+      # scan:
+      #   default: true
+      #   required: false
+      #   type: boolean
+jobs:
+  ####################################################################################################
+  # Image Build
+  ####################################################################################################
+  build-images:
+    runs-on: "docker-build-${{ inputs.node }}"
+    continue-on-error: true
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Clone required Repo
+        run: |
+          cd ${{ github.workspace }}/.github/workflows/docker/compose
+          # service=$(echo ${{ inputs.service }} | cut -d'_' -f1)
+          docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          echo ${docker_compose_yml}
+          if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then
+              git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4
+          fi
+          if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
+              git clone https://github.com/vllm-project/vllm.git vllm-openvino
+          fi
+          # echo "service=$service" >> $GITHUB_ENV
+
+      - name: Build Image
+        if: ${{ fromJSON(inputs.build) }}
+        uses: opea-project/validation/actions/image-build@main
+        with:
+          work_dir: ${{ github.workspace }}/
+          docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          registry: ${OPEA_IMAGE_REPO}opea
+          tag: ${{ inputs.tag }}
+  # ####################################################################################################
+  # # Trivy Scan
+  # ####################################################################################################
+  #   get-image-list:
+  #     needs: [build-images]
+  #     if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi' }}
+  #     runs-on: ubuntu-latest
+  #     outputs:
+  #       matrix: ${{ steps.scan-matrix.outputs.matrix }}
+  #     steps:
+  #       - name: Checkout out Repo
+  #         uses: actions/checkout@v4
+
+  #       - name: Set Matrix
+  #         id: scan-matrix
+  #         run: |
+  #           pip install yq
+  #           compose_path=${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
+  #           echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT
+
+  #   scan-images:
+  #     needs: [get-image-list, build-images]
+  #     if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi'}}
+  #     runs-on: "docker-build-${{ inputs.node }}"
+  #     strategy:
+  #       matrix:
+  #         image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
+  #       fail-fast: false
+  #     steps:
+  #       - name: Pull Image
+  #         run: |
+  #           docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+  #           echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV
+
+  #       - name: Scan Container
+  #         uses: opea-project/validation/actions/trivy-scan@main
+  #         with:
+  #           image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
+  #           output: ${{ matrix.image }}-scan.txt
+
+  #       - name: Cleanup
+  #         if: always()
+  #         run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+
+  #       - uses: actions/[email protected]
+  #         with:
+  #           name: ${{ matrix.image }}-scan
+  #           path: ${{ matrix.image }}-scan.txt
+  #           overwrite: true
+
+  ####################################################################################################
+  # Docker Compose Test
+  ####################################################################################################
+  test-service-compose:
+    needs: [build-images]
+    uses: ./.github/workflows/_run-docker-compose.yml
+    with:
+      tag: ${{ inputs.tag }}
+      service: ${{ inputs.service }}
+      hardware: ${{ inputs.node }}
+    secrets: inherit

.github/workflows/_get-test-matrix.yml

@@ -38,7 +38,10 @@ jobs:
         run: |
           set -xe
           if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
-            base_commit=${{ github.event.pull_request.base.sha }}
+            LATEST_COMMIT_SHA=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/opea-project/GenAIComps/commits?sha=main" | jq -r '.[0].sha')
+            echo "Latest commit SHA is $LATEST_COMMIT_SHA"
+            base_commit=$LATEST_COMMIT_SHA
           else
             base_commit=$(git rev-parse HEAD~1) # push event
           fi

.github/workflows/_run-docker-compose.yml

@@ -0,0 +1,111 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Image Build
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      registry:
+        description: Container Registry URL
+        required: false
+        default: ""
+        type: string
+      tag:
+        description: Container Tag
+        required: false
+        default: "latest"
+        type: string
+      service:
+        description: Example to test
+        required: true
+        type: string
+      hardware:
+        description: Hardware to run the test on
+        required: true
+        type: string
+jobs:
+  get-test-case:
+    runs-on: ubuntu-latest
+    outputs:
+      test_cases: ${{ steps.test-case-matrix.outputs.test_cases }}
+      CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+    steps:
+      - name: Get checkout ref
+        id: get-checkout-ref
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
+            CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
+          else
+            CHECKOUT_REF=${{ github.ref }}
+          fi
+          echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
+          echo "checkout ref ${CHECKOUT_REF}"
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+          fetch-depth: 0
+
+      - name: Get test matrix
+        shell: bash
+        id: test-case-matrix
+        run: |
+          set -x
+          service_l=$(echo ${{ inputs.service }} | tr '[:upper:]' '[:lower:]')
+          cd ${{ github.workspace }}/tests
+          test_cases=$(find . -type f -name "test_${service_l}*.sh" -print | cut -d/ -f2 | jq -R '.' | jq -sc '.')
+          echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
+
+  run-test:
+    needs: [get-test-case]
+    strategy:
+      matrix:
+        test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
+      fail-fast: false
+    runs-on: ${{ inputs.hardware }}
+    continue-on-error: true
+    steps:
+      - name: Clean up Working Directory
+        run: |
+          sudo rm -rf ${{github.workspace}}/*
+          docker system prune -f
+          docker rmi $(docker images --filter reference="*/*:comps" -q) || true
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.get-test-case.outputs.CHECKOUT_REF }}
+          fetch-depth: 0
+
+      - name: Run test
+        shell: bash
+        env:
+          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
+          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          PINECONE_KEY: ${{ secrets.PINECONE_KEY }}
+          service: ${{ inputs.service }}
+          hardware: ${{ inputs.hardware }}
+          test_case: ${{ matrix.test_case }}
+        run: |
+          cd ${{ github.workspace }}/tests
+          service=$(echo "${test_case}" | sed 's/test_\(.*\)\.sh/\1/')
+          echo "service=${service}" >> $GITHUB_ENV
+          if [ -f ${test_case} ]; then timeout 30m bash ${test_case}; else echo "Test script {${test_case}} not found, skip test!"; fi
+
+      - name: Clean up container
+        if: cancelled() || failure()
+        run: |
+          cid=$(docker ps -aq --filter "name=test-comps-*")
+          if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
+          docker system prune -f
+
+      - name: Publish pipeline artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.service }}
+          path: ${{ github.workspace }}/tests/*.log

.github/workflows/docker/compose/tts-compose.yaml

@@ -4,10 +4,10 @@
 # this file should be run in the root of the repo
 # images used by GenAIExamples: reranking-tei
 services:
-  asr:
+  tts:
     build:
       dockerfile: comps/tts/Dockerfile
-    image: ${REGISTRY}opea/asr:${TAG:-latest}
+    image: ${REGISTRY}opea/tts:${TAG:-latest}
   speecht5:
     build:
       dockerfile: comps/tts/speecht5/Dockerfile

.github/workflows/manual-comps-test.yml

@@ -0,0 +1,45 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Comps CD workflow on manual event
+on:
+  workflow_dispatch:
+    inputs:
+      services:
+        default: "asr"
+        description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus...]" #,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
+        required: true
+        type: string
+      build:
+        default: true
+        description: "Build test required images for Comps"
+        required: false
+        type: boolean
+
+permissions: read-all
+
+jobs:
+  get-test-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      services: ${{ steps.get-matrix.outputs.services }}
+    steps:
+      - name: Create Matrix
+        id: get-matrix
+        run: |
+          services=($(echo ${{ inputs.services }} | tr ',' ' '))
+          services_json=$(printf '%s\n' "${services[@]}" | sort -u | jq -R '.' | jq -sc '.')
+          echo "services=$services_json" >> $GITHUB_OUTPUT
+
+  run-services:
+    needs: [get-test-matrix]
+    strategy:
+      matrix:
+        service: ${{ fromJson(needs.get-test-matrix.outputs.services) }}
+      fail-fast: false
+    uses: ./.github/workflows/_comps-workflow.yml
+    with:
+      service: ${{ matrix.service }}
+      tag: "comps"
+      node: gaudi
+    secrets: inherit

.github/workflows/manual-freeze-images.yml

@@ -0,0 +1,43 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Freeze base images and 3rd part images on manual event
+
+on:
+  workflow_dispatch:
+
+jobs:
+  freeze-images:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref }}
+
+      - name: install skopeo
+        run: |
+          sudo apt update
+          sudo apt -y install skopeo
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Set up Git
+        run: |
+          git config --global user.name "NeuralChatBot"
+          git config --global user.email "[email protected]"
+          git remote set-url origin https://NeuralChatBot:"${{ secrets.ACTION_TOKEN }}"@github.com/opea-project/GenAIComps.git
+
+      - name: Run script
+        run: |
+          bash .github/workflows/scripts/freeze_images.sh
+
+      - name: Commit changes
+        run: |
+          git add .
+          git commit -s -m "Freeze base images tag"
+          git push

.github/workflows/manual-freeze-requirements.yml

@@ -0,0 +1,38 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Freeze-requirements
+
+on:
+  workflow_dispatch:
+
+jobs:
+  freeze-requirements:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Set up Git
+        run: |
+          git config --global user.name "NeuralChatBot"
+          git config --global user.email "[email protected]"
+          git remote set-url origin https://NeuralChatBot:"${{ secrets.ACTION_TOKEN }}"@github.com/opea-project/GenAIComps.git
+
+      - name: Run script
+        run: |
+          bash .github/workflows/scripts/freeze_requirements.sh
+
+      - name: Commit changes
+        run: |
+          git add .
+          git commit -s -m "Freeze requirements"
+          git push