Combine CI/CD docker compose. (#861)

* Combine CI/CD docker compose yaml files. Signed-off-by: ZePan110 <[email protected]> * Fix path issue in script Signed-off-by: ZePan110 <[email protected]> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Combine CI/CD docker compose yaml files. Signed-off-by: ZePan110 <[email protected]> * Fix path issue in script Signed-off-by: ZePan110 <[email protected]> * Fix path Signed-off-by: ZePan110 <[email protected]> * Combine CI/CD for .github/workflows/_comps-workflow.yml and .github/workflows/_get-image-list.yml Signed-off-by: ZePan110 <[email protected]> * Combine compose yaml file Signed-off-by: ZePan110 <[email protected]> * Remove CD file Signed-off-by: ZePan110 <[email protected]> * Add vllm-fork to CI. Signed-off-by: ZePan110 <[email protected]> * Fix issue Signed-off-by: ZePan110 <[email protected]> * Add CICD mode Signed-off-by: ZePan110 <[email protected]> * Add embedding-reranking-local to compose.yaml Signed-off-by: ZePan110 <[email protected]> * Change descriptions Signed-off-by: ZePan110 <[email protected]> * Fix script issue. Signed-off-by: ZePan110 <[email protected]> * Add new image. Signed-off-by: ZePan110 <[email protected]> * remove useless image from llms-compose.yaml Signed-off-by: ZePan110 <[email protected]> * Change folder name. Signed-off-by: ZePan110 <[email protected]> * Standardize the format of image names. Signed-off-by: ZePan110 <[email protected]> * Add commit print Signed-off-by: ZePan110 <[email protected]> * Standardize the format of image names Signed-off-by: ZePan110 <[email protected]> * Add vllm-arc to compose file. Signed-off-by: ZePan110 <[email protected]> * Restore test_agent_langchain_on_intel_hpu.sh Signed-off-by: ZePan110 <[email protected]> * Remove useless comments Signed-off-by: ZePan110 <[email protected]> --------- Signed-off-by: ZePan110 <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
opea-project · Nov 12, 2024 · 23c99c1 · 23c99c1
1 parent 37f3514
commit 23c99c1
Show file tree

Hide file tree

Showing 50 changed files with 252 additions and 267 deletions.
diff --git a/.github/workflows/_comps-workflow.yml b/.github/workflows/_comps-workflow.yml
@@ -24,7 +24,7 @@ on:
         type: boolean
       mode:
         default: "CD"
-        description: "Whether the test range is CI or CD"
+        description: "Whether the test range is CI, CD or CICD"
         required: false
         type: string
 
@@ -49,24 +49,32 @@ jobs:
         run: |
           cd ${{ github.workspace }}/.github/workflows/docker/compose
           # service=$(echo ${{ inputs.service }} | cut -d'_' -f1)
-          if [[ "${{ inputs.mode }}" == "CD" ]]; then
-            docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml
-          else
-            docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
-          fi
+          docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
           echo "docker_compose_path=${docker_compose_yml}" >> $GITHUB_OUTPUT
           if [ -f "$docker_compose_yml" ]; then
             echo "file_exists=true" >> $GITHUB_OUTPUT
           else
-            echo "There is no ${{ inputs.mode }} part of ${{ inputs.service }} that needs to be executed."
+            echo "The $docker_compose_yml does not exist!"
             echo "file_exists=false" >> $GITHUB_OUTPUT
           fi
 
           cd ${{ github.workspace }}
           if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
               git clone https://github.com/vllm-project/vllm.git vllm-openvino
-              cd ./vllm-openvino && git checkout v0.6.1 && cd ../
+              cd ./vllm-openvino && git checkout v0.6.1 && git rev-parse HEAD && cd ../
+          fi
+          if [[ $(grep -c "vllm-hpu:" ${docker_compose_yml}) != 0 ]]; then
+               git clone https://github.com/HabanaAI/vllm-fork.git vllm-fork
+               cd vllm-fork && git rev-parse HEAD && cd ../
           fi
+      - name: Get build list
+        id: get-build-list
+        env:
+          docker_compose_path: ${{ steps.get-yaml-path.outputs.docker_compose_path }}
+          mode: ${{ inputs.mode }}
+        run: |
+          build_list=$(bash ${{ github.workspace }}/.github/workflows/scripts/get_cicd_list.sh "${mode}" ${docker_compose_path})
+          echo "build_list=${build_list}" >> $GITHUB_OUTPUT
 
       - name: Build Image
         if: ${{ fromJSON(inputs.build) && steps.get-yaml-path.outputs.file_exists == 'true' }}
@@ -75,6 +83,7 @@ jobs:
           work_dir: ${{ github.workspace }}
           docker_compose_path: ${{ steps.get-yaml-path.outputs.docker_compose_path }}
           registry: ${OPEA_IMAGE_REPO}opea
+          service_list: ${{ steps.get-build-list.outputs.build_list }}
           tag: ${{ inputs.tag }}
 
   ####################################################################################################

diff --git a/.github/workflows/_get-image-list.yml b/.github/workflows/_get-image-list.yml
@@ -41,14 +41,10 @@ jobs:
               services=($(echo ${{ inputs.services }} | tr ',' ' '))
               for service in ${services[@]}
               do
-                  if [[ "${{ inputs.mode }}" == "CD" ]]; then
-                      docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose-cd.yaml
-                  else
-                      docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml
-                  fi
+                  docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml
                   if [ -f "$docker_compose_yml" ]; then
-                      images=$(cat $docker_compose_yml | yq -r '.[]' | jq 'keys' | jq -c '.')
-                      image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images}))
+                      build_list=$(bash ${{ github.workspace }}/.github/workflows/scripts/get_cicd_list.sh "${{ inputs.mode }}" $docker_compose_yml)
+                      image_list=$(echo $build_list| jq -R 'split(",")' | jq -s 'add')
                   fi
               done
           fi

diff --git a/...lows/docker/compose/agent-compose-cd.yaml → ...rkflows/docker/compose/agent-compose.yaml b/...lows/docker/compose/agent-compose-cd.yaml → ...rkflows/docker/compose/agent-compose.yaml
diff --git a/.../docker/compose/animation-compose-cd.yaml → ...ows/docker/compose/animation-compose.yaml b/.../docker/compose/animation-compose-cd.yaml → ...ows/docker/compose/animation-compose.yaml
@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # this file should be run in the root of the repo
-# images used by GenAIExamples: animation, wav2lip, wav2lip-gaudi
 services:
   animation:
     build:

diff --git a/.github/workflows/docker/compose/asr-compose.yaml b/.github/workflows/docker/compose/asr-compose.yaml
@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # this file should be run in the root of the repo
-# images used by GenAIExamples: asr,whisper,whisper-gaudi
 services:
   asr:
     build:

diff --git a/.github/workflows/docker/compose/dataprep-compose-cd.yaml b/.github/workflows/docker/compose/dataprep-compose-cd.yaml
diff --git a/.github/workflows/docker/compose/dataprep-compose.yaml b/.github/workflows/docker/compose/dataprep-compose.yaml
@@ -2,8 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # this file should be run in the root of the repo
-# images used by GenAIExamples: dataprep,dataprep-qdrant
-# others: dataprep-redis-llama-index,dataprep-on-ray-redis
 services:
   dataprep-redis:
     build:
@@ -25,3 +23,31 @@ services:
     build:
       dockerfile: comps/dataprep/multimodal/redis/langchain/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-multimodal-redis:${TAG:-latest}
+  dataprep-redis-llama-index:
+    build:
+      dockerfile: comps/dataprep/redis/llama_index/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-redis-llama-index:${TAG:-latest}
+  dataprep-milvus:
+    build:
+      dockerfile: comps/dataprep/milvus/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-milvus:${TAG:-latest}
+  dataprep-pgvector:
+    build:
+      dockerfile: comps/dataprep/pgvector/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-pgvector:${TAG:-latest}
+  dataprep-pinecone:
+    build:
+      dockerfile: comps/dataprep/pinecone/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-pinecone:${TAG:-latest}
+  dataprep-vdms:
+    build:
+      dockerfile: comps/dataprep/vdms/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest}
+  dataprep-neo4j:
+    build:
+      dockerfile: comps/dataprep/neo4j/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-neo4j:${TAG:-latest}
+  dataprep-neo4j-llamaindex:
+    build:
+      dockerfile: comps/dataprep/neo4j/llama_index/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep-neo4j-llamaindex:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/embeddings-compose-cd.yaml b/.github/workflows/docker/compose/embeddings-compose-cd.yaml
diff --git a/.github/workflows/docker/compose/embeddings-compose.yaml b/.github/workflows/docker/compose/embeddings-compose.yaml
@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # this file should be run in the root of the repo
-# images used by GenAIExamples: embedding-tei
 services:
   embedding-tei:
     build:
@@ -20,3 +19,27 @@ services:
     build:
       dockerfile: comps/embeddings/multimodal/multimodal_langchain/Dockerfile
     image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest}
+  embedding-langchain-mosec-endpoint:
+    build:
+      dockerfile: comps/embeddings/mosec/langchain/dependency/Dockerfile
+    image: ${REGISTRY:-opea}/embedding-langchain-mosec-endpoint:${TAG:-latest}
+  embedding-langchain-mosec:
+    build:
+      dockerfile: comps/embeddings/mosec/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/embedding-langchain-mosec:${TAG:-latest}
+  embedding-tei-llama-index:
+    build:
+      dockerfile: comps/embeddings/tei/llama_index/Dockerfile
+    image: ${REGISTRY:-opea}/embedding-tei-llama-index:${TAG:-latest}
+  embedding-multimodal-bridgetower-gaudi:
+    build:
+      dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile.intel_hpu
+    image: ${REGISTRY:-opea}/embedding-multimodal-bridgetower-gaudi:${TAG:-latest}
+  embedding-predictionguard:
+    build:
+      dockerfile: comps/embeddings/predictionguard/Dockerfile
+    image: ${REGISTRY:-opea}/embedding-predictionguard:${TAG:-latest}
+  embedding-reranking-local:
+    build:
+      dockerfile: comps/embeddings/tei/langchain/Dockerfile.dynamic_batching
+    image: ${REGISTRY:-opea}/embedding-reranking-local:${TAG:-latest}
diff --git a/...mpose/feedback_management-compose-cd.yaml → .../compose/feedback_management-compose.yaml b/...mpose/feedback_management-compose-cd.yaml → .../compose/feedback_management-compose.yaml
diff --git a/.github/workflows/docker/compose/finetuning-compose-cd.yaml b/.github/workflows/docker/compose/finetuning-compose-cd.yaml
diff --git a/.github/workflows/docker/compose/finetuning-compose.yaml b/.github/workflows/docker/compose/finetuning-compose.yaml
@@ -7,3 +7,7 @@ services:
     build:
       dockerfile: comps/finetuning/Dockerfile
     image: ${REGISTRY:-opea}/finetuning:${TAG:-latest}
+  finetuning-gaudi:
+    build:
+      dockerfile: comps/finetuning/Dockerfile.intel_hpu
+    image: ${REGISTRY:-opea}/finetuning-gaudi:${TAG:-latest}
diff --git a/.github/workflows/docker/compose/guardrails-compose-cd.yaml b/.github/workflows/docker/compose/guardrails-compose-cd.yaml
diff --git a/.github/workflows/docker/compose/guardrails-compose.yaml b/.github/workflows/docker/compose/guardrails-compose.yaml
@@ -2,15 +2,40 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # this file should be run in the root of the repo
-# images used by GenAIExamples: guardrails-tgi
-# others: guardrails-pii-detection
 services:
   guardrails-tgi:
     build:
       dockerfile: comps/guardrails/llama_guard/langchain/Dockerfile
     image: ${REGISTRY:-opea}/guardrails-tgi:${TAG:-latest}
-
+  guardrails-pii-detection:
+    build:
+      dockerfile: comps/guardrails/pii_detection/Dockerfile
+    image: ${REGISTRY:-opea}/guardrails-pii-detection:${TAG:-latest}
   guardrails-bias-detection:
     build:
-      dockerfile: comps/guardrails/llama_guard/langchain/Dockerfile
+      dockerfile: comps/guardrails/bias_detection/Dockerfile
     image: ${REGISTRY:-opea}/guardrails-bias-detection:${TAG:-latest}
+  guardrails-toxicity-detection:
+    build:
+      dockerfile: comps/guardrails/toxicity_detection/Dockerfile
+    image: ${REGISTRY:-opea}/guardrails-toxicity-detection:${TAG:-latest}
+  guardrails-wildguard:
+    build:
+      dockerfile: comps/guardrails/wildguard/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/guardrails-wildguard:${TAG:-latest}
+  guardrails-pii-detection-predictionguard:
+    build:
+      dockerfile: comps/guardrails/pii_detection/predictionguard/Dockerfile
+    image: ${REGISTRY:-opea}/guardrails-pii-predictionguard:${TAG:-latest}
+  guardrails-toxicity-detection-predictionguard:
+    build:
+      dockerfile: comps/guardrails/toxicity_detection/predictionguard/Dockerfile
+    image: ${REGISTRY:-opea}/guardrails-toxicity-predictionguard:${TAG:-latest}
+  guardrails-factuality-predictionguard:
+    build:
+      dockerfile: comps/guardrails/factuality/predictionguard/Dockerfile
+    image: ${REGISTRY:-opea}/guardrails-factuality-predictionguard:${TAG:-latest}
+  guardrails-injection-predictionguard:
+    build:
+      dockerfile: comps/guardrails/prompt_injection/predictionguard/Dockerfile
+    image: ${REGISTRY:-opea}/guardrails-injection-predictionguard:${TAG:-latest}
diff --git a/...ocker/compose/image2image-compose-cd.yaml → ...s/docker/compose/image2image-compose.yaml b/...ocker/compose/image2image-compose-cd.yaml → ...s/docker/compose/image2image-compose.yaml
@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # this file should be run in the root of the repo
-# images used by GenAIExamples: image2image,image2image-gaudi
 services:
   image2image:
     build:

diff --git a/...ocker/compose/image2video-compose-cd.yaml → ...s/docker/compose/image2video-compose.yaml b/...ocker/compose/image2video-compose-cd.yaml → ...s/docker/compose/image2video-compose.yaml
@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # this file should be run in the root of the repo
-# images used by GenAIExamples: image2video,svd,svd-gaudi
 services:
   image2video:
     build:

diff --git a/.../compose/intent_detection-compose-cd.yaml → ...ker/compose/intent_detection-compose.yaml b/.../compose/intent_detection-compose-cd.yaml → ...ker/compose/intent_detection-compose.yaml
diff --git a/.github/workflows/docker/compose/llms-compose-cd.yaml b/.github/workflows/docker/compose/llms-compose-cd.yaml