From 538e7c054627863323cdc15135b4599642daa00d Mon Sep 17 00:00:00 2001 From: chensuyue Date: Sun, 11 Aug 2024 14:39:07 +0800 Subject: [PATCH 01/11] unify workflow name format Signed-off-by: chensuyue --- ...t-test-matrix.yml => _get-test-matrix.yml} | 0 ...d-on-manual.yml => manual-image-build.yml} | 0 .../{code-scan.yml => mix-code-scan.yml} | 0 ...vice-test.yml => mix-megaservice-test.yml} | 2 +- .../{trellix.yml => mix-trellix.yml} | 0 ...test.yaml => pr-dockerfile-path-scan.yaml} | 0 ...vice-test.yml => pr-microservice-test.yml} | 4 +- ...build-on-push.yml => push-image-build.yml} | 2 +- .github/workflows/reuse-image-build.yml | 32 -------- ...-schedule.yml => schedule-image-build.yml} | 0 .../scripts/docker_images_build_push.sh | 77 ------------------- 11 files changed, 4 insertions(+), 113 deletions(-) rename .github/workflows/{reuse-get-test-matrix.yml => _get-test-matrix.yml} (100%) rename .github/workflows/{image-build-on-manual.yml => manual-image-build.yml} (100%) rename .github/workflows/{code-scan.yml => mix-code-scan.yml} (100%) rename .github/workflows/{megaservice-test.yml => mix-megaservice-test.yml} (98%) rename .github/workflows/{trellix.yml => mix-trellix.yml} (100%) rename .github/workflows/{test.yaml => pr-dockerfile-path-scan.yaml} (100%) rename .github/workflows/{microservice-test.yml => pr-microservice-test.yml} (95%) rename .github/workflows/{image-build-on-push.yml => push-image-build.yml} (98%) delete mode 100644 .github/workflows/reuse-image-build.yml rename .github/workflows/{image-build-on-schedule.yml => schedule-image-build.yml} (100%) delete mode 100644 .github/workflows/scripts/docker_images_build_push.sh diff --git a/.github/workflows/reuse-get-test-matrix.yml b/.github/workflows/_get-test-matrix.yml similarity index 100% rename from .github/workflows/reuse-get-test-matrix.yml rename to .github/workflows/_get-test-matrix.yml diff --git a/.github/workflows/image-build-on-manual.yml b/.github/workflows/manual-image-build.yml similarity index 100% rename from .github/workflows/image-build-on-manual.yml rename to .github/workflows/manual-image-build.yml diff --git a/.github/workflows/code-scan.yml b/.github/workflows/mix-code-scan.yml similarity index 100% rename from .github/workflows/code-scan.yml rename to .github/workflows/mix-code-scan.yml diff --git a/.github/workflows/megaservice-test.yml b/.github/workflows/mix-megaservice-test.yml similarity index 98% rename from .github/workflows/megaservice-test.yml rename to .github/workflows/mix-megaservice-test.yml index 7e55e2bf2..83c826cd7 100644 --- a/.github/workflows/megaservice-test.yml +++ b/.github/workflows/mix-megaservice-test.yml @@ -10,7 +10,7 @@ on: branches: [main] types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - - .github/workflows/megaservice-test.yml + - .github/workflows/mix-megaservice-test.yml - comps/cores/** - requirements.txt - setup.py diff --git a/.github/workflows/trellix.yml b/.github/workflows/mix-trellix.yml similarity index 100% rename from .github/workflows/trellix.yml rename to .github/workflows/mix-trellix.yml diff --git a/.github/workflows/test.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml similarity index 100% rename from .github/workflows/test.yaml rename to .github/workflows/pr-dockerfile-path-scan.yaml diff --git a/.github/workflows/microservice-test.yml b/.github/workflows/pr-microservice-test.yml similarity index 95% rename from .github/workflows/microservice-test.yml rename to .github/workflows/pr-microservice-test.yml index e40745f0b..79de02c1b 100644 --- a/.github/workflows/microservice-test.yml +++ b/.github/workflows/pr-microservice-test.yml @@ -12,7 +12,7 @@ on: - tests/** - "!**.md" - "!**.txt" - - .github/workflows/microservice-test.yml + - .github/workflows/pr-microservice-test.yml # If there is a new commit, the previous jobs will be canceled concurrency: @@ -21,7 +21,7 @@ concurrency: jobs: job1: - uses: ./.github/workflows/reuse-get-test-matrix.yml + uses: ./.github/workflows/_get-test-matrix.yml Microservice-test: needs: job1 diff --git a/.github/workflows/image-build-on-push.yml b/.github/workflows/push-image-build.yml similarity index 98% rename from .github/workflows/image-build-on-push.yml rename to .github/workflows/push-image-build.yml index a72d13a61..5472111dd 100644 --- a/.github/workflows/image-build-on-push.yml +++ b/.github/workflows/push-image-build.yml @@ -10,7 +10,7 @@ on: - comps/** - "!**.md" - "!**.txt" - - .github/workflows/image-build-on-push.yml + - .github/workflows/push-image-build.yml concurrency: group: ${{ github.workflow }}-${{ github.ref }}-on-push diff --git a/.github/workflows/reuse-image-build.yml b/.github/workflows/reuse-image-build.yml deleted file mode 100644 index e2ed6883b..000000000 --- a/.github/workflows/reuse-image-build.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -name: Image Build -permissions: read-all -on: - workflow_call: - inputs: - micro_service: - required: true - type: string - -jobs: - micro-image-build: - continue-on-error: true - strategy: - matrix: - node: [docker-build-xeon, docker-build-gaudi] - runs-on: ${{ matrix.node }} - steps: - - name: Checkout out Repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Building MicroService Docker Image - id: build-microservice-image - env: - micro_service: ${{ inputs.micro_service }} - hardware: ${{ matrix.node }} - run: | - bash .github/workflows/scripts/docker_images_build_push.sh ${micro_service} ${hardware} diff --git a/.github/workflows/image-build-on-schedule.yml b/.github/workflows/schedule-image-build.yml similarity index 100% rename from .github/workflows/image-build-on-schedule.yml rename to .github/workflows/schedule-image-build.yml diff --git a/.github/workflows/scripts/docker_images_build_push.sh b/.github/workflows/scripts/docker_images_build_push.sh deleted file mode 100644 index eaf4d78f3..000000000 --- a/.github/workflows/scripts/docker_images_build_push.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -set -xe - -WORKSPACE=$PWD -IMAGE_REPO=${IMAGE_REPO:-$OPEA_IMAGE_REPO} -IMAGE_TAG=${IMAGE_TAG:-latest} - -function docker_build() { - # docker_build - IMAGE_NAME=$1 - micro_service=$2 - dockerfile_path=${WORKSPACE}/comps/${micro_service} - if [[ "$IMAGE_NAME" == *"gaudi" ]]; then - dockerfile_name="Dockerfile_hpu" - else - dockerfile_name="Dockerfile" - fi - if [ -f "$dockerfile_path/$dockerfile_name" ]; then - DOCKERFILE_PATH="$dockerfile_path/$dockerfile_name" - elif [ -f "$dockerfile_path/docker/$dockerfile_name" ]; then - DOCKERFILE_PATH="$dockerfile_path/docker/$dockerfile_name" - else - echo "Dockerfile not found" - exit 1 - fi - echo "Building ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG using Dockerfile $DOCKERFILE_PATH" - - docker build --no-cache -t ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG -f $DOCKERFILE_PATH . - docker push ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG - docker rmi ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG -} - -micro_service=$1 -hardware=$(echo $2 | cut -d- -f3) -case ${micro_service} in - "asr"|"tts") - IMAGE_NAME="opea/${micro_service}" - ;; - "embeddings/langchain") - IMAGE_NAME="opea/embedding-tei" - ;; - "retrievers/langchain/redis") - IMAGE_NAME="opea/retriever-redis" - ;; - "reranks/tei") - IMAGE_NAME="opea/reranking-tei" - ;; - "llms/text-generation/tgi") - IMAGE_NAME="opea/llm-tgi" - ;; - "dataprep/redis/langchain") - IMAGE_NAME="opea/dataprep-redis" - ;; - "llms/summarization/tgi") - IMAGE_NAME="opea/llm-docsum-tgi" - ;; - "llms/faq-generation/tgi") - IMAGE_NAME="opea/llm-faqgen-tgi" - ;; - "web_retrievers/langchain/chroma") - IMAGE_NAME="opea/web-retriever-chroma" - ;; - "tts/speecht5") - if [ "${hardware}" == "gaudi" ]; then IMAGE_NAME="opea/speecht5-gaudi"; else IMAGE_NAME="opea/speecht5"; fi - ;; - "asr/whisper") - if [ "${hardware}" == "gaudi" ]; then IMAGE_NAME="opea/whisper-gaudi"; else IMAGE_NAME="opea/whisper"; fi - ;; - *) - echo "Not supported yet" - exit 0 - ;; -esac -docker_build "${IMAGE_NAME}" "${micro_service}" From 5470e6f939379d228188adbc8d28883dc84db23b Mon Sep 17 00:00:00 2001 From: chensuyue Date: Sun, 11 Aug 2024 15:12:16 +0800 Subject: [PATCH 02/11] rename test scripts Signed-off-by: chensuyue --- tests/{test_asr_whisper.sh => test_asr.sh} | 0 tests/{test_tts_speecht5.sh => test_tts.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/{test_asr_whisper.sh => test_asr.sh} (100%) rename tests/{test_tts_speecht5.sh => test_tts.sh} (100%) diff --git a/tests/test_asr_whisper.sh b/tests/test_asr.sh similarity index 100% rename from tests/test_asr_whisper.sh rename to tests/test_asr.sh diff --git a/tests/test_tts_speecht5.sh b/tests/test_tts.sh similarity index 100% rename from tests/test_tts_speecht5.sh rename to tests/test_tts.sh From e46aa971353f4b0234e3be6a376efcf58bdfd902 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Sun, 11 Aug 2024 17:23:50 +0800 Subject: [PATCH 03/11] add example test workflow Signed-off-by: chensuyue --- .github/workflows/pr-examples-test.yml | 57 +++++++++++++++ .github/workflows/pr-microservice-test.yml | 4 +- tests/test_workflow_chatqna.py | 80 ---------------------- 3 files changed, 60 insertions(+), 81 deletions(-) create mode 100644 .github/workflows/pr-examples-test.yml delete mode 100644 tests/test_workflow_chatqna.py diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml new file mode 100644 index 000000000..1ab68511a --- /dev/null +++ b/.github/workflows/pr-examples-test.yml @@ -0,0 +1,57 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Example-test + +on: + pull_request: + branches: [main] + types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped + paths: + - .github/workflows/pr-examples-test.yml + - comps/cores/** + - requirements.txt + - setup.py + +# If there is a new commit, the previous jobs will be canceled +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + MegaService: + runs-on: gaudi + steps: + - name: Clean Up Working Directory + run: | + sudo rm -rf ${{github.workspace}} || true + echo y | docker system prune + + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + ref: "refs/pull/${{ github.event.number }}/merge" + + - name: Run ChatQnA + run: | + git clone https://github.com/opea-project/GenAIExamples.git + cd GenAIExamples/ChatQnA/tests + GenAIComps_dir=${{github.workspace}} + sed -i "s#cd GenAIComps#cd ${GenAIComps_dir}#g" test_chatqna_on_gaudi.sh + sed -i "s#docker build -t#docker build --no-cache -q -t#g" test_chatqna_on_gaudi.sh + cat test_chatqna_on_gaudi.sh + bash test_chatqna_on_gaudi.sh + + - name: Clean up container + if: cancelled() || failure() + run: | + cd ${{ github.workspace }}/GenAIExamples/ChatQnA/docker/gaudi + docker compose stop && docker compose rm -f + echo y | docker system prune --all + + - name: Publish pipeline artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: "Examples-Test-Logs" + path: ${{ github.workspace }}/GenAIExamples/ChatQnA/tests/*.log diff --git a/.github/workflows/pr-microservice-test.yml b/.github/workflows/pr-microservice-test.yml index 79de02c1b..42e237e1e 100644 --- a/.github/workflows/pr-microservice-test.yml +++ b/.github/workflows/pr-microservice-test.yml @@ -31,7 +31,9 @@ jobs: continue-on-error: true steps: - name: Clean Up Working Directory - run: sudo rm -rf ${{github.workspace}}/* + run: | + sudo rm -rf ${{github.workspace}} + echo y | docker system prune - name: Checkout out Repo uses: actions/checkout@v4 diff --git a/tests/test_workflow_chatqna.py b/tests/test_workflow_chatqna.py deleted file mode 100644 index a2ea0f2d0..000000000 --- a/tests/test_workflow_chatqna.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -# - -import asyncio -import os - -from comps import ChatQnAGateway, MicroService, ServiceOrchestrator, ServiceType - -MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0") -MEGA_SERVICE_PORT = os.getenv("MEGA_SERVICE_PORT", 8888) -EMBEDDING_SERVICE_HOST_IP = os.getenv("EMBEDDING_SERVICE_HOST_IP", "0.0.0.0") -EMBEDDING_SERVICE_PORT = os.getenv("EMBEDDING_SERVICE_PORT", 6000) -RETRIEVER_SERVICE_HOST_IP = os.getenv("RETRIEVER_SERVICE_HOST_IP", "0.0.0.0") -RETRIEVER_SERVICE_PORT = os.getenv("RETRIEVER_SERVICE_PORT", 7000) -RERANK_SERVICE_HOST_IP = os.getenv("RERANK_SERVICE_HOST_IP", "0.0.0.0") -RERANK_SERVICE_PORT = os.getenv("RERANK_SERVICE_PORT", 8000) -LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0") -LLM_SERVICE_PORT = os.getenv("LLM_SERVICE_PORT", 9000) - - -class ChatQnAService: - def __init__(self, host="0.0.0.0", port=8000): - self.host = host - self.port = port - self.megaservice = ServiceOrchestrator() - - def add_remote_service(self): - embedding = MicroService( - name="embedding", - host=EMBEDDING_SERVICE_HOST_IP, - port=EMBEDDING_SERVICE_PORT, - endpoint="/v1/embeddings", - use_remote_service=True, - service_type=ServiceType.EMBEDDING, - ) - retriever = MicroService( - name="retriever", - host=RETRIEVER_SERVICE_HOST_IP, - port=RETRIEVER_SERVICE_PORT, - endpoint="/v1/retrieval", - use_remote_service=True, - service_type=ServiceType.RETRIEVER, - ) - rerank = MicroService( - name="rerank", - host=RERANK_SERVICE_HOST_IP, - port=RERANK_SERVICE_PORT, - endpoint="/v1/reranking", - use_remote_service=True, - service_type=ServiceType.RERANK, - ) - llm = MicroService( - name="llm", - host=LLM_SERVICE_HOST_IP, - port=LLM_SERVICE_PORT, - endpoint="/v1/chat/completions", - use_remote_service=True, - service_type=ServiceType.LLM, - ) - self.megaservice.add(embedding).add(retriever).add(rerank).add(llm) - self.megaservice.flow_to(embedding, retriever) - self.megaservice.flow_to(retriever, rerank) - self.megaservice.flow_to(rerank, llm) - self.gateway = ChatQnAGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port) - - async def schedule(self): - result_dict, runtime_graph = await self.megaservice.schedule( - initial_inputs={"text": "What is the revenue of Nike in 2023?"} - ) - print(result_dict) - - -if __name__ == "__main__": - chatqna = ChatQnAService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT) - chatqna.add_remote_service() - asyncio.run(chatqna.schedule()) From 76aeb0e127f92613ae35ba9e96b208547a42c612 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Sun, 11 Aug 2024 17:27:39 +0800 Subject: [PATCH 04/11] update workflow trigger Signed-off-by: chensuyue --- .github/workflows/pr-examples-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml index 1ab68511a..2215b17d6 100644 --- a/.github/workflows/pr-examples-test.yml +++ b/.github/workflows/pr-examples-test.yml @@ -10,8 +10,12 @@ on: paths: - .github/workflows/pr-examples-test.yml - comps/cores/** + - comps/embeddings/langchain/** + - comps/retrievers/langchain/redis/** + - comps/reranks/tei/** + - comps/llms/text-generation/tgi/** + - comps/dataprep/redis/langchain/** - requirements.txt - - setup.py # If there is a new commit, the previous jobs will be canceled concurrency: From c515b06c3b423aa48ae6e2dbcbe71cd88ce57db5 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Sun, 11 Aug 2024 23:29:21 +0800 Subject: [PATCH 05/11] fix some issue Signed-off-by: chensuyue --- .github/workflows/pr-examples-test.yml | 2 +- .github/workflows/pr-microservice-test.yml | 4 ++-- tests/test_asr.sh | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml index 2215b17d6..290e6d7f7 100644 --- a/.github/workflows/pr-examples-test.yml +++ b/.github/workflows/pr-examples-test.yml @@ -23,7 +23,7 @@ concurrency: cancel-in-progress: true jobs: - MegaService: + example-test: runs-on: gaudi steps: - name: Clean Up Working Directory diff --git a/.github/workflows/pr-microservice-test.yml b/.github/workflows/pr-microservice-test.yml index 42e237e1e..4d6c4c859 100644 --- a/.github/workflows/pr-microservice-test.yml +++ b/.github/workflows/pr-microservice-test.yml @@ -33,7 +33,7 @@ jobs: - name: Clean Up Working Directory run: | sudo rm -rf ${{github.workspace}} - echo y | docker system prune + docker system prune -f - name: Checkout out Repo uses: actions/checkout@v4 @@ -59,7 +59,7 @@ jobs: run: | cid=$(docker ps -aq --filter "name=test-comps-*") if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi - echo y | docker system prune --all + docker system prune -f - name: Publish pipeline artifact if: ${{ !cancelled() }} diff --git a/tests/test_asr.sh b/tests/test_asr.sh index 5e6e4a8c8..ee95c3c27 100644 --- a/tests/test_asr.sh +++ b/tests/test_asr.sh @@ -10,14 +10,14 @@ ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { cd $WORKPATH echo $(pwd) - docker build -t opea/whisper:latest -f comps/asr/whisper/Dockerfile . - docker build -t opea/asr:latest -f comps/asr/Dockerfile . + docker build -t opea/whisper:comps -f comps/asr/whisper/Dockerfile . + docker build -t opea/asr:comps -f comps/asr/Dockerfile . } function start_service() { unset http_proxy - docker run -d --name="test-comps-asr-whisper" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 7066:7066 --ipc=host opea/whisper:latest - docker run -d --name="test-comps-asr" -e ASR_ENDPOINT=http://$ip_address:7066 -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 9099:9099 --ipc=host opea/asr:latest + docker run -d --name="test-comps-asr-whisper" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 7066:7066 --ipc=host opea/whisper:comps + docker run -d --name="test-comps-asr" -e ASR_ENDPOINT=http://$ip_address:7066 -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 9099:9099 --ipc=host opea/asr:comps sleep 3m } From 5c25aca8459a02fb83ea1ec49eaea216aad28437 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Sun, 11 Aug 2024 23:39:37 +0800 Subject: [PATCH 06/11] fix bug Signed-off-by: chensuyue --- .github/workflows/pr-examples-test.yml | 2 +- .github/workflows/pr-microservice-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml index 290e6d7f7..1e5bba046 100644 --- a/.github/workflows/pr-examples-test.yml +++ b/.github/workflows/pr-examples-test.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Clean Up Working Directory run: | - sudo rm -rf ${{github.workspace}} || true + sudo rm -rf ${{github.workspace}}/* || true echo y | docker system prune - name: Checkout out Repo diff --git a/.github/workflows/pr-microservice-test.yml b/.github/workflows/pr-microservice-test.yml index 4d6c4c859..9abdce833 100644 --- a/.github/workflows/pr-microservice-test.yml +++ b/.github/workflows/pr-microservice-test.yml @@ -32,7 +32,7 @@ jobs: steps: - name: Clean Up Working Directory run: | - sudo rm -rf ${{github.workspace}} + sudo rm -rf ${{github.workspace}}/* docker system prune -f - name: Checkout out Repo From 7e70309c2e0af0363131fb73134cf092525a686a Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 12 Aug 2024 00:03:34 +0800 Subject: [PATCH 07/11] resolve the conflict between CI test Signed-off-by: chensuyue --- .github/workflows/pr-examples-test.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml index 1e5bba046..9522ab293 100644 --- a/.github/workflows/pr-examples-test.yml +++ b/.github/workflows/pr-examples-test.yml @@ -30,6 +30,7 @@ jobs: run: | sudo rm -rf ${{github.workspace}}/* || true echo y | docker system prune + docker rmi $(docker images --filter reference="*/*:comps" -q) || true - name: Checkout out Repo uses: actions/checkout@v4 @@ -39,11 +40,17 @@ jobs: - name: Run ChatQnA run: | git clone https://github.com/opea-project/GenAIExamples.git - cd GenAIExamples/ChatQnA/tests + cd ${{ github.workspace }}/GenAIExamples/ChatQnA/docker/gaudi + sed -i "s#:latest#:comps#g" compose.yaml + cat compose.yaml + + cd ${{ github.workspace }}/GenAIExamples/ChatQnA/tests GenAIComps_dir=${{github.workspace}} sed -i "s#cd GenAIComps#cd ${GenAIComps_dir}#g" test_chatqna_on_gaudi.sh sed -i "s#docker build -t#docker build --no-cache -q -t#g" test_chatqna_on_gaudi.sh + sed -i "s#:latest#:comps#g" test_chatqna_on_gaudi.sh cat test_chatqna_on_gaudi.sh + bash test_chatqna_on_gaudi.sh - name: Clean up container @@ -51,7 +58,8 @@ jobs: run: | cd ${{ github.workspace }}/GenAIExamples/ChatQnA/docker/gaudi docker compose stop && docker compose rm -f - echo y | docker system prune --all + docker system prune -f + docker rmi $(docker images --filter reference="*/*:comps" -q) || true - name: Publish pipeline artifact if: ${{ !cancelled() }} From 9bb62d3767c27b8f04fa01151fc58a264cf5c530 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 12 Aug 2024 00:22:00 +0800 Subject: [PATCH 08/11] update test machine Signed-off-by: chensuyue --- .github/workflows/pr-examples-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml index 9522ab293..a7c0dd934 100644 --- a/.github/workflows/pr-examples-test.yml +++ b/.github/workflows/pr-examples-test.yml @@ -24,7 +24,7 @@ concurrency: jobs: example-test: - runs-on: gaudi + runs-on: gaudi-01-3 steps: - name: Clean Up Working Directory run: | @@ -38,6 +38,8 @@ jobs: ref: "refs/pull/${{ github.event.number }}/merge" - name: Run ChatQnA + env: + HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} run: | git clone https://github.com/opea-project/GenAIExamples.git cd ${{ github.workspace }}/GenAIExamples/ChatQnA/docker/gaudi @@ -51,6 +53,7 @@ jobs: sed -i "s#:latest#:comps#g" test_chatqna_on_gaudi.sh cat test_chatqna_on_gaudi.sh + echo "Run test..." bash test_chatqna_on_gaudi.sh - name: Clean up container From e80f54e592317149c3a60a883a43e8ec9b354b6c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 11 Aug 2024 16:22:36 +0000 Subject: [PATCH 09/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/pr-examples-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml index a7c0dd934..823b39914 100644 --- a/.github/workflows/pr-examples-test.yml +++ b/.github/workflows/pr-examples-test.yml @@ -42,17 +42,17 @@ jobs: HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} run: | git clone https://github.com/opea-project/GenAIExamples.git - cd ${{ github.workspace }}/GenAIExamples/ChatQnA/docker/gaudi + cd ${{ github.workspace }}/GenAIExamples/ChatQnA/docker/gaudi sed -i "s#:latest#:comps#g" compose.yaml cat compose.yaml - + cd ${{ github.workspace }}/GenAIExamples/ChatQnA/tests GenAIComps_dir=${{github.workspace}} sed -i "s#cd GenAIComps#cd ${GenAIComps_dir}#g" test_chatqna_on_gaudi.sh sed -i "s#docker build -t#docker build --no-cache -q -t#g" test_chatqna_on_gaudi.sh sed -i "s#:latest#:comps#g" test_chatqna_on_gaudi.sh cat test_chatqna_on_gaudi.sh - + echo "Run test..." bash test_chatqna_on_gaudi.sh From ef2faff6c31aadae1c4cdc73e015c9aed1847d22 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 12 Aug 2024 01:34:27 +0800 Subject: [PATCH 10/11] update test workflow Signed-off-by: chensuyue --- .github/workflows/pr-examples-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml index a7c0dd934..11e8cb7d0 100644 --- a/.github/workflows/pr-examples-test.yml +++ b/.github/workflows/pr-examples-test.yml @@ -48,13 +48,14 @@ jobs: cd ${{ github.workspace }}/GenAIExamples/ChatQnA/tests GenAIComps_dir=${{github.workspace}} + sed -i '/GenAIComps.git/d' test_chatqna_on_gaudi.sh sed -i "s#cd GenAIComps#cd ${GenAIComps_dir}#g" test_chatqna_on_gaudi.sh sed -i "s#docker build -t#docker build --no-cache -q -t#g" test_chatqna_on_gaudi.sh sed -i "s#:latest#:comps#g" test_chatqna_on_gaudi.sh cat test_chatqna_on_gaudi.sh echo "Run test..." - bash test_chatqna_on_gaudi.sh + timeout 50m bash test_chatqna_on_gaudi.sh - name: Clean up container if: cancelled() || failure() @@ -62,7 +63,6 @@ jobs: cd ${{ github.workspace }}/GenAIExamples/ChatQnA/docker/gaudi docker compose stop && docker compose rm -f docker system prune -f - docker rmi $(docker images --filter reference="*/*:comps" -q) || true - name: Publish pipeline artifact if: ${{ !cancelled() }} From 705bb37a32158d447ea26126d5344b331a952522 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 12 Aug 2024 10:25:05 +0800 Subject: [PATCH 11/11] update test action Signed-off-by: chensuyue --- .github/workflows/pr-examples-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml index 23356b7a0..92354c5c2 100644 --- a/.github/workflows/pr-examples-test.yml +++ b/.github/workflows/pr-examples-test.yml @@ -4,7 +4,7 @@ name: Example-test on: - pull_request: + pull_request_target: branches: [main] types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: