Implement improved demo build pipeline (#1286)

.github/actions/get-changed-demos/action.yml

@@ -0,0 +1,63 @@
+name: 'Get Changed Demos'
+description: 'Get demos that have changed'
+inputs:
+  json:
+    description: Output as a JSON list
+    default: false
+outputs:
+  updated:
+    description: 'Names of new or updated demos'
+    value: ${{ steps.changed-files.outputs.all_changed_files }}
+  deleted:
+    description: 'Names of deleted demos'
+    value: ${{ steps.changed-files.outputs.deleted_files }}
+runs:
+  using: "composite"
+  steps:
+    - id: changed-files
+      uses: tj-actions/changed-files@v45
+      with:
+        # Exclude changes outside the current 
+        # directory and show path names 
+        # relative to it. NOTE: This 
+        # requires you to specify the 
+        # top-level directory via the `path` 
+        # input. 
+        diff_relative: true
+        # Output unique changed directories instead 
+        # of filenames. NOTE: This returns 
+        # `.` for changed files located 
+        # in the current working directory 
+        # which defaults to `$GITHUB_WORKSPACE`. 
+        dir_names: true
+        # Include only directories that have 
+        # been deleted as opposed to 
+        # directory names of files that 
+        # have been deleted in the 
+        # `deleted_files` output when `dir_names` is 
+        # set to `true`. 
+        dir_names_deleted_files_include_only_deleted_dirs: true
+        # Exclude the current directory represented 
+        # by `.` from the output 
+        # when `dir_names` is set to 
+        # `true`. 
+        dir_names_exclude_current_dir: true
+        # Separator used to split the 
+        # `dir_names_include_files` input
+        dir_names_include_files_separator: ' '
+        # Limit the directory output to 
+        # a maximum depth e.g `test/test1/test2` 
+        # with max depth of `2` 
+        # returns `test/test1`. 
+        dir_names_max_depth: 1
+        # Output list of changed files 
+        # in a JSON formatted string 
+        # which can be used for 
+        # matrix jobs. Example: https://github.com/tj-actions/changed-files/blob/main/.github/workflows/matrix-example.yml 
+        json: ${{ inputs.json }}
+        # Output renamed files as deleted 
+        # and added files. 
+        output_renamed_files_as_deleted_and_added: true
+        # Specify a relative path under 
+        # `$GITHUB_WORKSPACE` to locate the repository. 
+        path: 'demonstrations_v2'

.github/workflows/qml_pipeline_v2/deploy.py

@@ -0,0 +1,65 @@
+import requests
+from requests_auth_aws_sigv4 import AWSSigV4
+import argparse
+from pathlib import Path
+import sys
+import os
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+logger = logging.getLogger(__name__)
+
+DEPLOYMENT_URL_TEMPLATE = "{endpoint_url}/demo/{slug}?preview={preview}"
+
+parser = argparse.ArgumentParser(
+    prog="deploy",
+    description="Deploy demo zip files to the QML website.",
+)
+
+parser.add_argument(
+    "paths", help="The pattern to search for demo zip files.", type=Path, nargs="+"
+)
+parser.add_argument(
+    "-p",
+    "--preview",
+    type=str,
+    default="false",
+    choices=("true", "false"),
+    help="Whether to deploy to the preview site.",
+)
+
+
+def main():
+    args = parser.parse_args()
+    preview: str = args.preview
+    paths: list[Path] = args.paths
+
+    session = requests.Session()
+    session.auth = AWSSigV4("execute-api", region="us-east-1")
+
+    endpoint_url = os.environ["DEPLOYMENT_ENDPOINT_URL"]
+
+    for path in paths:
+        if not path.exists():
+            logger.error("Path '%s' does not exist.", path)
+            sys.exit(1)
+        elif not (path.is_file() and path.suffix == ".zip"):
+            logger.error("Path '%s' is not a zip file.", path)
+            sys.exit(1)
+
+        slug = path.stem
+        url = DEPLOYMENT_URL_TEMPLATE.format(
+            endpoint_url=endpoint_url, slug=slug, preview=preview
+        )
+        with open(path, "rb") as f:
+            logger.info("Deploying '%s' to '%s'", path, url)
+            try:
+                session.put(url, files={"file": f}).raise_for_status()
+            except requests.HTTPError:
+                logger.error("Failed to deploy '%s' to '%s'", path, url, exc_info=True)
+                sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/qml_pipeline_v2/requirements.txt

@@ -0,0 +1,2 @@
+requests==2.32.3
+requests-auth-aws-sigv4==0.7

.github/workflows/v2-build-demos.yml

@@ -0,0 +1,211 @@
+name: Build V2 Demos
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: Branch, tag or commit SHA to checkout
+        required: true
+        type: string
+      demo-names:
+        description: |
+          Instead of building all demos, only build the demos specified in this list.
+        required: false
+        type: string
+        default: ''
+      save-artifact:
+        description: |
+          Whether to save the built demos as an artifact.
+        required: false
+        type: boolean
+        default: true
+      artifact-name:
+        description: Name of the artifact containing the built demos (defaults to demo-build-{ref})
+        required: false
+        type: string
+        default: ''
+      keep-going:
+        description: Whether to keep going if a demo fails to build
+        required: false
+        type: boolean
+        default: false
+      quiet:
+        description: |
+          Whether to suppress output from the build process
+        required: false
+        type: boolean
+        default: false
+      batch_size:
+        description: |
+          Number of demos to build per job
+        type: number
+        default: 10
+
+    outputs:
+      artifact-name:
+        description: "Name of the artifact containing the built demos"
+        value: ${{ inputs.artifact-name }}
+
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: Branch, tag or commit SHA to checkout
+        required: true
+        type: string
+      demo-names:
+        description: |
+          Instead of building all demos, only build the demos specified in this list.
+        required: false
+        type: string
+        default: ''
+      save-artifact:
+        description: |
+          Whether to save the built demos as an artifact.
+        required: false
+        type: boolean
+        default: true
+      artifact-name:
+        description: Name of the artifact containing the built demos (defaults to demo-build-{ref})
+        required: true
+        type: string
+        default: ''
+      keep-going:
+        description: Whether to keep going if a demo fails to build
+        required: false
+        type: boolean
+        default: false
+      quiet:
+        description: |
+          Whether to suppress output from the build process
+        required: false
+        type: boolean
+        default: false
+      batch_size:
+        description: |
+          Number of demos to build per job
+        type: number
+        default: 10
+
+jobs:
+  generate-build-variables:
+    runs-on: ubuntu-latest
+    outputs:
+      artifact-name: ${{ steps.get-artifact-name.outputs.artifact-name }}
+      chunk-array: ${{ steps.get-chunk-array.outputs.chunk-array }}
+      chunk-indexes: ${{ steps.get-chunk-indexes.outputs.chunk-indexes }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 1
+
+      - name: Get artifact name
+        id: get-artifact-name
+        run: |
+          if [ -z ${{ inputs.artifact-name }} ]
+          then
+            printf "artifact-name=%s" "demo-build-${{ inputs.ref }}" >> $GITHUB_OUTPUT
+          else
+            printf "artifact-name=%s" "${{ inputs.artifact-name }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Get demo names
+        id: get-demo-names
+        run: |
+          if [ -z "${{ inputs.demo-names }}" ];
+          then
+            printf "demo-names=%s" "$(cd demonstrations_v2 && ls | tr '\n' ' ')" >> $GITHUB_OUTPUT
+          else
+            printf "demo-names=%s" "${{ inputs.demo-names }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create job chunk array
+        id: get-chunk-array
+        run: printf "chunk-array=%s" "$(jq -nc '[$ARGS.positional | _nwise(${{ inputs.batch_size }})]' --args ${{ steps.get-demo-names.outputs.demo-names }})" >> $GITHUB_OUTPUT
+
+      - name: Get chunk indexes
+        id: get-chunk-indexes
+        run: printf "chunk-indexes=%s" "$(jq -nc '${{ steps.get-chunk-array.outputs.chunk-array }} | [range(0, length)]')" >> $GITHUB_OUTPUT
+
+  build-demos:
+    runs-on: ubuntu-latest
+    needs: [generate-build-variables]
+    strategy:
+      matrix:
+        chunk-index: ${{ fromJson(needs.generate-build-variables.outputs.chunk-indexes) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 1
+
+      - name: Install pandoc and opencl
+        run: |
+          sudo apt-get install -y \
+            ocl-icd-opencl-dev \
+            pandoc 
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install . && poetry config warnings.export false
+
+      - name: Make demo name list
+        id: demo-name-list
+        run: printf "demo-names=%s" "$(jq -nc '${{ needs.generate-build-variables.outputs.chunk-array }}[${{ matrix.chunk-index }}][]' | tr -s '\n"' ' ')" >> $GITHUB_OUTPUT
+
+      - name: Build demos
+        run: |
+          qml build \
+            --format json \
+            --execute \
+            ${{ inputs.keep-going && '--keep-going' || '--no-keep-going' }} \
+            ${{ inputs.quiet && '--quiet' || '--no-quiet' }} \
+            ${{ steps.demo-name-list.outputs.demo-names }}
+      
+      - name: Upload artifacts
+        id: upload-artifacts
+        if: ${{ inputs.save-artifact }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ needs.generate-build-variables.outputs.artifact-name }}-c${{ matrix.chunk-index }}
+          path: _build/pack/*.zip
+          retention-days: 1
+
+  consolidate-artifacts:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.save-artifact }}
+    needs: [generate-build-variables, build-demos]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 1
+
+      - name: Download All Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: _build/pack
+          pattern: ${{ needs.generate-build-variables.outputs.artifact-name }}-c*
+          merge-multiple: true
+
+      - name: Upload consolidated artifact
+        id: upload-artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ needs.generate-build-variables.outputs.artifact-name }}
+          path: _build/pack/*.zip
+          retention-days: 1
+          overwrite: true
+
+      - name: Delete partial artifacts
+        uses: geekyeggo/delete-artifact@v5
+        with:
+            name: ${{ needs.generate-build-variables.outputs.artifact-name }}-c*
+
+