add upload_oci workflow

.github/workflows/build.yml

@@ -0,0 +1,303 @@
+name: build
+on:
+  workflow_call:
+    inputs:
+      version:
+        type: string
+        default: today
+      use_kms:
+        type: boolean
+        default: false
+    secrets:
+      secureboot_db_kms_arn:
+        required: false
+      aws_region:
+        required: false
+      aws_kms_role:
+        required: false
+      aws_oidc_session:
+        required: false
+    outputs:
+      version:
+        value: ${{ jobs.version.outputs.version }}
+jobs:
+  version:
+    name: resolve version
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # [email protected]
+        with:
+          submodules: true
+      - name: resolve version "${{ inputs.version }}"
+        id: version
+        run: echo version="$(bin/garden-version "${{ inputs.version }}")" | tee -a "$GITHUB_OUTPUT"
+  cert:
+    name: certificates
+    needs: version
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      # - uses: gardenlinux/workflow-telemetry-action@c75b594f552d305ffd5f9074637137bc343ba35e # pin@v2
+      #   with:
+      #     metric_frequency: 1
+      #     proc_trace_min_duration: 10
+      #     proc_trace_chart_max_count: 50
+      #     comment_on_pr: false
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # [email protected]
+        with:
+          submodules: true
+      - uses: ./.github/actions/setup
+      - name: set VERSION=${{ needs.version.outputs.version }}
+        run: |
+          cd $(dirname $(realpath bin/garden-version))/..
+          bin/garden-version "${{ needs.version.outputs.version }}" | tee VERSION
+          git update-index --assume-unchanged VERSION
+      - name: build certificates
+        if: ${{ ! inputs.use_kms }}
+        run: ./cert/build
+      - name: use kms backed certificates
+        if: ${{ inputs.use_kms }}
+        run: |
+          touch cert/gardenlinux-secureboot.db.arn
+          for f in secureboot.{{pk,null.pk,kek,db}.auth,db.{crt,arn}}; do
+            ln -sr "cert/gardenlinux-$f" "cert/$f"
+          done
+      - uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # pin@v3
+        with:
+          path: cert
+          key: cert-${{ github.run_id }}
+  base:
+    name: bootstrap stage
+    needs: version
+    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-latest-arm' || 'ubuntu-latest' }}
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [ amd64 ]
+    steps:
+      # - uses: gardenlinux/workflow-telemetry-action@c75b594f552d305ffd5f9074637137bc343ba35e # pin@v2
+      #   with:
+      #     metric_frequency: 1
+      #     proc_trace_min_duration: 10
+      #     proc_trace_chart_max_count: 50
+      #     comment_on_pr: false
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # [email protected]
+        with:
+          submodules: true
+      - uses: ./.github/actions/setup
+        with:
+          arch: "${{ matrix.arch }}"
+      - name: set VERSION=${{ needs.version.outputs.version }}
+        run: |
+          cd $(dirname $(realpath bin/garden-version))/..
+          bin/garden-version "${{ needs.version.outputs.version }}" | tee VERSION
+          git update-index --assume-unchanged VERSION
+      - name: build base-${{ matrix.arch }}
+        run: ./build base-${{ matrix.arch }}
+      - uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # pin@v3
+        with:
+          path: .build
+          key: base-${{ matrix.arch }}-${{ github.run_id }}
+  test_container:
+    name: container image for tests
+    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-latest-arm' || 'ubuntu-latest' }}
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [ amd64 ]
+    steps:
+      # - uses: gardenlinux/workflow-telemetry-action@c75b594f552d305ffd5f9074637137bc343ba35e # pin@v2
+      #   with:
+      #     metric_frequency: 1
+      #     proc_trace_min_duration: 10
+      #     proc_trace_chart_max_count: 50
+      #     comment_on_pr: false
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # [email protected]
+        with:
+          submodules: true
+      - uses: ./.github/actions/setup
+        with: 
+          arch: "${{ matrix.arch }}"
+      - name: build test container
+        run: |
+          OCI_IMAGE_TAG="$(./build --print-container-image)"
+          if [[ -f builder.dockerfile ]]; then
+            podman build -t "$OCI_IMAGE_TAG" -f builder.dockerfile .
+          fi
+          podman build --squash --tag test --build-arg base="$OCI_IMAGE_TAG" tests
+          podman save --format oci-archive test > test.oci
+      - name: upload test container
+        uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # pin@v3
+        with:
+          path: test.oci
+          key: test_container:${{ matrix.arch }}-${{ github.run_id }}
+
+  generate_matrix_images:
+    uses: ./.github/workflows/generate_matrix.yml
+    with: 
+      flags: '--exclude "bare-*" --no-arch --json-by-arch --build --test'
+  generate_matrix_bare:
+    uses: ./.github/workflows/generate_matrix.yml
+    with: 
+      flags: '--include-only "bare-*" --no-arch --json-by-arch --build --test'
+  images:
+    needs: [ version, cert, base, test_container, generate_matrix_images ]
+    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-latest-arm' || 'ubuntu-latest' }}
+    defaults:
+      run:
+        shell: bash
+    permissions:
+      id-token: write
+    environment: ${{ inputs.use_kms && 'oidc_aws_kms_secureboot_signing' || '' }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate_matrix_images.outputs.matrix) }}
+    steps:
+      # - uses: gardenlinux/workflow-telemetry-action@c75b594f552d305ffd5f9074637137bc343ba35e # pin@v2
+      #   with:
+      #     metric_frequency: 1
+      #     proc_trace_min_duration: 10
+      #     proc_trace_chart_max_count: 50
+      #     comment_on_pr: false
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # [email protected]
+        with:
+          submodules: true
+      - uses: ./.github/actions/setup
+        with: 
+          arch: "${{ matrix.arch }}"
+      - name: set VERSION=${{ needs.version.outputs.version }}
+        run: |
+          cd $(dirname $(realpath bin/garden-version))/..
+          bin/garden-version "${{ needs.version.outputs.version }}" | tee VERSION
+          git update-index --assume-unchanged VERSION
+      - name: load cert cache
+        uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # pin@v3
+        with:
+          path: cert
+          key: cert-${{ github.run_id }}
+          fail-on-cache-miss: true
+      - name: write secureboot db arn for kms backed certificates
+        if: ${{ inputs.use_kms }}
+        run: echo "${{ secrets.secureboot_db_kms_arn }}" > cert/gardenlinux-secureboot.db.arn
+      - name: configure aws credentials for kms signing
+        if: ${{ inputs.use_kms }}
+        uses: aws-actions/configure-aws-credentials@4fc4975a852c8cd99761e2de1f4ba73402e44dd9 # pin@v4
+        with:
+          role-to-assume: ${{ secrets.aws_kms_role }}
+          role-session-name: ${{ secrets.aws_oidc_session }}
+          aws-region: ${{ secrets.aws_region }}
+          role-duration-seconds: 14400
+      - name: load bootstrap stage cache
+        uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # pin@v3
+        with:
+          path: .build
+          key: base-${{ matrix.arch }}-${{ github.run_id }}
+          fail-on-cache-miss: true
+      - name: download test container cache
+        uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # pin@v3
+        with:
+          path: test.oci
+          key: test_container:${{ matrix.arch }}-${{ github.run_id }}
+          fail-on-cache-miss: true
+      - name: import test container image
+        run: |
+          podman load < test.oci
+          rm test.oci
+      - name: update bootstrap stage build artifact timestamps
+        run: |
+          t="$(date '+%s')"
+          find .build -exec touch -d "@$t" {} +
+      - name: build
+        run: ./build ${{ inputs.use_kms && '--kms' || '' }} ${{ matrix.flavor }}-${{ matrix.arch }}
+      - name: get cname
+        run: |
+          cname=$(basename "$(realpath ".build/${{ matrix.flavor}}-${{ matrix.arch }}")" .artifacts)
+          echo "cname=$cname" | tee -a "$GITHUB_ENV"
+      # ./test writes ${{ env.cname }}.chroot.test.log and puts it into .build/${{ env.cname }}.artifacts
+      - name: test
+        run: |
+          ./test --container-image test ${{ matrix.flavor }}-${{ matrix.arch }}
+      # chroot.test.xml is written in the entrypoint tests/init
+      - name: rename test results
+        if: always()
+        run: |
+          mv tests/chroot.test.xml .build/${{ env.cname }}.chroot.test.xml
+      - uses: pmeier/pytest-results-action@fc6576eced1f411ea48ab10e917d9cfce2960e29 # [email protected]
+        if: always()
+        with:
+          path: ".build/${{ env.cname }}.chroot.test.xml"
+      - name: add chroot.test.xml to build artifacts
+        run: echo "${{ env.cname }}.chroot.test.xml" >> ".build/${{ env.cname }}.artifacts"
+      - name: pack build artifacts for upload
+        run: tar -cSzvf "${{ env.cname }}.tar.gz" -C .build -T ".build/${{ env.cname }}.artifacts"
+      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # [email protected]
+        with:
+          name: "build-${{ env.cname }}"
+          path: "${{ env.cname }}.tar.gz"
+  bare_flavors:
+    needs: [ version, base, generate_matrix_bare ]
+    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-latest-arm' || 'ubuntu-latest' }}
+    defaults:
+      run:
+        shell: bash
+    if: needs.generate_matrix_bare.outputs.matrix != '{"include":[]}'
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate_matrix_bare.outputs.matrix) }}
+    steps:
+      # - uses: gardenlinux/workflow-telemetry-action@c75b594f552d305ffd5f9074637137bc343ba35e # pin@v2
+      #   with:
+      #     metric_frequency: 1
+      #     proc_trace_min_duration: 10
+      #     proc_trace_chart_max_count: 50
+      #     comment_on_pr: false
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # [email protected]
+        with:
+          submodules: true
+      - uses: ./.github/actions/setup
+        with:
+          arch: "${{ matrix.arch }}"
+      - name: set VERSION=${{ needs.version.outputs.version }}
+        run: |
+          cd $(dirname $(realpath bin/garden-version))/..
+          bin/garden-version "${{ needs.version.outputs.version }}" | tee VERSION
+          git update-index --assume-unchanged VERSION
+      - name: load bootstrap stage cache
+        uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # pin@v3
+        with:
+          path: .build
+          key: base-${{ matrix.arch }}-${{ github.run_id }}
+          fail-on-cache-miss: true
+      - name: update bootstrap stage build artifact timestamps
+        run: |
+          t="$(date '+%s')"
+          find .build -exec touch -d "@$t" {} +
+      - name: build
+        run: |
+          config=$(echo ${{ matrix.flavor }} | sed 's/^bare-//')
+          echo "config=$config" | tee -a "$GITHUB_ENV"
+          ./build_bare_flavors --arch "${{ matrix.arch }}" "${config}"
+          ls -la .build/bare_flavors
+      - name: test
+        run: |
+          image="$(podman load < .build/bare_flavors/${{ env.config }}-${{ matrix.arch }}.oci | awk '{ print $NF }')"
+          cd bare_flavors/${{ env.config }}/test
+          podman build -t test --build-arg image="$image" .
+          podman run --rm test
+      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # [email protected]
+        with:
+          name: "build-bare-${{ env.config }}-${{ matrix.arch }}"
+          path: ".build/bare_flavors/${{ env.config }}-${{ matrix.arch }}.oci"

.github/workflows/generate_matrix.yml

@@ -0,0 +1,40 @@
+on:
+  workflow_call:
+    inputs:
+      flags:
+        description: 'Flags passed to bin/flavors_parse.py'
+        type: string
+        required: true
+    outputs:
+      matrix:
+        description: 'The generated matrix'
+        value: ${{ jobs.generate_matrix.outputs.matrix }}
+
+jobs:
+  generate_matrix:
+    name: Generate Matrix
+    runs-on: 'ubuntu-latest'
+    defaults:
+      run:
+        shell: bash
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # [email protected]
+        with:
+          submodules: true
+      - id: matrix
+        run: |
+          FLAVORS_AMD64=$(./bin/flavors_parse.py ${{ inputs.flags }} | jq -r '.amd64 // []')
+          FLAVORS_ARM64=$(./bin/flavors_parse.py ${{ inputs.flags }} | jq -r '.arm64 // []')
+          MATRIX=$(jq -nc \
+            --argjson amd64 "$FLAVORS_AMD64" \
+            --argjson arm64 "$FLAVORS_ARM64" \
+            '{
+              include: (
+                ($amd64 | map({arch: "amd64", flavor: .})) +
+                ($arm64 | map({arch: "arm64", flavor: .}))
+              )
+            }'
+          )
+          echo "matrix=$MATRIX" | tee -a $GITHUB_OUTPUT

.github/workflows/nightly.yaml

@@ -19,6 +19,12 @@ jobs:
           submodules: "recursive"
   build:
     needs: [checkout]
-    uses: gardenlinux/gardenlinux/.github/workflows/build.yml@main
+    uses: ./.github/workflows/build.yml
     with:
       version: ${{ inputs.version || 'now' }}
+  upload_oci:
+    name: Run glcli to publish to OCI
+    needs: [ build ]
+    uses: ./.github/workflows/upload_oci.yml
+    with:
+      version: ${{ needs.build.outputs.version }}