From c9222a9d3c01b96c12d5b27c602e5c6a7ea088fc Mon Sep 17 00:00:00 2001 From: Michael FIG Date: Sat, 4 Jun 2022 13:03:51 -0600 Subject: [PATCH 1/4] ci(actions): fix up cache keys --- .github/actions/build-node/action.yml | 6 ------ .github/actions/restore-golang/action.yml | 5 +++-- .github/actions/restore-node/action.yml | 4 +++- .github/actions/save-node/action.yml | 4 +++- .github/workflows/deployment-test.yml | 1 - 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/actions/build-node/action.yml b/.github/actions/build-node/action.yml index 0deb186ca58..114bb1e4ad1 100644 --- a/.github/actions/build-node/action.yml +++ b/.github/actions/build-node/action.yml @@ -5,11 +5,6 @@ inputs: node-version: description: 'The version of Node.js to use' required: true - cache-built: - description: 'Cache the result of the build' - type: boolean - required: false - default: true runs: using: composite @@ -45,6 +40,5 @@ runs: run: yarn build - uses: ./.github/actions/save-node - if: inputs.cache-built == true with: node-version: ${{ inputs.node-version }} diff --git a/.github/actions/restore-golang/action.yml b/.github/actions/restore-golang/action.yml index 6b9051db7ce..ebe86344a9d 100644 --- a/.github/actions/restore-golang/action.yml +++ b/.github/actions/restore-golang/action.yml @@ -29,9 +29,10 @@ runs: uses: actions/cache@v1 with: path: ${{ env.GOPATH }}/pkg/mod - key: ${{ runner.os }}-${{ inputs.cache-key }}-go-${{ hashFiles('go.sum') }} + key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-key }}-go-built-${{ inputs.go-version }}-${{ hashFiles('go.sum') }} restore-keys: | - ${{ runner.os }}-${{ inputs.cache-key }}-go- + ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-key }}-go-built-${{ inputs.go-version }}- + ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-key }}-go-built- - name: go mod download run: go mod download shell: bash diff --git a/.github/actions/restore-node/action.yml b/.github/actions/restore-node/action.yml index 45ee70353d1..4cc74e87878 100644 --- a/.github/actions/restore-node/action.yml +++ b/.github/actions/restore-node/action.yml @@ -19,12 +19,14 @@ runs: - uses: actions/setup-node@v1 with: node-version: ${{ inputs.node-version }} + - uses: kenchan0130/actions-system-info@master + id: system-info - name: restore built files id: built uses: actions/cache@v1 with: path: . - key: ${{ runner.os }}-${{ inputs.node-version }}-built-${{ github.sha }} + key: ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-node-${{ inputs.node-version }}-built-${{ github.sha }} - name: yarn install --frozen-lockfile run: yarn install --frozen-lockfile shell: bash diff --git a/.github/actions/save-node/action.yml b/.github/actions/save-node/action.yml index aebab9071b4..6b01c7820c5 100644 --- a/.github/actions/save-node/action.yml +++ b/.github/actions/save-node/action.yml @@ -9,8 +9,10 @@ inputs: runs: using: composite steps: + - uses: kenchan0130/actions-system-info@master + id: system-info - name: cache build outputs uses: actions/cache@v1 with: path: . - key: ${{ runner.os }}-${{ inputs.node-version }}-built-${{ github.sha }} + key: ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-node-${{ inputs.node-version }}-built-${{ github.sha }} diff --git a/.github/workflows/deployment-test.yml b/.github/workflows/deployment-test.yml index 38ff63ab045..3b534620f54 100644 --- a/.github/workflows/deployment-test.yml +++ b/.github/workflows/deployment-test.yml @@ -38,7 +38,6 @@ jobs: - uses: ./.github/actions/build-node with: node-version: 14.x - cache-built: false # Select a branch on loadgen to test against by adding text to the body of the # pull request. For example: #loadgen-branch: user-123-update-foo From ef27822943327a834d22ce74e7e697661d0ddd2b Mon Sep 17 00:00:00 2001 From: Michael FIG Date: Sun, 5 Jun 2022 11:01:21 -0600 Subject: [PATCH 2/4] ci(build-node): remove now that `restore-node` is idempotent --- .github/actions/build-node/action.yml | 44 ----------------------- .github/actions/restore-golang/action.yml | 8 +++-- .github/workflows/after-merge.yml | 2 +- .github/workflows/deployment-test.yml | 4 +-- .github/workflows/test-all-packages.yml | 2 +- 5 files changed, 9 insertions(+), 51 deletions(-) delete mode 100644 .github/actions/build-node/action.yml diff --git a/.github/actions/build-node/action.yml b/.github/actions/build-node/action.yml deleted file mode 100644 index 114bb1e4ad1..00000000000 --- a/.github/actions/build-node/action.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Agoric SDK build Node.js -description: 'Build Node.js and cache built files' - -inputs: - node-version: - description: 'The version of Node.js to use' - required: true - -runs: - using: composite - steps: - - uses: actions/setup-node@v1 - with: - node-version: ${{ inputs.node-version }} - - name: cache node modules - uses: actions/cache@v1 - with: - path: ~/.cache/yarn - key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - # 'yarn install' must be done at the top level, to build all the - # cross-package symlinks - - name: yarn install - run: yarn install - shell: bash - # We do a diff instead of using --frozen-lockfile because - # - it doesn't detect deletions from yarn.lock (see https://github.com/Agoric/agoric-sdk/pull/4646#discussion_r814034842) - # - the diff provides diagnostic info of what changed - - name: fail if git checkout differs - run: git diff --exit-code - shell: bash - - name: check dependencies - run: yarn check-dependencies - shell: bash - # 'yarn build' loops over all workspaces - - name: yarn build - shell: bash - run: yarn build - - - uses: ./.github/actions/save-node - with: - node-version: ${{ inputs.node-version }} diff --git a/.github/actions/restore-golang/action.yml b/.github/actions/restore-golang/action.yml index ebe86344a9d..55c3ae35fdf 100644 --- a/.github/actions/restore-golang/action.yml +++ b/.github/actions/restore-golang/action.yml @@ -24,15 +24,17 @@ runs: - uses: actions/setup-go@v2 with: go-version: ${{ inputs.go-version }} + - uses: kenchan0130/actions-system-info@master + id: system-info - name: cache Go modules id: cache uses: actions/cache@v1 with: path: ${{ env.GOPATH }}/pkg/mod - key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-key }}-go-built-${{ inputs.go-version }}-${{ hashFiles('go.sum') }} + key: ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go-${{ inputs.go-version }}-${{ hashFiles('go.sum') }} restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-key }}-go-built-${{ inputs.go-version }}- - ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-key }}-go-built- + ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go-${{ inputs.go-version }}- + ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go- - name: go mod download run: go mod download shell: bash diff --git a/.github/workflows/after-merge.yml b/.github/workflows/after-merge.yml index c4b6d85a7ad..6d21a6390af 100644 --- a/.github/workflows/after-merge.yml +++ b/.github/workflows/after-merge.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'true' - - uses: ./.github/actions/build-node + - uses: ./.github/actions/restore-node with: node-version: ${{ matrix.node-version}} - name: notify on failure diff --git a/.github/workflows/deployment-test.yml b/.github/workflows/deployment-test.yml index 3b534620f54..99fd4e54911 100644 --- a/.github/workflows/deployment-test.yml +++ b/.github/workflows/deployment-test.yml @@ -30,12 +30,12 @@ jobs: steps: - uses: actions/checkout@v2 with: - clean: 'false' + submodules: 'true' - uses: ./.github/actions/restore-golang with: go-version: 1.17 cache-key: trusty - - uses: ./.github/actions/build-node + - uses: ./.github/actions/restore-node with: node-version: 14.x diff --git a/.github/workflows/test-all-packages.yml b/.github/workflows/test-all-packages.yml index 1ccf8ea6f48..13b4d705e3c 100644 --- a/.github/workflows/test-all-packages.yml +++ b/.github/workflows/test-all-packages.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'true' - - uses: ./.github/actions/build-node + - uses: ./.github/actions/restore-node with: node-version: ${{ matrix.node-version }} From 1fa944dbfda6d0dc6be09505aef65322ed60bf65 Mon Sep 17 00:00:00 2001 From: Michael FIG Date: Sat, 4 Jun 2022 22:57:45 -0600 Subject: [PATCH 3/4] ci(deployment-test): ensure test output is in /usr/src/agoric-sdk --- .github/actions/restore-golang/action.yml | 5 ++--- .github/workflows/deployment-test.yml | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/actions/restore-golang/action.yml b/.github/actions/restore-golang/action.yml index 55c3ae35fdf..4814f447c7c 100644 --- a/.github/actions/restore-golang/action.yml +++ b/.github/actions/restore-golang/action.yml @@ -31,10 +31,9 @@ runs: uses: actions/cache@v1 with: path: ${{ env.GOPATH }}/pkg/mod - key: ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go-${{ inputs.go-version }}-${{ hashFiles('go.sum') }} + key: ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go-${{ inputs.go-version }}-built-${{ hashFiles('go.sum') }} restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go-${{ inputs.go-version }}- - ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go- + ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go-${{ inputs.go-version }}-built- - name: go mod download run: go mod download shell: bash diff --git a/.github/workflows/deployment-test.yml b/.github/workflows/deployment-test.yml index 99fd4e54911..633edc1be66 100644 --- a/.github/workflows/deployment-test.yml +++ b/.github/workflows/deployment-test.yml @@ -67,25 +67,24 @@ jobs: path: testnet-load-generator ref: ${{steps.get-loadgen-branch.outputs.result}} - - name: Move repos under /usr/src where scripts expect them + - name: Put repos under /usr/src where scripts expect them run: | set -e sudo mv "$GITHUB_WORKSPACE/testnet-load-generator" /usr/src/testnet-load-generator - sudo mv "$GITHUB_WORKSPACE" /usr/src/agoric-sdk + sudo cp -a "$GITHUB_WORKSPACE" /usr/src/agoric-sdk ln -s /usr/src/agoric-sdk/packages/deployment/bin/ag-setup-cosmos /usr/local/bin/ag-setup-cosmos - ln -s /usr/src/agoric-sdk "$GITHUB_WORKSPACE" working-directory: / - name: Build cosmic-swingset dependencies run: | - # Some of our build relies on /usr/src/agoric-sdk set -e - cd /usr/src/agoric-sdk/packages/cosmic-swingset + cd packages/cosmic-swingset make install - working-directory: / + working-directory: /usr/src/agoric-sdk - run: sudo ./packages/deployment/scripts/install-deps.sh working-directory: /usr/src/agoric-sdk - run: /usr/src/agoric-sdk/packages/deployment/scripts/integration-test.sh + working-directory: /usr/src/agoric-sdk env: NETWORK_NAME: chaintest - name: capture results @@ -95,20 +94,21 @@ jobs: echo "NOW=$NOW" >> "$GITHUB_ENV" # Stop the chain from running. - /usr/src/agoric-sdk/packages/deployment/scripts/setup.sh play stop || true + packages/deployment/scripts/setup.sh play stop || true # Get the results. - /usr/src/agoric-sdk/packages/deployment/scripts/capture-integration-results.sh "${{ job.status == 'failure' }}" + packages/deployment/scripts/capture-integration-results.sh "${{ job.status == 'failure' }}" # Tear down the nodes. - echo yes | /usr/src/agoric-sdk/packages/deployment/scripts/setup.sh destroy || true + echo yes | packages/deployment/scripts/setup.sh destroy || true + working-directory: /usr/src/agoric-sdk env: NETWORK_NAME: chaintest - uses: actions/upload-artifact@v2 if: failure() with: name: deployment-test-results-${{ env.NOW }} - path: chaintest/results + path: /usr/src/agoric-sdk/chaintest/results - name: notify on failure if: failure() && github.event_name != 'pull_request' From c1de3a047b7312fa663571e23d356d2441d684f5 Mon Sep 17 00:00:00 2001 From: Michael FIG Date: Sun, 5 Jun 2022 16:11:48 -0600 Subject: [PATCH 4/4] ci: more actions cleanups and cache optimisation --- .github/actions/restore-golang/action.yml | 5 ----- .github/actions/restore-node/action.yml | 7 +++++++ .github/actions/save-node/action.yml | 18 ------------------ 3 files changed, 7 insertions(+), 23 deletions(-) delete mode 100644 .github/actions/save-node/action.yml diff --git a/.github/actions/restore-golang/action.yml b/.github/actions/restore-golang/action.yml index 4814f447c7c..e34f95be5de 100644 --- a/.github/actions/restore-golang/action.yml +++ b/.github/actions/restore-golang/action.yml @@ -5,11 +5,6 @@ inputs: go-version: description: 'The version of Go to use' required: true - cache-key: - description: pkg/mod cache key - required: false - type: string - default: latest runs: using: composite diff --git a/.github/actions/restore-node/action.yml b/.github/actions/restore-node/action.yml index 4cc74e87878..b489cefcf38 100644 --- a/.github/actions/restore-node/action.yml +++ b/.github/actions/restore-node/action.yml @@ -21,6 +21,13 @@ runs: node-version: ${{ inputs.node-version }} - uses: kenchan0130/actions-system-info@master id: system-info + - name: cache node modules + uses: actions/cache@v1 + with: + path: ~/.cache/yarn + key: ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-yarn- - name: restore built files id: built uses: actions/cache@v1 diff --git a/.github/actions/save-node/action.yml b/.github/actions/save-node/action.yml deleted file mode 100644 index 6b01c7820c5..00000000000 --- a/.github/actions/save-node/action.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Agoric SDK save Node.js build -description: 'Set up Node.js environment and restore cached built files' - -inputs: - node-version: - description: 'The version of Node.js we used' - required: true - -runs: - using: composite - steps: - - uses: kenchan0130/actions-system-info@master - id: system-info - - name: cache build outputs - uses: actions/cache@v1 - with: - path: . - key: ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-node-${{ inputs.node-version }}-built-${{ github.sha }}