start preparing build-runtime job (#127)

* Start moving things around

* Dancebox introduction

* few cosmetic changesd

* start preparing build-runtime job

* start testing

* Add job

* modify rust toolchain to see if that passes

* Revert "modify rust toolchain to see if that passes"

This reverts commit 0e78b6a.

* fix

* fmt

* tools to publish wasm

* remove publish runtime

* modify tools

* trigger on push

* fmt

* tsconfig

* github-utils

* wip

* try-with-tags

* revert and put it in correct place

* use new tag

* build runtime remove

* dummycollator for full node

* add back build runtime

* dancebox for orchestrator

* revert back

* get back to dispatch flow

* npm for pnpm

* trigger to see if it works

* Revert "npm for pnpm"

This reverts commit a832dce.

* Revert "Revert "npm for pnpm""

This reverts commit c88f3e3.

* Revert "Revert "Revert "npm for pnpm"""

This reverts commit 751421b.

* workflow dispatch

* fail job if docker image for sr-tool is not found

* Comments on  actions

* trigger on push

* rename

* Update cargo lock

* Revert "trigger on push"

This reverts commit 9fff59f.

* remove pnpm

* fix yml

.github/workflows/publish-runtime.yml

@@ -0,0 +1,154 @@
+name: Publish Runtime Draft
+
+# The code (like generate-release-body) will be taken from the tag versions, not master
+on:
+  workflow_dispatch:
+    inputs:
+      from:
+        description: tag (ex. runtime-53) to retrieve commit diff from
+        required: true
+      to:
+        description: tag (ex. runtime-155) to generate release note and srtool runtimes from
+        required: true
+
+jobs:
+  ####### Build runtimes with srtool #######
+
+  setup-scripts:
+    runs-on: self-hosted
+    steps:
+      ## Use scripts and tools from current branch
+      - uses: actions/checkout@v3
+      - name: Upload scripts
+        uses: actions/upload-artifact@v3
+        with:
+          name: original-scripts
+          path: scripts
+      - name: Upload tools
+        uses: actions/upload-artifact@v3
+        with:
+          name: original-tools
+          path: tools
+
+  build-srtool-runtimes:
+    needs: ["setup-scripts"]
+    runs-on: self-hosted
+    strategy:
+      matrix:
+        chain: ["dancebox"]
+        srtool_image:
+          - purestake/srtool
+        srtool_image_tag:
+          - 1.69.0
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.to }}
+      - name: Download original scripts
+        uses: actions/download-artifact@v3
+        with:
+          name: original-scripts
+          path: original-scripts
+      - name: Build & Push purestake/srtool image
+        if: github.repository == 'moondance-labs/tanssi'
+        run: |
+          docker pull "${{ matrix.srtool_image }}:${{ matrix.srtool_image_tag }}" && image_exists=true || image_exists=false
+
+          if [[ $image_exists = "false" ]]; then
+            exit 1
+          else
+            echo skiping build "${{ matrix.srtool_image }}:${{ matrix.srtool_image_tag }}", image already exists
+          fi
+      - name: Build runtime using "${{ matrix.srtool_image }}:${{ matrix.srtool_image_tag }}"
+        id: srtool_build
+        env:
+          GH_WORKFLOW_MATRIX_CHAIN: ${{ matrix.chain }}
+          GH_WORKFLOW_MATRIX_SRTOOL_IMAGE: ${{ matrix.srtool_image }}
+          GH_WORKFLOW_MATRIX_SRTOOL_IMAGE_TAG: ${{ matrix.srtool_image_tag }}
+        run: |
+          chmod u+x ./original-scripts/build-runtime-srtool.sh
+          ./original-scripts/build-runtime-srtool.sh
+      - name: Summary
+        run: |
+          echo '${{ steps.srtool_build.outputs.json }}' | jq . > ${{ matrix.chain }}-srtool-digest.json
+          cat ${{ matrix.chain }}-srtool-digest.json
+          cp ${{ steps.srtool_build.outputs.wasm_compressed }} ${{ matrix.chain }}-runtime.compact.compressed.wasm
+      - name: Archive Artifacts for ${{ matrix.chain }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.chain }}-runtime
+          path: |
+            ${{ matrix.chain }}-runtime.compact.compressed.wasm
+            ${{ matrix.chain }}-srtool-digest.json
+
+          
+  ####### Prepare the release draft #######
+  publish-draft-release:
+    runs-on: ubuntu-latest
+    needs: ["setup-scripts", "build-srtool-runtimes"]
+    outputs:
+      release_url: ${{ steps.create-release.outputs.html_url }}
+      asset_upload_url: ${{ steps.create-release.outputs.upload_url }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.to }}
+          fetch-depth: 0
+      - name: Download dancebox runtime
+        uses: actions/[email protected]
+        with:
+          name: dancebox-runtime
+          path: build     
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x  
+      - name: Download Original Tools
+        uses: actions/download-artifact@v3
+        with:
+          name: original-tools
+          path: original-tools 
+      - name: Generate release body
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        id: generate-release-body
+        working-directory: original-tools
+        run: |
+          yarn
+          yarn -s run ts-node github/generate-runtimes-body.ts --owner "${{ github.repository_owner }}" --repo "$(basename ${{ github.repository }})" --from "${{ github.event.inputs.from }}" --to "${{ github.event.inputs.to }}" --srtool-report-folder '../build/' > ../body.md      - name: Get runtime version
+      - name: Get runtime version
+        id: get-runtime-ver
+        run: |
+          runtime_dancebox_ver="$(cat ./runtime/dancebox/src/lib.rs | grep -o 'spec_version: [0-9]*' | tail -1 | grep -o '[0-9]*')"
+          echo "runtime_dancebox_ver=$runtime_dancebox_ver" >> $GITHUB_OUTPUT
+          mv build/dancebox-runtime.compact.compressed.wasm dancebox-runtime-${runtime_dancebox_ver}.wasm
+          mv build/dancebox-srtool-digest.json dancebox-runtime-${runtime_dancebox_ver}-srtool-digest.json
+      - name: Create draft release
+        id: create-release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.event.inputs.to }}
+          release_name: Runtime ${{ github.event.inputs.to }}
+          body_path: body.md
+          draft: true
+      - name: Upload dancebox wasm
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create-release.outputs.upload_url }}
+          asset_path: dancebox-runtime-${{ steps.get-runtime-ver.outputs.runtime_dancebox_ver }}.wasm
+          asset_name: dancebox-runtime-${{ steps.get-runtime-ver.outputs.runtime_dancebox_ver }}.wasm
+          asset_content_type: application/octet-stream
+      - name: Upload dancebox srtool digest
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create-release.outputs.upload_url }}
+          asset_path: dancebox-runtime-${{ steps.get-runtime-ver.outputs.runtime_dancebox_ver }}-srtool-digest.json
+          asset_name: dancebox-runtime-${{ steps.get-runtime-ver.outputs.runtime_dancebox_ver }}.srtool-digest.json
+          asset_content_type: application/json

scripts/build-runtime-srtool.sh

@@ -0,0 +1,44 @@
+
+# CARGO_NET_GIT_FETCH_WITH_CLI=true and --entrypoint /srtool/entrypoint.sh
+# are required to allow srtool to fetch from github private repositories
+
+# self-hosted runner uses user `maintenance` to match srtool `builder` user 1001
+# $(~/srtool/uid-gid-mapping.sh 1001 | xargs) is used to map the user and group
+
+# Docker command to generate JSON blob of the runtime
+CMD="docker run \
+  -i \
+  --rm \
+  $(~/srtool/uid-gid-mapping.sh 1001 | xargs) \
+  -e CARGO_NET_GIT_FETCH_WITH_CLI=true \
+  -e PACKAGE=${GH_WORKFLOW_MATRIX_CHAIN}-runtime \
+  -e RUNTIME_DIR=runtime/${GH_WORKFLOW_MATRIX_CHAIN} \
+  -v ${PWD}:/build \
+  -v /home/${USER}/srtool/.ssh:/home/builder/.ssh \
+  -v /home/${USER}/srtool/entrypoint.sh:/srtool/entrypoint.sh \
+  --entrypoint /srtool/entrypoint.sh \
+  ${GH_WORKFLOW_MATRIX_SRTOOL_IMAGE}:${GH_WORKFLOW_MATRIX_SRTOOL_IMAGE_TAG} \
+    build --app --json -cM"
+
+# Here we run the command and stream the output (JSON blob) to a variable
+stdbuf -oL $CMD | {
+  while IFS= read -r line
+  do
+      echo ║ $line
+      JSON="$line"
+  done
+
+  echo "json=$JSON" >> $GITHUB_OUTPUT
+
+  PROP=`echo $JSON | jq -r .runtimes.compact.prop`
+  echo "proposal_hash=$PROP" >> $GITHUB_OUTPUT
+
+  WASM=`echo $JSON | jq -r .runtimes.compact.wasm`
+  echo "wasm=$WASM" >> $GITHUB_OUTPUT
+
+  Z_WASM=`echo $JSON | jq -r .runtimes.compressed.wasm`
+  echo "wasm_compressed=$Z_WASM" >> $GITHUB_OUTPUT
+
+  IPFS=`echo $JSON | jq -r .runtimes.compact.ipfs`
+  echo "ipfs=$IPFS" >> $GITHUB_OUTPUT
+}

tools/github/generate-release-body.ts

@@ -0,0 +1,83 @@
+import { Octokit } from "octokit";
+import yargs from "yargs";
+import { getCommitAndLabels, getCompareLink } from "./github-utils";
+
+const BINARY_CHANGES_LABEL = "B5-clientnoteworthy";
+const BREAKING_CHANGES_LABEL = "D2-breaksapi";
+
+function capitalize(s) {
+  return s[0].toUpperCase() + s.slice(1);
+}
+
+async function main() {
+  const argv = yargs(process.argv.slice(2))
+    .usage("Usage: npm run ts-node github/generate-release-body.ts [args]")
+    .version("1.0.0")
+    .options({
+      from: {
+        type: "string",
+        describe: "previous tag to retrieve commits from",
+        required: true,
+      },
+      to: {
+        type: "string",
+        describe: "current tag being drafted",
+        required: true,
+      },
+      owner: {
+        type: "string",
+        describe: "Repository owner (Ex: moondance-labs)",
+        required: true,
+      },
+      repo: {
+        type: "string",
+        describe: "Repository name (Ex: dancebox)",
+        required: true,
+      },
+    })
+    .demandOption(["from", "to"])
+    .help().argv;
+
+  const octokit = new Octokit({
+    auth: process.env.GITHUB_TOKEN || undefined,
+  });
+
+  const previousTag = argv.from;
+  const newTag = argv.to;
+
+  const moduleLinks = ["substrate", "polkadot", "cumulus", "frontier"].map((repoName) => ({
+    name: repoName,
+    link: getCompareLink(repoName, previousTag, newTag),
+  }));
+
+  const { prByLabels } = await getCommitAndLabels(
+    octokit,
+    argv.owner,
+    argv.repo,
+    previousTag,
+    newTag
+  );
+  const filteredPr = prByLabels[BINARY_CHANGES_LABEL] || [];
+
+  const printPr = (pr) => {
+    if (pr.labels.includes(BREAKING_CHANGES_LABEL)) {
+      return "⚠️ " + pr.title + " (#" + pr.number + ")";
+    } else {
+      return pr.title + " (#" + pr.number + ")";
+    }
+  };
+
+  const template = `
+## Changes
+
+${filteredPr.map((pr) => `* ${printPr(pr)}`).join("\n")}
+
+## Dependency changes
+
+Tanssi: https://github.com/${argv.owner}/${argv.repo}/compare/${previousTag}...${newTag}
+${moduleLinks.map((modules) => `${capitalize(modules.name)}: ${modules.link}`).join("\n")}
+`;
+  console.log(template);
+}
+
+main();