test: yarn build cache trying things out #2

Signed-off-by: Peter Somogyvari <[email protected]>

package.json

@@ -25,7 +25,7 @@
     "reset:git": "git clean -f -X",
     "reset:yarn-lock": "yarn run init-registries && yarn install --update-checksums --force",
     "reset": "run-s reset:git reset:node-modules reset:yarn-lock configure",
-    "configure": "npm run init-registries && yarn install --mode=skip-build && yarn build:dev:backend",
+    "configure": "npm run init-registries && yarn install && yarn build:dev:backend",
     "install-yarn": "npm install --global [email protected]",
     "set-yarn-version": "yarn set version stable",
     "enable-corepack": "npm i -g corepack && corepack enable && corepack prepare [email protected] --activate",
@@ -264,36 +264,5 @@
     "web3-shh": {
       "built": false
     }
-  },
-  "peerDependenciesMeta": {
-    "@apollo/protobufjs": {},
-    "@nestjs/core": {},
-    "@openapitools/openapi-generator-cli": {},
-    "@trufflesuite/bigint-buffer": {},
-    "aws-sdk": {},
-    "bufferutil": {},
-    "cbor": {},
-    "classic-level": {},
-    "core-js": {},
-    "cpu-features": {},
-    "deasync": {},
-    "electron": {},
-    "es5-ext": {},
-    "esbuild": {},
-    "iso-constants": {},
-    "keccak": {},
-    "keytar": {},
-    "leveldown": {},
-    "nice-napi": {},
-    "pkcs11js": {},
-    "protobufjs": {},
-    "secp256k1": {},
-    "sqlite3": {},
-    "ssh2": {},
-    "truffle": {},
-    "utf-8-validate": {},
-    "web3": {},
-    "web3-bzz": {},
-    "web3-shh": {}
   }
 }

packages/cactus-cmd-api-server/src/test/typescript/unit/plugins/install-basic-plugin-ledger-connector-quorum-0-7-0.test.ts

@@ -22,7 +22,13 @@ import { DefaultApi as ApiServerApi } from "../../../../main/typescript/public-a
 
 const logLevel: LogLevelDesc = "TRACE";
 
-test("can import plugins at runtime (CLI)", async (t: Test) => {
+/**
+ * Skipping this test because we are switching to not hoisting dependencies
+ * and this made it so that the installation no longer works due to the latest
+ * quorum connector version up on npm (v1.1.3) being broken because it does not declare
+ * it's dependencies correctly (missing some of them).
+ */
+test.skip("can import plugins at runtime (CLI)", async (t: Test) => {
   // const pluginsPath = path.join(
   //   "/tmp/org/hyperledger/cactus/cmd-api-server/runtime-plugin-imports_test", // the dir path from the root
   //   uuidv4(), // then a random directory to ensure proper isolation

packages/cactus-plugin-htlc-eth-besu/foundry.toml

@@ -25,10 +25,10 @@ block_coinbase = '0x0000000000000000000000000000000000000000' # the address of `
 block_timestamp = 0                                           # the value of `block.timestamp` in tests
 block_difficulty = 0      
 fuzz = { runs = 256 }                                         # the number of fuzz runs for tests
-libs = ["../../node_modules"]
+libs = ["./node_modules"]
 remappings = [
-    "ds-test/=../../node_modules/ds-test/src/",
-    "forge-std/=../../node_modules/forge-std/src/",
+    "ds-test/=./node_modules/ds-test/src/",
+    "forge-std/=./node_modules/forge-std/src/",
 ]
 
 # Fuzzing CI

packages/cactus-plugin-ledger-connector-corda/package.json

@@ -74,7 +74,8 @@
     "@types/express": "4.17.13",
     "@types/multer": "1.4.7",
     "@types/node-ssh": "7.0.1",
-    "@types/temp": "0.9.1"
+    "@types/temp": "0.9.1",
+    "express": "4.17.3"
   },
   "engines": {
     "node": ">=10",

packages/cactus-plugin-ledger-connector-fabric/package.json

@@ -100,7 +100,7 @@
     "@types/temp": "0.9.1",
     "@types/uuid": "8.3.1",
     "fs-extra": "10.0.0",
-    "internal-ip": "7.0.0",
+    "internal-ip": "6.2.0",
     "socket.io": "4.4.1",
     "ws-wallet": "1.1.5"
   },

packages/cactus-plugin-ledger-connector-fabric/src/test/typescript/integration/fabric-v2-2-x/run-transaction-with-identities.test.ts

@@ -22,7 +22,7 @@ import {
   FabricTestLedgerV1,
   pruneDockerAllIfGithubAction,
 } from "@hyperledger/cactus-test-tooling";
-import { internalIpV4 } from "internal-ip";
+import { v4 as internalIpV4 } from "internal-ip";
 import axios from "axios";
 
 // test scenario

packages/cactus-plugin-ledger-connector-fabric/src/test/typescript/integration/fabric-v2-2-x/run-transaction-with-ws-ids.test.ts

@@ -22,7 +22,7 @@ import {
   WsTestServer,
   WS_IDENTITY_HTTP_PORT,
 } from "@hyperledger/cactus-test-tooling";
-import { internalIpV4 } from "internal-ip";
+import { v4 as internalIpV4 } from "internal-ip";
 import { WsWallet } from "ws-wallet";
 import { WsIdentityClient } from "ws-identity-client";
 

packages/cactus-plugin-ledger-connector-fabric/src/test/typescript/integration/identity-client.test.ts

@@ -1,5 +1,5 @@
 import axios from "axios";
-import { internalIpV4 } from "internal-ip";
+import { v4 as internalIpV4 } from "internal-ip";
 import {
   Containers,
   VaultTestServer,

packages/cactus-plugin-ledger-connector-iroha/package.json

@@ -78,7 +78,7 @@
     "@hyperledger/cactus-test-tooling": "2.0.0-alpha.1",
     "@types/express": "4.17.8",
     "@types/sanitize-html": "2.6.2",
-    "internal-ip": "7.0.0",
+    "internal-ip": "6.2.0",
     "socket.io": "4.4.1"
   },
   "engines": {

packages/cactus-plugin-ledger-connector-iroha/src/test/typescript/integration/generate-and-send-signed-transaction.test.ts

@@ -40,7 +40,7 @@ import express from "express";
 import bodyParser from "body-parser";
 import { AddressInfo } from "net";
 import { v4 as uuidv4 } from "uuid";
-import { internalIpV4 } from "internal-ip";
+import { v4 as internalIpV4 } from "internal-ip";
 import "jest-extended";
 
 import cryptoHelper from "iroha-helpers/lib/cryptoHelper";

packages/cactus-plugin-ledger-connector-iroha/src/test/typescript/integration/iroha-iroha-transfer-example.test.ts

@@ -2,7 +2,7 @@ import http from "http";
 import { AddressInfo } from "net";
 import test, { Test } from "tape-promise/tape";
 import { v4 as uuidv4 } from "uuid";
-import { internalIpV4 } from "internal-ip";
+import { v4 as internalIpV4 } from "internal-ip";
 import bodyParser from "body-parser";
 import express from "express";
 import {

packages/cactus-plugin-ledger-connector-iroha/src/test/typescript/integration/iroha-socketio-endpoint.test.ts

@@ -18,7 +18,7 @@ import {
   pruneDockerAllIfGithubAction,
 } from "@hyperledger/cactus-test-tooling";
 
-import { internalIpV4 } from "internal-ip";
+import { v4 as internalIpV4 } from "internal-ip";
 import { v4 as uuidv4 } from "uuid";
 import { RuntimeError } from "run-time-error";
 import cryptoHelper from "iroha-helpers/lib/cryptoHelper";

packages/cactus-plugin-ledger-connector-iroha/src/test/typescript/integration/openapi/openapi-validation.test.ts

@@ -2,7 +2,7 @@ import http from "http";
 import { AddressInfo } from "net";
 import test, { Test } from "tape-promise/tape";
 import { v4 as uuidv4 } from "uuid";
-import { internalIpV4 } from "internal-ip";
+import { v4 as internalIpV4 } from "internal-ip";
 import bodyParser from "body-parser";
 import express from "express";
 import { Server as SocketIoServer } from "socket.io";

packages/cactus-plugin-ledger-connector-iroha/src/test/typescript/integration/run-transaction-endpoint-v1.test.ts

@@ -2,7 +2,7 @@ import http from "http";
 import { AddressInfo } from "net";
 import test, { Test } from "tape-promise/tape";
 import { v4 as uuidv4 } from "uuid";
-import { internalIpV4 } from "internal-ip";
+import { v4 as internalIpV4 } from "internal-ip";
 import bodyParser from "body-parser";
 import express from "express";
 import { Server as SocketIoServer } from "socket.io";

tools/ci.sh

@@ -52,12 +52,34 @@ function checkOnlyDocumentation()
       ENDED_AT=`date +%s`
       runtime=$((ENDED_AT-STARTED_AT))
       echo "$(date +%FT%T%z) [CI] SUCCESS - runtime=$runtime seconds."
-      checkWorkTreeStatus
       exit 0
     fi
   done
 }
 
+function freeUpGitHubRunnerDiskSpace() {
+  # If we are running in a GitHub Actions runner, then free up 30 GB space by
+  # removing things we do not need such as the Android SDK and .NET.
+  #
+  # Huge thanks to Maxim Lobanov for the advice:
+  # https://github.com/actions/virtual-environments/issues/2606#issuecomment-772683150
+  #
+  # Why do this? Because we've been getting warnings about the runners being
+  # left with less than a hundred megabytes of disk space during the tests.
+  #
+  # This operation takes about 2 minutes to do and so is disabled by default to get better
+  # performance from the CI by default. It can be enabled on a per job basis via
+  # the env variables defined in the action's .yaml files.
+  #
+  if [ "${FREE_UP_GITHUB_RUNNER_DISK_SPACE_DISABLED:-true}" = "true" ]; then
+    echo "$(date +%FT%T%z) [CI] Freeing up GitHub Action Runner disk space disabled. Skipping..."
+  else
+    echo "$(date +%FT%T%z) [CI] Freeing up GitHub Action Runner disk space by deleting Android and .NET ..."
+    sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android
+    sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET
+  fi
+}
+
 function mainTask()
 {
   set -euxo pipefail
@@ -83,46 +105,31 @@ function mainTask()
   # Check if the modified files are only for documentation.
   checkOnlyDocumentation
 
-  # If we are running in a GitHub Actions runner, then free up 30 GB space by
-  # removing things we do not need such as the Android SDK and .NET.
-  #
-  # Huge thanks to Maxim Lobanov for the advice:
-  # https://github.com/actions/virtual-environments/issues/2606#issuecomment-772683150
-  #
-  # Why do this? Because we've been getting warnings about the runners being
-  # left with less than a hundred megabytes of disk space during the tests.
-  if [ "${GITHUB_ACTIONS:-false}" = "true" ]; then
-    echo 'Detected GitHub Action Runner deleting Android and .NET ...'
-    sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android
-    sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET
-  fi
+  # Can be turned ON/OFF via env var FREE_UP_GITHUB_RUNNER_DISK_SPACE_DISABLED=true/false
+  freeUpGitHubRunnerDiskSpace
 
-  dumpDiskUsageInfo
+  if [ "${DUMP_DISK_USAGE_INFO_DISABLED:-true}" = "true" ]; then
+    echo "$(date +%FT%T%z) [CI] dumpDiskUsageInfo disabled. Skipping..."
+  else
+    dumpDiskUsageInfo
+  fi
 
   docker --version
   docker-compose --version
   node --version
   npm --version
   java -version
-
-  # install npm 7 globally - needed because Node v12, v14 default to npm v6
-  npm install -g [email protected]
-  npm --version
   yarn --version
 
+  export NODE_OPTIONS=--max_old_space_size=5120
+
   ### COMMON
   cd $PROJECT_ROOT_DIR
 
-  npm run configure
-
-  # Obtains the major NodeJS version such as "12" from "v12.14.1"
-  # We only run the custom checks above v12 because the globby dependency's
-  # latest version is forcing us to use Ecmascript Modules which do not work
-  # on NodeJS 12 even with the additional flags passed in.
-  nodejs_version=`node --version | awk -v range=1 '{print substr($0,range+1,2)}'`
-  if [ "$nodejs_version" -gt "12" ]; then
-    echo "$(date +%FT%T%z) [CI] NodeJS is newer than v12, running custom checks..."
-    yarn run custom-checks
+  if [ "${CONFIGURE_DISABLED:-false}" = "true" ]; then
+    echo "$(date +%FT%T%z) [CI] npm run configure disabled. Skipping..."
+  else
+    npm run configure
   fi
 
   if [ "${TOOLS_VALIDATE_BUNDLE_NAMES_DISABLED:-false}" = "true" ]; then
@@ -143,18 +150,23 @@ function mainTask()
     yarn test:jest:all $JEST_TEST_PATTERN
   fi
 
-  dumpDiskUsageInfo
+  if [ "${DUMP_DISK_USAGE_INFO_DISABLED:-true}" = "true" ]; then
+    echo "$(date +%FT%T%z) [CI] dumpDiskUsageInfo disabled. Skipping..."
+  else
+    dumpDiskUsageInfo
+  fi
 
   if [ "${TAPE_TEST_RUNNER_DISABLED:-false}" = "true" ]; then
     echo "$(date +%FT%T%z) [CI] Tape test runner disabled. Skipping..."
   else
     yarn test:tap:all --bail $TAPE_TEST_PATTERN
   fi
 
-  dumpDiskUsageInfo
-
-  # The webpack production build needs more memory than the default allocation
-  export NODE_OPTIONS=--max_old_space_size=4096
+  if [ "${DUMP_DISK_USAGE_INFO_DISABLED:-true}" = "true" ]; then
+    echo "$(date +%FT%T%z) [CI] dumpDiskUsageInfo disabled. Skipping..."
+  else
+    dumpDiskUsageInfo
+  fi
 
   # We run the full build last because the tests don't need it so in the interest
   # of providing feedback about failing tests as early as possible we run the
@@ -166,10 +178,15 @@ function mainTask()
     yarn run build
   fi
 
+  if [ "${CHECK_WORK_TREE_STATUS_DISABLED:-true}" = "true" ]; then
+    echo "$(date +%FT%T%z) [CI] checkWorkTreeStatus disabled. Skipping..."
+  else
+    checkWorkTreeStatus
+  fi
+
   ENDED_AT=`date +%s`
   runtime=$((ENDED_AT-STARTED_AT))
   echo "$(date +%FT%T%z) [CI] SUCCESS - runtime=$runtime seconds."
-  checkWorkTreeStatus
   exit 0
 }
 

yarn.lock

@@ -5150,6 +5150,7 @@ __metadata:
     "@types/node-ssh": 7.0.1
     "@types/temp": 0.9.1
     axios: 0.21.4
+    express: 4.17.3
     express-openapi-validator: 3.10.0
     internal-ip: 6.2.0
     joi: 17.9.1
@@ -5229,7 +5230,7 @@ __metadata:
     form-data: 4.0.0
     fs-extra: 10.0.0
     http-status-codes: 2.1.4
-    internal-ip: 7.0.0
+    internal-ip: 6.2.0
     jsrsasign: 10.5.25
     multer: 1.4.5-lts.1
     ngo: 2.7.0
@@ -5324,7 +5325,7 @@ __metadata:
     express: 4.17.1
     fast-safe-stringify: 2.1.1
     grpc: 1.24.11
-    internal-ip: 7.0.0
+    internal-ip: 6.2.0
     iroha-helpers: 1.5.0
     key-encoder: 2.0.3
     openapi-types: 7.0.1
@@ -23111,18 +23112,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"internal-ip@npm:7.0.0":
-  version: 7.0.0
-  resolution: "internal-ip@npm:7.0.0"
-  dependencies:
-    default-gateway: ^6.0.3
-    ipaddr.js: ^2.0.1
-    is-ip: ^3.1.0
-    p-event: ^4.2.0
-  checksum: 058fafe81a7e4d2466474361b74fd712d986f553def881c7dff51fd7eb98573d03c44cab047dddf41473252406e66fa20ed23f6a1b76962b090eee26a22854b6
-  languageName: node
-  linkType: hard
-
 "internal-slot@npm:^1.0.4, internal-slot@npm:^1.0.5":
   version: 1.0.5
   resolution: "internal-slot@npm:1.0.5"