From 379fbe5d80dd8c13006486ddfdfead96c29e3f60 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Mon, 15 Nov 2021 07:56:36 +0300 Subject: [PATCH] 249 Separate evm_loader deployment from proxy-model deployment (#271) * switch to new evm_loader deployment * fix docker-compose file * fix deploy-test.sh * add error logging * Revert "add error logging" This reverts commit fcd00103bc94345218d4b51f3873aaa77929654a. * rename token owner file * fix docker-compose file * tests finally works * rename file * fix docker-compose-test * add evm_loader.log file as build artifact * Improve deploy sequence * use healthcheck in solana service * remove unnecessary dependence * add cleanup step * add solana_url into healthcheck explicitely * fix cleanup_docker function Co-authored-by: ivanl --- .buildkite/pipeline.yml | 1 + .buildkite/steps/deploy-test.sh | 25 ++++++++---- Dockerfile | 9 ++--- proxy/docker-compose-test.yml | 51 +++++++++++++++++++++-- proxy/run-proxy.sh | 71 --------------------------------- proxy/run-test-proxy.sh | 26 ++++++++++++ 6 files changed, 95 insertions(+), 88 deletions(-) create mode 100755 proxy/run-test-proxy.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f1ab50e2f..884c935d7 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -13,6 +13,7 @@ steps: - "proxy.log" - "solana.log" - "measurements.log" + - "evm_loader.log" - wait diff --git a/.buildkite/steps/deploy-test.sh b/.buildkite/steps/deploy-test.sh index 9ffd4f3e1..f79c8a3c5 100755 --- a/.buildkite/steps/deploy-test.sh +++ b/.buildkite/steps/deploy-test.sh @@ -35,15 +35,20 @@ UNISWAP_V2_CORE_IMAGE=neonlabsorg/uniswap-v2-core:stable # Refreshing uniswap-v2-core image is required to run .buildkite/steps/deploy-test.sh locally docker pull $UNISWAP_V2_CORE_IMAGE -docker-compose -f proxy/docker-compose-test.yml up -d - function cleanup_docker { - docker logs proxy >proxy.log 2>&1 - docker logs solana >solana.log 2>&1 - grep 'get_measurements' measurements.log - echo "Cleanup docker-compose..." + if docker logs proxy >proxy.log 2>&1; then + echo "proxy logs saved"; + grep 'get_measurements' measurements.log + fi + + if docker logs solana >solana.log 2>&1; then echo "solana logs saved"; fi + if docker logs evm_loader >evm_loader.log 2>&1; then echo "evm_loader logs saved"; fi + + echo "\nCleanup docker-compose..." docker-compose -f proxy/docker-compose-test.yml down --rmi 'all' echo "Cleanup docker-compose done." + echo "\nRemoving temporary data volumes..." + docker volume prune -f if grep '\[E\] get_measurements' + bash -c "create-test-accounts.sh 1 + && deploy-evm.sh + && cp evm_loader_id /var/deployment_data/ + && cp evm_loader-keypair.json /var/deployment_data/" postgres: container_name: postgres @@ -26,6 +52,12 @@ services: POSTGRES_USER: neon-proxy POSTGRES_PASSWORD: neon-proxy-pass hostname: postgres + healthcheck: + test: [ CMD-SHELL, "pg_isready" ] + interval: 5s + timeout: 10s + retries: 10 + start_period: 5s expose: - "5432" networks: @@ -35,19 +67,32 @@ services: container_name: proxy image: neonlabsorg/proxy:${REVISION} environment: + SOLANA_URL: http://solana:8899 POSTGRES_DB: neon-db POSTGRES_USER: neon-proxy POSTGRES_PASSWORD: neon-proxy-pass + NEW_USER_AIRDROP_AMOUNT: 100 CONFIG: ci hostname: proxy + depends_on: + postgres: + condition: service_healthy + evm_loader: + condition: service_completed_successfully ports: - 127.0.0.1:9090:9090 expose: - "9090" networks: - net - depends_on: - - postgres + volumes: + - deployment_data:/var/deployment_data + entrypoint: proxy/run-test-proxy.sh networks: net: + +volumes: + # provides data interchange between evm_loader and proxy + deployment_data: + diff --git a/proxy/run-proxy.sh b/proxy/run-proxy.sh index 262ed2e1e..103795d70 100755 --- a/proxy/run-proxy.sh +++ b/proxy/run-proxy.sh @@ -6,8 +6,6 @@ date if [ "$CONFIG" == "ci" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://solana:8899" - [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=deploy - [[ -z "$NEW_USER_AIRDROP_AMOUNT" ]] && export NEW_USER_AIRDROP_AMOUNT=100 [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=100000 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.5" [[ -z "$USE_COMBINED_START_CONTINUE" ]] && export USE_COMBINED_START_CONTINUE="NO" @@ -17,8 +15,6 @@ if [ "$CONFIG" == "ci" ]; then [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=10 elif [ "$CONFIG" == "local" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://localhost:8899" - [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=deploy - [[ -z "$NEW_USER_AIRDROP_AMOUNT" ]] && export NEW_USER_AIRDROP_AMOUNT=10 [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=0 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.9" [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=0 @@ -27,7 +23,6 @@ elif [ "$CONFIG" == "local" ]; then elif [ "$CONFIG" == "devnet" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="https://api.devnet.solana.com" [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU - [[ -z "$NEW_USER_AIRDROP_AMOUNT" ]] && export NEW_USER_AIRDROP_AMOUNT=0 [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=90000 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="10" [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=1 @@ -36,7 +31,6 @@ elif [ "$CONFIG" == "devnet" ]; then elif [ "$CONFIG" == "testnet" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="https://api.testnet.solana.com" [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU - [[ -z "$NEW_USER_AIRDROP_AMOUNT" ]] && export NEW_USER_AIRDROP_AMOUNT=0 [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=90000 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="15" [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE="1" @@ -49,73 +43,8 @@ fi [[ -z "$SOLANA_URL" ]] && echo "SOLANA_URL is not set" && exit 1 [[ -z "$EVM_LOADER" ]] && echo "EVM_LOADER is not set" && exit 1 -echo SOLANA_URL=$SOLANA_URL - solana config set -u $SOLANA_URL -solana config get - -for i in {1..30}; do - if solana cluster-version; then break; fi - sleep 2 -done - - -ADDRESS=$(solana address || echo "no wallet") - -if [ "$ADDRESS" == "no wallet" ]; then - solana-keygen new --no-passphrase -fi - -if ! solana account $(solana address); then - echo "airdropping..." - solana airdrop 1000 - # check that balance >= 10 otherwise airdroping by 1 SOL up to 10 - BALANCE=$(solana balance | tr '.' '\t'| tr '[:space:]' '\t' | cut -f1) - while [ "$BALANCE" -lt 10 ]; do - solana airdrop 1 - sleep 1 - BALANCE=$(solana balance | tr '.' '\t'| tr '[:space:]' '\t' | cut -f1) - done -fi - -solana address -solana balance - - -if [ "$EVM_LOADER" == "deploy" ]; then - echo "EVM_LOADER is set to load. A new Neon-evm will be deployed. deploying evm_loader..." - solana program deploy --upgrade-authority /spl/bin/evm_loader-keypair.json /spl/bin/evm_loader.so > evm_loader_id - export EVM_LOADER=$(cat evm_loader_id | sed '/Program Id: \([0-9A-Za-z]\+\)/,${s//\1/;b};s/^.*$//;$q1') - solana program dump "$EVM_LOADER" ./evm_loader.dump - /spl/bin/neon-cli --evm_loader="$EVM_LOADER" neon-elf-params ./evm_loader.dump -fi - -echo "EVM_LOADER=$EVM_LOADER" - - -echo "A new token will be created. Creating token..." -export ETH_TOKEN_MINT=$(/spl/bin/spl-token create-token --owner /spl/bin/test_token_owner -- /spl/bin/test_token_keypair | grep -Po 'Creating token \K[^\n]*') -echo "ETH_TOKEN_MINT=$ETH_TOKEN_MINT" - - -echo "A new collateral pool accounts will be created. Creating accounts..." -solana -k /spl/bin/collateral-pool-keypair.json airdrop 1000 -python3 /spl/bin/collateral_pool_generator.py /spl/bin/collateral-pool-keypair.json -export COLLATERAL_POOL_BASE=$(solana-keygen pubkey -f /spl/bin/collateral-pool-keypair.json) -echo "COLLATERAL_POOL_BASE=$COLLATERAL_POOL_BASE" - - -if [ "$NEW_USER_AIRDROP_AMOUNT" -gt 0 -a "$(spl-token balance "$ETH_TOKEN_MINT" || echo 0)" -eq 0 ]; then - echo 'Create balance and mint token' - TOKEN_ACCOUNT=$( (spl-token create-account "$ETH_TOKEN_MINT" || true) | grep -Po 'Creating account \K[^\n]*') - echo "TOKEN_ACCOUNT=$TOKEN_ACCOUNT" - spl-token mint "$ETH_TOKEN_MINT" $(("$NEW_USER_AIRDROP_AMOUNT"*1000)) --owner /spl/bin/test_token_owner -- "$TOKEN_ACCOUNT" -fi - -echo "NEW_USER_AIRDROP_AMOUNT=$NEW_USER_AIRDROP_AMOUNT" - - isArg() { case "$1" in "$2"|"$2="*) true;; *) false;; esac } EXTRA_ARGS_TIMEOUT=' --timeout 300' for val in $EXTRA_ARGS; do diff --git a/proxy/run-test-proxy.sh b/proxy/run-test-proxy.sh new file mode 100755 index 000000000..07bb99b49 --- /dev/null +++ b/proxy/run-test-proxy.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +if [ -z "$SOLANA_URL" ]; then + echo "SOLANA_URL is not set" + exit 1 +fi + +solana config set -u $SOLANA_URL + +echo "Dumping evm_loader and extracting ELF parameters" +export EVM_LOADER=$(cat /var/deployment_data/evm_loader_id | sed '/Program Id: \([0-9A-Za-z]\+\)/,${s//\1/;b};s/^.*$//;$q1') +solana program dump "$EVM_LOADER" ./evm_loader.dump +export $(/spl/bin/neon-cli --evm_loader="$EVM_LOADER" neon-elf-params ./evm_loader.dump) + +/spl/bin/create-test-accounts.sh 1 + +if [ "$(spl-token balance "$NEON_TOKEN_MINT" || echo 0)" -eq 0 ]; then + [[ -z "$NEW_USER_AIRDROP_AMOUNT" ]] && export NEW_USER_AIRDROP_AMOUNT=100 + echo "NEW_USER_AIRDROP_AMOUNT=$NEW_USER_AIRDROP_AMOUNT" + echo 'Create balance and mint token' + TOKEN_ACCOUNT=$( (spl-token create-account "$NEON_TOKEN_MINT" || true) | grep -Po 'Creating account \K[^\n]*') + echo "TOKEN_ACCOUNT=$TOKEN_ACCOUNT" + spl-token mint "$NEON_TOKEN_MINT" $(("$NEW_USER_AIRDROP_AMOUNT"*1000)) --owner /var/deployment_data/evm_loader-keypair.json -- "$TOKEN_ACCOUNT" +fi + +proxy/run-proxy.sh