diff --git a/.github/workflows/e2e-suite-windows.yml b/.github/workflows/e2e-suite-windows.yml index 0c347f74..c3e030a6 100644 --- a/.github/workflows/e2e-suite-windows.yml +++ b/.github/workflows/e2e-suite-windows.yml @@ -73,7 +73,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: make MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" testint + - run: make MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" test-int env: LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN_2 }} diff --git a/.github/workflows/e2e-suite.yml b/.github/workflows/e2e-suite.yml index e58b1fde..7c73e320 100644 --- a/.github/workflows/e2e-suite.yml +++ b/.github/workflows/e2e-suite.yml @@ -4,14 +4,14 @@ on: workflow_dispatch: inputs: use_minimal_test_account: - description: 'Use minimal test account' + description: 'Indicate whether to use a minimal test account with limited resources for testing. Defaults to "false"' required: false default: 'false' - module: - description: "The module from 'test/integration' to the target to be tested, e.g. 'cli, domains, events, etc'" + test_suite: + description: "Specify test suite to run from the 'tests/integration' directory. Examples: 'cli', 'domains', 'events', etc. If not provided, all suites are executed" required: false run_long_tests: - description: "Select True to run long tests, e.g. database, rebuild, etc" + description: "Select 'True' to include long-running tests (e.g., database provisioning, server rebuilds). Defaults to 'False'" required: false type: choice options: @@ -19,21 +19,21 @@ on: - "False" default: "False" sha: - description: 'The hash value of the commit.' + description: 'Specify commit hash to test. This value is mandatory to ensure the tests run against a specific commit' required: true default: '' pull_request_number: - description: 'The number of the PR. Ensure sha value is provided' + description: 'Specify pull request number associated with the commit. Optional, but recommended when providing a commit hash (sha)' required: false openapi_spec_url: - description: 'URL of the OpenAPI spec to use for the tests' + description: 'Specify URL of the OpenAPI specification file to use for testing. Useful for validating tests against a specific API version or custom specification' required: false default: '' python-version: - description: 'Specify Python version to use' + description: 'Specify the Python version to use for running tests. Leave empty to use the default Python version configured in the environment' required: false run-eol-python-version: - description: 'Run EOL python version?' + description: 'Indicates whether to run tests using an End-of-Life (EOL) Python version. Defaults to "false". Choose "true" to include tests for deprecated Python versions' required: false default: 'false' type: choice @@ -124,24 +124,18 @@ jobs: run: | timestamp=$(date +'%Y%m%d%H%M') report_filename="${timestamp}_cli_test_report.xml" - make testint TEST_ARGS="--junitxml=${report_filename}" MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" + make test-int TEST_ARGS="--junitxml=${report_filename}" TEST_SUITE="${{ inputs.test_suite }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" env: LINODE_CLI_TOKEN: ${{ env.LINODE_CLI_TOKEN }} - - name: Upload test results + - name: Upload Test Report as Artifact if: always() - run: | - filename=$(ls | grep -E '^[0-9]{12}_cli_test_report\.xml$') - python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/add_gha_info_to_xml.py \ - --branch_name "${GITHUB_REF#refs/*/}" \ - --gha_run_id "$GITHUB_RUN_ID" \ - --gha_run_number "$GITHUB_RUN_NUMBER" \ - --xmlfile "${filename}" - sync - python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/xml_to_obj.py "${filename}" - env: - LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }} - LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }} + uses: actions/upload-artifact@v4 + with: + name: test-report-file + if-no-files-found: ignore + path: '*.xml' + retention-days: 1 - name: Update PR Check Run uses: actions/github-script@v7 @@ -237,6 +231,51 @@ jobs: env: LINODE_CLI_TOKEN: ${{ env.LINODE_CLI_TOKEN }} + process-upload-report: + runs-on: ubuntu-latest + needs: [integration_tests] + if: always() && github.repository == 'linode/linode-cli' # Run even if integration tests fail and only on main repository + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: Download test report + uses: actions/download-artifact@v4 + with: + name: test-report-file + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Python dependencies + run: pip3 install requests wheel boto3==1.35.99 + + - name: Set release version env + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + + - name: Add variables and upload test results + if: always() + run: | + filename=$(ls | grep -E '^[0-9]{12}_cli_test_report\.xml$') + python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/add_gha_info_to_xml.py \ + --branch_name "${GITHUB_REF#refs/*/}" \ + --gha_run_id "$GITHUB_RUN_ID" \ + --gha_run_number "$GITHUB_RUN_NUMBER" \ + --xmlfile "${filename}" + sync + python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/xml_to_obj.py "${filename}" + env: + LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }} + LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }} + + notify-slack: runs-on: ubuntu-latest needs: [integration_tests] diff --git a/.github/workflows/nightly-smoke-tests.yml b/.github/workflows/nightly-smoke-tests.yml index 10cbad35..372de28c 100644 --- a/.github/workflows/nightly-smoke-tests.yml +++ b/.github/workflows/nightly-smoke-tests.yml @@ -40,7 +40,7 @@ jobs: - name: Run smoke tests id: smoke_tests run: | - make smoketest + make test-smoke env: LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }} diff --git a/Makefile b/Makefile index 04963b9d..27eb9355 100644 --- a/Makefile +++ b/Makefile @@ -2,15 +2,6 @@ # Makefile for more convenient building of the Linode CLI and its baked content # -# Test-related arguments -MODULE := -TEST_CASE_COMMAND := -TEST_ARGS := - -ifdef TEST_CASE -TEST_CASE_COMMAND = -k $(TEST_CASE) -endif - SPEC_VERSION ?= latest ifndef SPEC override SPEC = $(shell ./resolve_spec_url ${SPEC_VERSION}) @@ -66,8 +57,8 @@ clean: rm -f data-* rm -rf dist linode_cli.egg-info build -.PHONY: testunit -testunit: +.PHONY: test-unit +test-unit: @mkdir -p /tmp/linode/.config @orig_xdg_config_home=$${XDG_CONFIG_HOME:-}; \ export LINODE_CLI_TEST_MODE=1 XDG_CONFIG_HOME=/tmp/linode/.config; \ @@ -76,9 +67,14 @@ testunit: export XDG_CONFIG_HOME=$$orig_xdg_config_home; \ exit $$exit_code -.PHONY: testint -testint: - pytest tests/integration/${MODULE} ${TEST_CASE_COMMAND} ${TEST_ARGS} +# Integration Test Arguments +# TEST_SUITE: Optional, specify a test suite (e.g. domains), Default to run everything if not set +# TEST_CASE: Optional, specify a test case (e.g. 'test_create_a_domain') +# TEST_ARGS: Optional, additional arguments for pytest (e.g. '-v' for verbose mode) + +.PHONY: test-int +test-int: + pytest tests/integration/$(TEST_SUITE) $(if $(TEST_CASE),-k $(TEST_CASE)) $(TEST_ARGS) .PHONY: testall testall: @@ -86,7 +82,7 @@ testall: # Alias for unit; integration tests should be explicit .PHONY: test -test: testunit +test: test-unit .PHONY: black black: @@ -103,6 +99,6 @@ autoflake: .PHONY: format format: black isort autoflake -@PHONEY: smoketest -smoketest: +@PHONEY: test-smoke +test-smoke: pytest -m smoke tests/integration