From a40c65d6a08627aa2e6be612b3d45a7eacaa1550 Mon Sep 17 00:00:00 2001 From: Manasa Venkatakrishnan Date: Mon, 20 Mar 2023 13:04:17 -0700 Subject: [PATCH 1/6] Adding bdd tests to rdev --- .github/workflows/backend-bdd-tests.yml | 36 ------------------------- .github/workflows/deploy.yml | 13 --------- .github/workflows/push-rdev.yml | 24 +++++++++++++++++ 3 files changed, 24 insertions(+), 49 deletions(-) delete mode 100644 .github/workflows/backend-bdd-tests.yml diff --git a/.github/workflows/backend-bdd-tests.yml b/.github/workflows/backend-bdd-tests.yml deleted file mode 100644 index 784936051..000000000 --- a/.github/workflows/backend-bdd-tests.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Backend BDD Tests -on: - workflow_run: - workflows: [ 'Push Remote Dev', 'Deploy Changes to Cluster' ] - types: - - completed -jobs: - # Runs behave tests for backend code - bdd-tests: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v3 - with: - python-version: '3.9' - - - name: Download artifact - uses: actions/download-artifact@v3 - with: - name: napari_hub_dev_deployment - path: workflow_artifact - - - name: Fetch test prefix - run: echo "PREFIX=$(cat workflow_artifact/stack_name)" >> $GITHUB_ENV - - - name: Run tests - if: ${{ env.PREFIX != 'invalid' }} - working-directory: backend - run: | - pip install --upgrade pip - pip install -r test-requirements.txt - cd bdd_test && python -m pytest diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 0878b0ce8..a96a35492 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -71,16 +71,3 @@ jobs: tfe-token: ${{ secrets.TFE_TOKEN }} env: "staging" operation: "create-or-update" - update-artifacts: - name: Setup artifacts on environment deployed to - runs-on: ubuntu-latest - steps: - - name: Set workflow artifact file - run: | - mkdir -p workflow_artifact - echo $STACK_NAME > workflow_artifact/stack_name - - name: Upload stack name artifact - uses: actions/upload-artifact@v3 - with: - name: napari_hub_dev_deployment - path: workflow_artifact/stack_name diff --git a/.github/workflows/push-rdev.yml b/.github/workflows/push-rdev.yml index aae0e281f..07ac221eb 100644 --- a/.github/workflows/push-rdev.yml +++ b/.github/workflows/push-rdev.yml @@ -81,3 +81,27 @@ jobs: with: name: napari_hub_dev_deployment path: workflow_artifact/stack_name + # Runs behave tests for backend code + bdd-tests: + runs-on: ubuntu-latest + needs: create-update-dev + if: ${{ github.event_name == 'push' }} + + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: '3.9' + + - name: Fetch test prefix + run: echo $GITHUB_REF_NAME | tr '[:upper:]' '[:lower:]' | xargs -I {} -n 1 echo PREFIX={} >> $GITHUB_ENV + + - name: Run tests + working-directory: backend + run: | + pip install --upgrade pip + pip install -r test-requirements.txt + cd bdd_test && python -m pytest From 7b5ccd44593f2d146305e195699c54cfe60b78e6 Mon Sep 17 00:00:00 2001 From: Manasa Venkatakrishnan Date: Tue, 21 Mar 2023 13:04:38 -0700 Subject: [PATCH 2/6] fixing requirements --- backend/test-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/test-requirements.txt b/backend/test-requirements.txt index bf0d8ef31..d72b2d130 100644 --- a/backend/test-requirements.txt +++ b/backend/test-requirements.txt @@ -1,3 +1,3 @@ -pytest-bdd=6.1.1 -requests=2.28.2 -python-dateutil=2.8.2 +pytest-bdd==6.1.1 +requests==2.28.2 +python-dateutil==2.8.2 From 38adc4b684a4997da46297c04d127f3af861c903 Mon Sep 17 00:00:00 2001 From: Manasa Venkatakrishnan Date: Tue, 21 Mar 2023 13:10:22 -0700 Subject: [PATCH 3/6] fixing folder name --- .github/workflows/push-rdev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-rdev.yml b/.github/workflows/push-rdev.yml index 07ac221eb..28b5e8165 100644 --- a/.github/workflows/push-rdev.yml +++ b/.github/workflows/push-rdev.yml @@ -104,4 +104,4 @@ jobs: run: | pip install --upgrade pip pip install -r test-requirements.txt - cd bdd_test && python -m pytest + cd bdd_tests && python -m pytest From 5c24b371f9f06fbfb199bed7e6535af8b76ea2e8 Mon Sep 17 00:00:00 2001 From: Manasa Venkatakrishnan Date: Tue, 21 Mar 2023 14:02:50 -0700 Subject: [PATCH 4/6] Updating metrics tests for new format --- backend/bdd_tests/scenarios/metric.feature | 40 +++++++--- backend/bdd_tests/test_metric.py | 88 ++++++++++++++++------ 2 files changed, 92 insertions(+), 36 deletions(-) diff --git a/backend/bdd_tests/scenarios/metric.feature b/backend/bdd_tests/scenarios/metric.feature index 893fe9ca6..0b064b3e1 100644 --- a/backend/bdd_tests/scenarios/metric.feature +++ b/backend/bdd_tests/scenarios/metric.feature @@ -3,27 +3,43 @@ Feature: metrics Scenario: metrics api for valid plugin Given we call metrics api for napari-assistant Then response status is 200 - And it should have 12 entries for timeline - And it should have at least one non-zero installs in timeline - And it should have non-zero values for stats + And it should only have properties usage, maintenance + And it should have 12 entries for usage.timeline + And it should have at least one non-zero installs in usage.timeline + And it should have non-zero values for usage.stats +# And it should have 10 entries for maintenance.timeline + And it should have at least one non-zero commits in maintenance.timeline + And it should have non-zero values for maintenance.stats Scenario: metrics api for invalid plugin Given we call metrics api for foo Then response status is 200 - And it should have 12 entries for timeline - And it should have all zero installs in timeline - And it should have zero values for stats and timelines + And it should only have properties usage, maintenance + And it should have 12 entries for usage.timeline + And it should have all zero installs in usage.timeline + And it should have zero values for usage.stats +# And it should have 12 entries for maintenance.timeline + And it should have all zero commits in maintenance.timeline + And it should have zero values for maintenance.stats Scenario: metrics api for valid plugin with limit Given we call metrics api for napari-assistant with limit 5 Then response status is 200 - And it should have 5 entries for timeline - And it should have at least one non-zero installs in timeline - And it should have non-zero values for stats + And it should only have properties usage, maintenance + And it should have 5 entries for usage.timeline + And it should have at least one non-zero installs in usage.timeline + And it should have non-zero values for usage.stats +# And it should have 5 entries for maintenance.timeline + And it should have at least one non-zero commits in maintenance.timeline + And it should have non-zero values for maintenance.stats Scenario: metrics api for invalid plugin with limit Given we call metrics api for foo with limit 15 Then response status is 200 - And it should have 15 entries for timeline - And it should have all zero installs in timeline - And it should have zero values for stats and timelines + And it should only have properties usage, maintenance + And it should have 15 entries for usage.timeline + And it should have all zero installs in usage.timeline + And it should have zero values for usage.stats +# And it should have 15 entries for maintenance.timeline + And it should have all zero commits in maintenance.timeline + And it should have zero values for maintenance.stats diff --git a/backend/bdd_tests/test_metric.py b/backend/bdd_tests/test_metric.py index 088f1c86a..800192134 100644 --- a/backend/bdd_tests/test_metric.py +++ b/backend/bdd_tests/test_metric.py @@ -16,40 +16,80 @@ def call_metrics_with_plugin_name(plugin_name, context): call_api(context, f'/metrics/{plugin_name}') -@then(parsers.parse('it should have {limit:d} entries for timeline')) -def verify_timeline_limit(context, limit): - activity = context['response'].json()['activity'] - context['activity'] = activity - timeline = activity['timeline'] - assert len(timeline) == limit, f'actual size {len(timeline)} not equal to {limit}' - for i, date in enumerate(_generate_dates(limit)): - assert timeline[i]['timestamp'] == int(date) * 1000 +@then(parsers.parse('it should only have properties {properties_str}')) +def verify_response_properties(context, properties_str): + properties = properties_str.split(', ') + response = context['response'].json() + assert len(response) == len(properties) + for property in properties: + assert property in response, f'{property} not in response' + context[property] = response[property] + + +@then(parsers.parse('it should have {limit:d} entries for usage.timeline')) +def verify_usage_timeline_limit(context, limit): + _validate_timeline(context['usage']['timeline'], limit) + + +@then(parsers.parse('it should have {limit:d} entries for maintenance.timeline')) +def verify_maintenance_timeline_limit(context, limit): + _validate_timeline(context['maintenance']['timeline'], limit) + + +@then(parsers.parse('it should have at least one non-zero installs in usage.timeline')) +def verify_usage_timeline_has_any_non_zero_installs(context): + assert any(item['installs'] > 0 for item in context['usage']['timeline']) + +@then(parsers.parse('it should have at least one non-zero commits in maintenance.timeline')) +def verify_maintenance_timeline_has_any_non_zero_installs(context): + assert any(item['commits'] > 0 for item in context['maintenance']['timeline']) -@then(parsers.parse('it should have at least one non-zero installs in timeline')) -def verify_timeline_has_any_non_zero_installs(context): - assert any(item['installs'] > 0 for item in context['activity']['timeline']) +@then(parsers.parse('it should have non-zero values for usage.stats')) +def verify_it_has_non_zero_usage_stats(context): + stats = context['usage']['stats'] + assert stats['installs_in_last_30_days'] >= 0 + assert stats['total_installs'] > 0 -@then(parsers.parse('it should have non-zero values for stats')) -def verify_it_has_non_zero_activity_stats(context): - stats = context['activity']['stats'] - assert stats['installsInLast30Days'] >= 0 - assert stats['totalInstalls'] > 0 +@then(parsers.parse('it should have non-zero values for maintenance.stats')) +def verify_it_has_non_zero_maintenance_stats(context): + stats = context['maintenance']['stats'] + assert stats['latest_commit_timestamp'] is not None + assert stats['total_commits'] > 0 -@then(parsers.parse('it should have all zero installs in timeline')) -def verify_timeline_has_any_non_zero_installs(context): - assert all(item['installs'] == 0 for item in context['activity']['timeline']) +@then(parsers.parse('it should have all zero installs in usage.timeline')) +def verify_usage_timeline_has_any_non_zero_installs(context): + assert all(item['installs'] == 0 for item in context['usage']['timeline']) -@then(parsers.parse('it should have zero values for stats and timelines')) -def verify_it_has_zero_activity_data(context): - stats = context['activity']['stats'] - assert stats['installsInLast30Days'] == 0 - assert stats['totalInstalls'] == 0 + +@then(parsers.parse('it should have all zero commits in maintenance.timeline')) +def verify_maintenance_timeline_has_any_non_zero_commits(context): + assert all(item['commits'] == 0 for item in context['maintenance']['timeline']) + + +@then(parsers.parse('it should have zero values for usage.stats')) +def verify_it_has_zero_usage_data(context): + stats = context['usage']['stats'] + assert stats['installs_in_last_30_days'] == 0 + assert stats['total_installs'] == 0 + + +@then(parsers.parse('it should have zero values for maintenance.stats')) +def verify_it_has_zero_usage_data(context): + stats = context['maintenance']['stats'] + assert stats['latest_commit_timestamp'] is None + assert stats['total_commits'] == 0 def _generate_dates(limit): start_date = datetime.combine(datetime.now().replace(day=1), datetime.min.time()).replace(tzinfo=timezone.utc) return [(start_date - relativedelta(months=i)).timestamp() for i in range(limit, 0, -1)] + + +def _validate_timeline(timeline, limit): + assert len(timeline) == limit, f'actual size {len(timeline)} not equal to {limit}' + for i, date in enumerate(_generate_dates(limit)): + assert timeline[i]['timestamp'] == int(date) * 1000 From f79019a8484205d2c6633ad7c4d984418a2903b7 Mon Sep 17 00:00:00 2001 From: Manasa Venkatakrishnan Date: Wed, 22 Mar 2023 13:36:52 -0700 Subject: [PATCH 5/6] Removing artifact upload --- .github/workflows/push-rdev.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/push-rdev.yml b/.github/workflows/push-rdev.yml index 28b5e8165..329ab27dc 100644 --- a/.github/workflows/push-rdev.yml +++ b/.github/workflows/push-rdev.yml @@ -71,16 +71,6 @@ jobs: tfe-token: ${{ secrets.TFE_TOKEN }} env: "dev" - - name: Create workflow artifact - run: | - mkdir -p workflow_artifact - if [ ${{ github.event_name == 'delete' }} ]; then echo 'invalid'; else echo $STACK_NAME; fi > workflow_artifact/stack_name - - - name: Upload stack name artifact - uses: actions/upload-artifact@v3 - with: - name: napari_hub_dev_deployment - path: workflow_artifact/stack_name # Runs behave tests for backend code bdd-tests: runs-on: ubuntu-latest From 0c20dde9843789508a02c19ed762b87fe456f0d2 Mon Sep 17 00:00:00 2001 From: Manasa Venkatakrishnan Date: Wed, 22 Mar 2023 15:20:20 -0700 Subject: [PATCH 6/6] Removing tests for maintenance.timeline --- backend/bdd_tests/scenarios/metric.feature | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/bdd_tests/scenarios/metric.feature b/backend/bdd_tests/scenarios/metric.feature index 0b064b3e1..63a7e2602 100644 --- a/backend/bdd_tests/scenarios/metric.feature +++ b/backend/bdd_tests/scenarios/metric.feature @@ -7,7 +7,6 @@ Feature: metrics And it should have 12 entries for usage.timeline And it should have at least one non-zero installs in usage.timeline And it should have non-zero values for usage.stats -# And it should have 10 entries for maintenance.timeline And it should have at least one non-zero commits in maintenance.timeline And it should have non-zero values for maintenance.stats @@ -18,7 +17,6 @@ Feature: metrics And it should have 12 entries for usage.timeline And it should have all zero installs in usage.timeline And it should have zero values for usage.stats -# And it should have 12 entries for maintenance.timeline And it should have all zero commits in maintenance.timeline And it should have zero values for maintenance.stats @@ -29,7 +27,6 @@ Feature: metrics And it should have 5 entries for usage.timeline And it should have at least one non-zero installs in usage.timeline And it should have non-zero values for usage.stats -# And it should have 5 entries for maintenance.timeline And it should have at least one non-zero commits in maintenance.timeline And it should have non-zero values for maintenance.stats @@ -40,6 +37,5 @@ Feature: metrics And it should have 15 entries for usage.timeline And it should have all zero installs in usage.timeline And it should have zero values for usage.stats -# And it should have 15 entries for maintenance.timeline And it should have all zero commits in maintenance.timeline And it should have zero values for maintenance.stats