Behavioural test github workflow update

.github/workflows/deploy.yml

@@ -71,16 +71,3 @@ jobs:
           tfe-token: ${{ secrets.TFE_TOKEN }}
           env: "staging"
           operation: "create-or-update"
-  update-artifacts:
-    name: Setup artifacts on environment deployed to
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set workflow artifact file
-        run: |
-          mkdir -p workflow_artifact
-          echo $STACK_NAME > workflow_artifact/stack_name
-      - name: Upload stack name artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: napari_hub_dev_deployment
-          path: workflow_artifact/stack_name

.github/workflows/push-rdev.yml

@@ -71,13 +71,27 @@ jobs:
           tfe-token: ${{ secrets.TFE_TOKEN }}
           env: "dev"
 
-      - name: Create workflow artifact
-        run: | 
-          mkdir -p workflow_artifact
-          if [ ${{ github.event_name == 'delete' }} ]; then echo 'invalid'; else echo $STACK_NAME; fi > workflow_artifact/stack_name
+  # Runs behave tests for backend code
+  bdd-tests:
+    runs-on: ubuntu-latest
+    needs: create-update-dev
+    if: ${{ github.event_name == 'push' }}
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
 
-      - name: Upload stack name artifact
-        uses: actions/upload-artifact@v3
+      - name: Setup Python
+        uses: actions/setup-python@v3
         with:
-          name: napari_hub_dev_deployment
-          path: workflow_artifact/stack_name
+          python-version: '3.9'
+
+      - name: Fetch test prefix
+        run: echo $GITHUB_REF_NAME | tr '[:upper:]' '[:lower:]' | xargs -I {} -n 1 echo PREFIX={} >> $GITHUB_ENV
+
+      - name: Run tests
+        working-directory: backend
+        run: |
+          pip install --upgrade pip
+          pip install -r test-requirements.txt
+          cd bdd_tests && python -m pytest

backend/bdd_tests/scenarios/metric.feature

@@ -3,27 +3,39 @@ Feature: metrics
   Scenario: metrics api for valid plugin
     Given we call metrics api for napari-assistant
     Then response status is 200
-    And it should have 12 entries for timeline
-    And it should have at least one non-zero installs in timeline
-    And it should have non-zero values for stats
+    And it should only have properties usage, maintenance
+    And it should have 12 entries for usage.timeline
+    And it should have at least one non-zero installs in usage.timeline
+    And it should have non-zero values for usage.stats
+    And it should have at least one non-zero commits in maintenance.timeline
+    And it should have non-zero values for maintenance.stats
 
   Scenario: metrics api for invalid plugin
     Given we call metrics api for foo
     Then response status is 200
-    And it should have 12 entries for timeline
-    And it should have all zero installs in timeline
-    And it should have zero values for stats and timelines
+    And it should only have properties usage, maintenance
+    And it should have 12 entries for usage.timeline
+    And it should have all zero installs in usage.timeline
+    And it should have zero values for usage.stats
+    And it should have all zero commits in maintenance.timeline
+    And it should have zero values for maintenance.stats
 
   Scenario: metrics api for valid plugin with limit
     Given we call metrics api for napari-assistant with limit 5
     Then response status is 200
-    And it should have 5 entries for timeline
-    And it should have at least one non-zero installs in timeline
-    And it should have non-zero values for stats
+    And it should only have properties usage, maintenance
+    And it should have 5 entries for usage.timeline
+    And it should have at least one non-zero installs in usage.timeline
+    And it should have non-zero values for usage.stats
+    And it should have at least one non-zero commits in maintenance.timeline
+    And it should have non-zero values for maintenance.stats
 
   Scenario: metrics api for invalid plugin with limit
     Given we call metrics api for foo with limit 15
     Then response status is 200
-    And it should have 15 entries for timeline
-    And it should have all zero installs in timeline
-    And it should have zero values for stats and timelines
+    And it should only have properties usage, maintenance
+    And it should have 15 entries for usage.timeline
+    And it should have all zero installs in usage.timeline
+    And it should have zero values for usage.stats
+    And it should have all zero commits in maintenance.timeline
+    And it should have zero values for maintenance.stats

backend/bdd_tests/test_metric.py

@@ -16,40 +16,80 @@ def call_metrics_with_plugin_name(plugin_name, context):
     call_api(context, f'/metrics/{plugin_name}')
 
 
-@then(parsers.parse('it should have {limit:d} entries for timeline'))
-def verify_timeline_limit(context, limit):
-    activity = context['response'].json()['activity']
-    context['activity'] = activity
-    timeline = activity['timeline']
-    assert len(timeline) == limit, f'actual size {len(timeline)} not equal to {limit}'
-    for i, date in enumerate(_generate_dates(limit)):
-        assert timeline[i]['timestamp'] == int(date) * 1000
+@then(parsers.parse('it should only have properties {properties_str}'))
+def verify_response_properties(context, properties_str):
+    properties = properties_str.split(', ')
+    response = context['response'].json()
+    assert len(response) == len(properties)
+    for property in properties:
+        assert property in response, f'{property} not in response'
+        context[property] = response[property]
+
+
+@then(parsers.parse('it should have {limit:d} entries for usage.timeline'))
+def verify_usage_timeline_limit(context, limit):
+    _validate_timeline(context['usage']['timeline'], limit)
+
+
+@then(parsers.parse('it should have {limit:d} entries for maintenance.timeline'))
+def verify_maintenance_timeline_limit(context, limit):
+    _validate_timeline(context['maintenance']['timeline'], limit)
+
+
+@then(parsers.parse('it should have at least one non-zero installs in usage.timeline'))
+def verify_usage_timeline_has_any_non_zero_installs(context):
+    assert any(item['installs'] > 0 for item in context['usage']['timeline'])
+
 
+@then(parsers.parse('it should have at least one non-zero commits in maintenance.timeline'))
+def verify_maintenance_timeline_has_any_non_zero_installs(context):
+    assert any(item['commits'] > 0 for item in context['maintenance']['timeline'])
 
-@then(parsers.parse('it should have at least one non-zero installs in timeline'))
-def verify_timeline_has_any_non_zero_installs(context):
-    assert any(item['installs'] > 0 for item in context['activity']['timeline'])
 
+@then(parsers.parse('it should have non-zero values for usage.stats'))
+def verify_it_has_non_zero_usage_stats(context):
+    stats = context['usage']['stats']
+    assert stats['installs_in_last_30_days'] >= 0
+    assert stats['total_installs'] > 0
 
-@then(parsers.parse('it should have non-zero values for stats'))
-def verify_it_has_non_zero_activity_stats(context):
-    stats = context['activity']['stats']
-    assert stats['installsInLast30Days'] >= 0
-    assert stats['totalInstalls'] > 0
 
+@then(parsers.parse('it should have non-zero values for maintenance.stats'))
+def verify_it_has_non_zero_maintenance_stats(context):
+    stats = context['maintenance']['stats']
+    assert stats['latest_commit_timestamp'] is not None
+    assert stats['total_commits'] > 0
 
-@then(parsers.parse('it should have all zero installs in timeline'))
-def verify_timeline_has_any_non_zero_installs(context):
-    assert all(item['installs'] == 0 for item in context['activity']['timeline'])
 
+@then(parsers.parse('it should have all zero installs in usage.timeline'))
+def verify_usage_timeline_has_any_non_zero_installs(context):
+    assert all(item['installs'] == 0 for item in context['usage']['timeline'])
 
-@then(parsers.parse('it should have zero values for stats and timelines'))
-def verify_it_has_zero_activity_data(context):
-    stats = context['activity']['stats']
-    assert stats['installsInLast30Days'] == 0
-    assert stats['totalInstalls'] == 0
+
+@then(parsers.parse('it should have all zero commits in maintenance.timeline'))
+def verify_maintenance_timeline_has_any_non_zero_commits(context):
+    assert all(item['commits'] == 0 for item in context['maintenance']['timeline'])
+
+
+@then(parsers.parse('it should have zero values for usage.stats'))
+def verify_it_has_zero_usage_data(context):
+    stats = context['usage']['stats']
+    assert stats['installs_in_last_30_days'] == 0
+    assert stats['total_installs'] == 0
+
+
+@then(parsers.parse('it should have zero values for maintenance.stats'))
+def verify_it_has_zero_usage_data(context):
+    stats = context['maintenance']['stats']
+    assert stats['latest_commit_timestamp'] is None
+    assert stats['total_commits'] == 0
 
 
 def _generate_dates(limit):
     start_date = datetime.combine(datetime.now().replace(day=1), datetime.min.time()).replace(tzinfo=timezone.utc)
     return [(start_date - relativedelta(months=i)).timestamp() for i in range(limit, 0, -1)]
+
+
+def _validate_timeline(timeline, limit):
+    assert len(timeline) == limit, f'actual size {len(timeline)} not equal to {limit}'
+    for i, date in enumerate(_generate_dates(limit)):
+        assert timeline[i]['timestamp'] == int(date) * 1000

backend/test-requirements.txt

@@ -1,3 +1,3 @@
-pytest-bdd=6.1.1
-requests=2.28.2
-python-dateutil=2.8.2
+pytest-bdd==6.1.1
+requests==2.28.2
+python-dateutil==2.8.2