From 0937a62e68fa5f4be5165e873fdfbab4cf0151ca Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 2 Aug 2023 14:28:41 +0530 Subject: [PATCH 1/6] Add option to enhanced package data in API Signed-off-by: Tushar Goel --- packagedb/serializers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packagedb/serializers.py b/packagedb/serializers.py index 2b8a9afd..e744a6d2 100644 --- a/packagedb/serializers.py +++ b/packagedb/serializers.py @@ -161,6 +161,15 @@ class Meta: class PackageAPISerializer(HyperlinkedModelSerializer): + + def to_representation(self, instance): + data = super().to_representation(instance) + request = self.context["request"] + if hasattr(request, "query_params") and request.query_params.get("enhanced", False): + from packagedb.api import get_enhanced_package + data["enhanced_package"] = get_enhanced_package(instance) + return data + dependencies = DependentPackageSerializer(many=True) parties = PartySerializer(many=True) resources = HyperlinkedIdentityField(view_name='api:package-resources', lookup_field='uuid') From b34edf839d469bdca499fd007923a6ef99dd61f7 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 7 Aug 2023 19:50:38 +0530 Subject: [PATCH 2/6] Address review comments Signed-off-by: Tushar Goel --- packagedb/api.py | 6 +++ packagedb/serializers.py | 9 ---- packagedb/tests/test_api.py | 26 +++++++++-- .../package-filter_by_checksums-expected.json | 46 +++++++++++++++++++ 4 files changed, 75 insertions(+), 12 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index 43df7aa3..14150c13 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -522,6 +522,12 @@ def filter_by_checksums(self, request, *args, **kwargs): qs = Package.objects.filter(q) paginated_qs = self.paginate_queryset(qs) + get_enhanced_package_data = request.query_params.get('get_enhanced_package_data', False) + if get_enhanced_package_data: + enhanced_package_data = [get_enhanced_package(package=package) for package in qs] + enhanced_package_data = [package for package in enhanced_package_data if package] + paginated_response = self.get_paginated_response(enhanced_package_data) + return paginated_response serializer = PackageAPISerializer(paginated_qs, many=True, context={'request': request}) return self.get_paginated_response(serializer.data) diff --git a/packagedb/serializers.py b/packagedb/serializers.py index e744a6d2..2b8a9afd 100644 --- a/packagedb/serializers.py +++ b/packagedb/serializers.py @@ -161,15 +161,6 @@ class Meta: class PackageAPISerializer(HyperlinkedModelSerializer): - - def to_representation(self, instance): - data = super().to_representation(instance) - request = self.context["request"] - if hasattr(request, "query_params") and request.query_params.get("enhanced", False): - from packagedb.api import get_enhanced_package - data["enhanced_package"] = get_enhanced_package(instance) - return data - dependencies = DependentPackageSerializer(many=True) parties = PartySerializer(many=True) resources = HyperlinkedIdentityField(view_name='api:package-resources', lookup_field='uuid') diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index 8311e9e4..3418c2bc 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -265,6 +265,23 @@ def setUp(self): self.package3 = Package.objects.create(**self.package_data3) self.package3.refresh_from_db() + self.package_data4= { + 'type': 'jar', + 'namespace': 'sample', + 'name': 'Baz', + 'version': '90.123', + 'qualifiers': '', + 'subpath': '', + 'download_url': 'http://anothersample.com', + 'filename': 'Baz.zip', + 'sha1': 'testsha1-4', + 'md5': 'testmd5-3', + 'size': 100, + 'package_content': 5, + } + self.package4 = Package.objects.create(**self.package_data4) + self.package4.refresh_from_db() + self.test_url = 'http://testserver/api/packages/{}/' self.client = APIClient() @@ -272,7 +289,7 @@ def setUp(self): def test_package_api_list_endpoint(self): response = self.client.get('/api/packages/') self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(3, response.data.get('count')) + self.assertEqual(4, response.data.get('count')) def test_package_api_list_endpoint_filter(self): for key, value in self.package_data.items(): @@ -448,14 +465,17 @@ def test_package_api_filter_by_checksums(self): 'testsha1', 'testsha1-2', 'testsha1-3', + 'testsha1-4', ] data = { 'sha1': sha1s } + enhanced_response = self.client.post('/api/packages/filter_by_checksums/?get_enhanced_package_data=true', data=data) + self.assertEqual(1, len(enhanced_response.data['results'])) response = self.client.post('/api/packages/filter_by_checksums/', data=data) - self.assertEqual(3, response.data['count']) + self.assertEqual(4, response.data['count']) expected = self.get_test_loc('api/package-filter_by_checksums-expected.json') - self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources"], regen=False) + self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources"], regen=True) class PackageApiReindexingTestCase(JsonBasedTesting, TestCase): diff --git a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json index a01c743b..7bd514ec 100644 --- a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json +++ b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json @@ -136,5 +136,51 @@ "datasource_id":null, "file_references":[], "dependencies":[] + }, + { + "filename":"Baz.zip", + "package_sets":[], + "package_content":"binary", + "purl":"pkg:jar/sample/baz@90.123", + "type":"jar", + "namespace":"sample", + "name":"baz", + "version":"90.123", + "qualifiers":"", + "subpath":"", + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://anothersample.com", + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "size":100, + "md5":"testmd5-3", + "sha1":"testsha1-4", + "sha256":null, + "sha512":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "package_uid":"pkg:jar/sample/baz@90.123?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "file_references":[], + "dependencies":[] } ] \ No newline at end of file From d96a71925c8ecf797635b6242d11ccda9b70efe3 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 7 Aug 2023 19:56:10 +0530 Subject: [PATCH 3/6] Change regen to False Signed-off-by: Tushar Goel --- packagedb/tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index 3418c2bc..531d8f2b 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -475,7 +475,7 @@ def test_package_api_filter_by_checksums(self): response = self.client.post('/api/packages/filter_by_checksums/', data=data) self.assertEqual(4, response.data['count']) expected = self.get_test_loc('api/package-filter_by_checksums-expected.json') - self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources"], regen=True) + self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources"], regen=False) class PackageApiReindexingTestCase(JsonBasedTesting, TestCase): From d510cd7d4efaee9cfbfafd501d486c1dc0f4241d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 9 Aug 2023 01:02:39 +0530 Subject: [PATCH 4/6] Change get_enhanced_package logic and add tests Signed-off-by: Tushar Goel --- packagedb/api.py | 26 +- packagedb/tests/test_api.py | 76 +++++- ...cksums-enhanced-package-data-expected.json | 227 ++++++++++++++++++ .../package-filter_by_checksums-expected.json | 49 +++- 4 files changed, 354 insertions(+), 24 deletions(-) create mode 100644 packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json diff --git a/packagedb/api.py b/packagedb/api.py index 14150c13..26dc2cd2 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -522,11 +522,16 @@ def filter_by_checksums(self, request, *args, **kwargs): qs = Package.objects.filter(q) paginated_qs = self.paginate_queryset(qs) - get_enhanced_package_data = request.query_params.get('get_enhanced_package_data', False) - if get_enhanced_package_data: - enhanced_package_data = [get_enhanced_package(package=package) for package in qs] - enhanced_package_data = [package for package in enhanced_package_data if package] - paginated_response = self.get_paginated_response(enhanced_package_data) + enhance_package_data = request.query_params.get('enhance_package_data', False) + if enhance_package_data: + package_data = [] + for package in qs: + enhanced_package = get_enhanced_package(package=package) + if enhanced_package: + package_data.append(enhanced_package) + else: + package_data.append(package.to_dict()) + paginated_response = self.get_paginated_response(package_data) return paginated_response serializer = PackageAPISerializer(paginated_qs, many=True, context={'request': request}) return self.get_paginated_response(serializer.data) @@ -615,16 +620,9 @@ def _get_enhanced_package(package, packages): Return a mapping of package data based on `package` and Packages in `packages`. """ - mixing = False - package_data = {} + package_data = package.to_dict() for peer in packages: - if peer == package: - mixing = True - package_data = package.to_dict() - continue - if not mixing: - continue - if peer.package_content == package.package_content: + if peer.package_content >= package.package_content: # We do not want to mix data with peers of the same package content continue enhanced = False diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index 531d8f2b..3389ec2d 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -277,11 +277,72 @@ def setUp(self): 'sha1': 'testsha1-4', 'md5': 'testmd5-3', 'size': 100, - 'package_content': 5, + 'package_content': PackageContentType.BINARY, } self.package4 = Package.objects.create(**self.package_data4) self.package4.refresh_from_db() + self.package_data5= { + 'type': 'maven', + 'namespace': 'foot', + 'name': 'baz', + 'version': '90.123', + 'qualifiers': 'classifier=source', + 'subpath': '', + 'download_url': 'http://test-maven.com', + 'filename': 'Baz.zip', + 'sha1': 'testsha1-5', + 'md5': 'testmd5-11', + 'size': 100, + 'package_content': PackageContentType.SOURCE_ARCHIVE, + 'declared_license_expression': 'MIT', + } + + self.package5 = Package.objects.create(**self.package_data5) + self.package5.refresh_from_db() + + self.package_data6= { + 'type': 'maven', + 'namespace': 'fooo', + 'name': 'baz', + 'version': '90.123', + 'qualifiers': '', + 'subpath': '', + 'download_url': 'http://test-maven-11.com', + 'filename': 'Baz.zip', + 'sha1': 'testsha1-6', + 'md5': 'testmd5-11', + 'size': 100, + 'package_content': PackageContentType.BINARY, + } + + self.package6 = Package.objects.create(**self.package_data6) + self.package6.refresh_from_db() + + self.package_data7= { + 'type': 'github', + 'namespace': 'glue', + 'name': 'cat', + 'version': '90.123', + 'qualifiers': '', + 'subpath': '', + 'download_url': 'http://test-maven-111.com', + 'filename': 'Baz.zip', + 'sha1': 'testsha1-7', + 'md5': 'testmd5-11', + 'size': 100, + 'copyright': 'BACC', + 'package_content': PackageContentType.SOURCE_REPO, + } + + self.package7 = Package.objects.create(**self.package_data7) + self.package7.refresh_from_db() + + self.packageset_1 = PackageSet.objects.create() + self.packageset_1.packages.add(self.package6) + self.packageset_1.packages.add(self.package5) + self.packageset_1.packages.add(self.package7) + self.test_url = 'http://testserver/api/packages/{}/' self.client = APIClient() @@ -289,7 +350,7 @@ def setUp(self): def test_package_api_list_endpoint(self): response = self.client.get('/api/packages/') self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(4, response.data.get('count')) + self.assertEqual(7, response.data.get('count')) def test_package_api_list_endpoint_filter(self): for key, value in self.package_data.items(): @@ -466,16 +527,19 @@ def test_package_api_filter_by_checksums(self): 'testsha1-2', 'testsha1-3', 'testsha1-4', + 'testsha1-6', ] data = { 'sha1': sha1s } - enhanced_response = self.client.post('/api/packages/filter_by_checksums/?get_enhanced_package_data=true', data=data) - self.assertEqual(1, len(enhanced_response.data['results'])) + enhanced_response = self.client.post('/api/packages/filter_by_checksums/?enhance_package_data=true', data=data) + self.assertEqual(5, len(enhanced_response.data['results'])) + expected = self.get_test_loc('api/package-filter_by_checksums-enhanced-package-data-expected.json') + self.check_expected_results(enhanced_response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets",], regen=True) response = self.client.post('/api/packages/filter_by_checksums/', data=data) - self.assertEqual(4, response.data['count']) + self.assertEqual(5, response.data['count']) expected = self.get_test_loc('api/package-filter_by_checksums-expected.json') - self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources"], regen=False) + self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets",], regen=True) class PackageApiReindexingTestCase(JsonBasedTesting, TestCase): diff --git a/packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json b/packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json new file mode 100644 index 00000000..01a65ab6 --- /dev/null +++ b/packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json @@ -0,0 +1,227 @@ +[ + { + "type":"generic", + "namespace":"generic", + "name":"foo", + "version":"12.34", + "qualifiers":"test_qual=qual", + "subpath":"test_subpath", + "package_content":null, + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://example.com", + "size":101, + "md5":"testmd5", + "sha1":"testsha1", + "sha256":null, + "sha512":null, + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "dependencies":[], + "package_uid":"pkg:generic/generic/foo@12.34?test_qual=qual&uuid=fixed-uid-done-for-testing-5642512d1758#test_subpath", + "datasource_id":null, + "purl":"pkg:generic/generic/foo@12.34?test_qual=qual#test_subpath", + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "file_references":[] + }, + { + "type":"npm", + "namespace":"example", + "name":"bar", + "version":"56.78", + "qualifiers":"", + "subpath":"", + "package_content":null, + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://somethingelse.org", + "size":100, + "md5":"testmd5-2", + "sha1":"testsha1-2", + "sha256":null, + "sha512":null, + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "dependencies":[], + "package_uid":"pkg:npm/example/bar@56.78?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "purl":"pkg:npm/example/bar@56.78", + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "file_references":[] + }, + { + "type":"jar", + "namespace":"sample", + "name":"baz", + "version":"90.12", + "qualifiers":"", + "subpath":"", + "package_content":null, + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://anotherexample.com", + "size":100, + "md5":"testmd5-3", + "sha1":"testsha1-3", + "sha256":null, + "sha512":null, + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "dependencies":[], + "package_uid":"pkg:jar/sample/baz@90.12?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "purl":"pkg:jar/sample/baz@90.12", + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "file_references":[] + }, + { + "type":"jar", + "namespace":"sample", + "name":"baz", + "version":"90.123", + "qualifiers":"", + "subpath":"", + "package_content":"binary", + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://anothersample.com", + "size":100, + "md5":"testmd5-3", + "sha1":"testsha1-4", + "sha256":null, + "sha512":null, + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "dependencies":[], + "package_uid":"pkg:jar/sample/baz@90.123?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "purl":"pkg:jar/sample/baz@90.123", + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "file_references":[] + }, + { + "type":"maven", + "namespace":"fooo", + "name":"baz", + "version":"90.123", + "qualifiers":"", + "subpath":"", + "package_content":"binary", + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://test-maven-11.com", + "size":100, + "md5":"testmd5-11", + "sha1":"testsha1-6", + "sha256":null, + "sha512":null, + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "copyright":"BACC", + "holder":null, + "declared_license_expression":"MIT", + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{ + "enhanced_by":[ + "pkg:github/glue/cat@90.123", + "pkg:maven/foot/baz@90.123?classifier=source" + ] + }, + "dependencies":[], + "package_uid":"pkg:maven/fooo/baz@90.123?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "purl":"pkg:maven/fooo/baz@90.123", + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "file_references":[] + } +] \ No newline at end of file diff --git a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json index 7bd514ec..4fd07d81 100644 --- a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json +++ b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json @@ -1,7 +1,6 @@ [ { "filename":"Foo.zip", - "package_sets":[], "package_content":null, "purl":"pkg:generic/generic/foo@12.34?test_qual=qual#test_subpath", "type":"generic", @@ -47,7 +46,6 @@ }, { "filename":"Bar.zip", - "package_sets":[], "package_content":null, "purl":"pkg:npm/example/bar@56.78", "type":"npm", @@ -93,7 +91,6 @@ }, { "filename":"Baz.zip", - "package_sets":[], "package_content":null, "purl":"pkg:jar/sample/baz@90.12", "type":"jar", @@ -139,7 +136,6 @@ }, { "filename":"Baz.zip", - "package_sets":[], "package_content":"binary", "purl":"pkg:jar/sample/baz@90.123", "type":"jar", @@ -182,5 +178,50 @@ "datasource_id":null, "file_references":[], "dependencies":[] + }, + { + "filename":"Baz.zip", + "package_content":"binary", + "purl":"pkg:maven/fooo/baz@90.123", + "type":"maven", + "namespace":"fooo", + "name":"baz", + "version":"90.123", + "qualifiers":"", + "subpath":"", + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://test-maven-11.com", + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "size":100, + "md5":"testmd5-11", + "sha1":"testsha1-6", + "sha256":null, + "sha512":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "package_uid":"pkg:maven/fooo/baz@90.123?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "file_references":[], + "dependencies":[] } ] \ No newline at end of file From 4815a745e99c66457742babf39a8af45f3aafec6 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 9 Aug 2023 12:19:43 +0530 Subject: [PATCH 5/6] Address review comments Signed-off-by: Tushar Goel --- packagedb/api.py | 6 ++++-- packagedb/models.py | 6 ------ packagedb/tests/test_api.py | 13 +++++++------ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index 26dc2cd2..ce204132 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -503,7 +503,7 @@ def filter_by_checksums(self, request, *args, **kwargs): data = dict(request.data) unsupported_fields = [] for field, value in data.items(): - if field not in ('md5', 'sha1', 'sha256', 'sha512'): + if field not in ('md5', 'sha1', 'sha256', 'sha512', 'enhance_package_data'): unsupported_fields.append(field) if unsupported_fields: @@ -517,12 +517,14 @@ def filter_by_checksums(self, request, *args, **kwargs): for field, value in data.items(): # We create this intermediate dictionary so we can modify the field # name to have __in at the end + if field in ["enhance_package_data"]: + continue d = {f'{field}__in': value} q |= Q(**d) qs = Package.objects.filter(q) paginated_qs = self.paginate_queryset(qs) - enhance_package_data = request.query_params.get('enhance_package_data', False) + enhance_package_data = data.get("enhance_package_data", False) if enhance_package_data: package_data = [] for package in qs: diff --git a/packagedb/models.py b/packagedb/models.py index f4253f10..0f572536 100644 --- a/packagedb/models.py +++ b/packagedb/models.py @@ -1061,11 +1061,5 @@ def add_to_package_set(self, package): def get_package_set_members(self): """Return related Packages""" return self.packages.order_by( - 'type', - 'namespace', - 'name', - 'version', - 'qualifiers', - 'subpath', 'package_content', ) diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index 3389ec2d..e38f8a05 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -530,16 +530,17 @@ def test_package_api_filter_by_checksums(self): 'testsha1-6', ] data = { - 'sha1': sha1s + 'sha1': sha1s, } - enhanced_response = self.client.post('/api/packages/filter_by_checksums/?enhance_package_data=true', data=data) - self.assertEqual(5, len(enhanced_response.data['results'])) - expected = self.get_test_loc('api/package-filter_by_checksums-enhanced-package-data-expected.json') - self.check_expected_results(enhanced_response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets",], regen=True) response = self.client.post('/api/packages/filter_by_checksums/', data=data) self.assertEqual(5, response.data['count']) expected = self.get_test_loc('api/package-filter_by_checksums-expected.json') - self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets",], regen=True) + self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets",], regen=False) + data["enhance_package_data"] = True + enhanced_response = self.client.post('/api/packages/filter_by_checksums/', data=data) + self.assertEqual(5, len(enhanced_response.data['results'])) + expected = self.get_test_loc('api/package-filter_by_checksums-enhanced-package-data-expected.json') + self.check_expected_results(enhanced_response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets",], regen=False) class PackageApiReindexingTestCase(JsonBasedTesting, TestCase): From 5cc124ce544e3d998bd70904aa9415b9883511df Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 9 Aug 2023 13:12:36 -0700 Subject: [PATCH 6/6] Update get_enhanced_package * Return unenhanced package data for packages not in a package set or if they do not have package content * Refactor code in filter_by_checksums() Signed-off-by: Jono Yang --- packagedb/api.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/packagedb/api.py b/packagedb/api.py index ce204132..8ac1d7e1 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -513,30 +513,22 @@ def filter_by_checksums(self, request, *args, **kwargs): } return Response(response_data) + enhance_package_data = data.pop('enhance_package_data', False) q = Q() for field, value in data.items(): # We create this intermediate dictionary so we can modify the field # name to have __in at the end - if field in ["enhance_package_data"]: - continue d = {f'{field}__in': value} q |= Q(**d) qs = Package.objects.filter(q) paginated_qs = self.paginate_queryset(qs) - enhance_package_data = data.get("enhance_package_data", False) if enhance_package_data: - package_data = [] - for package in qs: - enhanced_package = get_enhanced_package(package=package) - if enhanced_package: - package_data.append(enhanced_package) - else: - package_data.append(package.to_dict()) - paginated_response = self.get_paginated_response(package_data) - return paginated_response - serializer = PackageAPISerializer(paginated_qs, many=True, context={'request': request}) - return self.get_paginated_response(serializer.data) + serialized_package_data = [get_enhanced_package(package=package) for package in paginated_qs] + else: + serializer = PackageAPISerializer(paginated_qs, many=True, context={'request': request}) + serialized_package_data = serializer.data + return self.get_paginated_response(serialized_package_data) UPDATEABLE_FIELDS = [ @@ -597,8 +589,16 @@ def get_enhanced_package(package): other packages in the same package_set. """ package_content = package.package_content - if package_content == PackageContentType.SOURCE_REPO: + in_package_sets = package.package_sets.count() > 0 + if ( + not in_package_sets + or not package_content + or package_content == PackageContentType.SOURCE_REPO + ): + # Return unenhanced package data for packages that are not in a package + # set or are source repo packages. # Source repo packages can't really be enhanced much further, datawise + # and we can't enhance a package that is not in a package set. return package.to_dict() if package_content in [PackageContentType.BINARY, PackageContentType.SOURCE_ARCHIVE]: # Binary packages can only be part of one set