From aa2091b3209b7c76d3f65eb65cc2e5bef91b0c73 Mon Sep 17 00:00:00 2001 From: Pierre Pirault Date: Tue, 24 Jan 2023 09:55:50 -0500 Subject: [PATCH 1/5] Update Pypi.dependencies after update to package manager api --- app/models/package_manager/base.rb | 5 +- app/models/package_manager/pypi.rb | 6 +- .../pypi_dependencies_requests.yml | 96 +++++++++++++++++++ spec/models/package_manager/pypi_spec.rb | 27 ++++++ 4 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 spec/fixtures/vcr_cassettes/pypi_dependencies_requests.yml diff --git a/app/models/package_manager/base.rb b/app/models/package_manager/base.rb index 5f9be12c3..c56860bc5 100644 --- a/app/models/package_manager/base.rb +++ b/app/models/package_manager/base.rb @@ -235,7 +235,10 @@ def self.save_dependencies(mapped_project, sync_version: :all) deps = begin dependencies(name, db_version.number, mapped_project) - rescue StandardError + rescue StandardError => e + Rails.logger.error( + "Error while trying to get dependencies for #{db_platform}/#{name}@#{db_version.number}: #{e.message}" + ) [] end diff --git a/app/models/package_manager/pypi.rb b/app/models/package_manager/pypi.rb index 18166f41a..4d7d1e9e3 100644 --- a/app/models/package_manager/pypi.rb +++ b/app/models/package_manager/pypi.rb @@ -123,14 +123,14 @@ def self.known_versions(name) &.index_by { |v| v[:number] } || {} end - def self.dependencies(name, version, _mapped_project) + def self.dependencies(name, version, _mapped_project = nil) api_response = get("https://pypi.org/pypi/#{name}/#{version}/json") deps = api_response.dig("info", "requires_dist") - source_info = api_response.dig("releases", version) + source_info = api_response.fetch("urls", []) Rails.logger.warn("Pypi sdist (no deps): #{name}") unless source_info.any? { |rel| rel["packagetype"] == "bdist_wheel" } deps.map do |dep| - name, version = dep.split + name, version = dep.split(/ /, 2) { project_name: name, requirements: version.nil? || version == ";" ? "*" : version.gsub(/\(|\)/, ""), diff --git a/spec/fixtures/vcr_cassettes/pypi_dependencies_requests.yml b/spec/fixtures/vcr_cassettes/pypi_dependencies_requests.yml new file mode 100644 index 000000000..0335f2fdf --- /dev/null +++ b/spec/fixtures/vcr_cassettes/pypi_dependencies_requests.yml @@ -0,0 +1,96 @@ +--- +http_interactions: +- request: + method: get + uri: https://pypi.org/pypi/requests/2.28.2/json + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v0.15.4 + Accept-Encoding: + - gzip,deflate,br + X-Datadog-Trace-Id: + - '2363087858382582356' + X-Datadog-Parent-Id: + - '4006652298076953106' + X-Datadog-Sampling-Priority: + - '1' + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + Access-Control-Allow-Headers: + - Content-Type, If-Match, If-Modified-Since, If-None-Match, If-Unmodified-Since + Access-Control-Allow-Methods: + - GET + Access-Control-Allow-Origin: + - "*" + Access-Control-Expose-Headers: + - X-PyPI-Last-Serial + Access-Control-Max-Age: + - '86400' + Cache-Control: + - max-age=900, public + Content-Security-Policy: + - base-uri 'self'; block-all-mixed-content; connect-src 'self' https://api.github.com/repos/ + fastly-insights.com *.fastly-insights.com *.ethicalads.io https://api.pwnedpasswords.com + https://cdn.jsdelivr.net/npm/mathjax@3.2.2/es5/sre/mathmaps/ https://2p66nmmycsj3.statuspage.io; + default-src 'none'; font-src 'self' fonts.gstatic.com; form-action 'self' + https://checkout.stripe.com; frame-ancestors 'none'; frame-src 'none'; img-src + 'self' https://warehouse-camo.ingress.cmh1.psfhosted.org/ www.google-analytics.com + *.fastly-insights.com *.ethicalads.io; script-src 'self' www.googletagmanager.com + www.google-analytics.com *.fastly-insights.com *.ethicalads.io 'sha256-U3hKDidudIaxBDEzwGJApJgPEf2mWk6cfMWghrAa6i0=' + https://cdn.jsdelivr.net/npm/mathjax@3.2.2/ 'sha256-1CldwzdEg2k1wTmf7s5RWVd7NMXI/7nxxjJM2C4DqII=' + 'sha256-0POaN8stWYQxhzjKS+/eOfbbJ/u4YHO5ZagJvLpMypo='; style-src 'self' fonts.googleapis.com + *.ethicalads.io 'sha256-2YHqZokjiizkHi1Zt+6ar0XJ0OeEy/egBnlm+MDMtrM=' 'sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' + 'sha256-JLEjeN9e5dGsz5475WyRaoA4eQOdNPxDIeUhclnJDCE=' 'sha256-mQyxHEuwZJqpxCw3SLmc4YOySNKXunyu2Oiz1r3/wAE=' + 'sha256-OCf+kv5Asiwp++8PIevKBYSgnNLNUZvxAp4a7wMLuKA=' 'sha256-h5LOiLhk6wiJrGsG5ItM0KimwzWQH/yAcmoJDJL//bY='; + worker-src *.fastly-insights.com + Content-Type: + - application/json + Etag: + - '"uwMvOAUCDFUE81ox51egSA"' + Referrer-Policy: + - origin-when-cross-origin + Server: + - nginx/1.13.9 + X-Pypi-Last-Serial: + - '16406303' + Accept-Ranges: + - bytes + Date: + - Mon, 23 Jan 2023 20:30:04 GMT + X-Served-By: + - cache-iad-kiad7000109-IAD + X-Cache: + - HIT + X-Cache-Hits: + - '7' + X-Timer: + - S1674505804.395177,VS0,VE0 + Vary: + - Accept-Encoding + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Frame-Options: + - deny + X-Xss-Protection: + - 1; mode=block + X-Content-Type-Options: + - nosniff + X-Permitted-Cross-Domain-Policies: + - none + Content-Length: + - '6303' + body: + encoding: ASCII-8BIT + string: !binary |- + eyJpbmZvIjp7ImF1dGhvciI6Iktlbm5ldGggUmVpdHoiLCJhdXRob3JfZW1haWwiOiJtZUBrZW5uZXRocmVpdHoub3JnIiwiYnVndHJhY2tfdXJsIjpudWxsLCJjbGFzc2lmaWVycyI6WyJEZXZlbG9wbWVudCBTdGF0dXMgOjogNSAtIFByb2R1Y3Rpb24vU3RhYmxlIiwiRW52aXJvbm1lbnQgOjogV2ViIEVudmlyb25tZW50IiwiSW50ZW5kZWQgQXVkaWVuY2UgOjogRGV2ZWxvcGVycyIsIkxpY2Vuc2UgOjogT1NJIEFwcHJvdmVkIDo6IEFwYWNoZSBTb2Z0d2FyZSBMaWNlbnNlIiwiTmF0dXJhbCBMYW5ndWFnZSA6OiBFbmdsaXNoIiwiT3BlcmF0aW5nIFN5c3RlbSA6OiBPUyBJbmRlcGVuZGVudCIsIlByb2dyYW1taW5nIExhbmd1YWdlIDo6IFB5dGhvbiIsIlByb2dyYW1taW5nIExhbmd1YWdlIDo6IFB5dGhvbiA6OiAzIiwiUHJvZ3JhbW1pbmcgTGFuZ3VhZ2UgOjogUHl0aG9uIDo6IDMgOjogT25seSIsIlByb2dyYW1taW5nIExhbmd1YWdlIDo6IFB5dGhvbiA6OiAzLjEwIiwiUHJvZ3JhbW1pbmcgTGFuZ3VhZ2UgOjogUHl0aG9uIDo6IDMuMTEiLCJQcm9ncmFtbWluZyBMYW5ndWFnZSA6OiBQeXRob24gOjogMy43IiwiUHJvZ3JhbW1pbmcgTGFuZ3VhZ2UgOjogUHl0aG9uIDo6IDMuOCIsIlByb2dyYW1taW5nIExhbmd1YWdlIDo6IFB5dGhvbiA6OiAzLjkiLCJQcm9ncmFtbWluZyBMYW5ndWFnZSA6OiBQeXRob24gOjogSW1wbGVtZW50YXRpb24gOjogQ1B5dGhvbiIsIlByb2dyYW1taW5nIExhbmd1YWdlIDo6IFB5dGhvbiA6OiBJbXBsZW1lbnRhdGlvbiA6OiBQeVB5IiwiVG9waWMgOjogSW50ZXJuZXQgOjogV1dXL0hUVFAiLCJUb3BpYyA6OiBTb2Z0d2FyZSBEZXZlbG9wbWVudCA6OiBMaWJyYXJpZXMiXSwiZGVzY3JpcHRpb24iOiIjIFJlcXVlc3RzXG5cbioqUmVxdWVzdHMqKiBpcyBhIHNpbXBsZSwgeWV0IGVsZWdhbnQsIEhUVFAgbGlicmFyeS5cblxuYGBgcHl0aG9uXG4+Pj4gaW1wb3J0IHJlcXVlc3RzXG4+Pj4gciA9IHJlcXVlc3RzLmdldCgnaHR0cHM6Ly9odHRwYmluLm9yZy9iYXNpYy1hdXRoL3VzZXIvcGFzcycsIGF1dGg9KCd1c2VyJywgJ3Bhc3MnKSlcbj4+PiByLnN0YXR1c19jb2RlXG4yMDBcbj4+PiByLmhlYWRlcnNbJ2NvbnRlbnQtdHlwZSddXG4nYXBwbGljYXRpb24vanNvbjsgY2hhcnNldD11dGY4J1xuPj4+IHIuZW5jb2Rpbmdcbid1dGYtOCdcbj4+PiByLnRleHRcbid7XCJhdXRoZW50aWNhdGVkXCI6IHRydWUsIC4uLidcbj4+PiByLmpzb24oKVxueydhdXRoZW50aWNhdGVkJzogVHJ1ZSwgLi4ufVxuYGBgXG5cblJlcXVlc3RzIGFsbG93cyB5b3UgdG8gc2VuZCBIVFRQLzEuMSByZXF1ZXN0cyBleHRyZW1lbHkgZWFzaWx5LiBUaGVyZeKAmXMgbm8gbmVlZCB0byBtYW51YWxseSBhZGQgcXVlcnkgc3RyaW5ncyB0byB5b3VyIFVSTHMsIG9yIHRvIGZvcm0tZW5jb2RlIHlvdXIgYFBVVGAgJiBgUE9TVGAgZGF0YSDigJQgYnV0IG5vd2FkYXlzLCBqdXN0IHVzZSB0aGUgYGpzb25gIG1ldGhvZCFcblxuUmVxdWVzdHMgaXMgb25lIG9mIHRoZSBtb3N0IGRvd25sb2FkZWQgUHl0aG9uIHBhY2thZ2VzIHRvZGF5LCBwdWxsaW5nIGluIGFyb3VuZCBgMzBNIGRvd25sb2FkcyAvIHdlZWtg4oCUIGFjY29yZGluZyB0byBHaXRIdWIsIFJlcXVlc3RzIGlzIGN1cnJlbnRseSBbZGVwZW5kZWQgdXBvbl0oaHR0cHM6Ly9naXRodWIuY29tL3BzZi9yZXF1ZXN0cy9uZXR3b3JrL2RlcGVuZGVudHM/cGFja2FnZV9pZD1VR0ZqYTJGblpTMDFOekE0T1RFeE5nJTNEJTNEKSBieSBgMSwwMDAsMDAwK2AgcmVwb3NpdG9yaWVzLiBZb3UgbWF5IGNlcnRhaW5seSBwdXQgeW91ciB0cnVzdCBpbiB0aGlzIGNvZGUuXG5cblshW0Rvd25sb2Fkc10oaHR0cHM6Ly9wZXB5LnRlY2gvYmFkZ2UvcmVxdWVzdHMvbW9udGgpXShodHRwczovL3BlcHkudGVjaC9wcm9qZWN0L3JlcXVlc3RzKVxuWyFbU3VwcG9ydGVkIFZlcnNpb25zXShodHRwczovL2ltZy5zaGllbGRzLmlvL3B5cGkvcHl2ZXJzaW9ucy9yZXF1ZXN0cy5zdmcpXShodHRwczovL3B5cGkub3JnL3Byb2plY3QvcmVxdWVzdHMpXG5bIVtDb250cmlidXRvcnNdKGh0dHBzOi8vaW1nLnNoaWVsZHMuaW8vZ2l0aHViL2NvbnRyaWJ1dG9ycy9wc2YvcmVxdWVzdHMuc3ZnKV0oaHR0cHM6Ly9naXRodWIuY29tL3BzZi9yZXF1ZXN0cy9ncmFwaHMvY29udHJpYnV0b3JzKVxuXG4jIyBJbnN0YWxsaW5nIFJlcXVlc3RzIGFuZCBTdXBwb3J0ZWQgVmVyc2lvbnNcblxuUmVxdWVzdHMgaXMgYXZhaWxhYmxlIG9uIFB5UEk6XG5cbmBgYGNvbnNvbGVcbiQgcHl0aG9uIC1tIHBpcCBpbnN0YWxsIHJlcXVlc3RzXG5gYGBcblxuUmVxdWVzdHMgb2ZmaWNpYWxseSBzdXBwb3J0cyBQeXRob24gMy43Ky5cblxuIyMgU3VwcG9ydGVkIEZlYXR1cmVzICYgQmVzdOKAk1ByYWN0aWNlc1xuXG5SZXF1ZXN0cyBpcyByZWFkeSBmb3IgdGhlIGRlbWFuZHMgb2YgYnVpbGRpbmcgcm9idXN0IGFuZCByZWxpYWJsZSBIVFRQ4oCTc3BlYWtpbmcgYXBwbGljYXRpb25zLCBmb3IgdGhlIG5lZWRzIG9mIHRvZGF5LlxuXG4tIEtlZXAtQWxpdmUgJiBDb25uZWN0aW9uIFBvb2xpbmdcbi0gSW50ZXJuYXRpb25hbCBEb21haW5zIGFuZCBVUkxzXG4tIFNlc3Npb25zIHdpdGggQ29va2llIFBlcnNpc3RlbmNlXG4tIEJyb3dzZXItc3R5bGUgVExTL1NTTCBWZXJpZmljYXRpb25cbi0gQmFzaWMgJiBEaWdlc3QgQXV0aGVudGljYXRpb25cbi0gRmFtaWxpYXIgYGRpY3Rg4oCTbGlrZSBDb29raWVzXG4tIEF1dG9tYXRpYyBDb250ZW50IERlY29tcHJlc3Npb24gYW5kIERlY29kaW5nXG4tIE11bHRpLXBhcnQgRmlsZSBVcGxvYWRzXG4tIFNPQ0tTIFByb3h5IFN1cHBvcnRcbi0gQ29ubmVjdGlvbiBUaW1lb3V0c1xuLSBTdHJlYW1pbmcgRG93bmxvYWRzXG4tIEF1dG9tYXRpYyBob25vcmluZyBvZiBgLm5ldHJjYFxuLSBDaHVua2VkIEhUVFAgUmVxdWVzdHNcblxuIyMgQVBJIFJlZmVyZW5jZSBhbmQgVXNlciBHdWlkZSBhdmFpbGFibGUgb24gW1JlYWQgdGhlIERvY3NdKGh0dHBzOi8vcmVxdWVzdHMucmVhZHRoZWRvY3MuaW8pXG5cblshW1JlYWQgdGhlIERvY3NdKGh0dHBzOi8vcmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbS9wc2YvcmVxdWVzdHMvbWFpbi9leHQvc3MucG5nKV0oaHR0cHM6Ly9yZXF1ZXN0cy5yZWFkdGhlZG9jcy5pbylcblxuIyMgQ2xvbmluZyB0aGUgcmVwb3NpdG9yeVxuXG5XaGVuIGNsb25pbmcgdGhlIFJlcXVlc3RzIHJlcG9zaXRvcnksIHlvdSBtYXkgbmVlZCB0byBhZGQgdGhlIGAtY1xuZmV0Y2guZnNjay5iYWRUaW1lem9uZT1pZ25vcmVgIGZsYWcgdG8gYXZvaWQgYW4gZXJyb3IgYWJvdXQgYSBiYWQgY29tbWl0IChzZWVcblt0aGlzIGlzc3VlXShodHRwczovL2dpdGh1Yi5jb20vcHNmL3JlcXVlc3RzL2lzc3Vlcy8yNjkwKSBmb3IgbW9yZSBiYWNrZ3JvdW5kKTpcblxuYGBgc2hlbGxcbmdpdCBjbG9uZSAtYyBmZXRjaC5mc2NrLmJhZFRpbWV6b25lPWlnbm9yZSBodHRwczovL2dpdGh1Yi5jb20vcHNmL3JlcXVlc3RzLmdpdFxuYGBgXG5cbllvdSBjYW4gYWxzbyBhcHBseSB0aGlzIHNldHRpbmcgdG8geW91ciBnbG9iYWwgR2l0IGNvbmZpZzpcblxuYGBgc2hlbGxcbmdpdCBjb25maWcgLS1nbG9iYWwgZmV0Y2guZnNjay5iYWRUaW1lem9uZSBpZ25vcmVcbmBgYFxuXG4tLS1cblxuWyFbS2VubmV0aCBSZWl0el0oaHR0cHM6Ly9yYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tL3BzZi9yZXF1ZXN0cy9tYWluL2V4dC9rci5wbmcpXShodHRwczovL2tlbm5ldGhyZWl0ei5vcmcpIFshW1B5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uXShodHRwczovL3Jhdy5naXRodWJ1c2VyY29udGVudC5jb20vcHNmL3JlcXVlc3RzL21haW4vZXh0L3BzZi5wbmcpXShodHRwczovL3d3dy5weXRob24ub3JnL3BzZilcblxuXG4iLCJkZXNjcmlwdGlvbl9jb250ZW50X3R5cGUiOiJ0ZXh0L21hcmtkb3duIiwiZG9jc191cmwiOm51bGwsImRvd25sb2FkX3VybCI6IiIsImRvd25sb2FkcyI6eyJsYXN0X2RheSI6LTEsImxhc3RfbW9udGgiOi0xLCJsYXN0X3dlZWsiOi0xfSwiaG9tZV9wYWdlIjoiaHR0cHM6Ly9yZXF1ZXN0cy5yZWFkdGhlZG9jcy5pbyIsImtleXdvcmRzIjoiIiwibGljZW5zZSI6IkFwYWNoZSAyLjAiLCJtYWludGFpbmVyIjoiIiwibWFpbnRhaW5lcl9lbWFpbCI6IiIsIm5hbWUiOiJyZXF1ZXN0cyIsInBhY2thZ2VfdXJsIjoiaHR0cHM6Ly9weXBpLm9yZy9wcm9qZWN0L3JlcXVlc3RzLyIsInBsYXRmb3JtIjpudWxsLCJwcm9qZWN0X3VybCI6Imh0dHBzOi8vcHlwaS5vcmcvcHJvamVjdC9yZXF1ZXN0cy8iLCJwcm9qZWN0X3VybHMiOnsiRG9jdW1lbnRhdGlvbiI6Imh0dHBzOi8vcmVxdWVzdHMucmVhZHRoZWRvY3MuaW8iLCJIb21lcGFnZSI6Imh0dHBzOi8vcmVxdWVzdHMucmVhZHRoZWRvY3MuaW8iLCJTb3VyY2UiOiJodHRwczovL2dpdGh1Yi5jb20vcHNmL3JlcXVlc3RzIn0sInJlbGVhc2VfdXJsIjoiaHR0cHM6Ly9weXBpLm9yZy9wcm9qZWN0L3JlcXVlc3RzLzIuMjguMi8iLCJyZXF1aXJlc19kaXN0IjpbImNoYXJzZXQtbm9ybWFsaXplciAoPDQsPj0yKSIsImlkbmEgKDw0LD49Mi41KSIsInVybGxpYjMgKDwxLjI3LD49MS4yMS4xKSIsImNlcnRpZmkgKD49MjAxNy40LjE3KSIsIlB5U29ja3MgKCE9MS41LjcsPj0xLjUuNikgOyBleHRyYSA9PSAnc29ja3MnIiwiY2hhcmRldCAoPDYsPj0zLjAuMikgOyBleHRyYSA9PSAndXNlX2NoYXJkZXRfb25fcHkzJyJdLCJyZXF1aXJlc19weXRob24iOiI+PTMuNywgPDQiLCJzdW1tYXJ5IjoiUHl0aG9uIEhUVFAgZm9yIEh1bWFucy4iLCJ2ZXJzaW9uIjoiMi4yOC4yIiwieWFua2VkIjpmYWxzZSwieWFua2VkX3JlYXNvbiI6bnVsbH0sImxhc3Rfc2VyaWFsIjoxNjQwNjMwMywidXJscyI6W3siY29tbWVudF90ZXh0IjoiIiwiZGlnZXN0cyI6eyJibGFrZTJiXzI1NiI6ImQyZjQyNzRkMWRiZTk2YjQxY2Y0ZTBlZmI3MGNiY2VkMjc4ZmZkNjFiNWM3YmI3MDMzOGI2MmFmOTRjY2IyNWIiLCJtZDUiOiJlODkyZmJmMDNjYzNkNTY2Y2U1ODk3MGIyYzk0MzA3MCIsInNoYTI1NiI6IjY0Mjk5ZjQ5MDkyMjNkYTc0NzYyMmMwMzBiNzgxYzBkNzgxMWUzNTljMzcxMjRiNGJkMzY4ZmI4YzY1MThiYWEifSwiZG93bmxvYWRzIjotMSwiZmlsZW5hbWUiOiJyZXF1ZXN0cy0yLjI4LjItcHkzLW5vbmUtYW55LndobCIsImhhc19zaWciOmZhbHNlLCJtZDVfZGlnZXN0IjoiZTg5MmZiZjAzY2MzZDU2NmNlNTg5NzBiMmM5NDMwNzAiLCJwYWNrYWdldHlwZSI6ImJkaXN0X3doZWVsIiwicHl0aG9uX3ZlcnNpb24iOiJweTMiLCJyZXF1aXJlc19weXRob24iOiI+PTMuNywgPDQiLCJzaXplIjo2MjgyMiwidXBsb2FkX3RpbWUiOiIyMDIzLTAxLTEyVDE2OjI0OjUyIiwidXBsb2FkX3RpbWVfaXNvXzg2MDEiOiIyMDIzLTAxLTEyVDE2OjI0OjUyLjI0MTI1OVoiLCJ1cmwiOiJodHRwczovL2ZpbGVzLnB5dGhvbmhvc3RlZC5vcmcvcGFja2FnZXMvZDIvZjQvMjc0ZDFkYmU5NmI0MWNmNGUwZWZiNzBjYmNlZDI3OGZmZDYxYjVjN2JiNzAzMzhiNjJhZjk0Y2NiMjViL3JlcXVlc3RzLTIuMjguMi1weTMtbm9uZS1hbnkud2hsIiwieWFua2VkIjpmYWxzZSwieWFua2VkX3JlYXNvbiI6bnVsbH0seyJjb21tZW50X3RleHQiOiIiLCJkaWdlc3RzIjp7ImJsYWtlMmJfMjU2IjoiOWRlZTM5MTA3NmY1OTM3ZjBhOGNkZjVlNTNiNzAxZmZjOTE3NTNlODdiMDdkNjZiYWU0YTA5YWE2NzE4OTdiZiIsIm1kNSI6IjA5Yjc1MmUwYjBhNjcyZDgwNWFlNTQ0NTVjMTI4ZDQyIiwic2hhMjU2IjoiOThiMWIyNzgyZTNjNmM0OTA0OTM4Yjg0YzBlYjkzMjcyMTA2OWRmZGI5MTM0MzEzYmVmZjdjODNjMmRmMjRiZiJ9LCJkb3dubG9hZHMiOi0xLCJmaWxlbmFtZSI6InJlcXVlc3RzLTIuMjguMi50YXIuZ3oiLCJoYXNfc2lnIjpmYWxzZSwibWQ1X2RpZ2VzdCI6IjA5Yjc1MmUwYjBhNjcyZDgwNWFlNTQ0NTVjMTI4ZDQyIiwicGFja2FnZXR5cGUiOiJzZGlzdCIsInB5dGhvbl92ZXJzaW9uIjoic291cmNlIiwicmVxdWlyZXNfcHl0aG9uIjoiPj0zLjcsIDw0Iiwic2l6ZSI6MTA4MjA2LCJ1cGxvYWRfdGltZSI6IjIwMjMtMDEtMTJUMTY6MjQ6NTQiLCJ1cGxvYWRfdGltZV9pc29fODYwMSI6IjIwMjMtMDEtMTJUMTY6MjQ6NTQuNjg3NzcwWiIsInVybCI6Imh0dHBzOi8vZmlsZXMucHl0aG9uaG9zdGVkLm9yZy9wYWNrYWdlcy85ZC9lZS8zOTEwNzZmNTkzN2YwYThjZGY1ZTUzYjcwMWZmYzkxNzUzZTg3YjA3ZDY2YmFlNGEwOWFhNjcxODk3YmYvcmVxdWVzdHMtMi4yOC4yLnRhci5neiIsInlhbmtlZCI6ZmFsc2UsInlhbmtlZF9yZWFzb24iOm51bGx9XSwidnVsbmVyYWJpbGl0aWVzIjpbXX0K + http_version: + recorded_at: Mon, 23 Jan 2023 20:30:04 GMT +recorded_with: VCR 4.0.0 diff --git a/spec/models/package_manager/pypi_spec.rb b/spec/models/package_manager/pypi_spec.rb index 17359cf17..044e89acc 100644 --- a/spec/models/package_manager/pypi_spec.rb +++ b/spec/models/package_manager/pypi_spec.rb @@ -119,4 +119,31 @@ expect(described_class.deprecation_info('foo')).to eq({is_deprecated: true, message: "Development Status :: 7 - Inactive"}) end end + + describe ".dependencies" do + it "returns the dependencies of a particular version" do + VCR.use_cassette("pypi_dependencies_requests", record: :once) do + expect( + described_class.dependencies("requests", "2.28.2") + ).to match_array( + [ + ["charset-normalizer", "<4,>=2"], + ["idna", "<4,>=2.5"], + ["urllib3", "<1.27,>=1.21.1"], + ["certifi", ">=2017.4.17"], + ["PySocks", "!=1.5.7,>=1.5.6 ; extra == 'socks'"], + ["chardet", "<6,>=3.0.2 ; extra == 'use_chardet_on_py3'"] + ].map do |name, requirements| + { + project_name: name, + requirements: requirements, + kind: "runtime", + optional: false, + platform: "Pypi" + } + end + ) + end + end + end end From 15ac575438317a883430b7e59291dc8fdfae3b4c Mon Sep 17 00:00:00 2001 From: Pierre Pirault Date: Tue, 24 Jan 2023 20:16:00 -0500 Subject: [PATCH 2/5] Implement smarter parsing logic according to PEP508 specification --- app/models/package_manager/pypi.rb | 12 +++++- spec/models/package_manager/pypi_spec.rb | 53 ++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/app/models/package_manager/pypi.rb b/app/models/package_manager/pypi.rb index 4d7d1e9e3..fc6dac172 100644 --- a/app/models/package_manager/pypi.rb +++ b/app/models/package_manager/pypi.rb @@ -11,6 +11,8 @@ class Pypi < Base ENTIRE_PACKAGE_CAN_BE_DEPRECATED = true SUPPORTS_SINGLE_VERSION_UPDATE = true PYPI_PRERELEASE = /(a|b|rc|dev)[0-9]+$/.freeze + # Adapted from https://peps.python.org/pep-0508/#names + PYPI_508_NAME_REGEX = /^([A-Z0-9][A-Z0-9._-]*[A-Z0-9])/i def self.package_link(db_project, version = nil) # NB PEP 503: "All URLs which respond with an HTML5 page MUST end with a / and the repository SHOULD redirect the URLs without a / to add a / to the end." @@ -123,6 +125,14 @@ def self.known_versions(name) &.index_by { |v| v[:number] } || {} end + # Simply parses out the name of a PEP 508 Dependency specification: https://peps.python.org/pep-0508/ + # Leaves the rest as-is with any leading semicolons or spaces stripped + def self.parse_pep_508_dep_spec(dep) + name, requirement = dep.split(PYPI_508_NAME_REGEX, 2).last(2).map(&:strip) + requirement = requirement&.sub(/^[\s;]*/, "") || "" + return name, requirement + end + def self.dependencies(name, version, _mapped_project = nil) api_response = get("https://pypi.org/pypi/#{name}/#{version}/json") deps = api_response.dig("info", "requires_dist") @@ -130,7 +140,7 @@ def self.dependencies(name, version, _mapped_project = nil) Rails.logger.warn("Pypi sdist (no deps): #{name}") unless source_info.any? { |rel| rel["packagetype"] == "bdist_wheel" } deps.map do |dep| - name, version = dep.split(/ /, 2) + name, version = parse_pep_508_dep_spec(dep) { project_name: name, requirements: version.nil? || version == ";" ? "*" : version.gsub(/\(|\)/, ""), diff --git a/spec/models/package_manager/pypi_spec.rb b/spec/models/package_manager/pypi_spec.rb index 044e89acc..9b6caaf1b 100644 --- a/spec/models/package_manager/pypi_spec.rb +++ b/spec/models/package_manager/pypi_spec.rb @@ -145,5 +145,58 @@ ) end end + + # Copied from the tests of https://peps.python.org/pep-0508/#complete-grammar + [ + ["A", "A", ""], + ["A.B-C_D", "A.B-C_D", ""], + ["aa", "aa", ""], + ["name", "name", ""], + ["name<=1", "name", "<=1"], + ["name>=3", "name", ">=3"], + ["name>=3,<2", "name", ">=3,<2"], + ["name@http://foo.com", "name", "@http://foo.com"], + [ + "name [fred,bar] @ http://foo.com ; python_version=='2.7'", + "name", + "[fred,bar] @ http://foo.com ; python_version=='2.7'" + ], + [ + "name[quux, strange];python_version<'2.7' and platform_version=='2'", + "name", + "[quux, strange];python_version<'2.7' and platform_version=='2'" + ], + [ + "name; os_name=='a' or os_name=='b'", + "name", + "os_name=='a' or os_name=='b'" + ], + [ + "name; os_name=='a' and os_name=='b' or os_name=='c'", + "name", + "os_name=='a' and os_name=='b' or os_name=='c'" + ], + [ + "name; os_name=='a' and (os_name=='b' or os_name=='c')", + "name", + "os_name=='a' and (os_name=='b' or os_name=='c')" + ], + [ + "name; os_name=='a' or os_name=='b' and os_name=='c'", + "name", + "os_name=='a' or os_name=='b' and os_name=='c'" + ], + [ + "name; (os_name=='a' or os_name=='b') and os_name=='c'", + "name", + "(os_name=='a' or os_name=='b') and os_name=='c'" + ], + ].each do |test, expected_name, expected_requirement| + it "#{test} should be parsed correctly" do + expect( + PackageManager::Pypi.parse_pep_508_dep_spec(test) + ).to eq([expected_name, expected_requirement]) + end + end end end From ef5746ac8f170c55ae74477f4a1ab9a9701cb79e Mon Sep 17 00:00:00 2001 From: Pierre Pirault Date: Tue, 24 Jan 2023 20:19:40 -0500 Subject: [PATCH 3/5] Fix name of constant --- app/models/package_manager/pypi.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/package_manager/pypi.rb b/app/models/package_manager/pypi.rb index fc6dac172..761b036cb 100644 --- a/app/models/package_manager/pypi.rb +++ b/app/models/package_manager/pypi.rb @@ -12,7 +12,7 @@ class Pypi < Base SUPPORTS_SINGLE_VERSION_UPDATE = true PYPI_PRERELEASE = /(a|b|rc|dev)[0-9]+$/.freeze # Adapted from https://peps.python.org/pep-0508/#names - PYPI_508_NAME_REGEX = /^([A-Z0-9][A-Z0-9._-]*[A-Z0-9])/i + PEP_508_NAME_REGEX = /^([A-Z0-9][A-Z0-9._-]*[A-Z0-9])/i def self.package_link(db_project, version = nil) # NB PEP 503: "All URLs which respond with an HTML5 page MUST end with a / and the repository SHOULD redirect the URLs without a / to add a / to the end." @@ -128,7 +128,7 @@ def self.known_versions(name) # Simply parses out the name of a PEP 508 Dependency specification: https://peps.python.org/pep-0508/ # Leaves the rest as-is with any leading semicolons or spaces stripped def self.parse_pep_508_dep_spec(dep) - name, requirement = dep.split(PYPI_508_NAME_REGEX, 2).last(2).map(&:strip) + name, requirement = dep.split(PEP_508_NAME_REGEX, 2).last(2).map(&:strip) requirement = requirement&.sub(/^[\s;]*/, "") || "" return name, requirement end From afb53496a77a1893f2b6120ba5d63a4ee004e4d9 Mon Sep 17 00:00:00 2001 From: Pierre Pirault Date: Wed, 25 Jan 2023 16:32:25 -0500 Subject: [PATCH 4/5] Fixes to dependency specification name regex and variable renames --- app/models/package_manager/pypi.rb | 8 ++++---- spec/models/package_manager/pypi_spec.rb | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/app/models/package_manager/pypi.rb b/app/models/package_manager/pypi.rb index 761b036cb..a681a5f0b 100644 --- a/app/models/package_manager/pypi.rb +++ b/app/models/package_manager/pypi.rb @@ -12,7 +12,7 @@ class Pypi < Base SUPPORTS_SINGLE_VERSION_UPDATE = true PYPI_PRERELEASE = /(a|b|rc|dev)[0-9]+$/.freeze # Adapted from https://peps.python.org/pep-0508/#names - PEP_508_NAME_REGEX = /^([A-Z0-9][A-Z0-9._-]*[A-Z0-9])/i + PEP_508_NAME_REGEX = /^([A-Z0-9][A-Z0-9._-]*[A-Z0-9]|[A-Z0-9])/i def self.package_link(db_project, version = nil) # NB PEP 503: "All URLs which respond with an HTML5 page MUST end with a / and the repository SHOULD redirect the URLs without a / to add a / to the end." @@ -140,10 +140,10 @@ def self.dependencies(name, version, _mapped_project = nil) Rails.logger.warn("Pypi sdist (no deps): #{name}") unless source_info.any? { |rel| rel["packagetype"] == "bdist_wheel" } deps.map do |dep| - name, version = parse_pep_508_dep_spec(dep) + dep_name, requirements = parse_pep_508_dep_spec(dep) { - project_name: name, - requirements: version.nil? || version == ";" ? "*" : version.gsub(/\(|\)/, ""), + project_name: dep_name, + requirements: requirements.blank? ? "*" : requirements, kind: "runtime", optional: false, platform: self.name.demodulize, diff --git a/spec/models/package_manager/pypi_spec.rb b/spec/models/package_manager/pypi_spec.rb index 9b6caaf1b..05a6dca47 100644 --- a/spec/models/package_manager/pypi_spec.rb +++ b/spec/models/package_manager/pypi_spec.rb @@ -127,12 +127,12 @@ described_class.dependencies("requests", "2.28.2") ).to match_array( [ - ["charset-normalizer", "<4,>=2"], - ["idna", "<4,>=2.5"], - ["urllib3", "<1.27,>=1.21.1"], - ["certifi", ">=2017.4.17"], - ["PySocks", "!=1.5.7,>=1.5.6 ; extra == 'socks'"], - ["chardet", "<6,>=3.0.2 ; extra == 'use_chardet_on_py3'"] + ["charset-normalizer", "(<4,>=2)"], + ["idna", "(<4,>=2.5)"], + ["urllib3", "(<1.27,>=1.21.1)"], + ["certifi", "(>=2017.4.17)"], + ["PySocks", "(!=1.5.7,>=1.5.6) ; extra == 'socks'"], + ["chardet", "(<6,>=3.0.2) ; extra == 'use_chardet_on_py3'"] ].map do |name, requirements| { project_name: name, @@ -149,6 +149,7 @@ # Copied from the tests of https://peps.python.org/pep-0508/#complete-grammar [ ["A", "A", ""], + ["A>=3", "A", ">=3"], ["A.B-C_D", "A.B-C_D", ""], ["aa", "aa", ""], ["name", "name", ""], From 4f46915680b66b255870f2f92ba02e3869f66f9f Mon Sep 17 00:00:00 2001 From: Pierre Pirault Date: Wed, 25 Jan 2023 17:01:53 -0500 Subject: [PATCH 5/5] single character packages now correctly parse with blank requirements rather than nil --- app/models/package_manager/pypi.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/package_manager/pypi.rb b/app/models/package_manager/pypi.rb index a681a5f0b..2d3cbc74f 100644 --- a/app/models/package_manager/pypi.rb +++ b/app/models/package_manager/pypi.rb @@ -129,7 +129,7 @@ def self.known_versions(name) # Leaves the rest as-is with any leading semicolons or spaces stripped def self.parse_pep_508_dep_spec(dep) name, requirement = dep.split(PEP_508_NAME_REGEX, 2).last(2).map(&:strip) - requirement = requirement&.sub(/^[\s;]*/, "") || "" + requirement = requirement.sub(/^[\s;]*/, "") return name, requirement end