diff --git a/src/fetchcode/package.py b/src/fetchcode/package.py index 1b0f18d..e8c30a3 100644 --- a/src/fetchcode/package.py +++ b/src/fetchcode/package.py @@ -71,6 +71,21 @@ def get_pypi_codeview_url(project_urls): return code_view_url +def get_npm_field_url(value): + """ + Return a URL from an npm ``repository`` or ``bugs`` field. + + The registry stores these as either a URL string or an object with a ``url``. + """ + if isinstance(value, str): + return value or None + if isinstance(value, dict): + url = value.get("url") + if isinstance(url, str): + return url or None + return None + + @router.route("pkg:cargo/.*") def get_cargo_data_from_purl(purl): """ @@ -121,28 +136,37 @@ def get_npm_data_from_purl(purl): purl = PackageURL.from_string(purl) base_path = "http://registry.npmjs.org" name = purl.name + namespace = purl.namespace + # Scoped packages are published as @scope/name. Keep namespace and name + # separate so the emitted Package PURL stays pkg:npm/%40scope/name. + registry_name = f"{namespace}/{name}" if namespace else name version = purl.version - api_url = f"{base_path}/{name}" + api_url = f"{base_path}/{registry_name}" response = get_response(api_url) - vcs_data = response.get("repository") or {} - bugs = response.get("bugs") or {} - download_url = f"{base_path}/{name}/-/{name}-{version}.tgz" if version else None - vcs_url = vcs_data.get("url") - bug_tracking_url = bugs.get("url") + download_url = f"{base_path}/{registry_name}/-/{name}-{version}.tgz" if version else None + vcs_url = get_npm_field_url(response.get("repository")) + bug_tracking_url = get_npm_field_url(response.get("bugs")) license = response.get("license") homepage_url = response.get("homepage") - versions = response.get("versions", []) + versions = response.get("versions") or {} for num in versions: version = versions[num] - version_purl = PackageURL(type=purl.type, name=name, version=version.get("version")) - repository = version.get("repository") or {} - bugs = response.get("bugs") or {} + if not isinstance(version, dict): + continue + version_purl = PackageURL( + type=purl.type, + namespace=namespace, + name=name, + version=version.get("version"), + ) dist = version.get("dist") or {} - vcs_url = repository.get("url") + if not isinstance(dist, dict): + dist = {} + vcs_url = get_npm_field_url(version.get("repository")) download_url = dist.get("tarball") - bug_tracking_url = bugs.get("url") + bug_tracking_url = get_npm_field_url(version.get("bugs") or response.get("bugs")) declared_license = license if purl.version and version_purl.version != purl.version: diff --git a/tests/test_package.py b/tests/test_package.py index be24b58..e1e22b5 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -59,15 +59,85 @@ def test_cargo_packages(mock_get): check_packages(packages, expected_data) +SCOPED_NPM_REGISTRY = { + "name": "@angular/core", + "repository": {"url": "git+https://github.com/angular/angular.git"}, + "bugs": {"url": "https://github.com/angular/angular/issues"}, + "license": "MIT", + "homepage": "https://angular.io", + "versions": { + "1.0.0": { + "version": "1.0.0", + "repository": {"type": "git", "url": "git+https://github.com/angular/angular.git"}, + "dist": {"tarball": "https://registry.npmjs.org/@angular/core/-/core-1.0.0.tgz"}, + }, + "1.0.1": { + "version": "1.0.1", + "repository": {"type": "git", "url": "git+https://github.com/angular/angular.git"}, + "dist": {"tarball": "https://registry.npmjs.org/@angular/core/-/core-1.0.1.tgz"}, + }, + }, +} + + @mock.patch("fetchcode.package.get_response") -def test_npm_packages(mock_get): - side_effect = [load_json("tests/data/npm_mock_data.json")] - purl = "pkg:npm/express" - expected_data = "tests/data/npm.json" - mock_get.side_effect = side_effect - packages = list(info(purl)) +def test_npm_scoped_packages(mock_get): + mock_get.return_value = SCOPED_NPM_REGISTRY + packages = list(info("pkg:npm/%40angular/core")) + + mock_get.assert_called_once_with("http://registry.npmjs.org/@angular/core") + assert [p.version for p in packages] == ["1.0.0", "1.0.1"] + assert all(p.namespace == "@angular" and p.name == "core" for p in packages) + assert packages[0].purl == "pkg:npm/%40angular/core@1.0.0" + assert packages[0].api_url == "http://registry.npmjs.org/@angular/core" + assert packages[0].download_url == "https://registry.npmjs.org/@angular/core/-/core-1.0.0.tgz" + assert packages[0].homepage_url == "https://angular.io" + assert packages[0].bug_tracking_url == "https://github.com/angular/angular/issues" + assert packages[0].vcs_url == "git+https://github.com/angular/angular.git" + assert packages[0].declared_license == "MIT" - check_packages(packages, expected_data) + +@mock.patch("fetchcode.package.get_response") +def test_npm_scoped_package_with_version(mock_get): + mock_get.return_value = SCOPED_NPM_REGISTRY + packages = list(info("pkg:npm/%40angular/core@1.0.1")) + + mock_get.assert_called_once_with("http://registry.npmjs.org/@angular/core") + assert len(packages) == 1 + package = packages[0] + assert package.namespace == "@angular" + assert package.name == "core" + assert package.version == "1.0.1" + assert package.purl == "pkg:npm/%40angular/core@1.0.1" + assert package.download_url == "https://registry.npmjs.org/@angular/core/-/core-1.0.1.tgz" + + +@mock.patch("fetchcode.package.get_response") +def test_npm_string_repository_and_bugs(mock_get): + mock_get.return_value = { + "name": "@babel/core", + "repository": "https://github.com/babel/babel.git", + "bugs": "https://github.com/babel/babel/issues", + "license": "MIT", + "homepage": "https://babeljs.io", + "versions": { + "7.0.0": { + "version": "7.0.0", + "repository": "https://github.com/babel/babel.git", + "bugs": "https://github.com/babel/babel/issues", + "dist": {"tarball": "https://registry.npmjs.org/@babel/core/-/core-7.0.0.tgz"}, + } + }, + } + packages = list(info("pkg:npm/%40babel/core@7.0.0")) + + assert len(packages) == 1 + package = packages[0] + assert package.namespace == "@babel" + assert package.name == "core" + assert package.vcs_url == "https://github.com/babel/babel.git" + assert package.bug_tracking_url == "https://github.com/babel/babel/issues" + assert package.download_url == "https://registry.npmjs.org/@babel/core/-/core-7.0.0.tgz" @mock.patch("fetchcode.package.get_response")