diff --git a/pyproject.toml b/pyproject.toml index a21129c..92b2d82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "pytz", "XlsxWriter", "PyYAML", - "fosslight_util>=2.2.2", + "fosslight_util>=2.2.8", "python-magic-bin; sys_platform == 'win32'", "python-magic; 'darwin' in sys_platform", "python-magic; 'linux' in sys_platform", diff --git a/src/fosslight_binary/_jar_analysis.py b/src/fosslight_binary/_jar_analysis.py index d97c82c..4f41fbb 100644 --- a/src/fosslight_binary/_jar_analysis.py +++ b/src/fosslight_binary/_jar_analysis.py @@ -7,11 +7,13 @@ import hashlib import logging import os +import re import tempfile import zipfile import defusedxml.ElementTree as ET import requests import fosslight_util.constant as constant +from fosslight_util._get_downloadable_url import _maven_repo_bases_for from fosslight_util.get_pom_license import get_license_from_pom from fosslight_util.oss_item import OssItem @@ -20,8 +22,10 @@ _CENTRAL_SEARCH_URL = "https://search.maven.org/solrsearch/select" _REQUEST_TIMEOUT = 10 # seconds – used for HEAD / POM download _CENTRAL_SEARCH_TIMEOUT = 2.5 # seconds – tight timeout for Search API (retried on timeout) +_MAVEN_JAR_HTTP_TIMEOUT = (2, 2) # match Util probe timeouts for multi-repo jar checks _MAX_RETRY = 3 # maximum Central API retry attempts per JAR _central_network_warned = False # Flag to suppress repeated network-unavailable warnings within one run +_COORD_TOKEN = re.compile(r'[A-Za-z0-9._+-]+') # shape of a Maven groupId / artifactId / version def _sha1_of_file(filepath): @@ -123,10 +127,18 @@ def _search_central_by_sha1(sha1, timeout=None): if not docs: return {}, False doc = docs[0] + groupId = doc.get("g", "") + artifactId = doc.get("a", "") + version = doc.get("v") or doc.get("latestVersion", "") + + if not (groupId and artifactId and version): + logger.debug(f"Maven Central returned an incomplete document for {sha1}: {doc}") + return {}, False + return { - "groupId": doc.get("g", ""), - "artifactId": doc.get("a", ""), - "version": doc.get("v") or doc.get("latestVersion", ""), + "groupId": groupId, + "artifactId": artifactId, + "version": version, }, False except requests.exceptions.Timeout: logger.debug(f"Maven Central SHA-1 search timed out ({sha1}) – will retry") @@ -167,34 +179,63 @@ def _download_pom_to_tempfile(group_id, artifact_id, version, timeout=None): return None, any_timeout -def _build_central_jar_url(group_id, artifact_id, version): - if not (group_id and artifact_id and version): - return "" - group_path = group_id.replace('.', '/') - return f"https://repo1.maven.org/maven2/{group_path}/{artifact_id}/{version}/{artifact_id}-{version}.jar" +def _is_maven_coordinate(*parts): + """True only if every part could be a Maven coordinate token. + + MANIFEST.MF vendor fields are display names ("The Apache Software + Foundation", "Google, Inc.", "%bundleVendor"), not groupIds. Rejecting them + keeps bogus URLs out of the report and avoids HEAD requests that can only 404. + """ + return all(p and _COORD_TOKEN.fullmatch(p) for p in parts) -def _exists_in_central(group_id, artifact_id, version): - if not (group_id and artifact_id and version): - return False - url = _build_central_jar_url(group_id, artifact_id, version) +def _maven_jar_url_exists(url, timeout=None): + """True if a Maven host serves this jar URL (HEAD, with GET fallback).""" + _timeout = timeout if timeout is not None else _MAVEN_JAR_HTTP_TIMEOUT try: - resp = requests.head(url, timeout=_REQUEST_TIMEOUT, allow_redirects=True) - return resp.status_code == 200 + resp = requests.head(url, timeout=_timeout, allow_redirects=True) + if resp.status_code == 200: + return True + # Some Maven hosts reject HEAD; fall back to a streamed GET (Util pattern). + if resp.status_code in (403, 405, 501): + resp = requests.get(url, stream=True, allow_redirects=True, timeout=_timeout) + try: + return resp.status_code == 200 + finally: + resp.close() except Exception as ex: if _is_network_error(ex): - _warn_network_once(f"existence check: {group_id}:{artifact_id}:{version}") + _warn_network_once(f"jar existence check: {url}") else: - logger.debug(f"Central existence check failed ({group_id}:{artifact_id}:{version}): {ex}") - return False + logger.debug(f"Maven jar existence check failed ({url}): {ex}") + return False + +def _find_jar_download_url(group_id, artifact_id, version): + """Return the first known-repo URL that hosts ``{artifact}-{version}.jar``. + Uses FOSSLight Util's repository order (group hints + MAVEN_REPOSITORY_BASES) + so non-Central artifacts (Google, Confluent, Spring, …) can still fill + Download Location. + """ + if not _is_maven_coordinate(group_id, artifact_id, version): + return "" + group_path = group_id.replace('.', '/') + jar_name = f"{artifact_id}-{version}.jar" + for repo_base in _maven_repo_bases_for(group_path): + url = f"{repo_base}/{group_path}/{artifact_id}/{version}/{jar_name}" + if _maven_jar_url_exists(url): + logger.debug(f"Maven jar found: {url}") + return url + return "" -def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central=False): + +def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central_search=False): groupId = artifactId = version = project_url = license_str = '' confirmed_in_central = False + trusted_coordinates = False source = '' - if skip_central: + if skip_central_search: central_info = {} timed_out = False else: @@ -219,6 +260,7 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central groupId, artifactId, version, project_url = g2, a2, v2, url2 source = 'pom.xml' confirmed_in_central = True + trusted_coordinates = True if pom_tmp_path: try: @@ -239,6 +281,7 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central groupId, artifactId, version = c_groupId, c_artifactId, c_version source = 'Maven Central' confirmed_in_central = True + trusted_coordinates = True tmp_path, timed_out = _download_pom_to_tempfile( groupId, artifactId, version, timeout=search_timeout) @@ -276,6 +319,7 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central if g2 or a2: groupId, artifactId, version, project_url = g2, a2, v2, url2 source = 'pom.xml' + trusted_coordinates = True if pom_tmp_path: try: @@ -291,8 +335,6 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central os.remove(pom_tmp_path) except Exception: pass - if not confirmed_in_central and not skip_central: - confirmed_in_central = _exists_in_central(groupId, artifactId, version) if not (groupId and artifactId): g3, a3, v3, url3 = _read_manifest_from_jar(jar_path) @@ -300,14 +342,20 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central groupId = g3 artifactId = a3 version = version or v3 + confirmed_in_central = False project_url = project_url or url3 source = 'MANIFEST.MF' + trusted_coordinates = False if not (groupId or artifactId): return None, False + if confirmed_in_central or trusted_coordinates: + dl_url = _find_jar_download_url(groupId, artifactId, version) + else: + dl_url = "" + oss_name = f"{groupId}:{artifactId}" if groupId and artifactId else (artifactId or groupId) - dl_url = _build_central_jar_url(groupId, artifactId, version) if confirmed_in_central else "" oss = OssItem(oss_name, version, license_str, dl_url) oss.comment = source @@ -374,7 +422,7 @@ def analyze_jar_file(path_to_find_bin, path_to_exclude): logger.warning( f"{rel_path}: Maven Central API timed out after {_MAX_RETRY} attempts" " – falling back to JAR internals") - result, _ = _process_one_jar(jar_path, rel_path, sha1, skip_central=True) + result, _ = _process_one_jar(jar_path, rel_path, sha1, skip_central_search=True) if result is not None: _store_jar_result(jar_items, sha1, result) continue