From e7a0d4e1c3477ff97942904295ccfb206e123cc4 Mon Sep 17 00:00:00 2001 From: bjk7119 Date: Thu, 6 Aug 2026 17:04:09 +0900 Subject: [PATCH 1/5] fix(jar): keep download location when central search api times out --- src/fosslight_binary/_jar_analysis.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/fosslight_binary/_jar_analysis.py b/src/fosslight_binary/_jar_analysis.py index d97c82c..995d3cd 100644 --- a/src/fosslight_binary/_jar_analysis.py +++ b/src/fosslight_binary/_jar_analysis.py @@ -189,12 +189,12 @@ def _exists_in_central(group_id, artifact_id, version): return False -def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central=False): +def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central_search=False): groupId = artifactId = version = project_url = license_str = '' confirmed_in_central = False source = '' - if skip_central: + if skip_central_search: central_info = {} timed_out = False else: @@ -291,7 +291,9 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central os.remove(pom_tmp_path) except Exception: pass - if not confirmed_in_central and not skip_central: + # The existence check hits repo1.maven.org, not the Search API, so it + # still runs when the Search API was skipped for timing out. + if not confirmed_in_central: confirmed_in_central = _exists_in_central(groupId, artifactId, version) if not (groupId and artifactId): @@ -374,7 +376,7 @@ def analyze_jar_file(path_to_find_bin, path_to_exclude): logger.warning( f"{rel_path}: Maven Central API timed out after {_MAX_RETRY} attempts" " – falling back to JAR internals") - result, _ = _process_one_jar(jar_path, rel_path, sha1, skip_central=True) + result, _ = _process_one_jar(jar_path, rel_path, sha1, skip_central_search=True) if result is not None: _store_jar_result(jar_items, sha1, result) continue From 81edc6dfe1ee07033bfaa668f14f1ff4e8806603 Mon Sep 17 00:00:00 2001 From: bjk7119 Date: Fri, 7 Aug 2026 10:21:47 +0900 Subject: [PATCH 2/5] fix(jar): check central existence against manifest-derived coordinates --- src/fosslight_binary/_jar_analysis.py | 28 ++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/fosslight_binary/_jar_analysis.py b/src/fosslight_binary/_jar_analysis.py index 995d3cd..e092311 100644 --- a/src/fosslight_binary/_jar_analysis.py +++ b/src/fosslight_binary/_jar_analysis.py @@ -7,6 +7,7 @@ import hashlib import logging import os +import re import tempfile import zipfile import defusedxml.ElementTree as ET @@ -22,6 +23,7 @@ _CENTRAL_SEARCH_TIMEOUT = 2.5 # seconds – tight timeout for Search API (retried on timeout) _MAX_RETRY = 3 # maximum Central API retry attempts per JAR _central_network_warned = False # Flag to suppress repeated network-unavailable warnings within one run +_COORD_TOKEN = re.compile(r'[A-Za-z0-9._+-]+') # shape of a Maven groupId / artifactId / version def _sha1_of_file(filepath): @@ -167,17 +169,27 @@ def _download_pom_to_tempfile(group_id, artifact_id, version, timeout=None): return None, any_timeout +def _is_maven_coordinate(*parts): + """True only if every part could be a Maven coordinate token. + + MANIFEST.MF vendor fields are display names ("The Apache Software + Foundation", "Google, Inc.", "%bundleVendor"), not groupIds. Rejecting them + keeps bogus URLs out of the report and avoids HEAD requests that can only 404. + """ + return all(p and _COORD_TOKEN.fullmatch(p) for p in parts) + + def _build_central_jar_url(group_id, artifact_id, version): - if not (group_id and artifact_id and version): + if not _is_maven_coordinate(group_id, artifact_id, version): return "" group_path = group_id.replace('.', '/') return f"https://repo1.maven.org/maven2/{group_path}/{artifact_id}/{version}/{artifact_id}-{version}.jar" def _exists_in_central(group_id, artifact_id, version): - if not (group_id and artifact_id and version): - return False url = _build_central_jar_url(group_id, artifact_id, version) + if not url: + return False try: resp = requests.head(url, timeout=_REQUEST_TIMEOUT, allow_redirects=True) return resp.status_code == 200 @@ -291,10 +303,6 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central os.remove(pom_tmp_path) except Exception: pass - # The existence check hits repo1.maven.org, not the Search API, so it - # still runs when the Search API was skipped for timing out. - if not confirmed_in_central: - confirmed_in_central = _exists_in_central(groupId, artifactId, version) if not (groupId and artifactId): g3, a3, v3, url3 = _read_manifest_from_jar(jar_path) @@ -308,6 +316,12 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central if not (groupId or artifactId): return None, False + # Runs against the final coordinates, so MANIFEST.MF-derived ones are checked + # too. This queries repo1.maven.org rather than the Search API, so it still + # applies when the Search API was skipped for timing out. + if not confirmed_in_central: + confirmed_in_central = _exists_in_central(groupId, artifactId, version) + oss_name = f"{groupId}:{artifactId}" if groupId and artifactId else (artifactId or groupId) dl_url = _build_central_jar_url(groupId, artifactId, version) if confirmed_in_central else "" From 7f5533ee11e5272680740529ada22ba0dc716cc6 Mon Sep 17 00:00:00 2001 From: bjk7119 Date: Fri, 7 Aug 2026 13:04:27 +0900 Subject: [PATCH 3/5] fix(jar): confirm central existence only for trusted maven coordinates --- src/fosslight_binary/_jar_analysis.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/fosslight_binary/_jar_analysis.py b/src/fosslight_binary/_jar_analysis.py index e092311..1a004ee 100644 --- a/src/fosslight_binary/_jar_analysis.py +++ b/src/fosslight_binary/_jar_analysis.py @@ -204,6 +204,9 @@ def _exists_in_central(group_id, artifact_id, version): def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central_search=False): groupId = artifactId = version = project_url = license_str = '' confirmed_in_central = False + # True only while the coordinates come from a pom.xml or the SHA-1 search. + # MANIFEST.MF fields are display metadata and must never be looked up. + trusted_coordinates = False source = '' if skip_central_search: @@ -231,6 +234,7 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central groupId, artifactId, version, project_url = g2, a2, v2, url2 source = 'pom.xml' confirmed_in_central = True + trusted_coordinates = True if pom_tmp_path: try: @@ -251,6 +255,7 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central groupId, artifactId, version = c_groupId, c_artifactId, c_version source = 'Maven Central' confirmed_in_central = True + trusted_coordinates = True tmp_path, timed_out = _download_pom_to_tempfile( groupId, artifactId, version, timeout=search_timeout) @@ -288,6 +293,7 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central if g2 or a2: groupId, artifactId, version, project_url = g2, a2, v2, url2 source = 'pom.xml' + trusted_coordinates = True if pom_tmp_path: try: @@ -312,14 +318,17 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central version = version or v3 project_url = project_url or url3 source = 'MANIFEST.MF' + # Overwrites any pom-derived groupId/artifactId, so whatever trust + # they carried no longer applies to this coordinate triple. + trusted_coordinates = False if not (groupId or artifactId): return None, False - # Runs against the final coordinates, so MANIFEST.MF-derived ones are checked - # too. This queries repo1.maven.org rather than the Search API, so it still - # applies when the Search API was skipped for timing out. - if not confirmed_in_central: + # Runs against the final coordinates, and only when they are real Maven + # coordinates. This queries repo1.maven.org rather than the Search API, so it + # still applies when the Search API was skipped for timing out. + if not confirmed_in_central and trusted_coordinates: confirmed_in_central = _exists_in_central(groupId, artifactId, version) oss_name = f"{groupId}:{artifactId}" if groupId and artifactId else (artifactId or groupId) From 958d10d2cab29d256e0290fc6999d5cf82175a8d Mon Sep 17 00:00:00 2001 From: bjk7119 Date: Fri, 7 Aug 2026 14:37:07 +0900 Subject: [PATCH 4/5] fix(jar): reject incomplete central responses and manifest-confirmed urls --- src/fosslight_binary/_jar_analysis.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/fosslight_binary/_jar_analysis.py b/src/fosslight_binary/_jar_analysis.py index 1a004ee..d980bfb 100644 --- a/src/fosslight_binary/_jar_analysis.py +++ b/src/fosslight_binary/_jar_analysis.py @@ -125,10 +125,18 @@ def _search_central_by_sha1(sha1, timeout=None): if not docs: return {}, False doc = docs[0] + groupId = doc.get("g", "") + artifactId = doc.get("a", "") + version = doc.get("v") or doc.get("latestVersion", "") + + if not (groupId and artifactId and version): + logger.debug(f"Maven Central returned an incomplete document for {sha1}: {doc}") + return {}, False + return { - "groupId": doc.get("g", ""), - "artifactId": doc.get("a", ""), - "version": doc.get("v") or doc.get("latestVersion", ""), + "groupId": groupId, + "artifactId": artifactId, + "version": version, }, False except requests.exceptions.Timeout: logger.debug(f"Maven Central SHA-1 search timed out ({sha1}) – will retry") @@ -204,8 +212,6 @@ def _exists_in_central(group_id, artifact_id, version): def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central_search=False): groupId = artifactId = version = project_url = license_str = '' confirmed_in_central = False - # True only while the coordinates come from a pom.xml or the SHA-1 search. - # MANIFEST.MF fields are display metadata and must never be looked up. trusted_coordinates = False source = '' @@ -316,18 +322,14 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central groupId = g3 artifactId = a3 version = version or v3 + confirmed_in_central = False project_url = project_url or url3 source = 'MANIFEST.MF' - # Overwrites any pom-derived groupId/artifactId, so whatever trust - # they carried no longer applies to this coordinate triple. trusted_coordinates = False if not (groupId or artifactId): return None, False - # Runs against the final coordinates, and only when they are real Maven - # coordinates. This queries repo1.maven.org rather than the Search API, so it - # still applies when the Search API was skipped for timing out. if not confirmed_in_central and trusted_coordinates: confirmed_in_central = _exists_in_central(groupId, artifactId, version) From 1b82bfcb9a7946d934ca57dff57f6c3de837c7c8 Mon Sep 17 00:00:00 2001 From: bjk7119 Date: Mon, 24 Aug 2026 14:27:01 +0900 Subject: [PATCH 5/5] feat(jar): resolve download location via util multi-repo probe --- pyproject.toml | 2 +- src/fosslight_binary/_jar_analysis.py | 59 ++++++++++++++++++--------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a21129c..92b2d82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "pytz", "XlsxWriter", "PyYAML", - "fosslight_util>=2.2.2", + "fosslight_util>=2.2.8", "python-magic-bin; sys_platform == 'win32'", "python-magic; 'darwin' in sys_platform", "python-magic; 'linux' in sys_platform", diff --git a/src/fosslight_binary/_jar_analysis.py b/src/fosslight_binary/_jar_analysis.py index d980bfb..4f41fbb 100644 --- a/src/fosslight_binary/_jar_analysis.py +++ b/src/fosslight_binary/_jar_analysis.py @@ -13,6 +13,7 @@ import defusedxml.ElementTree as ET import requests import fosslight_util.constant as constant +from fosslight_util._get_downloadable_url import _maven_repo_bases_for from fosslight_util.get_pom_license import get_license_from_pom from fosslight_util.oss_item import OssItem @@ -21,6 +22,7 @@ _CENTRAL_SEARCH_URL = "https://search.maven.org/solrsearch/select" _REQUEST_TIMEOUT = 10 # seconds – used for HEAD / POM download _CENTRAL_SEARCH_TIMEOUT = 2.5 # seconds – tight timeout for Search API (retried on timeout) +_MAVEN_JAR_HTTP_TIMEOUT = (2, 2) # match Util probe timeouts for multi-repo jar checks _MAX_RETRY = 3 # maximum Central API retry attempts per JAR _central_network_warned = False # Flag to suppress repeated network-unavailable warnings within one run _COORD_TOKEN = re.compile(r'[A-Za-z0-9._+-]+') # shape of a Maven groupId / artifactId / version @@ -187,26 +189,44 @@ def _is_maven_coordinate(*parts): return all(p and _COORD_TOKEN.fullmatch(p) for p in parts) -def _build_central_jar_url(group_id, artifact_id, version): - if not _is_maven_coordinate(group_id, artifact_id, version): - return "" - group_path = group_id.replace('.', '/') - return f"https://repo1.maven.org/maven2/{group_path}/{artifact_id}/{version}/{artifact_id}-{version}.jar" - - -def _exists_in_central(group_id, artifact_id, version): - url = _build_central_jar_url(group_id, artifact_id, version) - if not url: - return False +def _maven_jar_url_exists(url, timeout=None): + """True if a Maven host serves this jar URL (HEAD, with GET fallback).""" + _timeout = timeout if timeout is not None else _MAVEN_JAR_HTTP_TIMEOUT try: - resp = requests.head(url, timeout=_REQUEST_TIMEOUT, allow_redirects=True) - return resp.status_code == 200 + resp = requests.head(url, timeout=_timeout, allow_redirects=True) + if resp.status_code == 200: + return True + # Some Maven hosts reject HEAD; fall back to a streamed GET (Util pattern). + if resp.status_code in (403, 405, 501): + resp = requests.get(url, stream=True, allow_redirects=True, timeout=_timeout) + try: + return resp.status_code == 200 + finally: + resp.close() except Exception as ex: if _is_network_error(ex): - _warn_network_once(f"existence check: {group_id}:{artifact_id}:{version}") + _warn_network_once(f"jar existence check: {url}") else: - logger.debug(f"Central existence check failed ({group_id}:{artifact_id}:{version}): {ex}") - return False + logger.debug(f"Maven jar existence check failed ({url}): {ex}") + return False + + +def _find_jar_download_url(group_id, artifact_id, version): + """Return the first known-repo URL that hosts ``{artifact}-{version}.jar``. + Uses FOSSLight Util's repository order (group hints + MAVEN_REPOSITORY_BASES) + so non-Central artifacts (Google, Confluent, Spring, …) can still fill + Download Location. + """ + if not _is_maven_coordinate(group_id, artifact_id, version): + return "" + group_path = group_id.replace('.', '/') + jar_name = f"{artifact_id}-{version}.jar" + for repo_base in _maven_repo_bases_for(group_path): + url = f"{repo_base}/{group_path}/{artifact_id}/{version}/{jar_name}" + if _maven_jar_url_exists(url): + logger.debug(f"Maven jar found: {url}") + return url + return "" def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central_search=False): @@ -330,11 +350,12 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central if not (groupId or artifactId): return None, False - if not confirmed_in_central and trusted_coordinates: - confirmed_in_central = _exists_in_central(groupId, artifactId, version) + if confirmed_in_central or trusted_coordinates: + dl_url = _find_jar_download_url(groupId, artifactId, version) + else: + dl_url = "" oss_name = f"{groupId}:{artifactId}" if groupId and artifactId else (artifactId or groupId) - dl_url = _build_central_jar_url(groupId, artifactId, version) if confirmed_in_central else "" oss = OssItem(oss_name, version, license_str, dl_url) oss.comment = source