diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dd15acb..b4ad1fd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,6 +57,7 @@ jobs: runs-on: ubuntu-latest environment: release-identity permissions: + actions: read contents: read steps: - name: Checkout exact protected main @@ -68,6 +69,7 @@ jobs: - name: Bind the request to the reviewed release state id: release env: + GH_TOKEN: ${{ github.token }} REQUESTED_SOURCE_COMMIT: ${{ inputs.source_commit }} REQUESTED_VERSION: ${{ inputs.version }} run: | @@ -88,6 +90,9 @@ jobs: git fetch --no-tags origin \ '+refs/heads/main:refs/remotes/origin/main' test "$(git rev-parse refs/remotes/origin/main)" = "$REQUESTED_SOURCE_COMMIT" + python scripts/verify_release_ci.py \ + --repository "$GITHUB_REPOSITORY" \ + --sha "$REQUESTED_SOURCE_COMMIT" python scripts/verify_release_lock.py project_version=$(python - <<'PY' @@ -258,6 +263,9 @@ jobs: - name: Verify the public distribution boundary run: python scripts/check_source_boundary.py --require-dist + - name: Verify third-party notices + run: python scripts/check_third_party_notices.py --require-dist + - name: Refuse conflicting immutable PyPI files env: RELEASE_TAG: ${{ github.ref_name }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 61d66f7..914ff0b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,6 +61,7 @@ jobs: run: | uv build --no-sources python scripts/check_source_boundary.py --require-dist + python scripts/check_third_party_notices.py --require-dist - name: Run tests run: | diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..a360416 --- /dev/null +++ b/NOTICE @@ -0,0 +1,46 @@ +OpenAdapt Evals third-party notices + +VAGEN +===== + +OpenAdapt Evals contains modified copies of two VAGEN abstract environment +interfaces: + +Source repository: https://github.com/RAGEN-AI/VAGEN +Source commit: fe4b11db336bb9474aa5b30651460caeb598f97f + +Upstream path: vagen/envs/gym_base_env.py +Upstream SHA-256: 62e820752a244df252f9a57c5d86a1f7ea2b5d74688a51e4484bf53800414aeb +Distributed path: openadapt_evals/adapters/_vendored/gym_base_env.py +Distributed SHA-256: e50f5ddd09da49bcb8dd944c6140dd7dab1fd54877d65f78a28045049f132ceb +Modification status: Modified for local type annotations and lint rules. + +Upstream path: vagen/envs/gym_image_env.py +Upstream SHA-256: 89ab3991c8517e60eb25a90401d7a07260f5f7276a22e2ec9f598f40aba336ce +Distributed path: openadapt_evals/adapters/_vendored/gym_image_env.py +Distributed SHA-256: 637e132044ab385b27b7ec0310e0cf0a02fce7572ea8b78776a3b15a12ccf15e +Modification status: Modified for local type annotations and lint rules. + +License: MIT + +MIT License + +Copyright (c) 2025 RAGEN.AI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pyproject.toml b/pyproject.toml index ed729e2..5d00736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ description = "Evaluation infrastructure for GUI agent benchmarks" readme = "README.md" requires-python = ">=3.10" license = "MIT" +license-files = ["LICENSE", "NOTICE"] authors = [ {name = "Richard Abrich", email = "richard@openadapt.ai"} ] diff --git a/scripts/check_third_party_notices.py b/scripts/check_third_party_notices.py new file mode 100644 index 0000000..4894211 --- /dev/null +++ b/scripts/check_third_party_notices.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Verify vendored-code provenance and notices in source and distributions.""" + +from __future__ import annotations + +import argparse +import hashlib +import tarfile +import zipfile +from pathlib import Path, PurePosixPath + +ROOT = Path(__file__).resolve().parents[1] +NOTICE_PATH = "NOTICE" +SOURCE_REPOSITORY = "https://github.com/RAGEN-AI/VAGEN" +SOURCE_COMMIT = "fe4b11db336bb9474aa5b30651460caeb598f97f" +VENDORED_FILES = { + "openadapt_evals/adapters/_vendored/gym_base_env.py": { + "upstream_path": "vagen/envs/gym_base_env.py", + "upstream_sha256": "62e820752a244df252f9a57c5d86a1f7ea2b5d74688a51e4484bf53800414aeb", + "distributed_sha256": "e50f5ddd09da49bcb8dd944c6140dd7dab1fd54877d65f78a28045049f132ceb", + }, + "openadapt_evals/adapters/_vendored/gym_image_env.py": { + "upstream_path": "vagen/envs/gym_image_env.py", + "upstream_sha256": "89ab3991c8517e60eb25a90401d7a07260f5f7276a22e2ec9f598f40aba336ce", + "distributed_sha256": "637e132044ab385b27b7ec0310e0cf0a02fce7572ea8b78776a3b15a12ccf15e", + }, +} + + +class NoticeError(RuntimeError): + """A vendored source or required notice is absent or inconsistent.""" + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def verify_repository(root: Path = ROOT) -> bytes: + """Verify the checked-in notice and each vendored file it inventories.""" + + notice_path = root / NOTICE_PATH + try: + notice = notice_path.read_bytes() + notice_text = notice.decode("utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise NoticeError(f"cannot read {notice_path}: {exc}") from exc + + required = ( + SOURCE_REPOSITORY, + SOURCE_COMMIT, + "Copyright (c) 2025 RAGEN.AI", + "The above copyright notice and this permission notice shall be included", + ) + for value in required: + if value not in notice_text: + raise NoticeError(f"{NOTICE_PATH} is missing {value!r}") + + for path, provenance in VENDORED_FILES.items(): + try: + payload = (root / path).read_bytes() + except OSError as exc: + raise NoticeError(f"cannot read vendored file {path}: {exc}") from exc + actual = _sha256(payload) + expected = provenance["distributed_sha256"] + if actual != expected: + raise NoticeError( + f"vendored file {path} has SHA-256 {actual}, expected {expected}; " + f"update its provenance before release" + ) + for value in ( + path, + provenance["upstream_path"], + provenance["upstream_sha256"], + expected, + ): + if value not in notice_text: + raise NoticeError(f"{NOTICE_PATH} does not inventory {value!r}") + return notice + + +def _verify_wheel(path: Path, notice: bytes) -> None: + with zipfile.ZipFile(path) as archive: + notice_names = [ + name + for name in archive.namelist() + if PurePosixPath(name).name == NOTICE_PATH + and ".dist-info/licenses" in PurePosixPath(name).as_posix() + ] + if len(notice_names) != 1: + raise NoticeError(f"{path.name} must contain one dist-info/licenses/NOTICE") + if archive.read(notice_names[0]) != notice: + raise NoticeError(f"{path.name} contains a changed NOTICE") + for vendored_path, provenance in VENDORED_FILES.items(): + try: + payload = archive.read(vendored_path) + except KeyError as exc: + raise NoticeError(f"{path.name} is missing {vendored_path}") from exc + if _sha256(payload) != provenance["distributed_sha256"]: + raise NoticeError(f"{path.name} contains an unrecorded {vendored_path}") + + +def _verify_sdist(path: Path, notice: bytes) -> None: + with tarfile.open(path, "r:gz") as archive: + members = [member for member in archive.getmembers() if member.isfile()] + notice_members = [ + member + for member in members + if len(PurePosixPath(member.name).parts) == 2 + and PurePosixPath(member.name).name == NOTICE_PATH + ] + if len(notice_members) != 1: + raise NoticeError(f"{path.name} must contain one top-level NOTICE") + notice_stream = archive.extractfile(notice_members[0]) + if notice_stream is None or notice_stream.read() != notice: + raise NoticeError(f"{path.name} contains a changed NOTICE") + root_name = PurePosixPath(notice_members[0].name).parts[0] + by_name = {member.name: member for member in members} + for vendored_path, provenance in VENDORED_FILES.items(): + member_name = f"{root_name}/{vendored_path}" + member = by_name.get(member_name) + if member is None: + raise NoticeError(f"{path.name} is missing {vendored_path}") + stream = archive.extractfile(member) + if stream is None or _sha256(stream.read()) != provenance["distributed_sha256"]: + raise NoticeError(f"{path.name} contains an unrecorded {vendored_path}") + + +def verify_distributions(directory: Path, notice: bytes) -> None: + """Require and verify one or more wheels and source distributions.""" + + wheels = sorted(directory.glob("*.whl")) + sdists = sorted(directory.glob("*.tar.gz")) + if not wheels or not sdists: + raise NoticeError("distribution directory must contain a wheel and an sdist") + for path in wheels: + _verify_wheel(path, notice) + for path in sdists: + _verify_sdist(path, notice) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--directory", type=Path) + parser.add_argument("--require-dist", action="store_true") + arguments = parser.parse_args() + root = arguments.root.resolve() + try: + notice = verify_repository(root) + if arguments.require_dist: + directory = (arguments.directory or root / "dist").resolve() + verify_distributions(directory, notice) + except (NoticeError, OSError, tarfile.TarError, zipfile.BadZipFile) as exc: + print(f"FATAL: {exc}") + return 1 + scope = "source and distributions" if arguments.require_dist else "source" + print(f"OK: third-party notices match the {scope} files.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_release_ci.py b/scripts/verify_release_ci.py new file mode 100644 index 0000000..5fce935 --- /dev/null +++ b/scripts/verify_release_ci.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Require a successful main-branch test workflow for an exact release SHA.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable, Mapping +from typing import Any + +GITHUB_API_VERSION = "2022-11-28" +TEST_WORKFLOW_NAME = "test" +TEST_WORKFLOW_PATH = ".github/workflows/test.yml" +_REPOSITORY_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +_SHA_RE = re.compile(r"^[0-9a-f]{40}$") + +JSONFetcher = Callable[[str, Mapping[str, str]], Mapping[str, Any]] + + +class ReleaseCIError(RuntimeError): + """The exact release commit does not have successful test evidence.""" + + +class GitHubJSONFetcher: + """Read one bounded GitHub API response and reject every query error.""" + + def __init__(self, token: str) -> None: + if not token: + raise ReleaseCIError("GH_TOKEN is required") + self._token = token + + def __call__(self, endpoint: str, params: Mapping[str, str]) -> Mapping[str, Any]: + query = urllib.parse.urlencode(params) + request = urllib.request.Request( + f"https://api.github.com{endpoint}?{query}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self._token}", + "User-Agent": "openadapt-evals-release-ci-gate", + "X-GitHub-Api-Version": GITHUB_API_VERSION, + }, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + if response.status < 200 or response.status >= 300: + raise ReleaseCIError(f"GitHub API returned HTTP {response.status}") + payload = json.load(response) + except ReleaseCIError: + raise + except ( + urllib.error.URLError, + TimeoutError, + OSError, + json.JSONDecodeError, + UnicodeDecodeError, + ) as exc: + raise ReleaseCIError(f"GitHub API request failed: {exc}") from exc + if not isinstance(payload, dict): + raise ReleaseCIError("GitHub API response is not an object") + return payload + + +def require_successful_test_run( + fetch_json: JSONFetcher, + *, + repository: str, + sha: str, +) -> int: + """Return the latest exact test run ID only when that run succeeded.""" + + if not _REPOSITORY_RE.fullmatch(repository): + raise ReleaseCIError(f"invalid GitHub repository: {repository!r}") + if not _SHA_RE.fullmatch(sha): + raise ReleaseCIError(f"invalid Git commit SHA: {sha!r}") + + payload = fetch_json( + f"/repos/{repository}/actions/workflows/test.yml/runs", + { + "branch": "main", + "event": "push", + "head_sha": sha, + "per_page": "100", + }, + ) + runs = payload.get("workflow_runs") + if not isinstance(runs, list) or not all(isinstance(run, dict) for run in runs): + raise ReleaseCIError("GitHub API response has an invalid workflow_runs list") + + exact_runs = [ + run + for run in runs + if run.get("head_sha") == sha + and run.get("head_branch") == "main" + and run.get("event") == "push" + and run.get("name") == TEST_WORKFLOW_NAME + and run.get("path") == TEST_WORKFLOW_PATH + ] + if not exact_runs: + raise ReleaseCIError(f"no exact-SHA test workflow run exists for {sha}") + + try: + latest = max( + exact_runs, + key=lambda run: (str(run.get("created_at", "")), int(run.get("id", 0))), + ) + except (TypeError, ValueError) as exc: + raise ReleaseCIError("the exact-SHA test workflow run has an invalid id") from exc + + run_id = latest.get("id") + if not isinstance(run_id, int) or run_id <= 0: + raise ReleaseCIError("the exact-SHA test workflow run has an invalid id") + status = latest.get("status") + conclusion = latest.get("conclusion") + if status != "completed": + raise ReleaseCIError(f"exact-SHA test workflow run {run_id} is {status!r}/{conclusion!r}") + if conclusion != "success": + raise ReleaseCIError(f"exact-SHA test workflow run {run_id} concluded {conclusion!r}") + return run_id + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--repository", required=True) + parser.add_argument("--sha", required=True) + args = parser.parse_args() + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if not token: + raise SystemExit("GH_TOKEN or GITHUB_TOKEN is required") + try: + run_id = require_successful_test_run( + GitHubJSONFetcher(token), + repository=args.repository, + sha=args.sha, + ) + except ReleaseCIError as exc: + raise SystemExit(f"REFUSED: {exc}") from exc + print(f"The exact-SHA test workflow succeeded in run {run_id}.") + + +if __name__ == "__main__": + main() diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 28e3116..1f2fe8f 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -104,6 +104,15 @@ def test_release_configuration_is_fail_closed() -> None: assert "test \"$GITHUB_REF\" = 'refs/heads/main'" in workflow assert 'test "$GITHUB_SHA" = "$REQUESTED_SOURCE_COMMIT"' in workflow assert "refs/remotes/origin/main" in workflow + create_tag_job = workflow.split(" create-release-tag:", 1)[1].split(" publish-pypi:", 1)[0] + assert "permissions:\n actions: read\n contents: read" in create_tag_job + assert "python scripts/verify_release_ci.py" in create_tag_job + assert "GH_TOKEN: ${{ github.token }}" in create_tag_job + assert '--repository "$GITHUB_REPOSITORY"' in create_tag_job + assert '--sha "$REQUESTED_SOURCE_COMMIT"' in create_tag_job + assert create_tag_job.index("python scripts/verify_release_ci.py") < create_tag_job.index( + "id: release-app" + ) assert 'git tag -a "$RELEASE_TAG" "$SOURCE_COMMIT"' in workflow assert "refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}" in workflow app_pushes = [ @@ -162,6 +171,6 @@ def test_all_third_party_actions_are_commit_pinned() -> None: action_pattern = re.compile(r"uses:\s*([^\s@]+)@([^\s#]+)") for path in (ROOT / ".github" / "workflows").glob("*.yml"): for action, action_ref in action_pattern.findall(path.read_text(encoding="utf-8")): - assert re.fullmatch(r"[0-9a-f]{40}", action_ref), ( - f"{path.name}: {action}@{action_ref} is not pinned to a commit" - ) + assert re.fullmatch( + r"[0-9a-f]{40}", action_ref + ), f"{path.name}: {action}@{action_ref} is not pinned to a commit" diff --git a/tests/test_third_party_notices.py b/tests/test_third_party_notices.py new file mode 100644 index 0000000..0d83295 --- /dev/null +++ b/tests/test_third_party_notices.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import importlib.util +import io +import shutil +import tarfile +import zipfile +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check_third_party_notices.py" +SPEC = importlib.util.spec_from_file_location("check_third_party_notices", SCRIPT) +assert SPEC and SPEC.loader +notices = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(notices) + + +def _files() -> dict[str, bytes]: + return {path: (ROOT / path).read_bytes() for path in notices.VENDORED_FILES} + + +def _write_wheel( + path: Path, + notice: bytes, + files: dict[str, bytes], + *, + include_notice: bool = True, +) -> None: + with zipfile.ZipFile(path, "w") as archive: + if include_notice: + archive.writestr("openadapt_evals-0.0.0.dist-info/licenses/NOTICE", notice) + for name, payload in files.items(): + archive.writestr(name, payload) + + +def _write_sdist( + path: Path, + notice: bytes, + files: dict[str, bytes], + *, + include_notice: bool = True, +) -> None: + members = dict(files) + if include_notice: + members[notices.NOTICE_PATH] = notice + with tarfile.open(path, "w:gz") as archive: + for name, payload in members.items(): + info = tarfile.TarInfo(f"openadapt_evals-0.0.0/{name}") + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + + +def test_repository_notice_inventories_exact_vendored_files() -> None: + notices.verify_repository() + + +def test_changed_vendored_file_requires_a_provenance_update(tmp_path: Path) -> None: + (tmp_path / notices.NOTICE_PATH).write_bytes((ROOT / notices.NOTICE_PATH).read_bytes()) + for path in notices.VENDORED_FILES: + destination = tmp_path / path + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / path, destination) + first = tmp_path / next(iter(notices.VENDORED_FILES)) + first.write_bytes(first.read_bytes() + b"\n") + + with pytest.raises(notices.NoticeError, match="update its provenance"): + notices.verify_repository(tmp_path) + + +def test_built_archives_carry_the_exact_notice_and_vendored_files(tmp_path: Path) -> None: + notice = notices.verify_repository() + files = _files() + _write_wheel(tmp_path / "package.whl", notice, files) + _write_sdist(tmp_path / "package.tar.gz", notice, files) + + notices.verify_distributions(tmp_path, notice) + + +@pytest.mark.parametrize("kind", ["wheel", "sdist"]) +def test_built_archive_without_notice_is_refused(tmp_path: Path, kind: str) -> None: + notice = notices.verify_repository() + files = _files() + if kind == "wheel": + _write_wheel(tmp_path / "package.whl", notice, files, include_notice=False) + _write_sdist(tmp_path / "package.tar.gz", notice, files) + else: + _write_wheel(tmp_path / "package.whl", notice, files) + _write_sdist(tmp_path / "package.tar.gz", notice, files, include_notice=False) + + with pytest.raises(notices.NoticeError, match="must contain one"): + notices.verify_distributions(tmp_path, notice) + + +def test_release_runs_the_notice_gate_on_built_archives() -> None: + workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8") + test_workflow = (ROOT / ".github" / "workflows" / "test.yml").read_text(encoding="utf-8") + metadata = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + + assert 'license-files = ["LICENSE", "NOTICE"]' in metadata + source_boundary = workflow.index("python scripts/check_source_boundary.py --require-dist") + notice_boundary = workflow.index("python scripts/check_third_party_notices.py --require-dist") + publish = workflow.index("pypa/gh-action-pypi-publish@") + assert source_boundary < notice_boundary < publish + assert "python scripts/check_third_party_notices.py --require-dist" in test_workflow diff --git a/tests/test_verify_release_ci.py b/tests/test_verify_release_ci.py new file mode 100644 index 0000000..83719d6 --- /dev/null +++ b/tests/test_verify_release_ci.py @@ -0,0 +1,103 @@ +"""Tests for the exact-SHA release CI gate.""" + +from __future__ import annotations + +import urllib.error +from collections.abc import Mapping +from typing import Any + +import pytest + +from scripts.verify_release_ci import ( + GitHubJSONFetcher, + ReleaseCIError, + require_successful_test_run, +) + +REPOSITORY = "OpenAdaptAI/openadapt-evals" +SHA = "a" * 40 +RUN_ID = 12345 + + +def _run( + *, + sha: str = SHA, + status: str = "completed", + conclusion: str | None = "success", + run_id: int = RUN_ID, +) -> dict[str, Any]: + return { + "id": run_id, + "name": "test", + "path": ".github/workflows/test.yml", + "head_branch": "main", + "head_sha": sha, + "event": "push", + "status": status, + "conclusion": conclusion, + "created_at": "2026-09-03T00:00:00Z", + } + + +class FakeGitHub: + def __init__(self, runs: list[dict[str, Any]]) -> None: + self.runs = runs + self.calls: list[tuple[str, dict[str, str]]] = [] + + def __call__(self, endpoint: str, params: Mapping[str, str]) -> Mapping[str, Any]: + self.calls.append((endpoint, dict(params))) + return {"workflow_runs": self.runs} + + +def _require(fake: FakeGitHub) -> int: + return require_successful_test_run(fake, repository=REPOSITORY, sha=SHA) + + +def test_accepts_latest_successful_exact_sha_test_run() -> None: + older = {**_run(run_id=100), "created_at": "2026-09-02T00:00:00Z"} + fake = FakeGitHub([older, _run()]) + + assert _require(fake) == RUN_ID + assert fake.calls == [ + ( + "/repos/OpenAdaptAI/openadapt-evals/actions/workflows/test.yml/runs", + { + "branch": "main", + "event": "push", + "head_sha": SHA, + "per_page": "100", + }, + ) + ] + + +def test_rejects_missing_test_run() -> None: + with pytest.raises(ReleaseCIError, match="no exact-SHA"): + _require(FakeGitHub([])) + + +@pytest.mark.parametrize("status", ["queued", "in_progress", "waiting", "pending", "requested"]) +def test_rejects_pending_test_run(status: str) -> None: + with pytest.raises(ReleaseCIError, match=f"is '{status}'/None"): + _require(FakeGitHub([_run(status=status, conclusion=None)])) + + +@pytest.mark.parametrize("conclusion", ["failure", "cancelled", "timed_out"]) +def test_rejects_failed_test_run(conclusion: str) -> None: + with pytest.raises(ReleaseCIError, match=f"concluded '{conclusion}'"): + _require(FakeGitHub([_run(conclusion=conclusion)])) + + +def test_rejects_sha_mismatched_run_even_if_successful() -> None: + with pytest.raises(ReleaseCIError, match="no exact-SHA"): + _require(FakeGitHub([_run(sha="b" * 40)])) + + +def test_rejects_api_error(monkeypatch: pytest.MonkeyPatch) -> None: + def fail_request(*args: Any, **kwargs: Any) -> None: + raise urllib.error.URLError("simulated API error") + + monkeypatch.setattr("scripts.verify_release_ci.urllib.request.urlopen", fail_request) + + with pytest.raises(ReleaseCIError, match="GitHub API request failed"): + require_successful_test_run(GitHubJSONFetcher("token"), repository=REPOSITORY, sha=SHA)