Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion repo_cache/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@ score-repo-cache sync --org eclipse-score --repo score --repo score_tools
`sync` clones each selected repository's default branch into
`~/.cache/repo-cache/<org>/<name>` (override with `--cache-dir`), or fetches
and resets an existing checkout back to a clean state if it was already
cloned there.
cloned there. Repositories with no Git references are reported as empty and
do not make the command fail; checkout, authentication, and other operational
errors remain failures.

## Library

Expand All @@ -51,6 +53,8 @@ from repo_cache import default_cache_directory, sync_org
report = sync_org(org="eclipse-score", cache_dir=default_cache_directory())
for outcome in report.failures:
print(outcome.repository.name, outcome.error)
for outcome in report.empty_repositories:
print(outcome.repository.name, "is empty")
```

## Bazel
Expand Down
1 change: 1 addition & 0 deletions repo_cache/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from .src.checkout import restore_synced_default_branch as restore_synced_default_branch
from .src.checkout import sync_default_branch as sync_default_branch
from .src.errors import CommandError as CommandError
from .src.errors import EmptyRepositoryError as EmptyRepositoryError
from .src.errors import RepoCacheError as RepoCacheError
from .src.github import ensure_authenticated as ensure_authenticated
from .src.github import list_repositories as list_repositories
Expand Down
51 changes: 47 additions & 4 deletions repo_cache/src/checkout.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from urllib.parse import urlparse

from .command import run_command
from .errors import CommandError
from .errors import CommandError, EmptyRepositoryError

_DEFAULT_BRANCH_REF = "refs/repo-cache/default"

Expand All @@ -30,9 +30,25 @@ def sync_default_branch(*, repository: str, branch: str, destination: Path) -> N

if (destination / ".git").is_dir():
_verify_cached_remote(repository=repository, checkout=destination)
run_command(
["git", "-C", str(destination), "fetch", "--depth", "1", "origin", branch]
)
try:
run_command(
[
"git",
"-C",
str(destination),
"fetch",
"--depth",
"1",
"origin",
branch,
]
)
except CommandError as exc:
if _repository_is_empty(repository):
raise EmptyRepositoryError(
f"repository has no Git references: {repository}"
) from exc
raise
run_command(
[
"git",
Expand All @@ -54,6 +70,8 @@ def sync_default_branch(*, repository: str, branch: str, destination: Path) -> N
raise CommandError(
f"checkout cache path exists but is not a Git repository: {destination}"
)
if _repository_is_empty(repository):
raise EmptyRepositoryError(f"repository has no Git references: {repository}")
destination.parent.mkdir(parents=True, exist_ok=True)
run_command(
[
Expand All @@ -74,6 +92,31 @@ def sync_default_branch(*, repository: str, branch: str, destination: Path) -> N
)


def _repository_is_empty(repository: str) -> bool:
"""Check for Git refs without replacing the original sync error."""

try:
output = run_command(
[
"gh",
"api",
f"/repos/{repository}/git/refs?per_page=1",
"--jq",
"length",
]
)
except CommandError as exc:
# GitHub reports an empty repository as HTTP 409. Other probe errors
# should leave the original clone/fetch error as the useful diagnosis.
return "Git Repository is empty." in str(exc)
try:
return int(output.strip()) == 0
except ValueError as exc:
raise CommandError(
f"gh returned an invalid Git reference count for {repository}"
) from exc


def _verify_cached_remote(*, repository: str, checkout: Path) -> None:
expected_url = run_command(
["gh", "repo", "view", repository, "--json", "url", "--jq", ".url"]
Expand Down
9 changes: 7 additions & 2 deletions repo_cache/src/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,9 +137,14 @@ def _run_sync(args: argparse.Namespace) -> int:
)
for outcome in report.failures:
print(f"error: {outcome.repository.name}: {outcome.error}", file=sys.stderr)
synced = len(report.outcomes) - len(report.failures)
for outcome in report.empty_repositories:
print(f"empty: {outcome.repository.name}", file=sys.stderr)
synced = (
len(report.outcomes) - len(report.failures) - len(report.empty_repositories)
)
print(
f"Synchronized {synced}/{len(report.outcomes)} checkout(s) at {cache_dir / args.org}"
f"Synchronized {synced}/{len(report.outcomes)} checkout(s) at "
f"{cache_dir / args.org} ({len(report.empty_repositories)} empty)"
)
return 2 if report.failures else 0

Expand Down
4 changes: 4 additions & 0 deletions repo_cache/src/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,7 @@ def __init__(self, message: object) -> None:

class CommandError(RepoCacheError):
"""A required `gh` or `git` command failed or was unavailable."""


class EmptyRepositoryError(RepoCacheError):
"""A repository has no Git references and therefore no checkout to sync."""
18 changes: 16 additions & 2 deletions repo_cache/src/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from pathlib import Path

from .checkout import sync_default_branch
from .errors import RepoCacheError, redact_sensitive_text
from .errors import EmptyRepositoryError, RepoCacheError, redact_sensitive_text
from .github import ensure_authenticated, list_repositories
from .models import Repository

Expand All @@ -37,6 +37,7 @@ class SyncOutcome:
repository: Repository
checkout: Path
error: str | None = None
empty: bool = False


@dataclass(frozen=True)
Expand All @@ -51,6 +52,10 @@ class SyncReport:
def failures(self) -> tuple[SyncOutcome, ...]:
return tuple(outcome for outcome in self.outcomes if outcome.error)

@property
def empty_repositories(self) -> tuple[SyncOutcome, ...]:
return tuple(outcome for outcome in self.outcomes if outcome.empty)


def sync_org(
*,
Expand All @@ -66,6 +71,8 @@ def sync_org(
Raises RepoCacheError for an authentication failure or an unknown `repos`
name. Per-repository sync failures are captured in `SyncOutcome.error`
rather than raised, so one broken repository does not abort the rest.
Empty repositories are reported in `SyncReport.empty_repositories` instead
of being treated as failures.
"""

if workers < 1:
Expand Down Expand Up @@ -106,7 +113,9 @@ def sync_org(
)

outcomes: dict[str, SyncOutcome] = {
repository.name: SyncOutcome(repository, cache_dir / org / repository.name)
repository.name: SyncOutcome(
repository, cache_dir / org / repository.name, empty=True
)
for repository in selected_repositories
if repository.default_branch is None
}
Expand All @@ -133,6 +142,11 @@ def sync_org(
checkout = cache_dir / org / repository.name
try:
future.result()
except EmptyRepositoryError:
outcomes[repository.name] = SyncOutcome(
repository, checkout, empty=True
)
status = "empty"
except (RepoCacheError, OSError) as exc:
error = redact_sensitive_text(str(exc) or exc.__class__.__name__)
outcomes[repository.name] = SyncOutcome(repository, checkout, error)
Expand Down
101 changes: 100 additions & 1 deletion repo_cache/tests/test_checkout.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import pytest

from repo_cache.src.checkout import restore_synced_default_branch, sync_default_branch
from repo_cache.src.errors import CommandError
from repo_cache.src.errors import CommandError, EmptyRepositoryError


def test_cached_checkout_rejects_a_different_origin(
Expand Down Expand Up @@ -51,6 +51,8 @@ def test_sync_default_branch_clones_a_missing_checkout(

def record(command: list[str]) -> str:
commands.append(command)
if command[:2] == ["gh", "api"]:
return "1\n"
return ""

monkeypatch.setattr("repo_cache.src.checkout.run_command", record)
Expand All @@ -60,6 +62,13 @@ def record(command: list[str]) -> str:
)

assert commands == [
[
"gh",
"api",
"/repos/owner/repository/git/refs?per_page=1",
"--jq",
"length",
],
[
"gh",
"repo",
Expand All @@ -83,6 +92,96 @@ def record(command: list[str]) -> str:
]


def test_sync_default_branch_reports_an_empty_repository(
monkeypatch, tmp_path: Path
) -> None:
destination = tmp_path / "checkout"
commands: list[list[str]] = []

def record(command: list[str]) -> str:
commands.append(command)
raise CommandError(
"gh api /repos/owner/empty/git/refs: Git Repository is empty. (HTTP 409)"
)

monkeypatch.setattr("repo_cache.src.checkout.run_command", record)

with pytest.raises(EmptyRepositoryError, match="has no Git references"):
sync_default_branch(
repository="owner/empty", branch="main", destination=destination
)

assert commands == [
[
"gh",
"api",
"/repos/owner/empty/git/refs?per_page=1",
"--jq",
"length",
]
]


def test_sync_default_branch_preserves_a_non_empty_repository_failure(
Comment thread
AlexanderLanin marked this conversation as resolved.
monkeypatch, tmp_path: Path
) -> None:
destination = tmp_path / "checkout"

def record(command: list[str]) -> str:
if command[:2] == ["gh", "api"]:
return "1\n"
if command[:3] == ["gh", "repo", "clone"]:
raise CommandError("gh repo clone: remote branch not found")
return ""

monkeypatch.setattr("repo_cache.src.checkout.run_command", record)

with pytest.raises(CommandError, match="remote branch not found"):
sync_default_branch(
repository="owner/non-empty", branch="main", destination=destination
)


@pytest.mark.parametrize(
("refs_count", "expected_exception", "message"),
[
("0\n", EmptyRepositoryError, "has no Git references"),
("1\n", CommandError, "fetch failed"),
],
)
def test_sync_default_branch_handles_a_cached_fetch_failure(
monkeypatch,
tmp_path: Path,
refs_count: str,
expected_exception: type[Exception],
message: str,
) -> None:
destination = tmp_path / "checkout"
(destination / ".git").mkdir(parents=True)
commands: list[list[str]] = []

def record(command: list[str]) -> str:
commands.append(command)
if command[:4] == ["gh", "repo", "view", "owner/repository"]:
return "https://github.com/owner/repository\n"
if command[-3:] == ["remote", "get-url", "origin"]:
return "git@github.com:owner/repository.git\n"
if command[3:4] == ["fetch"]:
raise CommandError("git fetch failed")
if command[:2] == ["gh", "api"]:
return refs_count
return ""

monkeypatch.setattr("repo_cache.src.checkout.run_command", record)

with pytest.raises(expected_exception, match=message):
sync_default_branch(
repository="owner/repository", branch="main", destination=destination
)

assert commands[-1][:2] == ["gh", "api"]


def test_restore_synced_default_branch_never_fetches(
monkeypatch, tmp_path: Path
) -> None:
Expand Down
21 changes: 21 additions & 0 deletions repo_cache/tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,27 @@ def test_sync_succeeds_with_zero_exit_code(monkeypatch, tmp_path: Path) -> None:
assert exit_code == 0


def test_sync_reports_empty_repositories_without_a_failure(
monkeypatch, tmp_path: Path, capsys
) -> None:
repository = Repository("empty", "main")
report = SyncReport(
org="acme",
cache_dir=tmp_path,
outcomes=(SyncOutcome(repository, tmp_path / "acme" / "empty", empty=True),),
)
monkeypatch.setattr(cli_module, "sync_org", lambda **_: report)

exit_code = cli_module.main(
["sync", "--org", "acme", "--cache-dir", str(tmp_path), "--quiet"]
)

captured = capsys.readouterr()
assert exit_code == 0
assert "empty: empty" in captured.err
assert "Synchronized 0/1 checkout(s)" in captured.out


def test_main_reports_repo_cache_errors_without_a_traceback(
monkeypatch, capsys
) -> None:
Expand Down
26 changes: 24 additions & 2 deletions repo_cache/tests/test_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import pytest

from repo_cache.src import sync as sync_module
from repo_cache.src.errors import RepoCacheError
from repo_cache.src.errors import EmptyRepositoryError, RepoCacheError
from repo_cache.src.models import Repository
from repo_cache.src.sync import SyncOutcome, sync_org

Expand Down Expand Up @@ -90,8 +90,30 @@ def fail_sync(**_: object) -> None:
report = sync_org(org="acme", cache_dir=tmp_path)

assert report.outcomes == (
SyncOutcome(repositories[0], tmp_path / "acme" / "empty", None),
SyncOutcome(repositories[0], tmp_path / "acme" / "empty", empty=True),
)
assert report.failures == ()
assert report.empty_repositories == report.outcomes


def test_sync_org_reports_empty_checkout_without_marking_it_as_a_failure(
monkeypatch, tmp_path: Path
) -> None:
repositories = (Repository("empty", "main"), Repository("fine", "main"))
_stub_listing(monkeypatch, repositories)

def fake_sync(*, repository: str, branch: str, destination: Path) -> None:
if repository.endswith("empty"):
raise EmptyRepositoryError("repository has no Git references")

monkeypatch.setattr(sync_module, "sync_default_branch", fake_sync)

report = sync_org(org="acme", cache_dir=tmp_path)

assert report.failures == ()
assert [outcome.repository.name for outcome in report.empty_repositories] == [
"empty"
]


def test_sync_org_records_a_sync_failure_without_aborting_others(
Expand Down
2 changes: 1 addition & 1 deletion repo_policy_sync/src/reporting.py
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@ def _summary_lines(report: RunReport) -> list[str]:
("Repositories", f"{summary.repositories} selected", ""),
(" ✅ synchronized", str(summary.synchronized), ""),
(" ⚠ sync failed", str(summary.sync_failures), ""),
(" ⏭ skipped (no default branch)", str(summary.skipped), ""),
(" ⏭ skipped (no usable checkout)", str(summary.skipped), ""),
("Policy evaluations", str(summary.evaluations), ""),
_summary_row(" ✅", summary.compliant, summary.evaluations),
_summary_row(
Expand Down
Loading