From 5cb472946a75e393c319f51b4844f3372a99c1aa Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 14 Aug 2026 22:40:59 -0600 Subject: [PATCH 01/29] Add incremental (ref-addressed) branch indexing alongside snapshot indexing Adds a `update: ` source knob (`snapshot` default, `incremental` branch-only) so a fast-moving branch that deploys off `main` can stay current with delta updates instead of a full commit-addressed re-index on every HEAD advance. - New `git.ref_key` field on every content doc: the bare commit SHA for snapshot content, or `{host}~{org}~{repo}~{ref}` for incremental content (no `git.commit` of its own). - A new refs "join doc" kind, `_id = git.ref_key`, carrying the citable commit for both modes (snapshot: one per commit; incremental: one per branch, holding the live HEAD). - Delta indexer for incremental branches: `git diff --name-status` between the previously completed commit and the new tip drives which paths are deleted/reindexed; a missing diff base (force-push, GC'd, first index) triggers a full branch-namespace rebuild. - Two-phase incremental publication (`indexing` -> `ready`/`failed`) so a crash mid-update leaves the prior commit and content in place. - Every Agent Builder content tool now runs one universal query shape -- `WHERE git.ref_key == ?git_ref_key | LOOKUP JOIN sourcerer-refs ON git.ref_key` -- replacing the old `git_commit` wildcard param with a required, exact `git_ref_key`. - One-time, default-on upgrade backfill (`--no-backfill` to opt out): stamps `git.ref_key`/ `update_mode` onto pre-existing snapshot content, migrates the files/lines/refs index mappings in place, and creates the missing per-commit join docs. - A post-index uniqueness gate verifies every content `git.ref_key` resolves to exactly one refs join doc, exiting non-zero on any violation. - Docs (`README.md`, `AGENTS.md`, `sourcerer.example.yml`) and the ref-resolution skill updated for the new `update:` knob and the universal join query. Verified end-to-end against a local-dev cluster: a pre-upgrade snapshot baseline (`elastic/sourcerer@v2.5.0`) upgrades in place, a new incremental source (`elastic/serverless-gitops@main`) indexes ref-addressed and is idempotent on re-run, the universal join query resolves a commit for both modes, and the uniqueness gate exits 0 for both. --- AGENTS.md | 67 ++- README.md | 30 ++ sourcerer.example.yml | 15 + src/sourcerer/cli.py | 14 +- src/sourcerer/commands/index/command.py | 226 +++++++- src/sourcerer/commands/index/documents.py | 228 +++++++- src/sourcerer/commands/index/git.py | 107 ++++ src/sourcerer/commands/index/markers.py | 488 +++++++++++++++++- src/sourcerer/commands/index/selection.py | 4 +- src/sourcerer/config.py | 23 +- .../sourcerer.code.grep.yml | 15 +- .../sourcerer.code.search.yml | 15 +- .../sourcerer.files.cat.yml | 15 +- .../sourcerer.files.head.yml | 15 +- .../sourcerer.files.ls.yml | 15 +- .../sourcerer.files.read_lines.yml | 15 +- .../sourcerer.files.tail.yml | 15 +- .../sourcerer.files.tree.yml | 15 +- .../sourcerer.files.wc.yml | 15 +- .../sourcerer.refs.list.yml | 2 +- .../index_templates/sourcerer-v2-files.json | 14 + .../index_templates/sourcerer-v2-lines.json | 14 + .../index_templates/sourcerer-v2-refs.json | 3 + src/sourcerer/progress.py | 4 + src/sourcerer/queries.py | 61 +++ src/sourcerer/skills/ref-resolution/SKILL.md | 20 +- src/sourcerer/utils.py | 13 + tests/test_agent_builder_tools.py | 24 + tests/test_backfill.py | 177 +++++++ tests/test_cli_index.py | 44 +- tests/test_config.py | 34 +- tests/test_documents.py | 72 ++- tests/test_git_changes.py | 166 ++++++ tests/test_incremental_index.py | 145 ++++++ tests/test_markers.py | 198 ++++++- tests/test_uniqueness_gate.py | 93 ++++ tests/test_utils.py | 19 +- 37 files changed, 2363 insertions(+), 77 deletions(-) create mode 100644 tests/test_backfill.py create mode 100644 tests/test_git_changes.py create mode 100644 tests/test_incremental_index.py create mode 100644 tests/test_uniqueness_gate.py diff --git a/AGENTS.md b/AGENTS.md index bb5c40f..51c17ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ Commands: include them. - `sourcerer index / [-b ] [-t ] [-c ]` (single-repo path defaults to `git.host` = `github`) -- `sourcerer index --config [--prune] [--dry-run]` +- `sourcerer index --config [--prune] [--dry-run] [--no-backfill]` - `sourcerer prune [--config ] [--dry-run]` (config-driven retention prune is skipped without `--config`; the orphan sweep always runs) - `sourcerer mcp-proxy [-e ]` (run a stdio MCP proxy that forwards to the Kibana @@ -57,6 +57,34 @@ config's `sources:` is a YAML list, one entry per (host, org, repo, ref_type). S | `match` | yes | For `branch`/`tag`: pattern string or list of patterns matched against ref names (version DSL + glob), a ref matches if any pattern hits. For `commit`: a commit SHA/prefix string or list of them (see below). | | `since` | no | Index-side inclusion floor: the earliest commit to start indexing from. See below. Not valid for `git.ref_type: commit`. | | `retain` | no | Retention policy (see below). Omit to keep forever. For `git.ref_type: commit`, only `age` is valid. | +| `update` | no | `snapshot` (default) or `incremental` (branch-only). See below. | + +#### `update: ` (snapshot vs. incremental) + +`snapshot` (default): content is commit-addressed, as always -- `git.ref_key` on every content +doc equals its `git.commit`, and a HEAD advance on a branch indexes a whole new snapshot under +the new commit. + +`incremental` (branch-only; rejects `since`/`retain` -- there is no per-commit history for either +to apply to): content is ref-addressed instead. `git.ref_key` is +`{host}~{org}~{repo}~{ref}` and carries no `git.commit` of its own; the branch's live commit +lives only on its refs join doc (`_id = git.ref_key`). A HEAD advance runs `git diff +--name-status` between the previously-completed commit and the new tip and only deletes/ +reindexes the paths git reports changed -- add/modify/delete/rename/copy -- instead of +reindexing the whole tree. A missing diff base (force-push, GC'd, or the first index) rebuilds +the whole branch namespace. The refs join doc publishes `status: indexing` before any content +change and `status: ready` (with the new commit) only after the deletes/indexes/refresh all +succeed, so a crash mid-update leaves the prior commit and content in place. + +```yaml +- git: + host: github + org: elastic + repo: serverless-gitops + ref_type: branch + match: main + update: incremental +``` #### `git.ref_type: commit` (pinning an explicit commit) @@ -347,6 +375,43 @@ creates one index/shard per commit — see `specs/sourcerer-yml.md` for the cave resolve it to a commit via the refs index (the `sourcerer.refs.list` tool), then filter content by `git.host` + `git.commit`. +### `git.ref_key` and the universal join query + +Every content doc (file and line, both `update` modes) carries a `git.ref_key` keyword field: +the bare commit SHA for `snapshot` content, or `{host}~{org}~{repo}~{ref}` for `incremental` +content (see `update: ` above; `build_ref_key` in `src/sourcerer/utils.py`). A second, +distinct kind of `sourcerer-v2-refs` document -- a **refs join doc**, `_id = git.ref_key` +(exactly one per key) -- carries the citable `git.commit`: one per commit for snapshot content, +one per branch (holding the live HEAD) for incremental content. This is a different id space +from the hashed, append-only `build_ref_id` ref-name markers described above (those still drive +`since`/retention history and are untouched by this). + +Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) therefore runs the +same query shape regardless of mode, with no `update_mode` conditional: + +```esql +FROM sourcerer-lines +| WHERE git.ref_key == ?git_ref_key AND ... +| LOOKUP JOIN sourcerer-refs ON git.ref_key +``` + +`git_ref_key` is a required, exact-match param (no wildcards) -- resolve a ref to it first (see +`src/sourcerer/skills/ref-resolution/SKILL.md`): a snapshot ref resolves to its commit and uses +that commit directly as `git_ref_key`; an incremental branch builds `{host}~{org}~{repo}~{ref}` +directly, no commit resolution needed. The join adds/overwrites `git.commit` on every row, so +snapshot content (which already carries its own, identical `git.commit`) is unaffected and +incremental content (which has none) gets it from the join. + +### Upgrade backfill (`--no-backfill`) + +`sourcerer index` runs a one-time, idempotent upgrade backfill by default on every invocation: +an `_update_by_query` stamps `git.ref_key = git.commit` + `update_mode: snapshot` onto +pre-existing snapshot content that predates this feature, the refs index's mapping is +re-applied to the existing physical index (a template change alone only affects indices +created afterward), and a snapshot refs join doc is created for every already-indexed commit +that lacks one. Pass `--no-backfill` to skip it. Safe to run every time: a repeat run touches +nothing (see `backfill_repo` in `src/sourcerer/commands/index/markers.py`). + ## Releases `pyproject.toml` is the source of truth for the project version. Release version changes diff --git a/README.md b/README.md index 9068d22..116e0cc 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,36 @@ Make sure you have [uv](https://docs.astral.sh/uv/) and [git](https://git-scm.co The [`sourcerer.yml` specification](specs/sourcerer-yml.md) has the full reference of fields supported by the configuration file. +### Snapshot vs. incremental indexing (`update: `) + +Each source can set `update: snapshot` (the default) or `update: incremental` (branch-only). +Both modes write content that carries a `git.ref_key`, and every Agent Builder tool query and +skill resolves a commit the same way regardless of mode: +`WHERE git.ref_key == ?git_ref_key | LOOKUP JOIN sourcerer-refs ON git.ref_key`. + +- **`snapshot`** (default): content is commit-addressed, exactly as before. `git.ref_key` is the + commit SHA itself, so every ref (branch, tag, or pinned commit) that resolves to the same + commit collapses to one snapshot. A moving branch's HEAD advance indexes a brand-new snapshot + under the new commit. +- **`incremental`** (branch-only): content is ref-addressed instead. `git.ref_key` is + `{host}~{org}~{repo}~{ref}` and content carries no `git.commit` of its own -- the branch's + current commit lives only on its refs join doc, resolved via the join above. A HEAD advance + re-indexes only the files `git diff --name-status` reports changed (add/modify/delete/rename), + not the whole tree, so staying current on a fast-moving branch (e.g. GitOps/IaC repos that + deploy off `main`) is cheap. `since` and `retain` don't apply to an incremental source (there is + no per-commit history to filter or retain) and are rejected if given. + +```yaml +sources: +- git: { host: "github", org: "elastic", repo: "serverless-gitops", ref_type: "branch" } + match: "main" + update: incremental +``` + +Upgrading from a pre-`ref_key` install is automatic and invisible: every `index` run backfills +pre-existing snapshot content in place (idempotent -- a repeat run changes nothing) unless you +pass `--no-backfill`. + ### Cloning with SSH By default, `sourcerer` clones repos using HTTPS. You can override this in `sourcerer.yml` by setting the `urls.clone` of a Git host to an SSH-compatible URL template. diff --git a/sourcerer.example.yml b/sourcerer.example.yml index 242af3d..49ccec8 100644 --- a/sourcerer.example.yml +++ b/sourcerer.example.yml @@ -184,6 +184,21 @@ sources: retain: count: 5 +# Incremental (ref-addressed) update mode -- branch-only. Instead of a new commit-addressed +# snapshot on every HEAD advance, content is keyed by git.ref_key = "{host}~{org}~{repo}~{ref}" +# and stays in place: a HEAD advance re-indexes only the files `git diff` reports changed +# (a delta update), rather than the whole tree. Good for a fast-moving branch that deploys off +# main, where staying current matters more than retaining per-commit history. `since` and +# `retain` are not meaningful here (there is no per-commit history to filter/retain -- see +# specs/incremental-indexing.md) and are rejected if given. +- git: + host: github + org: elastic + repo: serverless-gitops + ref_type: branch + match: main + update: incremental # default: snapshot + # Feature/fix branches as of a week ago; keep the newest commit, prune > 1 month. - git: host: github diff --git a/src/sourcerer/cli.py b/src/sourcerer/cli.py index db4fb4a..74237da 100755 --- a/src/sourcerer/cli.py +++ b/src/sourcerer/cli.py @@ -268,10 +268,18 @@ def setup(url, api_key, username, password, kb_url, config_path, include_experim "run (skip re-indexing it); older markers are treated as stuck and re-indexed. Also " "drives the schedule gate's stuck-run detection. Duration like 30m, 1h, 6h, 1d. Default 1h.", ) +@click.option( + "--no-backfill", + is_flag=True, + default=False, + help="Skip the one-time upgrade backfill that stamps git.ref_key/update_mode onto " + "pre-existing snapshot content and migrates the refs index (default: run it, idempotently, " + "on every invocation).", +) @env_option @insecure_option @auth_options -def index(repo_spec, branch, tag, commit, config_path, force, quiet, cache_dir, ephemeral, prune, dry_run, retry_window, url, api_key, username, password, insecure): +def index(repo_spec, branch, tag, commit, config_path, force, quiet, cache_dir, ephemeral, prune, dry_run, retry_window, no_backfill, url, api_key, username, password, insecure): """Index a remote GitHub repo's git-tracked files into Elasticsearch. Provide a REPO_SPEC ('//') for a single repo, or --config to index multiple @@ -284,7 +292,7 @@ def index(repo_spec, branch, tag, commit, config_path, force, quiet, cache_dir, if config_path: if repo_spec or branch or tag or commit: raise click.UsageError("--config cannot be combined with REPO_SPEC or -b/-t/-c") - index_cmd.run_config(config_path, url, api_key, username, password, force, quiet, cache_dir, ephemeral, prune, dry_run, retry_window=retry_window, insecure=insecure) + index_cmd.run_config(config_path, url, api_key, username, password, force, quiet, cache_dir, ephemeral, prune, dry_run, retry_window=retry_window, insecure=insecure, no_backfill=no_backfill) else: if prune: raise click.UsageError("--prune requires --config (there is no retention policy for a single ref)") @@ -292,7 +300,7 @@ def index(repo_spec, branch, tag, commit, config_path, force, quiet, cache_dir, raise click.UsageError("--dry-run requires --config") if not repo_spec: raise click.UsageError("provide a REPO_SPEC ('//') or --config") - index_cmd.run(repo_spec, branch, tag, commit, url, api_key, username, password, force, quiet, cache_dir, ephemeral, retry_window=retry_window, insecure=insecure) + index_cmd.run(repo_spec, branch, tag, commit, url, api_key, username, password, force, quiet, cache_dir, ephemeral, retry_window=retry_window, insecure=insecure, no_backfill=no_backfill) @cli.command() diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index 808c3b7..5d86444 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -11,6 +11,8 @@ # Standard packages import datetime +import json +import pathlib import subprocess import sys import threading @@ -26,10 +28,11 @@ from ...planner import Marker, plan_repo from ...progress import ProgressReporter, Unit, make_reporter from ...indices import files_index, lines_index +from ...queries import check_ref_key_uniqueness from ...utils import ES_ERRORS, make_client from ..prune import command as prune_cmd from ..prune.execute import delete_commit_from_indices -from .documents import index_repo +from .documents import index_incremental_paths, index_repo from .git import ( checkout_branch, checkout_ref, @@ -37,6 +40,7 @@ count_tracked_files, default_branch, list_branch_commits, + plan_changes, prepared_repo, ref_dates, resolve_cache_root, @@ -44,9 +48,12 @@ _rev_info, ) from .markers import ( - build_ref_id, commits_with_content, content_present, fully_indexed_counts, - markers_status_by_id, _needs_index, pre_clone_skip, recorded_routing, - should_index, write_indexing_marker, write_ref_marker, + backfill_repo, build_ref_id, commits_with_content, content_present, + count_incremental_branch_docs, delete_incremental_branch, delete_incremental_paths, + fully_indexed_counts, markers_status_by_id, _needs_index, pre_clone_skip, + read_incremental_ref, recorded_routing, refresh_incremental_content, should_index, + write_incremental_failed, write_incremental_indexing, write_incremental_ready, + write_indexing_marker, write_ref_marker, write_snapshot_join_doc, ) from .report import dry_run_config from .schedule import filter_config_by_schedule @@ -54,6 +61,47 @@ from .selection import _effective_since_floor, _load_config, _resolve_entry +# The index template files, reused by the upgrade backfill to migrate the mapping of EXISTING +# physical indices (a put_index_template change alone only affects indices created afterward). +_INDEX_TEMPLATES_DIR = pathlib.Path(__file__).resolve().parents[2] / "elastic" / "index_templates" + + +def _load_template_mapping(name: str) -> dict | None: + try: + body = json.loads((_INDEX_TEMPLATES_DIR / name).read_text()) + except OSError: + return None + return body.get("template", {}).get("mappings") + + +def _load_refs_mapping() -> dict | None: + return _load_template_mapping("sourcerer-v2-refs.json") + + +def _load_files_mapping() -> dict | None: + return _load_template_mapping("sourcerer-v2-files.json") + + +def _load_lines_mapping() -> dict | None: + return _load_template_mapping("sourcerer-v2-lines.json") + + +def _run_uniqueness_gate(es: Elasticsearch, host: str, org: str, repo: str) -> bool: + """Post-index uniqueness gate (INV-011): every distinct `git.ref_key` in this repo's + content must resolve to exactly one `sourcerer-v2-refs` join doc. Prints the offending + ref_key(s) to stderr and returns False on any violation; True (silent) when the invariant + holds.""" + offending = check_ref_key_uniqueness(es, host, org, repo) + if offending: + click.echo( + f"Error: {host}/{org}/{repo}: {len(offending)} git.ref_key value(s) missing or " + f"duplicated in sourcerer-v2-refs: {', '.join(offending)}", + err=True, + ) + return False + return True + + def _branch_has_since(branch_name: str, cfg) -> bool: """True if any selector for this branch has a date/age/commit/ref `since` that would trigger a history walk. Version-based `since` (a tag name used as a floor for tag @@ -212,6 +260,10 @@ def index_ref_in_dir( # data that the prune stale-location sweep reclaims. write_ref_marker(es, host, org, repo, ref_type, ref_for_id, commit_sha, commit_date_iso, files_count, lines_count, index_level=level, index_suffix=suffix) + # Every snapshot unit -- whether freshly indexed or reusing a sibling's already-indexed + # content -- must have its `_id = commit` refs join doc so the universal join query resolves + # a commit for this content regardless of which ref reached it (INV-004). + write_snapshot_join_doc(es, host, org, repo, ref_type, ref_for_id, commit_sha, commit_date_iso) if migrating: # Reconstruct the OLD index name from the prior marker's routing and drop this commit's # stale copy there. Commit-safety (another surviving ref sharing the commit) is respected @@ -225,6 +277,97 @@ def index_ref_in_dir( reporter.finish(unit, status, files_count, lines_count) +def index_incremental_branch_in_dir( + es: Elasticsearch, + host: str, + org: str, + repo: str, + repo_dir, + branch: str, + force: bool = False, + reporter: ProgressReporter | None = None, + unit: Unit | None = None, +) -> None: + """Advance one incremental (ref-addressed) branch source in an already-cloned `repo_dir`. + + Reads the branch's prior completed commit (its refs join doc, `_id = ref_key`), checks out + the fetched branch tip, and either: + - does nothing (already at the completed commit and not `--force`), + - does a full rebuild (first index, `--force`, or a missing diff base -- INV-007): delete + the whole branch namespace, then index every currently-tracked path, or + - does a delta update: `git diff --name-status` (via `plan_changes`) between the prior and + new commit, deleting only the paths git reports removed/changed and (re)indexing only the + paths git reports added/changed (INV-008 -- scoped by the exact `ref_key`, never a whole + namespace sweep). + The refs join doc is published `indexing` before any mutation and `ready` only after the + content deletes/indexes and a refresh all succeed (INV-006); a raised exception instead + records `write_incremental_failed` and leaves the completed pointer untouched, then + re-raises so the caller's per-unit error handling reports it. + """ + if reporter is None: + reporter = ProgressReporter() + if unit is None: + unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", update="incremental") + + reporter.set_stage(unit, "checkout") + checkout_branch(repo_dir, branch) + new_sha = resolve_commit(repo_dir) + commit_date_iso = commit_date(repo_dir) + + prior = read_incremental_ref(es, host, org, repo, branch) + old_sha = None if force else (prior.get("git", {}).get("commit") if prior else None) + + if old_sha == new_sha and not force: + reporter.finish(unit, "skipped") + return + + level = unit.index_level + suffix = unit.index_suffix + + reporter.set_stage(unit, "indexing") + write_incremental_indexing(es, host, org, repo, branch, completed_commit=old_sha, + target_commit=new_sha, prior=prior) + try: + full_rebuild = old_sha is None or force + if not full_rebuild: + plan = plan_changes(repo_dir, old_sha, new_sha) + full_rebuild = plan.base_missing + + if full_rebuild: + delete_incremental_branch(es, host, org, repo, branch, index_level=level, index_suffix=suffix) + reporter.set_total_files(unit, count_tracked_files(repo_dir)) + index_incremental_paths( + es, host, org, repo, repo_dir, branch, None, + on_progress=lambda f, l: reporter.update_counts(unit, f, l), + index_level=level, index_suffix=suffix, + ) + else: + delete_incremental_paths(es, host, org, repo, branch, plan.delete_paths, + index_level=level, index_suffix=suffix) + reporter.set_total_files(unit, len(plan.index_paths)) + index_incremental_paths( + es, host, org, repo, repo_dir, branch, plan.index_paths, + on_progress=lambda f, l: reporter.update_counts(unit, f, l), + index_level=level, index_suffix=suffix, + ) + + refresh_incremental_content(es, host, org, repo, index_level=level, index_suffix=suffix) + files_count, lines_count = count_incremental_branch_docs( + es, host, org, repo, branch, index_level=level, index_suffix=suffix, + ) + write_incremental_ready(es, host, org, repo, branch, new_sha, commit_date_iso, + files_count, lines_count) + except KeyboardInterrupt: + write_incremental_failed(es, host, org, repo, branch, completed_commit=old_sha, + target_commit=new_sha, error="interrupted", prior=prior) + raise + except Exception as e: + write_incremental_failed(es, host, org, repo, branch, completed_commit=old_sha, + target_commit=new_sha, error=str(e), prior=prior) + raise + reporter.finish(unit, "indexed", files_count, lines_count) + + def index_one( es: Elasticsearch, host: str, @@ -299,6 +442,7 @@ def run( ephemeral: bool = False, retry_window: datetime.timedelta | None = None, insecure: bool = False, + no_backfill: bool = False, ) -> None: parts = repo_spec.split("/", 2) if len(parts) != 3 or not all(parts): @@ -322,6 +466,12 @@ def run( es = make_client(url, api_key, username, password, insecure=insecure) cache_root = None if ephemeral else resolve_cache_root(cache_dir) + if not no_backfill: + backfill_repo( + es, host, org, repo, refs_mapping=_load_refs_mapping(), + files_mapping=_load_files_mapping(), lines_mapping=_load_lines_mapping(), + ) + kind = "branch" if branch else "tag" if tag else "commit" if commit else "default" unit = Unit(host=host, org=org, repo=repo, ref=branch or tag or commit, kind=kind) reporter = make_reporter(quiet) @@ -346,6 +496,9 @@ def run( reporter.finish(unit, "error", detail=f"Elasticsearch request failed: {e}") sys.exit(1) + if not _run_uniqueness_gate(es, host, org, repo): + sys.exit(1) + def run_config( config_path: str, @@ -361,6 +514,7 @@ def run_config( dry_run: bool = False, retry_window: datetime.timedelta | None = None, insecure: bool = False, + no_backfill: bool = False, ) -> None: """ Index every (repo, ref) the config selects. First list the remote branches and tags for @@ -386,6 +540,19 @@ def run_config( es = make_client(url, api_key, username, password, insecure=insecure) cache_root = None if ephemeral else resolve_cache_root(cache_dir) + # One-time upgrade backfill (default-on; --no-backfill opts out; skipped on --dry-run, + # which promises no ES writes). Runs once per distinct (host, org, repo) in the config, + # before the schedule gate, so it applies regardless of which sources are due this tick. + if not no_backfill and not dry_run: + refs_mapping = _load_refs_mapping() + files_mapping = _load_files_mapping() + lines_mapping = _load_lines_mapping() + for repo_cfg in config.repos: + backfill_repo( + es, repo_cfg.host, repo_cfg.org, repo_cfg.repo, refs_mapping=refs_mapping, + files_mapping=files_mapping, lines_mapping=lines_mapping, + ) + # Schedule gate: determine which sources are due for indexing based on their configured # schedule and the refs index's record of when they were last indexed. Sources with no # schedule (or schedule "* * * * *") are always due; others are skipped until their next @@ -478,6 +645,49 @@ def process_group(item: tuple[tuple[str, str, str], list[Unit]]) -> None: return (host, org, repo), group = item clone_url = hosts[host].clone_url(org, repo) + + # Incremental branch units are split from the snapshot pre-clone/skip/retention flow + # entirely: no cohort retention, no `since` history walk, no commit-addressed content + # reuse -- each is a standalone two-phase delta update against its own prior state + # (see index_incremental_branch_in_dir). Processed here, before the snapshot-only + # `group` continues below with incremental units filtered out. + incremental_units = [u for u in group if u.update == "incremental"] + group = [u for u in group if u.update != "incremental"] + for unit in incremental_units: + reporter.start(unit) + if incremental_units: + try: + with prepared_repo(host, org, repo, clone_url, cache_root, ephemeral) as repo_dir: + if repo_dir is None: + for unit in incremental_units: + reporter.finish( + unit, "locked", + detail="another sourcerer run holds this repo's cache lock", + ) + else: + for unit in incremental_units: + if _aborted.is_set(): + break + try: + index_incremental_branch_in_dir( + es, host, org, repo, repo_dir, unit.ref, force, reporter, unit, + ) + except KeyboardInterrupt: + break + except (FileNotFoundError, subprocess.CalledProcessError, + ValueError, *ES_ERRORS) as e: + with failures_lock: + failures += 1 + reporter.finish(unit, "error", detail=str(e)) + except (FileNotFoundError, subprocess.CalledProcessError, ValueError) as e: + for unit in incremental_units: + if unit.status is None: + with failures_lock: + failures += 1 + reporter.finish(unit, "error", detail=str(e)) + if not group: + return + # 2a. Cheap pre-clone skip for the whole group (no clone yet). # # Batched approach: for branch/tag units that carry a remote_sha from Phase 1's @@ -727,6 +937,14 @@ def process_group(item: tuple[tuple[str, str, str], list[Unit]]) -> None: # errors are handled inside process_group and counted in `failures`. list(pool.map(process_group, groups.items())) + # Post-index uniqueness gate (INV-011), one distinct repo at a time, skipped on abort (the + # plan is incomplete). Every offending repo's ref_key(s) are reported before exiting. + if not _aborted.is_set(): + distinct_repos = {(c.host, c.org, c.repo) for c in entries} + for host, org, repo in sorted(distinct_repos): + if not _run_uniqueness_gate(es, host, org, repo): + failures += 1 + # Prune only after ALL indexing is complete, so a ref newly indexed this run is present in # the refs index before it's scored for retention (e.g. it can be the cohort-newest that # supersedes an older sibling). Skipped on abort: the plan is incomplete, so its retention diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index 9f0d9f2..42fee80 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -18,7 +18,7 @@ # App packages from ...indices import files_index, lines_index -from ...utils import make_doc_id +from ...utils import build_ref_key, make_doc_id from .git import get_symlink_paths, iter_tracked_files from .runtime import _aborted, _tuning @@ -91,7 +91,11 @@ def build_file_doc( "org": org, "repo": repo, "commit": commit_sha, + # Snapshot ref_key is the bare commit -- the content is addressed by commit, so the + # commit itself is the stable join key (see build_ref_key for the incremental shape). + "ref_key": commit_sha, }, + "update_mode": "snapshot", "file": file_fields, } # Content identity is (host, org, repo, commit, path): the same blob reached via any ref @@ -138,7 +142,9 @@ def iter_line_docs( "org": org, "repo": repo, "commit": commit_sha, + "ref_key": commit_sha, }, + "update_mode": "snapshot", "file": file_fields, } for line_num, line_content in enumerate(content.splitlines(), start=1): @@ -146,6 +152,117 @@ def iter_line_docs( yield _id, {**base, "line": {"number": line_num, "content": line_content}} +def build_incremental_file_doc( + host: str, + org: str, + repo: str, + ref: str, + rel_path: str, + abs_path: pathlib.Path, + *, + binary: bool = False, + is_symlink: bool | None = None, + target_path: str | None = None, + target_size: int | None = None, +) -> tuple[str, dict]: + """Ref-addressed (incremental) file doc: no `git.commit`; `git.ref_key` is the tilde-joined + `build_ref_key(host, org, repo, ref)` and `_id` is stable across commits (derived from the + branch name, not the commit), so a modified file's doc overwrites in place on the next + HEAD advance rather than minting a new id.""" + p = pathlib.PurePosixPath(rel_path) + directory = "" if str(p.parent) == "." else str(p.parent) + extension = p.suffix.lstrip(".") or None + _is_symlink = abs_path.is_symlink() if is_symlink is None else is_symlink + size = abs_path.lstat().st_size + file_fields: dict = { + "path": rel_path, + "directory": directory, + "name": p.name, + "extension": extension, + "size": size, + } + attrs = file_attributes(abs_path, binary=binary, is_symlink=_is_symlink) + if attrs: + file_fields["attributes"] = attrs + if _is_symlink: + if target_path is None: + try: + target_path = os.readlink(abs_path) + except OSError: + pass + if target_path is not None: + file_fields["target_path"] = target_path + if target_size is None: + try: + target_size = abs_path.stat().st_size + except OSError: + pass + if target_size is not None: + file_fields["target_size"] = target_size + doc = { + "git": { + "host": host, + "org": org, + "repo": repo, + "ref": ref, + "ref_type": "branch", + "ref_key": build_ref_key(host, org, repo, ref), + }, + "update_mode": "incremental", + "file": file_fields, + } + _id = make_doc_id(host, org, repo, "branch", ref, rel_path) + return _id, doc + + +def iter_incremental_line_docs( + host: str, + org: str, + repo: str, + ref: str, + rel_path: str, + content: str, + *, + size: int | None = None, + target_path: str | None = None, + target_size: int | None = None, + attributes: list[str] | None = None, +) -> Iterator[tuple[str, dict]]: + """Ref-addressed (incremental) line docs -- same shape as `build_incremental_file_doc`.""" + p = pathlib.PurePosixPath(rel_path) + directory = "" if str(p.parent) == "." else str(p.parent) + extension = p.suffix.lstrip(".") or None + file_fields: dict = { + "path": rel_path, + "directory": directory, + "name": p.name, + "extension": extension, + } + if size is not None: + file_fields["size"] = size + if target_path is not None: + file_fields["target_path"] = target_path + if target_size is not None: + file_fields["target_size"] = target_size + if attributes is not None: + file_fields["attributes"] = attributes + base = { + "git": { + "host": host, + "org": org, + "repo": repo, + "ref": ref, + "ref_type": "branch", + "ref_key": build_ref_key(host, org, repo, ref), + }, + "update_mode": "incremental", + "file": file_fields, + } + for line_num, line_content in enumerate(content.splitlines(), start=1): + _id = make_doc_id(host, org, repo, "branch", ref, rel_path, str(line_num)) + yield _id, {**base, "line": {"number": line_num, "content": line_content}} + + # Per-(repo, commit, tag) context for the worker processes, set once per pool by _init_worker # so only the (small) file path crosses the process boundary on each task. _WORKER_CTX: dict = {} @@ -164,6 +281,21 @@ def _init_worker( _WORKER_CTX.update( host=host, org=org, repo=repo, commit_sha=commit_sha, repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, + mode="snapshot", + ) + + +def _init_worker_incremental( + host: str, org: str, repo: str, ref: str, repo_dir: str, symlink_paths: frozenset[str] = frozenset(), + index_level: str = "repo", index_suffix: str | None = None, +) -> None: + """Same as `_init_worker`, but for the incremental (ref-addressed) path: `ref` replaces + `commit_sha` and `mode` routes `_build_one_file_actions` to the incremental doc builders.""" + signal.signal(signal.SIGINT, signal.SIG_IGN) + _WORKER_CTX.update( + host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), + symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, + mode="incremental", ) @@ -184,11 +316,15 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: text. Runs in a worker process (see _init_worker for the shared context). Mirrors the old inline generator -- a binary file or one that can't be read yields only its file doc.""" ctx = _WORKER_CTX - host, org, repo, commit_sha = ctx["host"], ctx["org"], ctx["repo"], ctx["commit_sha"] + incremental = ctx.get("mode", "snapshot") == "incremental" + host, org, repo = ctx["host"], ctx["org"], ctx["repo"] + commit_sha = ctx.get("ref") if incremental else ctx["commit_sha"] level, suffix = ctx.get("index_level", "repo"), ctx.get("index_suffix") abs_path = ctx["repo_dir"] / rel_path - f_index = files_index(host, org, repo, commit_sha, level, suffix) - l_index = lines_index(host, org, repo, commit_sha, level, suffix) + # Incremental content is ref-addressed (no commit-level index name); the physical index name + # never includes the commit for this path since ref-keyed docs don't carry git.commit. + f_index = files_index(host, org, repo, None if incremental else commit_sha, level, suffix) + l_index = lines_index(host, org, repo, None if incremental else commit_sha, level, suffix) # Read the file's bytes once: detect binary from the first 8 KB, and (if text) decode the # same buffer for line splitting -- no second read of the file. Binary flag must be computed # before build_file_doc so it reaches file.attributes. @@ -228,7 +364,8 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: else: git_target_path = None git_target_size = None - file_id, file_doc = build_file_doc( + doc_builder = build_incremental_file_doc if incremental else build_file_doc + file_id, file_doc = doc_builder( host, org, repo, commit_sha, rel_path, abs_path, binary=binary, is_symlink=True if is_git_symlink else None, target_path=git_target_path, @@ -239,7 +376,8 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: return actions content = raw.decode("utf-8", errors="surrogateescape") ff = file_doc["file"] - for line_id, line_doc in iter_line_docs( + line_iter = iter_incremental_line_docs if incremental else iter_line_docs + for line_id, line_doc in line_iter( host, org, repo, commit_sha, rel_path, content, size=ff["size"], target_path=ff.get("target_path"), @@ -340,3 +478,81 @@ def generate_actions(): if on_progress is not None: on_progress(files_count, lines_count) return files_count, lines_count + + +def index_incremental_paths( + es: Elasticsearch, + host: str, + org: str, + repo: str, + repo_dir: pathlib.Path, + ref: str, + rel_paths: list[str] | None = None, + on_progress: Callable[[int, int], None] | None = None, + index_level: str = "repo", + index_suffix: str | None = None, +) -> tuple[int, int]: + """Index a set of paths for an incremental (ref-addressed) branch source. + + `rel_paths=None` walks the whole checked-out tree (first index / full rebuild, e.g. when a + diff base is unavailable). A given `rel_paths` list indexes only those paths -- the delta + indexer's changed/added set (see `commands/index/git.py:plan_changes`), which is what makes + an incremental HEAD advance only touch the files git reports changed. Deletions for removed + paths are the caller's responsibility (see `markers.delete_by_ref_key`) since they need no + doc generation. Mirrors `index_repo`'s worker-pool ingest loop. + """ + files_count = 0 + lines_count = 0 + f_index = files_index(host, org, repo, None, index_level, index_suffix) + + t = _tuning() + symlink_paths = get_symlink_paths(repo_dir) + paths = list(iter_tracked_files(repo_dir)) if rel_paths is None else list(rel_paths) + with ProcessPoolExecutor( + max_workers=max(1, t.index_workers), + initializer=_init_worker_incremental, + initargs=(host, org, repo, ref, str(repo_dir), symlink_paths, index_level, index_suffix), + ) as executor: + def _batched(items: Iterator[str], n: int) -> Iterator[list[str]]: + it = iter(items) + while batch := list(islice(it, n)): + yield batch + + def generate_actions(): + batches = _batched(iter(paths), t.index_worker_chunksize) + max_inflight = max(1, t.index_workers) * 2 + inflight: deque = deque() + for batch in islice(batches, max_inflight): + inflight.append(executor.submit(build_file_actions, batch)) + while inflight: + file_actions = inflight.popleft().result() + for batch in islice(batches, 1): + inflight.append(executor.submit(build_file_actions, batch)) + yield from file_actions + + processed = 0 + try: + for _ok, info in es_parallel_bulk( + es, + generate_actions(), + thread_count=t.bulk_threads, + chunk_size=t.bulk_chunk_size, + max_chunk_bytes=t.bulk_max_bytes, + queue_size=t.bulk_queue_size, + ): + if _aborted.is_set(): + raise KeyboardInterrupt + meta = next(iter(info.values())) if info else {} + if meta.get("_index") == f_index: + files_count += 1 + else: + lines_count += 1 + processed += 1 + if on_progress is not None and processed % 1000 == 0: + on_progress(files_count, lines_count) + except KeyboardInterrupt: + executor.shutdown(wait=False, cancel_futures=True) + raise + if on_progress is not None: + on_progress(files_count, lines_count) + return files_count, lines_count diff --git a/src/sourcerer/commands/index/git.py b/src/sourcerer/commands/index/git.py index 73683fb..5878a52 100644 --- a/src/sourcerer/commands/index/git.py +++ b/src/sourcerer/commands/index/git.py @@ -14,6 +14,7 @@ import tempfile import time from collections.abc import Iterator +from dataclasses import dataclass, field # App packages from ...queries import _parse_dt @@ -347,6 +348,112 @@ def count_tracked_files(repo_dir: pathlib.Path) -> int: return sum(1 for _ in iter_tracked_files(repo_dir)) +@dataclass +class ChangePlan: + """A pure plan for turning one incremental branch update (old commit -> new commit) into + Elasticsearch work. `delete_paths` are the prior file/line docs to remove synchronously; + `index_paths` are the current tree paths to (re)index. A modified or type-changed file + appears in BOTH (delete its stale docs, then re-index every current line). `base_missing` + is True when the old commit object is unavailable locally -- the caller must then fall back + to full branch-namespace reconciliation instead of trusting an empty diff (INV-007).""" + + delete_paths: list[str] = field(default_factory=list) + index_paths: list[str] = field(default_factory=list) + base_missing: bool = False + + +def base_commit_available(repo_dir: pathlib.Path, old_sha: str) -> bool: + """True if `old_sha` resolves to a commit object present in the local clone. Uses + `git cat-file -e ^{commit}` so a tag or partial object still fails closed. A False + here means the diff base is gone (e.g. force-push, shallow clone) and the caller must + rebuild rather than treat the missing base as an empty diff.""" + try: + subprocess.run( + ["git", "-C", str(repo_dir), "cat-file", "-e", f"{old_sha}^{{commit}}"], + check=True, + capture_output=True, + ) + except (subprocess.CalledProcessError, OSError): + return False + return True + + +def _dedupe(paths: list[str]) -> list[str]: + """Order-preserving de-duplication (dict keeps first-seen order).""" + return list(dict.fromkeys(paths)) + + +def _parse_name_status_z(raw: bytes) -> tuple[list[str], list[str]]: + """Parse `git diff --name-status -z` output into (delete_paths, index_paths). + + The `-z` stream is a flat run of NUL-terminated tokens: a status token, then one path + (add/modify/delete/type-change) or two paths (rename/copy: old then new). Parsing by NUL + boundary -- never by whitespace -- preserves spaces, tabs, and unusual bytes in paths. + Mapping (rename detection is an optimization; delete+add of the same paths is equivalent): + A (add) -> index new + M (modify) -> delete + index (replace at file granularity) + T (type change) -> delete + index + D (delete) -> delete + R (rename) -> delete old + index new + C (copy) -> index new (source is unchanged, stays indexed) + An unrecognized status fails safe as delete + index of its path.""" + tokens = raw.split(b"\x00") + # Each real token is NUL-terminated, so a well-formed stream ends in an empty tail token; + # drop trailing empties so a complete record's last path isn't misread as "present but + # empty" and a truncated record is detected by running past the end. + while tokens and tokens[-1] == b"": + tokens.pop() + delete_paths: list[str] = [] + index_paths: list[str] = [] + i = 0 + n = len(tokens) + while i < n: + status = tokens[i] + if not status: + i += 1 + continue + code = status.decode("utf-8", errors="surrogateescape")[0] + if code in ("R", "C"): + if i + 2 >= n: + break # truncated record + old = tokens[i + 1].decode("utf-8", errors="surrogateescape") + new = tokens[i + 2].decode("utf-8", errors="surrogateescape") + i += 3 + if code == "R": + delete_paths.append(old) + index_paths.append(new) + else: + if i + 1 >= n: + break # truncated record + path = tokens[i + 1].decode("utf-8", errors="surrogateescape") + i += 2 + if code == "A": + index_paths.append(path) + elif code == "D": + delete_paths.append(path) + else: # M, T, or an unknown status -> replace the whole file + delete_paths.append(path) + index_paths.append(path) + return _dedupe(delete_paths), _dedupe(index_paths) + + +def plan_changes(repo_dir: pathlib.Path, old_sha: str, new_sha: str) -> ChangePlan: + """Build the ChangePlan for advancing an incremental branch from `old_sha` to `new_sha`. + Returns a `base_missing` plan (no paths) when the old commit is unavailable locally; the + caller then rebuilds the branch namespace. Otherwise runs a NUL-delimited name-status diff + with rename (-M) and copy (-C) detection and maps each record (see _parse_name_status_z).""" + if not base_commit_available(repo_dir, old_sha): + return ChangePlan(base_missing=True) + result = subprocess.run( + ["git", "-C", str(repo_dir), "diff", "--name-status", "-z", "-M", "-C", + old_sha, new_sha], + check=True, + capture_output=True, + ) + delete_paths, index_paths = _parse_name_status_z(result.stdout) + return ChangePlan(delete_paths=delete_paths, index_paths=index_paths) + + def _ls_remote(url: str, *patterns: str, flags: tuple[str, ...] = ()) -> str | None: """ Run `git ls-remote` against a remote without cloning. The URL must precede the ref diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index d3082af..de19fce 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -11,8 +11,8 @@ from elasticsearch import Elasticsearch, NotFoundError # App packages -from ...indices import FILES_ALIAS, REFS_ALIAS, REFS_INDEX, files_index -from ...utils import make_doc_id +from ...indices import FILES_ALIAS, LINES_ALIAS, REFS_ALIAS, REFS_INDEX, files_index, lines_index +from ...utils import build_ref_key, make_doc_id from .git import resolve_remote @@ -536,6 +536,490 @@ def pre_clone_skip( return False, ref_for_id, remote_sha +# --- refs join docs (git.ref_key), keyed by `_id = ref_key` ------------------------------- +# One document per `ref_key` (INV-004): snapshot content's join doc lives at `_id = `; +# an incremental branch's single join doc lives at `_id = {host}~{org}~{repo}~{ref}` and its +# `git.commit` is the branch's live HEAD, advanced only by a two-phase indexing -> ready +# publication (INV-006). These are a DISTINCT id space from `build_ref_id`'s hashed, append-only +# ref-name markers above (untouched -- they still drive `since`/retention history); a join doc's +# `_id` is a plain, unhashed `ref_key` string, which a `build_ref_id` hash can never collide with. + +ERROR_MAX_LEN = 2000 # bound stored failure text so a giant git/ES error can't bloat the doc + + +def write_snapshot_join_doc( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref_type: str, + ref: str, + commit_sha: str, + commit_date_iso: str | None, + refresh: bool = False, +) -> None: + """Write (or idempotently re-write) the snapshot refs join doc: `_id = commit_sha`, + `git.ref_key = commit_sha`, `git.commit = commit_sha` (the commit is its own citable + identity). `ref`/`ref_type` record the ref that produced this write (informational only -- + multiple refs resolving to the same commit all write the same doc, so re-writes are a + no-op change).""" + doc = { + "git": { + "ref_key": commit_sha, + "host": host, + "org": org, + "repo": repo, + "ref": ref, + "ref_type": ref_type, + "commit": commit_sha, + "commit_date": commit_date_iso, + }, + "update_mode": "snapshot", + "status": "complete", + } + es.index(index=REFS_INDEX, id=commit_sha, document=doc, refresh=refresh) + + +def read_incremental_ref(es: Elasticsearch, host: str, org: str, repo: str, ref: str) -> dict | None: + """The branch's incremental join doc `_source`, or None if never indexed. A real-time GET + (by `_id = ref_key`), so it reflects the last write even without a refresh.""" + try: + return es.get(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref))["_source"] + except NotFoundError: + return None + + +def _now_iso() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def _build_incremental_join_doc( + host: str, + org: str, + repo: str, + ref: str, + *, + status: str, + commit: str | None, + target_commit: str | None = None, + commit_date_iso: str | None = None, + files_count: int = 0, + lines_count: int = 0, + indexed_at: str | None = None, + indexing_started_at: str | None = None, + failed_at: str | None = None, + error: str | None = None, +) -> dict: + return { + "git": { + "ref_key": build_ref_key(host, org, repo, ref), + "host": host, + "org": org, + "repo": repo, + "ref": ref, + "ref_type": "branch", + "commit": commit, + "target_commit": target_commit, + "commit_date": commit_date_iso, + }, + "update_mode": "incremental", + "status": status, + "files_count": files_count, + "lines_count": lines_count, + "indexed_at": indexed_at, + "indexing_started_at": indexing_started_at, + "failed_at": failed_at, + "error": error[:ERROR_MAX_LEN] if error else None, + } + + +def write_incremental_indexing( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref: str, + completed_commit: str | None, + target_commit: str, + prior: dict | None = None, + refresh: bool = False, +) -> None: + """Publish `status: indexing`: the completed pointer (`git.commit`) stays at the LAST + completed SHA (or None on a first index) while `git.target_commit` advertises the candidate + SHA the run is advancing to. A failed run never overwrites `git.commit` with `target_commit` + (INV-006) -- only `write_incremental_ready` does that, after delete+index+refresh succeed.""" + prior = prior or {} + pg = prior.get("git", {}) + doc = _build_incremental_join_doc( + host, org, repo, ref, + status="indexing", + commit=completed_commit, + target_commit=target_commit, + commit_date_iso=pg.get("commit_date"), + files_count=prior.get("files_count", 0), + lines_count=prior.get("lines_count", 0), + indexed_at=prior.get("indexed_at"), + indexing_started_at=_now_iso(), + failed_at=prior.get("failed_at"), + error=prior.get("error"), + ) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref), document=doc, refresh=refresh) + + +def write_incremental_ready( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref: str, + commit: str, + commit_date_iso: str | None, + files_count: int, + lines_count: int, + refresh: bool = True, +) -> None: + """Publish `status: ready` at the NEW completed commit, clearing `target_commit` and any + prior failure fields. This is the pointer-advancing publication boundary (INV-006): callers + must delete+index+refresh the content indices FIRST, then call this.""" + doc = _build_incremental_join_doc( + host, org, repo, ref, + status="ready", + commit=commit, + target_commit=None, + commit_date_iso=commit_date_iso, + files_count=files_count, + lines_count=lines_count, + indexed_at=_now_iso(), + indexing_started_at=None, + failed_at=None, + error=None, + ) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref), document=doc, refresh=refresh) + + +def write_incremental_failed( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref: str, + completed_commit: str | None, + target_commit: str | None, + error: str, + prior: dict | None = None, + refresh: bool = False, +) -> None: + """Record a failed update WITHOUT advancing the completed pointer: status stays `indexing`, + `git.commit` remains the last completed SHA, and a bounded `error`/`failed_at` are stored for + diagnosis (INV-006). The next run retries old -> current and clears these on success.""" + prior = prior or {} + pg = prior.get("git", {}) + doc = _build_incremental_join_doc( + host, org, repo, ref, + status="indexing", + commit=completed_commit, + target_commit=target_commit, + commit_date_iso=pg.get("commit_date"), + files_count=prior.get("files_count", 0), + lines_count=prior.get("lines_count", 0), + indexed_at=prior.get("indexed_at"), + indexing_started_at=prior.get("indexing_started_at") or _now_iso(), + failed_at=_now_iso(), + error=error, + ) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref), document=doc, refresh=refresh) + + +def _delete_by_query_sync(es: Elasticsearch, index: str, query: dict, refresh: bool) -> None: + """Synchronous delete-by-query used by the incremental path. Unlike the async prune + deletion, this waits for completion so a subsequent re-index can't race a still-running + delete, and uses `conflicts="proceed"` so a concurrent version bump doesn't abort the batch. + A missing index (first index, before any content exists) is ignored.""" + try: + es.delete_by_query( + index=index, + query=query, + wait_for_completion=True, + conflicts="proceed", + refresh=refresh, + ignore_unavailable=True, + allow_no_indices=True, + ) + except NotFoundError: + pass + + +def delete_incremental_paths( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref: str, + paths, + index_level: str = "repo", + index_suffix: str | None = None, + refresh: bool = False, +) -> None: + """Synchronously delete the file and line docs for `paths` on this exact branch. Scoped by + the exact `git.ref_key` (a single keyword term, so one branch's docs can never bleed into + another's -- INV-008) plus a `file.path` terms filter, never a wildcard. A no-op for an + empty path set.""" + paths = list(paths) + if not paths: + return + ref_key = build_ref_key(host, org, repo, ref) + query = { + "bool": { + "filter": [ + {"term": {"git.ref_key": ref_key}}, + {"terms": {"file.path": paths}}, + ] + } + } + for index in ( + files_index(host, org, repo, None, index_level, index_suffix), + lines_index(host, org, repo, None, index_level, index_suffix), + ): + _delete_by_query_sync(es, index, query, refresh) + + +def delete_incremental_branch( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref: str, + index_level: str = "repo", + index_suffix: str | None = None, + refresh: bool = False, +) -> None: + """Delete EVERY incremental content doc for this branch (full namespace), scoped by the + exact `git.ref_key` (INV-008). Used for the initial index and the missing-diff-base rebuild + (INV-007).""" + ref_key = build_ref_key(host, org, repo, ref) + query = {"bool": {"filter": [{"term": {"git.ref_key": ref_key}}]}} + for index in ( + files_index(host, org, repo, None, index_level, index_suffix), + lines_index(host, org, repo, None, index_level, index_suffix), + ): + _delete_by_query_sync(es, index, query, refresh) + + +def count_incremental_branch_docs( + es: Elasticsearch, host: str, org: str, repo: str, ref: str, + index_level: str = "repo", index_suffix: str | None = None, +) -> tuple[int, int]: + """Authoritative (files, lines) totals for a branch's current incremental view, counted by + exact `git.ref_key`. Call AFTER refreshing the content indices so counts reflect the just- + applied deletes and indexes -- these become the ready marker's files_count/lines_count. + Returns 0 for an index that does not exist yet.""" + ref_key = build_ref_key(host, org, repo, ref) + query = {"bool": {"filter": [{"term": {"git.ref_key": ref_key}}]}} + + def _count(index: str) -> int: + try: + return int(es.count(index=index, query=query)["count"]) + except NotFoundError: + return 0 + + return ( + _count(files_index(host, org, repo, None, index_level, index_suffix)), + _count(lines_index(host, org, repo, None, index_level, index_suffix)), + ) + + +def refresh_incremental_content( + es: Elasticsearch, host: str, org: str, repo: str, + index_level: str = "repo", index_suffix: str | None = None, +) -> None: + """Make the branch's just-written incremental content visible before the ready pointer is + published (INV-006: content refresh precedes the final refs write). Best-effort over + missing indices.""" + es.indices.refresh( + index=[ + files_index(host, org, repo, None, index_level, index_suffix), + lines_index(host, org, repo, None, index_level, index_suffix), + ], + ignore_unavailable=True, + allow_no_indices=True, + ) + + +# --- one-time upgrade backfill (default-on; --no-backfill opts out) ----------------------- +# Stamps `git.ref_key`/`update_mode` onto pre-existing snapshot content that predates this +# feature, migrates the refs index mapping, and creates the missing `_id = commit` join docs +# (INV-009/INV-010). Safe to run on every `index` invocation: both the content update and the +# join-doc creation are no-ops the second time. + +def backfill_snapshot_ref_keys(es: Elasticsearch, host: str, org: str, repo: str) -> int: + """Idempotent `_update_by_query` stamping `git.ref_key = git.commit` + `update_mode: + "snapshot"` onto this repo's content docs that lack `git.ref_key` (pre-upgrade data). + Returns the total number of docs updated across the files and lines aliases; 0 on a repeat + run (INV-009) since the `must_not: exists` filter then matches nothing.""" + query = { + "bool": { + "filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + ], + "must_not": [{"exists": {"field": "git.ref_key"}}], + } + } + script = { + "source": ( + "ctx._source.git.ref_key = ctx._source.git.commit; " + "ctx._source.update_mode = 'snapshot';" + ), + "lang": "painless", + } + total = 0 + for index in (FILES_ALIAS, LINES_ALIAS): + try: + resp = es.update_by_query( + index=index, query=query, script=script, + wait_for_completion=True, conflicts="proceed", refresh=True, + ignore_unavailable=True, allow_no_indices=True, + ) + total += int(resp.get("updated", 0)) + except NotFoundError: + pass + return total + + +def distinct_commits_for_repo(es: Elasticsearch, host: str, org: str, repo: str) -> set[str]: + """Every distinct `git.commit` present in this repo's content, via a terms aggregation. + Used by the backfill to find every already-indexed snapshot commit that needs a refs join + doc (INV-010). Returns an empty set when the files index doesn't exist yet.""" + query = { + "bool": { + "filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + ] + } + } + try: + resp = es.search( + index=FILES_ALIAS, size=0, query=query, + aggs={"commits": {"terms": {"field": "git.commit", "size": 10000}}}, + ) + except NotFoundError: + return set() + return {b["key"] for b in resp["aggregations"]["commits"]["buckets"]} + + +def commits_with_join_doc(es: Elasticsearch, commits: set[str]) -> set[str]: + """The subset of `commits` that already have a `_id = commit` refs join doc. A batched + ids lookup, the join-doc analogue of `commits_with_content`.""" + if not commits: + return set() + try: + resp = es.search( + index=REFS_ALIAS, size=len(commits), query={"ids": {"values": sorted(commits)}}, + source_includes=[], + ) + except NotFoundError: + return set() + return {hit["_id"] for hit in resp["hits"]["hits"]} + + +def backfill_refs_join_docs(es: Elasticsearch, host: str, org: str, repo: str) -> int: + """Ensure a snapshot `_id = commit` join doc exists for every distinct content commit in + this repo (INV-010). For each missing commit, an existing `build_ref_id` marker (if any) + supplies the ref/ref_type/commit_date it was originally indexed under; falls back to + generic values if none is found (the content is authoritative either way -- the join doc's + ref/ref_type are informational). Returns the number of join docs created; 0 on a repeat run + (INV-009).""" + commits = distinct_commits_for_repo(es, host, org, repo) + if not commits: + return 0 + missing = commits - commits_with_join_doc(es, commits) + created = 0 + for commit_sha in missing: + query = { + "bool": { + "filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.commit": commit_sha}}, + {"term": {"status": "complete"}}, + ] + } + } + try: + resp = es.search(index=REFS_ALIAS, size=1, query=query) + except NotFoundError: + resp = {"hits": {"hits": []}} + hits = resp["hits"]["hits"] + if hits: + src_git = hits[0]["_source"].get("git", {}) + ref = src_git.get("ref") or commit_sha + ref_type = src_git.get("ref_type") or "commit" + commit_date_iso = src_git.get("commit_date") + else: + ref, ref_type, commit_date_iso = commit_sha, "commit", None + write_snapshot_join_doc(es, host, org, repo, ref_type, ref, commit_sha, commit_date_iso) + created += 1 + if created: + # Refresh so a uniqueness gate run immediately afterward (see command._run_uniqueness_gate) + # sees every join doc just created rather than racing the refs index's refresh interval. + try: + es.indices.refresh(index=REFS_INDEX) + except NotFoundError: + pass + return created + + +def apply_refs_index_mapping(es: Elasticsearch, mapping: dict) -> None: + """Apply an updated mapping to the physical REFS_INDEX. A `put_index_template` change alone + (see `setup`) only affects indices created AFTER the change -- an existing repo's refs index + predates the `git.ref_key` field and needs its mapping updated explicitly so the field is + typed as intended rather than dynamically guessed on first write. A no-op if the index + doesn't exist yet.""" + try: + es.indices.put_mapping(index=REFS_INDEX, properties=mapping.get("properties", {})) + except NotFoundError: + pass + + +def apply_content_index_mapping(es: Elasticsearch, files_mapping: dict, lines_mapping: dict) -> None: + """Apply the updated files/lines template mappings to every EXISTING physical content index + behind the read aliases. This must run BEFORE `backfill_snapshot_ref_keys` writes + `git.ref_key`/`update_mode` onto pre-existing content: an index created before this feature + has no explicit mapping for those fields, so the first `_update_by_query` write would + otherwise fall back to ES's dynamic string mapping (`text`, no fielddata) instead of the + `keyword` type the template defines -- silently breaking every later `git.ref_key` + aggregation/sort/exact-match query. `put_mapping` against an alias updates every backing + index it resolves to. A no-op if neither alias has any backing index yet.""" + for alias, mapping in ((FILES_ALIAS, files_mapping), (LINES_ALIAS, lines_mapping)): + try: + es.indices.put_mapping(index=alias, properties=mapping.get("properties", {})) + except NotFoundError: + pass + + +def backfill_repo( + es: Elasticsearch, host: str, org: str, repo: str, refs_mapping: dict | None = None, + files_mapping: dict | None = None, lines_mapping: dict | None = None, +) -> dict: + """Run the full one-time upgrade for one repo: apply the updated content/refs index + mappings (once, if given -- must happen BEFORE the content update so the new fields land + typed correctly rather than dynamically guessed), stamp `ref_key`/`update_mode` onto + pre-existing snapshot content (idempotent), and create a join doc for every already-indexed + commit that lacks one. Returns a small summary dict for reporting; every field is 0 on a + repeat run (INV-009).""" + if files_mapping is not None and lines_mapping is not None: + apply_content_index_mapping(es, files_mapping, lines_mapping) + if refs_mapping is not None: + apply_refs_index_mapping(es, refs_mapping) + updated = backfill_snapshot_ref_keys(es, host, org, repo) + created = backfill_refs_join_docs(es, host, org, repo) + return {"content_updated": updated, "join_docs_created": created} + + def resolve_head(es: Elasticsearch, host: str, org: str, repo: str, ref_type: str, ref: str) -> dict | None: """The current marker for a ref: the newest commit_date among its (possibly many) markers. For a branch with retained history this is its live tip; for a tag it's the diff --git a/src/sourcerer/commands/index/selection.py b/src/sourcerer/commands/index/selection.py index 2a4f755..b04f998 100644 --- a/src/sourcerer/commands/index/selection.py +++ b/src/sourcerer/commands/index/selection.py @@ -46,7 +46,7 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: seen.add((rt, prefix)) units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=prefix, kind=rt, - index_level=sel.index_level, index_suffix=sel.index_suffix, + index_level=sel.index_level, index_suffix=sel.index_suffix, update=sel.update, )) continue if rt not in fetched: @@ -67,7 +67,7 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=name, kind=rt, remote_sha=ref_map[name], - index_level=sel.index_level, index_suffix=sel.index_suffix, + index_level=sel.index_level, index_suffix=sel.index_suffix, update=sel.update, )) failed_kinds = sorted(k for k, v in fetched.items() if v is None) diff --git a/src/sourcerer/config.py b/src/sourcerer/config.py index d4a853c..d6939df 100644 --- a/src/sourcerer/config.py +++ b/src/sourcerer/config.py @@ -287,6 +287,9 @@ class Selector: # route to different indices (e.g. kibana release tags -> ~repo, deploy tags -> ~repo^deploy). index_level: str = "repo" # "host" | "org" | "repo" | "commit" index_suffix: str | None = None # appended as ^{suffix}; None == no suffix + # sources[i].update: "snapshot" (default, commit-addressed content) or "incremental" + # (ref-addressed content, branch-only -- see specs/incremental-indexing.md). + update: str = "snapshot" def matches(self, ref_type: str, ref: str) -> Version | None: if self.ref_type != ref_type: @@ -408,6 +411,7 @@ def _parse_commit_match(raw: dict, ctx: str) -> list[str]: _GIT_KEYS = {"host", "org", "repo", "ref_type"} _INDEX_LEVELS = ("host", "org", "repo", "commit") +_UPDATE_MODES = ("snapshot", "incremental") # A suffix goes into a physical index name after a `^`, so it must be safe as an index-name # segment: the same characters forbidden in a host id, plus the `^` we use as the suffix delimiter. _FORBIDDEN_SUFFIX_CHARS = _FORBIDDEN_HOST_CHARS | {"^"} @@ -475,11 +479,26 @@ def _parse_git_scope(raw: dict, ctx: str) -> tuple[str, str, str, str]: def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: """Parse one `sources[i]` entry into (host, org, repo, Selector). The ref_type comes from the `git` block; `match`/`since`/`retain` are top-level siblings.""" - unknown = set(raw) - {"git", "match", "since", "retain", "schedule", "index"} + unknown = set(raw) - {"git", "match", "since", "retain", "schedule", "index", "update"} if unknown: raise ValueError(f"{ctx}: unknown keys {sorted(unknown)}") host, org, repo, ref_type = _parse_git_scope(raw, ctx) + update = raw.get("update", "snapshot") + if update not in _UPDATE_MODES: + raise ValueError(f"{ctx} update: must be one of {list(_UPDATE_MODES)} (got {update!r})") + if update == "incremental": + if ref_type != "branch": + raise ValueError(f"{ctx} update: 'incremental' is only valid for git.ref_type: branch " + f"(got ref_type {ref_type!r})") + # An incremental branch maintains a single mutable ref-addressed view with no per-commit + # history for retention to trim and no inclusion floor to apply -- both since and retain + # are meaningless here (see specs/incremental-indexing.md). + if raw.get("since") is not None: + raise ValueError(f"{ctx}: 'update: incremental' cannot be combined with 'since'") + if raw.get("retain") is not None: + raise ValueError(f"{ctx}: 'update: incremental' cannot be combined with 'retain'") + if ref_type == "commit": # A pinned commit has no enumerable name to pattern-match against (see selection.py), # so `match` holds literal SHA/prefix strings instead of version.py DSL patterns, and @@ -531,7 +550,7 @@ def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: selector = Selector(ref_type=ref_type, raw_patterns=patterns, compiled=compiled, since=since, retain=retain, levels=levels, schedule=schedule, - index_level=index_level, index_suffix=index_suffix) + index_level=index_level, index_suffix=index_suffix, update=update) return host, org, repo, selector diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml index 7f67840..7402b95 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml @@ -9,10 +9,16 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND git.ref_key == ?git_ref_key AND file.path LIKE ?file_path AND line.content RLIKE ?regex + // Resolve the commit for this ref_key via the universal join -- identical for + // snapshot and incremental content. Snapshot content already carries its own + // git.commit (equal to git.ref_key), so the join's value simply overwrites it in + // place; incremental content has no git.commit of its own, so the join supplies it. + | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -39,11 +45,10 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit: + git_ref_key: type: string - description: Filter by git commit(s) (supports * wildcards) - optional: true - defaultValue: "*" + description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + optional: false file_path: type: string description: File path(s) to grep (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml index 7986bf5..5549dd2 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml @@ -9,10 +9,16 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND git.ref_key == ?git_ref_key AND file.path LIKE ?file_path AND MATCH(line.content.text, ?q) + // Resolve the commit for this ref_key via the universal join -- identical for + // snapshot and incremental content. Snapshot content already carries its own + // git.commit (equal to git.ref_key), so the join's value simply overwrites it in + // place; incremental content has no git.commit of its own, so the join supplies it. + | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -39,11 +45,10 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit: + git_ref_key: type: string - description: Filter by git commit(s) (supports * wildcards) - optional: true - defaultValue: "*" + description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + optional: false file_path: type: string description: File path(s) to search (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml index 95d0d0b..0de5965 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml @@ -9,9 +9,15 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND git.ref_key == ?git_ref_key AND file.path LIKE ?file_path + // Resolve the commit for this ref_key via the universal join -- identical for + // snapshot and incremental content. Snapshot content already carries its own + // git.commit (equal to git.ref_key), so the join's value simply overwrites it in + // place; incremental content has no git.commit of its own, so the join supplies it. + | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -57,11 +63,10 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit: + git_ref_key: type: string - description: Filter by git commit(s) (supports * wildcards) - optional: true - defaultValue: "*" + description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + optional: false file_path: type: string description: File path(s) to cat (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml index 7879a3c..4a0994a 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml @@ -9,9 +9,15 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND git.ref_key == ?git_ref_key AND file.path LIKE ?file_path + // Resolve the commit for this ref_key via the universal join -- identical for + // snapshot and incremental content. Snapshot content already carries its own + // git.commit (equal to git.ref_key), so the join's value simply overwrites it in + // place; incremental content has no git.commit of its own, so the join supplies it. + | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -56,11 +62,10 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit: + git_ref_key: type: string - description: Filter by git commit(s) (supports * wildcards) - optional: true - defaultValue: "*" + description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + optional: false file_path: type: string description: File path(s) to head (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml index 0a68227..76c0558 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml @@ -9,7 +9,13 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND git.ref_key == ?git_ref_key + + // Resolve the commit for this ref_key via the universal join -- identical for + // snapshot and incremental content. Snapshot content already carries its own + // git.commit (equal to git.ref_key), so the join's value simply overwrites it in + // place; incremental content has no git.commit of its own, so the join supplies it. + | LOOKUP JOIN sourcerer-refs ON git.ref_key // Enforce glob depth for * and ** on file.path. // Without this, "src/*/Job.java" would match files at any depth, @@ -91,11 +97,10 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit: + git_ref_key: type: string - description: Filter by git commit(s) (supports * wildcards) - optional: true - defaultValue: "*" + description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + optional: false file_path: type: string description: File path to ls (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml index e97eaac..3864cee 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml @@ -9,11 +9,17 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND git.ref_key == ?git_ref_key AND file.path LIKE ?file_path AND line.number >= ?line_number_start AND line.number <= ?line_number_end + // Resolve the commit for this ref_key via the universal join -- identical for + // snapshot and incremental content. Snapshot content already carries its own + // git.commit (equal to git.ref_key), so the join's value simply overwrites it in + // place; incremental content has no git.commit of its own, so the join supplies it. + | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -59,11 +65,10 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit: + git_ref_key: type: string - description: Filter by git commit(s) (supports * wildcards) - optional: true - defaultValue: "*" + description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + optional: false file_path: type: string description: File path(s) to read (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml index 4cf802a..7249386 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml @@ -9,9 +9,15 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND git.ref_key == ?git_ref_key AND file.path LIKE ?file_path + // Resolve the commit for this ref_key via the universal join -- identical for + // snapshot and incremental content. Snapshot content already carries its own + // git.commit (equal to git.ref_key), so the join's value simply overwrites it in + // place; incremental content has no git.commit of its own, so the join supplies it. + | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -56,11 +62,10 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit: + git_ref_key: type: string - description: Filter by git commit(s) (supports * wildcards) - optional: true - defaultValue: "*" + description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + optional: false file_path: type: string description: File path(s) to tail (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml index f030602..454e161 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml @@ -9,7 +9,13 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND git.ref_key == ?git_ref_key + + // Resolve the commit for this ref_key via the universal join -- identical for + // snapshot and incremental content. Snapshot content already carries its own + // git.commit (equal to git.ref_key), so the join's value simply overwrites it in + // place; incremental content has no git.commit of its own, so the join supplies it. + | LOOKUP JOIN sourcerer-refs ON git.ref_key // Split each file path into its segments | EVAL _segs = SPLIT(file.path, "/") @@ -180,11 +186,10 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit: + git_ref_key: type: string - description: Filter by git commit(s) (supports * wildcards) - optional: true - defaultValue: "*" + description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + optional: false file_path: type: string description: Directory to tree, e.g. src or src/sourcerer. The tree is rooted there and recurses; depth is controlled by L, not by this pattern. May also be a glob to filter what is listed (e.g. src/*.xml lists only XML files under src, at any depth). A single * already matches across directories, so ** is never required. diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml index b729f41..33bce5f 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml @@ -9,9 +9,15 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND git.ref_key == ?git_ref_key AND file.path LIKE ?file_path + // Resolve the commit for this ref_key via the universal join -- identical for + // snapshot and incremental content. Snapshot content already carries its own + // git.commit (equal to git.ref_key), so the join's value simply overwrites it in + // place; incremental content has no git.commit of its own, so the join supplies it. + | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -171,11 +177,10 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit: + git_ref_key: type: string - description: Filter by git commit(s) (supports * wildcards) - optional: true - defaultValue: "*" + description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + optional: false file_path: type: string description: File path(s) to count (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml index 68fb3ed..62294b4 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml @@ -16,7 +16,7 @@ configuration: // Format the response | SORT indexed_at DESC - | KEEP git.host, git.org, git.repo, git.ref, git.ref_type, git.commit, git.commit_date, status, files_count, lines_count, indexed_at, indexing_started_at + | KEEP git.host, git.org, git.repo, git.ref, git.ref_type, git.ref_key, git.commit, git.commit_date, status, files_count, lines_count, indexed_at, indexing_started_at | LIMIT 1000000 params: git_host: diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v2-files.json b/src/sourcerer/elastic/index_templates/sourcerer-v2-files.json index 9b0719b..5ce16ab 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v2-files.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v2-files.json @@ -55,9 +55,23 @@ "commit": { "type": "keyword", "normalizer": "lowercase" + }, + "ref_key": { + "type": "keyword" + }, + "ref": { + "type": "keyword" + }, + "ref_type": { + "type": "keyword", + "normalizer": "lowercase" } } }, + "update_mode": { + "type": "keyword", + "normalizer": "lowercase" + }, "file": { "properties": { "path": { diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v2-lines.json b/src/sourcerer/elastic/index_templates/sourcerer-v2-lines.json index 83410e0..1af54ff 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v2-lines.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v2-lines.json @@ -97,9 +97,23 @@ "commit": { "type": "keyword", "normalizer": "lowercase" + }, + "ref_key": { + "type": "keyword" + }, + "ref": { + "type": "keyword" + }, + "ref_type": { + "type": "keyword", + "normalizer": "lowercase" } } }, + "update_mode": { + "type": "keyword", + "normalizer": "lowercase" + }, "file": { "properties": { "path": { diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v2-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v2-refs.json index 0ee8455..322a57f 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v2-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v2-refs.json @@ -59,6 +59,9 @@ }, "commit_date": { "type": "date" + }, + "ref_key": { + "type": "keyword" } } }, diff --git a/src/sourcerer/progress.py b/src/sourcerer/progress.py index 7ef8b97..88ee5ee 100644 --- a/src/sourcerer/progress.py +++ b/src/sourcerer/progress.py @@ -67,6 +67,10 @@ class Unit: # unit's content docs are written to; defaults reproduce the historical repo-level name. index_level: str = "repo" index_suffix: str | None = None + # sources[i].update carried from the selector that emitted this unit: "snapshot" (default, + # commit-addressed) or "incremental" (ref-addressed, branch-only). Routes the unit to the + # incremental delta-index path instead of the snapshot pre-clone/skip/retention flow. + update: str = "snapshot" @property def label(self) -> str: diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index 4c3a580..5e95a59 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -244,6 +244,67 @@ def content_indices_for_commit( return sorted(names) +def enumerate_content_ref_keys(es: Elasticsearch, host: str, org: str, repo: str) -> set[str]: + """Every distinct `git.ref_key` present in this repo's content (files + lines aliases), via + a paginated composite aggregation scoped to (host, org, repo). Feeds the post-upgrade + uniqueness gate (INV-011): every value this returns must resolve to exactly one + `sourcerer-v2-refs` join doc. Returns an empty set if neither alias has any matching docs.""" + filters = [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + ] + out: set[str] = set() + for index in (FILES_ALIAS, LINES_ALIAS): + after: dict | None = None + while True: + composite: dict = { + "size": _COMPOSITE_PAGE_SIZE, + "sources": [{"ref_key": {"terms": {"field": "git.ref_key"}}}], + } + if after is not None: + composite["after"] = after + try: + resp = es.search( + index=index, size=0, + query={"bool": {"filter": filters}}, + aggs={"keys": {"composite": composite}}, + ) + except NotFoundError: + break + agg = resp["aggregations"]["keys"] + buckets = agg["buckets"] + if not buckets: + break + for b in buckets: + out.add(b["key"]["ref_key"]) + after = agg.get("after_key") + if after is None: + break + return out + + +def check_ref_key_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> list[str]: + """The post-upgrade uniqueness gate (INV-011): every distinct `git.ref_key` in this repo's + content must resolve to EXACTLY ONE `sourcerer-v2-refs` join doc. Returns the sorted list of + offending ref_keys (missing entirely, or matched by more than one join doc) -- empty means + the invariant holds. A single aggregation query counts join docs per ref_key; a key absent + from the buckets has zero matches (missing).""" + ref_keys = enumerate_content_ref_keys(es, host, org, repo) + if not ref_keys: + return [] + try: + resp = es.search( + index=REFS_ALIAS, size=0, + query={"terms": {"git.ref_key": sorted(ref_keys)}}, + aggs={"keys": {"terms": {"field": "git.ref_key", "size": len(ref_keys)}}}, + ) + counts = {b["key"]: b["doc_count"] for b in resp["aggregations"]["keys"]["buckets"]} + except NotFoundError: + counts = {} + return sorted(key for key in ref_keys if counts.get(key, 0) != 1) + + def resolve_content_commit( es: Elasticsearch, host: str, org: str, repo: str, prefix: str, ) -> set[str]: diff --git a/src/sourcerer/skills/ref-resolution/SKILL.md b/src/sourcerer/skills/ref-resolution/SKILL.md index d116faf..bd31ccd 100644 --- a/src/sourcerer/skills/ref-resolution/SKILL.md +++ b/src/sourcerer/skills/ref-resolution/SKILL.md @@ -52,5 +52,21 @@ one per historical commit. Resolve "branch as of date D" like this: If only one marker exists for the branch (tip-only indexing, no `since`), state that historical snapshots are unavailable for that branch. -## Pinning the commit -Once a ref resolves to a `git.commit`, use that commit in every subsequent `sourcerer.code.*` and `sourcerer.files.*` call for that repo and ref. Re-invoke this skill only when the question introduces a new or additional ref. +## Pinning the ref_key +Every content query (`sourcerer.code.*` and `sourcerer.files.*`) takes the same single param, +`git_ref_key`, and runs the identical universal join query underneath +(`WHERE git.ref_key == ?git_ref_key | LOOKUP JOIN sourcerer-refs ON git.ref_key`) regardless of +whether the source is indexed as `snapshot` or `incremental`. Derive it once a ref is resolved: + +- **Snapshot** (the default; most tags and one-off branch indexes): resolve the ref to its + `git.commit` as above, then use that **commit SHA directly** as `git_ref_key` -- for snapshot + content, `git.ref_key == git.commit`. +- **Incremental** (a branch source configured with `update: incremental` in `sourcerer.yml`; + `refs.list` surfaces its join doc with `update_mode: incremental` and no separate per-commit + history): build the `git_ref_key` directly as `{host}~{org}~{repo}~{ref}` (host/org/repo + lowercased, ref case-preserved, `~`-joined) -- no commit resolution needed, since the key names + the branch itself and the join always resolves it to the CURRENT indexed commit at query time. + +Use the resulting `git_ref_key` in every subsequent content call for that repo and ref, and read +the resolved `git.commit` back from the join for citations. Re-invoke this skill only when the +question introduces a new or additional ref. diff --git a/src/sourcerer/utils.py b/src/sourcerer/utils.py index 6617722..b979d72 100644 --- a/src/sourcerer/utils.py +++ b/src/sourcerer/utils.py @@ -17,6 +17,19 @@ ID_DIGEST_SIZE = 16 +def build_ref_key(host: str, org: str, repo: str, ref: str) -> str: + """Deterministic incremental ref_key: `{host}~{org}~{repo}~{ref}` (host/org/repo lowercased, + ref case-preserved). + + `~` is safe as a delimiter because it is illegal in git ref names (see + `git check-ref-format`) and is already the index-name segment delimiter used for + host/org/repo elsewhere (see `indices.py`), so it cannot collide with any of the joined + values. Snapshot content instead uses the bare commit SHA as its `ref_key` -- this helper + is only for the incremental (ref-addressed) shape. + """ + return "~".join((host.lower(), org.lower(), repo.lower(), ref)) + + def make_doc_id(*parts: str) -> str: """Deterministic, URL-safe document id: BLAKE2b hex of the NUL-joined fields. diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index cfed713..fea78c3 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -40,6 +40,30 @@ def test_git_host_filtered_before_git_org(): f"{tid} filters git.org before git.host" +_CONTENT_TOOL_IDS = ( + "sourcerer.code.search", "sourcerer.code.grep", + "sourcerer.files.cat", "sourcerer.files.head", "sourcerer.files.tail", + "sourcerer.files.ls", "sourcerer.files.tree", + "sourcerer.files.read_lines", "sourcerer.files.wc", +) + + +def test_content_tools_use_universal_ref_key_join_query(): + # INV-005: every content tool's WHERE runs the identical `git.ref_key == ?git_ref_key` + # shape with no update_mode/mode conditional, and joins sourcerer-refs on git.ref_key to + # resolve the commit for both snapshot and incremental content. + tools = _tools() + for tid in _CONTENT_TOOL_IDS: + query = tools[tid]["configuration"]["query"] + params = tools[tid]["configuration"]["params"] + assert "update_mode" not in query, f"{tid} query has an update_mode conditional" + assert "git.ref_key == ?git_ref_key" in query, f"{tid} missing the exact ref_key filter" + assert "| LOOKUP JOIN sourcerer-refs ON git.ref_key" in query, f"{tid} missing the universal join" + assert "git_commit" not in params, f"{tid} still has the old git_commit param" + assert params["git_ref_key"]["optional"] is False + assert "defaultValue" not in params["git_ref_key"] + + def test_output_keeps_git_host(): # Every tool that KEEPs git.org must also KEEP git.host (before it), so host reaches output. for tid, tool in _tools().items(): diff --git a/tests/test_backfill.py b/tests/test_backfill.py new file mode 100644 index 0000000..33d1190 --- /dev/null +++ b/tests/test_backfill.py @@ -0,0 +1,177 @@ +"""Tests for the one-time upgrade backfill in sourcerer.commands.index.markers: stamping +git.ref_key/update_mode onto pre-existing snapshot content, migrating the refs index mapping, +and creating missing snapshot join docs. Every ES call is mocked (INV-009/INV-010).""" + +# Standard packages +from unittest.mock import MagicMock + +# Third-party packages +from elastic_transport import ApiResponseMeta, HttpHeaders +from elasticsearch import NotFoundError + +# App packages +from sourcerer.commands.index.markers import ( + apply_content_index_mapping, + apply_refs_index_mapping, + backfill_refs_join_docs, + backfill_repo, + backfill_snapshot_ref_keys, + commits_with_join_doc, + distinct_commits_for_repo, +) +from sourcerer.indices import FILES_ALIAS, LINES_ALIAS, REFS_INDEX + +FULL_SHA = "cfefb3b2378ccbadefa7c8f4f9e21b3a1d2e5f60" + + +def _not_found() -> NotFoundError: + meta = ApiResponseMeta(status=404, http_version="1.1", headers=HttpHeaders({}), duration=0.0, node=None) + return NotFoundError("index_not_found_exception", meta, None) + + +class TestBackfillSnapshotRefKeys: + def test_scoped_to_repo_and_missing_ref_key(self): + es = MagicMock() + es.update_by_query.return_value = {"updated": 3} + total = backfill_snapshot_ref_keys(es, "github", "acme", "widgets") + assert total == 6 # 3 (files) + 3 (lines) + assert es.update_by_query.call_count == 2 + indices = {c.kwargs["index"] for c in es.update_by_query.call_args_list} + assert indices == {FILES_ALIAS, LINES_ALIAS} + for call in es.update_by_query.call_args_list: + query = call.kwargs["query"] + assert {"term": {"git.host": "github"}} in query["bool"]["filter"] + assert {"exists": {"field": "git.ref_key"}} in query["bool"]["must_not"] + + def test_second_run_is_a_no_op(self): + # Idempotency (INV-009): once every doc has ref_key, the must_not:exists filter + # matches nothing, so a repeat run updates 0 docs. + es = MagicMock() + es.update_by_query.return_value = {"updated": 0} + assert backfill_snapshot_ref_keys(es, "github", "acme", "widgets") == 0 + + def test_missing_index_is_ignored(self): + es = MagicMock() + es.update_by_query.side_effect = _not_found() + assert backfill_snapshot_ref_keys(es, "github", "acme", "widgets") == 0 + + +class TestDistinctCommitsForRepo: + def test_returns_bucket_keys(self): + es = MagicMock() + es.search.return_value = {"aggregations": {"commits": {"buckets": [ + {"key": "aaa"}, {"key": "bbb"}, + ]}}} + assert distinct_commits_for_repo(es, "github", "acme", "widgets") == {"aaa", "bbb"} + + def test_missing_index_returns_empty_set(self): + es = MagicMock() + es.search.side_effect = _not_found() + assert distinct_commits_for_repo(es, "github", "acme", "widgets") == set() + + +class TestCommitsWithJoinDoc: + def test_empty_input_short_circuits(self): + es = MagicMock() + assert commits_with_join_doc(es, set()) == set() + es.search.assert_not_called() + + def test_returns_hit_ids(self): + es = MagicMock() + es.search.return_value = {"hits": {"hits": [{"_id": "aaa"}]}} + assert commits_with_join_doc(es, {"aaa", "bbb"}) == {"aaa"} + + +class TestBackfillRefsJoinDocs: + def test_creates_join_doc_for_missing_commit(self): + es = MagicMock() + es.search.side_effect = [ + # distinct_commits_for_repo + {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, + # commits_with_join_doc -- no existing join docs + {"hits": {"hits": []}}, + # per-missing-commit lookup for a build_ref_id marker (none found) + {"hits": {"hits": []}}, + ] + created = backfill_refs_join_docs(es, "github", "acme", "widgets") + assert created == 1 + assert es.index.call_args.kwargs["id"] == FULL_SHA + assert es.index.call_args.kwargs["document"]["git"]["ref_key"] == FULL_SHA + + def test_second_run_creates_nothing(self): + # INV-009/INV-010: every commit already has a join doc -> no-op. + es = MagicMock() + es.search.side_effect = [ + {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, + {"hits": {"hits": [{"_id": FULL_SHA}]}}, + ] + assert backfill_refs_join_docs(es, "github", "acme", "widgets") == 0 + es.index.assert_not_called() + + def test_no_commits_short_circuits(self): + es = MagicMock() + es.search.return_value = {"aggregations": {"commits": {"buckets": []}}} + assert backfill_refs_join_docs(es, "github", "acme", "widgets") == 0 + es.index.assert_not_called() + + def test_refreshes_refs_index_when_docs_created(self): + # A uniqueness-gate run immediately afterward must see the just-created join doc + # rather than racing the refs index's refresh interval. + es = MagicMock() + es.search.side_effect = [ + {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, + {"hits": {"hits": []}}, + {"hits": {"hits": []}}, + ] + backfill_refs_join_docs(es, "github", "acme", "widgets") + assert es.indices.refresh.call_args.kwargs["index"] == REFS_INDEX + + def test_no_refresh_when_nothing_created(self): + es = MagicMock() + es.search.side_effect = [ + {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, + {"hits": {"hits": [{"_id": FULL_SHA}]}}, + ] + backfill_refs_join_docs(es, "github", "acme", "widgets") + es.indices.refresh.assert_not_called() + + +class TestApplyContentIndexMapping: + def test_puts_mapping_on_both_aliases(self): + es = MagicMock() + apply_content_index_mapping( + es, + {"properties": {"git": {"properties": {"ref_key": {"type": "keyword"}}}}}, + {"properties": {"git": {"properties": {"ref_key": {"type": "keyword"}}}}}, + ) + indices = {c.kwargs["index"] for c in es.indices.put_mapping.call_args_list} + assert indices == {FILES_ALIAS, LINES_ALIAS} + + def test_missing_index_is_ignored(self): + es = MagicMock() + es.indices.put_mapping.side_effect = _not_found() + apply_content_index_mapping(es, {"properties": {}}, {"properties": {}}) # no raise + + +class TestApplyRefsIndexMapping: + def test_puts_mapping_on_refs_index(self): + es = MagicMock() + apply_refs_index_mapping(es, {"properties": {"git": {"properties": {"ref_key": {"type": "keyword"}}}}}) + assert es.indices.put_mapping.call_args.kwargs["index"] == REFS_INDEX + + def test_missing_index_is_ignored(self): + es = MagicMock() + es.indices.put_mapping.side_effect = _not_found() + apply_refs_index_mapping(es, {"properties": {}}) # no raise + + +class TestBackfillRepo: + def test_second_run_is_fully_idempotent(self): + es = MagicMock() + es.update_by_query.return_value = {"updated": 0} + es.search.return_value = {"hits": {"hits": []}, "aggregations": {"commits": {"buckets": []}}} + summary = backfill_repo( + es, "github", "acme", "widgets", refs_mapping={"properties": {}}, + files_mapping={"properties": {}}, lines_mapping={"properties": {}}, + ) + assert summary == {"content_updated": 0, "join_docs_created": 0} diff --git a/tests/test_cli_index.py b/tests/test_cli_index.py index d80fe58..a37f3d7 100644 --- a/tests/test_cli_index.py +++ b/tests/test_cli_index.py @@ -68,7 +68,7 @@ def test_insecure_env_var_true_resolves_to_true(self): def fake_run(repo_spec, branch, tag, commit, url, api_key, username, password, force=False, quiet=False, cache_dir=None, ephemeral=False, - retry_window=None, insecure=False): + retry_window=None, insecure=False, no_backfill=False): captured["insecure"] = insecure with patch("sourcerer.commands.index.command.run", side_effect=fake_run): @@ -86,7 +86,7 @@ def test_insecure_env_var_absent_resolves_to_false(self): def fake_run(repo_spec, branch, tag, commit, url, api_key, username, password, force=False, quiet=False, cache_dir=None, ephemeral=False, - retry_window=None, insecure=False): + retry_window=None, insecure=False, no_backfill=False): captured["insecure"] = insecure with patch("sourcerer.commands.index.command.run", side_effect=fake_run): @@ -96,3 +96,43 @@ def fake_run(repo_spec, branch, tag, commit, url, api_key, username, password, ], env={}, catch_exceptions=False) assert captured.get("insecure") is False + + +class TestNoBackfillOption: + def test_help_shows_no_backfill(self): + runner = CliRunner() + result = runner.invoke(index, ["--help"]) + assert result.exit_code == 0 + assert "--no-backfill" in result.output + + def test_no_backfill_flag_forwarded_to_run(self): + runner = CliRunner() + captured = {} + + def fake_run(repo_spec, branch, tag, commit, url, api_key, username, password, + force=False, quiet=False, cache_dir=None, ephemeral=False, + retry_window=None, insecure=False, no_backfill=False): + captured["no_backfill"] = no_backfill + + with patch("sourcerer.commands.index.command.run", side_effect=fake_run): + runner.invoke(index, [ + "--url", "http://es:9200", "--no-backfill", "github/org/repo", + ], catch_exceptions=False) + + assert captured.get("no_backfill") is True + + def test_default_is_backfill_enabled(self): + runner = CliRunner() + captured = {} + + def fake_run(repo_spec, branch, tag, commit, url, api_key, username, password, + force=False, quiet=False, cache_dir=None, ephemeral=False, + retry_window=None, insecure=False, no_backfill=False): + captured["no_backfill"] = no_backfill + + with patch("sourcerer.commands.index.command.run", side_effect=fake_run): + runner.invoke(index, [ + "--url", "http://es:9200", "github/org/repo", + ], catch_exceptions=False) + + assert captured.get("no_backfill") is False diff --git a/tests/test_config.py b/tests/test_config.py index 83736aa..643e19b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -53,7 +53,7 @@ def _git(host="github", org="acme", repo="widgets", ref_type="branch"): def _source(host="github", org="acme", repo="widgets", ref_type="branch", - match="main", since=None, retain=None, omit_match=False): + match="main", since=None, retain=None, omit_match=False, update=None): src = {"git": _git(host, org, repo, ref_type)} if not omit_match: src["match"] = match @@ -61,6 +61,8 @@ def _source(host="github", org="acme", repo="widgets", ref_type="branch", src["since"] = since if retain is not None: src["retain"] = retain + if update is not None: + src["update"] = update return src @@ -189,6 +191,36 @@ def test_versioned_patterns_agreeing_on_levels_is_fine(self): assert cfg.repos[0].selectors[0].levels == ("major", "minor", "patch") +class TestParseUpdateMode: + def test_default_is_snapshot(self): + cfg = _cfg([_source()]) + assert cfg.repos[0].selectors[0].update == "snapshot" + + def test_incremental_accepted_on_branch(self): + cfg = _cfg([_source(ref_type="branch", update="incremental")]) + assert cfg.repos[0].selectors[0].update == "incremental" + + def test_incremental_rejected_on_tag(self): + with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): + _cfg([_source(ref_type="tag", match="v1.0.0", update="incremental")]) + + def test_incremental_rejected_on_commit(self): + with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): + _cfg([_source(ref_type="commit", match="cfefb3b", update="incremental")]) + + def test_invalid_mode_raises(self): + with pytest.raises(ValueError, match="must be one of"): + _cfg([_source(update="bogus")]) + + def test_incremental_with_since_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'since'"): + _cfg([_source(ref_type="branch", update="incremental", since={"age": "1y"})]) + + def test_incremental_with_retain_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'retain'"): + _cfg([_source(ref_type="branch", update="incremental", retain={"count": 5})]) + + class TestParseCommitSource: def test_full_sha_accepted(self): cfg = _cfg([_source(ref_type="commit", match="a" * 40)]) diff --git a/tests/test_documents.py b/tests/test_documents.py index 0f29d41..fd96476 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -14,7 +14,9 @@ _build_one_file_actions, build_file_actions, build_file_doc, + build_incremental_file_doc, file_attributes, + iter_incremental_line_docs, iter_line_docs, ) from sourcerer.indices import files_index, lines_index @@ -27,7 +29,14 @@ def _set_worker_ctx(host: str, org: str, repo: str, commit_sha: str, repo_dir, s # should leave behind for the rest of the pytest session. documents._WORKER_CTX.update( host=host, org=org, repo=repo, commit_sha=commit_sha, repo_dir=pathlib.Path(repo_dir), - symlink_paths=symlink_paths, + symlink_paths=symlink_paths, mode="snapshot", + ) + + +def _set_worker_ctx_incremental(host: str, org: str, repo: str, ref: str, repo_dir, symlink_paths=frozenset()) -> None: + documents._WORKER_CTX.update( + host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), + symlink_paths=symlink_paths, mode="incremental", ) @@ -69,7 +78,9 @@ def test_git_fields(self, tmp_path): p = tmp_path / "a.txt" p.write_text("hello") _id, doc = build_file_doc("github", "acme", "widgets", "deadbeef", "a.txt", p) - assert doc["git"] == {"host": "github", "org": "acme", "repo": "widgets", "commit": "deadbeef"} + assert doc["git"] == {"host": "github", "org": "acme", "repo": "widgets", "commit": "deadbeef", + "ref_key": "deadbeef"} + assert doc["update_mode"] == "snapshot" def test_host_changes_id(self, tmp_path): p = tmp_path / "a.txt" @@ -120,6 +131,12 @@ def test_broken_symlink_has_target_path_but_no_target_size(self, tmp_path): class TestIterLineDocs: + def test_snapshot_ref_key_and_update_mode(self): + docs = list(iter_line_docs("github", "acme", "widgets", "deadbeef", "a.txt", "one")) + _id, doc = docs[0] + assert doc["git"]["ref_key"] == "deadbeef" + assert doc["update_mode"] == "snapshot" + def test_line_numbering_starts_at_one(self): docs = list(iter_line_docs("github", "acme", "widgets", "deadbeef", "a.txt", "one\ntwo\nthree")) numbers = [d["line"]["number"] for _id, d in docs] @@ -162,6 +179,57 @@ def test_no_optional_fields_when_omitted(self): assert "attributes" not in d["file"] +class TestIncrementalDocs: + def test_ref_key_is_tilde_joined(self, tmp_path): + p = tmp_path / "a.txt" + p.write_text("hello") + _id, doc = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + assert doc["git"]["ref_key"] == "github~acme~widgets~main" + + def test_no_commit_field(self, tmp_path): + p = tmp_path / "a.txt" + p.write_text("hello") + _id, doc = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + assert "commit" not in doc["git"] + + def test_update_mode_incremental(self, tmp_path): + p = tmp_path / "a.txt" + p.write_text("hello") + _id, doc = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + assert doc["update_mode"] == "incremental" + + def test_id_stable_across_commits(self, tmp_path): + # The whole point of ref-addressing: the id does not depend on the commit, only the + # ref, so a modified file's doc overwrites in place rather than minting a new id. + p = tmp_path / "a.txt" + p.write_text("hello") + id1, _ = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + p.write_text("hello world -- content changed, same ref/path") + id2, _ = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + assert id1 == id2 + + def test_id_differs_from_snapshot_id(self, tmp_path): + p = tmp_path / "a.txt" + p.write_text("hello") + snap_id, _ = build_file_doc("github", "acme", "widgets", "deadbeef", "a.txt", p) + incr_id, _ = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + assert snap_id != incr_id + + def test_line_docs_ref_key_and_no_commit(self): + docs = list(iter_incremental_line_docs("github", "acme", "widgets", "main", "a.txt", "one\ntwo")) + for _id, d in docs: + assert d["git"]["ref_key"] == "github~acme~widgets~main" + assert "commit" not in d["git"] + assert d["update_mode"] == "incremental" + + def test_worker_ctx_routes_to_incremental_builders(self, tmp_path): + (tmp_path / "a.txt").write_text("one\ntwo\n") + _set_worker_ctx_incremental("github", "acme", "widgets", "main", tmp_path) + actions = _build_one_file_actions("a.txt") + assert actions[0]["_source"]["git"]["ref_key"] == "github~acme~widgets~main" + assert "commit" not in actions[0]["_source"]["git"] + + class TestFileAttributes: def test_plain_file_has_no_attributes(self, tmp_path): p = tmp_path / "a.txt" diff --git a/tests/test_git_changes.py b/tests/test_git_changes.py new file mode 100644 index 0000000..ea47fe4 --- /dev/null +++ b/tests/test_git_changes.py @@ -0,0 +1,166 @@ +"""Tests for the NUL-safe Git diff planner in sourcerer.commands.index.git. + +Two layers: + 1. Integration against a real temporary Git repository for the common change kinds (add, + modify, delete, rename, type change) and awkward path bytes (spaces, tabs, Unicode). + 2. Pure-parser tests over crafted `-z` byte streams for copy/rename records and truncation, + which are hard to elicit deterministically from git itself. +""" + +# Standard packages +import os +import subprocess + +# Third-party packages +import pytest + +# App packages +from sourcerer.commands.index.git import ( + ChangePlan, + _parse_name_status_z, + base_commit_available, + plan_changes, +) + + +def _git(repo, *args, env=None): + subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, env=env) + + +def _commit_all(repo, message): + env = { + **os.environ, + "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@e", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@e", + } + _git(repo, "add", "-A", env=env) + _git(repo, "commit", "-m", message, env=env) + out = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + check=True, capture_output=True, text=True, env=env, + ) + return out.stdout.strip() + + +@pytest.fixture +def repo(tmp_path): + _git(tmp_path, "init") + _git(tmp_path, "config", "user.email", "t@e") + _git(tmp_path, "config", "user.name", "t") + return tmp_path + + +class TestBaseCommitAvailable: + def test_plan_changes_base_available_for_present_commit(self, repo): + (repo / "a.txt").write_text("hi\n") + sha = _commit_all(repo, "init") + assert base_commit_available(repo, sha) is True + + def test_plan_changes_base_unavailable_for_absent_commit(self, repo): + (repo / "a.txt").write_text("hi\n") + _commit_all(repo, "init") + assert base_commit_available(repo, "0" * 40) is False + + +class TestPlanChangesIntegration: + def test_plan_changes_base_missing_returns_flagged_plan(self, repo): + (repo / "a.txt").write_text("hi\n") + new = _commit_all(repo, "init") + plan = plan_changes(repo, "0" * 40, new) + assert plan.base_missing is True + assert plan.delete_paths == [] and plan.index_paths == [] + + def test_plan_changes_add_modify_delete(self, repo): + (repo / "keep.txt").write_text("keep\n") + (repo / "gone.txt").write_text("gone\n") + (repo / "mod.txt").write_text("v1\n") + old = _commit_all(repo, "init") + (repo / "gone.txt").unlink() + (repo / "mod.txt").write_text("incremental\n") + (repo / "new.txt").write_text("new\n") + new = _commit_all(repo, "change") + plan = plan_changes(repo, old, new) + assert set(plan.index_paths) == {"mod.txt", "new.txt"} + assert set(plan.delete_paths) == {"gone.txt", "mod.txt"} + + def test_plan_changes_rename(self, repo): + (repo / "old_name.txt").write_text("stable content here\nline two\n") + old = _commit_all(repo, "init") + _git(repo, "mv", "old_name.txt", "new_name.txt") + new = _commit_all(repo, "rename") + plan = plan_changes(repo, old, new) + assert "old_name.txt" in plan.delete_paths + assert "new_name.txt" in plan.index_paths + assert "old_name.txt" not in plan.index_paths + + def test_plan_changes_type_change_file_to_symlink(self, repo): + (repo / "target.txt").write_text("target\n") + (repo / "thing").write_text("regular\n") + old = _commit_all(repo, "init") + (repo / "thing").unlink() + os.symlink("target.txt", repo / "thing") + new = _commit_all(repo, "typechange") + plan = plan_changes(repo, old, new) + # A type change replaces the whole file: delete the stale docs and re-index. + assert "thing" in plan.delete_paths + assert "thing" in plan.index_paths + + def test_plan_changes_awkward_paths_preserved(self, repo): + (repo / "seed.txt").write_text("seed\n") + old = _commit_all(repo, "init") + weird = "dir with spaces/tab\tfile.txt" + unicode_path = "café/naïve.txt" + (repo / "dir with spaces").mkdir() + (repo / weird).write_text("x\n") + (repo / "café").mkdir() + (repo / unicode_path).write_text("y\n") + new = _commit_all(repo, "add weird paths") + plan = plan_changes(repo, old, new) + assert weird in plan.index_paths + assert unicode_path in plan.index_paths + + +class TestParseNameStatusZ: + def test_plan_changes_parser_copy_indexes_destination_only(self): + raw = b"C100\x00src.txt\x00dst.txt\x00" + delete, index = _parse_name_status_z(raw) + assert delete == [] + assert index == ["dst.txt"] + + def test_plan_changes_parser_rename_deletes_source_indexes_destination(self): + raw = b"R100\x00from.txt\x00to.txt\x00" + delete, index = _parse_name_status_z(raw) + assert delete == ["from.txt"] + assert index == ["to.txt"] + + def test_plan_changes_parser_mixed_stream(self): + raw = ( + b"A\x00added.txt\x00" + b"M\x00mod.txt\x00" + b"D\x00del.txt\x00" + b"R096\x00old.txt\x00new.txt\x00" + ) + delete, index = _parse_name_status_z(raw) + assert delete == ["mod.txt", "del.txt", "old.txt"] + assert index == ["added.txt", "mod.txt", "new.txt"] + + def test_plan_changes_parser_dedupe_preserves_order(self): + raw = b"M\x00a.txt\x00M\x00a.txt\x00" + delete, index = _parse_name_status_z(raw) + assert delete == ["a.txt"] + assert index == ["a.txt"] + + def test_plan_changes_parser_truncated_rename_record_ignored(self): + raw = b"R100\x00only_one_path.txt\x00" + delete, index = _parse_name_status_z(raw) + assert delete == [] and index == [] + + def test_plan_changes_parser_empty_stream(self): + assert _parse_name_status_z(b"") == ([], []) + + +class TestChangePlanDefaults: + def test_plan_changes_defaults(self): + plan = ChangePlan() + assert plan.delete_paths == [] and plan.index_paths == [] + assert plan.base_missing is False diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py new file mode 100644 index 0000000..92d2a77 --- /dev/null +++ b/tests/test_incremental_index.py @@ -0,0 +1,145 @@ +"""Tests for the incremental (ref-addressed) branch orchestration in +sourcerer.commands.index.command.index_incremental_branch_in_dir: the two-phase +indexing -> ready publication, full rebuild vs delta update selection, and failure handling. +Every ES call and every git/documents side effect is mocked/patched -- these are orchestration +tests, not an end-to-end index run (see specs/incremental-indexing.md Task 16 for that).""" + +# Standard packages +from unittest.mock import MagicMock, patch + +# App packages +from sourcerer.commands.index.command import index_incremental_branch_in_dir +from sourcerer.commands.index.git import ChangePlan +from sourcerer.progress import ProgressReporter, Unit + +OLD = "1111111111111111111111111111111111111111" +NEW = "2222222222222222222222222222222222222222" + + +def _patch_common(prior=None, plan=None): + """Patch every git/documents/markers side effect index_incremental_branch_in_dir calls, + returning the patcher context managers as a dict of MagicMocks keyed by name.""" + patchers = { + "checkout_branch": patch("sourcerer.commands.index.command.checkout_branch"), + "resolve_commit": patch("sourcerer.commands.index.command.resolve_commit", return_value=NEW), + "commit_date": patch("sourcerer.commands.index.command.commit_date", return_value="2026-01-01T00:00:00+00:00"), + "read_incremental_ref": patch("sourcerer.commands.index.command.read_incremental_ref", return_value=prior), + "plan_changes": patch("sourcerer.commands.index.command.plan_changes", return_value=plan or ChangePlan()), + "delete_incremental_branch": patch("sourcerer.commands.index.command.delete_incremental_branch"), + "delete_incremental_paths": patch("sourcerer.commands.index.command.delete_incremental_paths"), + "index_incremental_paths": patch("sourcerer.commands.index.command.index_incremental_paths", return_value=(3, 30)), + "count_tracked_files": patch("sourcerer.commands.index.command.count_tracked_files", return_value=3), + "refresh_incremental_content": patch("sourcerer.commands.index.command.refresh_incremental_content"), + "count_incremental_branch_docs": patch( + "sourcerer.commands.index.command.count_incremental_branch_docs", return_value=(3, 30) + ), + "write_incremental_indexing": patch("sourcerer.commands.index.command.write_incremental_indexing"), + "write_incremental_ready": patch("sourcerer.commands.index.command.write_incremental_ready"), + "write_incremental_failed": patch("sourcerer.commands.index.command.write_incremental_failed"), + } + mocks = {name: p.start() for name, p in patchers.items()} + return patchers, mocks + + +def _stop(patchers): + for p in patchers.values(): + p.stop() + + +class TestIncrementalIndexFirstRun: + def test_first_index_does_full_rebuild(self): + patchers, mocks = _patch_common(prior=None) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + update="incremental") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + mocks["delete_incremental_branch"].assert_called_once() + mocks["index_incremental_paths"].assert_called_once() + # rel_paths (4th positional after repo_dir/branch) is None -> full tree walk. + call_args = mocks["index_incremental_paths"].call_args + assert call_args[0][6] is None + mocks["delete_incremental_paths"].assert_not_called() + mocks["write_incremental_ready"].assert_called_once() + assert mocks["write_incremental_ready"].call_args[0][5] == NEW + finally: + _stop(patchers) + + +class TestIncrementalIndexDeltaRun: + def test_second_run_indexes_only_changed_paths(self): + prior = {"git": {"commit": OLD}} + plan = ChangePlan(delete_paths=["gone.txt"], index_paths=["new.txt"]) + patchers, mocks = _patch_common(prior=prior, plan=plan) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + update="incremental") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + mocks["delete_incremental_branch"].assert_not_called() + mocks["delete_incremental_paths"].assert_called_once() + assert mocks["delete_incremental_paths"].call_args[0][5] == ["gone.txt"] + mocks["index_incremental_paths"].assert_called_once() + call_args = mocks["index_incremental_paths"].call_args + assert call_args[0][6] == ["new.txt"] + mocks["write_incremental_ready"].assert_called_once() + finally: + _stop(patchers) + + def test_missing_diff_base_triggers_full_rebuild(self): + prior = {"git": {"commit": OLD}} + plan = ChangePlan(base_missing=True) + patchers, mocks = _patch_common(prior=prior, plan=plan) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + update="incremental") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + mocks["delete_incremental_branch"].assert_called_once() + call_args = mocks["index_incremental_paths"].call_args + assert call_args[0][6] is None + finally: + _stop(patchers) + + def test_no_change_skips_entirely(self): + prior = {"git": {"commit": NEW}} # already at the new (checked-out) commit + patchers, mocks = _patch_common(prior=prior) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + update="incremental") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + mocks["index_incremental_paths"].assert_not_called() + mocks["write_incremental_indexing"].assert_not_called() + mocks["write_incremental_ready"].assert_not_called() + assert unit.status == "skipped" + finally: + _stop(patchers) + + +class TestIncrementalIndexFailure: + def test_failed_run_does_not_advance_commit(self): + prior = {"git": {"commit": OLD}} + plan = ChangePlan(delete_paths=[], index_paths=["a.txt"]) + patchers, mocks = _patch_common(prior=prior, plan=plan) + mocks["index_incremental_paths"].side_effect = RuntimeError("bulk failed") + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + update="incremental") + try: + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + assert False, "expected RuntimeError to propagate" + except RuntimeError: + pass + mocks["write_incremental_ready"].assert_not_called() + mocks["write_incremental_failed"].assert_called_once() + # The completed pointer stays at OLD -- a failed run must not advance it (INV-006). + assert mocks["write_incremental_failed"].call_args.kwargs["completed_commit"] == OLD + finally: + _stop(patchers) diff --git a/tests/test_markers.py b/tests/test_markers.py index f85136b..efc77c6 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -11,10 +11,28 @@ # App packages from sourcerer.commands.index.markers import ( - build_ref_id, commit_prefix_indexed, commits_with_content, fully_indexed_counts, - markers_status_by_id, _needs_index, _parse_marker_started, pre_clone_skip, should_index, + ERROR_MAX_LEN, + build_ref_id, + commit_prefix_indexed, + commits_with_content, + count_incremental_branch_docs, + delete_incremental_branch, + delete_incremental_paths, + fully_indexed_counts, + markers_status_by_id, + _needs_index, + _parse_marker_started, + pre_clone_skip, + read_incremental_ref, + should_index, + write_incremental_failed, + write_incremental_indexing, + write_incremental_ready, + write_ref_marker, + write_snapshot_join_doc, ) -from sourcerer.indices import FILES_ALIAS, REFS_ALIAS +from sourcerer.indices import FILES_ALIAS, REFS_ALIAS, REFS_INDEX +from sourcerer.utils import build_ref_key FULL_SHA = "cfefb3b2378ccbadefa7c8f4f9e21b3a1d2e5f60" @@ -376,3 +394,177 @@ def test_host_changes_result(self): assert fully_indexed_counts(es, "gitlab", "acme", "widgets", FULL_SHA) == (7, 900) call = es.search.call_args.kwargs assert {"term": {"git.host": "gitlab"}} in call["query"]["bool"]["filter"] + + +OLD = "1111111111111111111111111111111111111111" +NEW = "2222222222222222222222222222222222222222" + + +def _indexed_doc(es): + return es.index.call_args.kwargs["document"] + + +class TestSnapshotJoinDoc: + def test_join_doc_id_and_ref_key_are_the_commit(self): + es = MagicMock() + write_snapshot_join_doc(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None) + call = es.index.call_args.kwargs + assert call["id"] == OLD + assert call["index"] == REFS_INDEX + doc = call["document"] + assert doc["git"]["ref_key"] == OLD + assert doc["git"]["commit"] == OLD + assert doc["update_mode"] == "snapshot" + assert doc["status"] == "complete" + + def test_join_doc_idempotent_rewrite_same_id(self): + first = MagicMock() + second = MagicMock() + write_snapshot_join_doc(first, "github", "acme", "widgets", "branch", "main", OLD, None) + write_snapshot_join_doc(second, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None) + assert first.index.call_args.kwargs["id"] == second.index.call_args.kwargs["id"] + + +class TestIncrementalRefKeyIdentity: + def test_id_is_ref_key_not_a_hash(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "main", OLD, NEW) + assert es.index.call_args.kwargs["id"] == build_ref_key("github", "acme", "widgets", "main") + + def test_stable_across_calls_commit_independent(self): + a = build_ref_key("github", "acme", "widgets", "main") + b = build_ref_key("github", "acme", "widgets", "main") + assert a == b # one document per branch, no commit folded in + + +class TestWriteIncrementalIndexing: + def test_incremental_marker_preserves_completed_commit_and_exposes_target(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "main", + completed_commit=OLD, target_commit=NEW) + doc = _indexed_doc(es) + assert doc["status"] == "indexing" + assert doc["git"]["commit"] == OLD # completed pointer unchanged (INV-006) + assert doc["git"]["target_commit"] == NEW + assert doc["update_mode"] == "incremental" + assert es.index.call_args.kwargs["index"] == REFS_INDEX + + def test_incremental_marker_first_index_has_no_completed_commit(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "main", + completed_commit=None, target_commit=NEW) + assert _indexed_doc(es)["git"]["commit"] is None + + def test_incremental_marker_carries_prior_counts(self): + es = MagicMock() + prior = {"files_count": 12, "lines_count": 340, "git": {"commit_date": "2026-01-01T00:00:00+00:00"}} + write_incremental_indexing(es, "github", "acme", "widgets", "main", OLD, NEW, prior=prior) + doc = _indexed_doc(es) + assert doc["files_count"] == 12 and doc["lines_count"] == 340 + assert doc["git"]["commit_date"] == "2026-01-01T00:00:00+00:00" + + +class TestWriteIncrementalReady: + def test_incremental_marker_advances_commit_and_clears_target_and_error(self): + es = MagicMock() + write_incremental_ready(es, "github", "acme", "widgets", "main", commit=NEW, + commit_date_iso="2026-02-02T00:00:00+00:00", + files_count=5, lines_count=99) + doc = _indexed_doc(es) + assert doc["status"] == "ready" + assert doc["git"]["commit"] == NEW # advances only after a successful run (INV-006) + assert doc["git"]["target_commit"] is None + assert doc["error"] is None and doc["failed_at"] is None + assert doc["files_count"] == 5 and doc["lines_count"] == 99 + assert es.index.call_args.kwargs["refresh"] is True # publication boundary + + +class TestWriteIncrementalFailed: + def test_incremental_marker_keeps_status_indexing_and_retains_old_pointer(self): + es = MagicMock() + write_incremental_failed(es, "github", "acme", "widgets", "main", completed_commit=OLD, + target_commit=NEW, error="boom") + doc = _indexed_doc(es) + assert doc["status"] == "indexing" # not advanced -- a failed run leaves the prior state + assert doc["git"]["commit"] == OLD + assert doc["git"]["target_commit"] == NEW + assert doc["error"] == "boom" + assert doc["failed_at"] is not None + + def test_incremental_marker_error_text_is_bounded(self): + es = MagicMock() + write_incremental_failed(es, "github", "acme", "widgets", "main", OLD, NEW, error="x" * 5000) + assert len(_indexed_doc(es)["error"]) == ERROR_MAX_LEN + + +class TestReadIncrementalRef: + def test_returns_source(self): + es = MagicMock() + es.get.return_value = {"_source": {"status": "ready", "git": {"commit": NEW}}} + assert read_incremental_ref(es, "github", "acme", "widgets", "main") == ( + {"status": "ready", "git": {"commit": NEW}} + ) + assert es.get.call_args.kwargs["id"] == build_ref_key("github", "acme", "widgets", "main") + + def test_missing_returns_none(self): + es = MagicMock() + es.get.side_effect = _not_found() + assert read_incremental_ref(es, "github", "acme", "widgets", "main") is None + + +class TestDeleteIncrementalPaths: + def test_empty_paths_is_noop(self): + es = MagicMock() + delete_incremental_paths(es, "github", "acme", "widgets", "main", []) + es.delete_by_query.assert_not_called() + + def test_scoped_to_exact_ref_key_and_paths(self): + es = MagicMock() + delete_incremental_paths(es, "github", "acme", "widgets", "main", ["a.txt", "b.txt"]) + assert es.delete_by_query.call_count == 2 # files + lines indices + for call in es.delete_by_query.call_args_list: + query = call.kwargs["query"] + assert {"term": {"git.ref_key": build_ref_key("github", "acme", "widgets", "main")}} in ( + query["bool"]["filter"] + ) + assert {"terms": {"file.path": ["a.txt", "b.txt"]}} in query["bool"]["filter"] + + def test_missing_index_is_ignored(self): + es = MagicMock() + es.delete_by_query.side_effect = _not_found() + delete_incremental_paths(es, "github", "acme", "widgets", "main", ["a.txt"]) # no raise + + +class TestDeleteIncrementalBranch: + def test_scoped_to_exact_ref_key_only(self): + es = MagicMock() + delete_incremental_branch(es, "github", "acme", "widgets", "main") + assert es.delete_by_query.call_count == 2 + for call in es.delete_by_query.call_args_list: + query = call.kwargs["query"] + assert query["bool"]["filter"] == [ + {"term": {"git.ref_key": build_ref_key("github", "acme", "widgets", "main")}} + ] + + def test_isolated_from_another_branch(self): + # Two incremental branches indexed; deleting one's docs must never scope to the other's + # ref_key (INV-008) -- asserted here at the query-construction level. + es_a = MagicMock() + es_b = MagicMock() + delete_incremental_branch(es_a, "github", "acme", "widgets", "main") + delete_incremental_branch(es_b, "github", "acme", "widgets", "dev") + query_a = es_a.delete_by_query.call_args_list[0].kwargs["query"] + query_b = es_b.delete_by_query.call_args_list[0].kwargs["query"] + assert query_a != query_b + + +class TestCountIncrementalBranchDocs: + def test_missing_index_returns_zero(self): + es = MagicMock() + es.count.side_effect = _not_found() + assert count_incremental_branch_docs(es, "github", "acme", "widgets", "main") == (0, 0) + + def test_returns_counts(self): + es = MagicMock() + es.count.return_value = {"count": 5} + assert count_incremental_branch_docs(es, "github", "acme", "widgets", "main") == (5, 5) diff --git a/tests/test_uniqueness_gate.py b/tests/test_uniqueness_gate.py new file mode 100644 index 0000000..aee9db4 --- /dev/null +++ b/tests/test_uniqueness_gate.py @@ -0,0 +1,93 @@ +"""Tests for the post-upgrade uniqueness gate: sourcerer.queries.check_ref_key_uniqueness +(INV-011) and its command.py wiring in _run_uniqueness_gate. Every ES call is mocked.""" + +# Standard packages +from unittest.mock import MagicMock + +# Third-party packages +from elastic_transport import ApiResponseMeta, HttpHeaders +from elasticsearch import NotFoundError + +# App packages +from sourcerer.commands.index.command import _run_uniqueness_gate +from sourcerer.queries import check_ref_key_uniqueness, enumerate_content_ref_keys + + +def _not_found() -> NotFoundError: + meta = ApiResponseMeta(status=404, http_version="1.1", headers=HttpHeaders({}), duration=0.0, node=None) + return NotFoundError("index_not_found_exception", meta, None) + + +class TestEnumerateContentRefKeys: + def test_collects_keys_across_both_aliases(self): + es = MagicMock() + es.search.side_effect = [ + {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "aaa"}}]}}}, # files + {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "bbb"}}]}}}, # lines + ] + assert enumerate_content_ref_keys(es, "github", "acme", "widgets") == {"aaa", "bbb"} + + def test_missing_index_contributes_nothing(self): + es = MagicMock() + es.search.side_effect = _not_found() + assert enumerate_content_ref_keys(es, "github", "acme", "widgets") == set() + + +class TestCheckRefKeyUniqueness: + def test_clean_repo_returns_empty(self): + es = MagicMock() + es.search.side_effect = [ + # enumerate_content_ref_keys: files then lines + {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "aaa"}}]}}}, + {"aggregations": {"keys": {"buckets": []}}}, + # uniqueness count query + {"aggregations": {"keys": {"buckets": [{"key": "aaa", "doc_count": 1}]}}}, + ] + assert check_ref_key_uniqueness(es, "github", "acme", "widgets") == [] + + def test_missing_join_doc_is_offending(self): + es = MagicMock() + es.search.side_effect = [ + {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "aaa"}}]}}}, + {"aggregations": {"keys": {"buckets": []}}}, + {"aggregations": {"keys": {"buckets": []}}}, # no join doc at all + ] + assert check_ref_key_uniqueness(es, "github", "acme", "widgets") == ["aaa"] + + def test_duplicate_join_doc_is_offending(self): + es = MagicMock() + es.search.side_effect = [ + {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "aaa"}}]}}}, + {"aggregations": {"keys": {"buckets": []}}}, + {"aggregations": {"keys": {"buckets": [{"key": "aaa", "doc_count": 2}]}}}, + ] + assert check_ref_key_uniqueness(es, "github", "acme", "widgets") == ["aaa"] + + def test_no_content_short_circuits(self): + es = MagicMock() + es.search.side_effect = [ + {"aggregations": {"keys": {"buckets": []}}}, + {"aggregations": {"keys": {"buckets": []}}}, + ] + assert check_ref_key_uniqueness(es, "github", "acme", "widgets") == [] + + +class TestRunUniquenessGate: + def test_passes_silently_when_clean(self): + es = MagicMock() + es.search.side_effect = [ + {"aggregations": {"keys": {"buckets": []}}}, + {"aggregations": {"keys": {"buckets": []}}}, + ] + assert _run_uniqueness_gate(es, "github", "acme", "widgets") is True + + def test_fails_and_reports_on_violation(self, capsys): + es = MagicMock() + es.search.side_effect = [ + {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "aaa"}}]}}}, + {"aggregations": {"keys": {"buckets": []}}}, + {"aggregations": {"keys": {"buckets": []}}}, + ] + assert _run_uniqueness_gate(es, "github", "acme", "widgets") is False + captured = capsys.readouterr() + assert "aaa" in captured.err diff --git a/tests/test_utils.py b/tests/test_utils.py index debadce..8521071 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch # App packages -from sourcerer.utils import make_doc_id, make_client +from sourcerer.utils import make_doc_id, make_client, build_ref_key class TestMakeDocId: @@ -33,6 +33,23 @@ def test_non_utf8_parts_round_trip_via_surrogateescape(self): assert make_doc_id("acme", "widgets", weird) == make_doc_id("acme", "widgets", weird) +class TestBuildRefKey: + def test_ref_key_incremental_shape(self): + assert build_ref_key("github", "elastic", "sourcerer", "main") == ( + "github~elastic~sourcerer~main" + ) + + def test_ref_key_lowercases_host_org_repo_preserves_ref_case(self): + assert build_ref_key("GitHub", "Elastic", "Sourcerer", "Feature/Mixed-Case") == ( + "github~elastic~sourcerer~Feature/Mixed-Case" + ) + + def test_ref_key_deterministic(self): + assert build_ref_key("github", "acme", "widgets", "main") == build_ref_key( + "github", "acme", "widgets", "main" + ) + + class TestMakeClient: """make_client TLS behaviour — Elasticsearch constructor is patched, no real connection.""" From e6bc3799e53edc30c94737c6b528820ac769a5ac Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 14 Aug 2026 22:45:02 -0600 Subject: [PATCH 02/29] Fix code-review findings - `sourcerer.refs.list`'s default `status: "complete"` filter silently omitted every `update: incremental` branch, whose join doc status is `"ready"`/`"indexing"`, never `"complete"`. The default (no-arg) call now also matches `status == "ready"`. - Reject `update: incremental` combined with `index.level: commit` at config-parse time. Incremental content carries no `git.commit`, so a commit-level index name (which requires one) could never be built, previously failing at runtime on every indexing attempt instead of being rejected up front. --- src/sourcerer/config.py | 4 ++++ .../agent_builder_tools/sourcerer.refs.list.yml | 8 ++++++-- tests/test_agent_builder_tools.py | 10 ++++++++++ tests/test_config.py | 12 +++++++++++- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/sourcerer/config.py b/src/sourcerer/config.py index d6939df..a2a9184 100644 --- a/src/sourcerer/config.py +++ b/src/sourcerer/config.py @@ -547,6 +547,10 @@ def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: index_level, index_suffix = "repo", None if raw.get("index") is not None: index_level, index_suffix = _parse_index(raw["index"], ctx) + if update == "incremental" and index_level == "commit": + # Incremental content carries no git.commit of its own (see build_ref_key), so a + # commit-level index name -- which requires a commit sha -- can never be built for it. + raise ValueError(f"{ctx}: 'update: incremental' cannot be combined with 'index.level: commit'") selector = Selector(ref_type=ref_type, raw_patterns=patterns, compiled=compiled, since=since, retain=retain, levels=levels, schedule=schedule, diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml index 62294b4..ed7b653 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml @@ -12,7 +12,11 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status LIKE ?status + // An incremental branch's join doc is never "complete" (only "ready"/"indexing" -- + // see markers.write_incremental_ready/write_incremental_indexing), so the default + // ?status == "complete" must also surface "ready" docs, or the default (no-arg) call + // this skill documents would silently omit every incremental branch from the results. + AND (status LIKE ?status OR (?status == "complete" AND status == "ready")) // Format the response | SORT indexed_at DESC @@ -51,6 +55,6 @@ configuration: defaultValue: "*" status: type: string - description: Filter by ref status. "complete" = fully indexed (default); "indexing" = currently being indexed; "*" = all statuses. + description: Filter by ref status. "complete" = fully indexed (default; also includes incremental branches, whose join doc status is "ready" rather than "complete"); "indexing" = currently being indexed; "*" = all statuses. optional: true defaultValue: "complete" diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index fea78c3..1f06d56 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -64,6 +64,16 @@ def test_content_tools_use_universal_ref_key_join_query(): assert "defaultValue" not in params["git_ref_key"] +def test_refs_list_default_status_also_surfaces_incremental_ready(): + # Incremental join docs are never status:"complete" (only "ready"/"indexing" -- see + # markers.write_incremental_ready/write_incremental_indexing), so the default (no-arg) + # call must not silently omit every incremental branch. + tool = _tools()["sourcerer.refs.list"] + query = tool["configuration"]["query"] + assert 'status == "ready"' in query + assert tool["configuration"]["params"]["status"]["defaultValue"] == "complete" + + def test_output_keeps_git_host(): # Every tool that KEEPs git.org must also KEEP git.host (before it), so host reaches output. for tid, tool in _tools().items(): diff --git a/tests/test_config.py b/tests/test_config.py index 643e19b..6f21ec9 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -53,7 +53,7 @@ def _git(host="github", org="acme", repo="widgets", ref_type="branch"): def _source(host="github", org="acme", repo="widgets", ref_type="branch", - match="main", since=None, retain=None, omit_match=False, update=None): + match="main", since=None, retain=None, omit_match=False, update=None, index=None): src = {"git": _git(host, org, repo, ref_type)} if not omit_match: src["match"] = match @@ -63,6 +63,8 @@ def _source(host="github", org="acme", repo="widgets", ref_type="branch", src["retain"] = retain if update is not None: src["update"] = update + if index is not None: + src["index"] = index return src @@ -220,6 +222,14 @@ def test_incremental_with_retain_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'retain'"): _cfg([_source(ref_type="branch", update="incremental", retain={"count": 5})]) + def test_incremental_with_commit_level_index_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'index.level: commit'"): + _cfg([_source(ref_type="branch", update="incremental", index={"level": "commit"})]) + + def test_incremental_with_repo_level_index_is_fine(self): + cfg = _cfg([_source(ref_type="branch", update="incremental", index={"level": "repo"})]) + assert cfg.repos[0].selectors[0].update == "incremental" + class TestParseCommitSource: def test_full_sha_accepted(self): From a11972dc5d6cffefbd73ef9015fdb3db0d784fa7 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Sat, 15 Aug 2026 00:10:23 -0600 Subject: [PATCH 03/29] Stop exposing git.ref_key as an agent-facing concept git.ref_key is a storage/join implementation detail (how a content doc finds its refs join doc via LOOKUP JOIN) -- it never needed to be something an agent constructs or passes as a query param. Replace the `git_ref_key` param on all 9 Agent Builder content tools with `git_ref`: an exact commit SHA or branch/tag name, matched via `(git.commit == ?git_ref OR git.ref == ?git_ref)` before the LOOKUP JOIN, which still resolves `git.ref_key` internally to attach the citable commit. `sourcerer.refs.list` no longer surfaces `git.ref_key` in its output either. This restores the pre-incremental mental model for an agent: resolve a ref via `refs.list`, then pass the resolved value straight through to a content tool -- no new field to learn, no construction, no per-mode branching in the agent's own reasoning. Docs (AGENTS.md, README.md, ref-resolution SKILL.md) and tests updated accordingly. --- AGENTS.md | 21 ++++++--- README.md | 6 +-- .../sourcerer.code.grep.yml | 26 ++++++++--- .../sourcerer.code.search.yml | 26 ++++++++--- .../sourcerer.files.cat.yml | 26 ++++++++--- .../sourcerer.files.head.yml | 26 ++++++++--- .../sourcerer.files.ls.yml | 26 ++++++++--- .../sourcerer.files.read_lines.yml | 26 ++++++++--- .../sourcerer.files.tail.yml | 26 ++++++++--- .../sourcerer.files.tree.yml | 28 ++++++++---- .../sourcerer.files.wc.yml | 26 ++++++++--- .../sourcerer.refs.list.yml | 6 ++- src/sourcerer/skills/ref-resolution/SKILL.md | 44 +++++++++++-------- tests/test_agent_builder_tools.py | 35 +++++++++++---- 14 files changed, 246 insertions(+), 102 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 51c17ad..9c2d0c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -387,19 +387,26 @@ from the hashed, append-only `build_ref_id` ref-name markers described above (th `since`/retention history and are untouched by this). Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) therefore runs the -same query shape regardless of mode, with no `update_mode` conditional: +same query shape regardless of mode, with no `update_mode` conditional -- and `git.ref_key` is +NEVER an agent-facing param, only the internal join field: ```esql FROM sourcerer-lines -| WHERE git.ref_key == ?git_ref_key AND ... +| WHERE ... AND (git.commit == ?git_ref OR git.ref == ?git_ref) | LOOKUP JOIN sourcerer-refs ON git.ref_key +| WHERE git.commit LIKE ?git_commit ``` -`git_ref_key` is a required, exact-match param (no wildcards) -- resolve a ref to it first (see -`src/sourcerer/skills/ref-resolution/SKILL.md`): a snapshot ref resolves to its commit and uses -that commit directly as `git_ref_key`; an incremental branch builds `{host}~{org}~{repo}~{ref}` -directly, no commit resolution needed. The join adds/overwrites `git.commit` on every row, so -snapshot content (which already carries its own, identical `git.commit`) is unaffected and +`git_ref` is a required, exact-match param (no wildcards) -- resolve a ref first (see +`src/sourcerer/skills/ref-resolution/SKILL.md`), then pass through whatever it resolved to: a +snapshot ref's commit SHA, or an incremental branch's plain name (e.g. `main`) -- no construction, +no `ref_key` involved. The tool matches `git_ref` against whichever field the row actually +carries (`git.commit` for snapshot, `git.ref` for incremental), so the same param and the same +query shape work for both without the caller knowing which mode it is. `git_commit` is optional +(default `"*"`) and filters the commit the join resolves -- it lets a caller assert the branch it +resolved `git_ref` against hasn't since advanced; a no-op for a commit-scoped query, since +`git.commit` already equals `git_ref` there. The join adds/overwrites `git.commit` on every row, +so snapshot content (which already carries its own, identical `git.commit`) is unaffected and incremental content (which has none) gets it from the join. ### Upgrade backfill (`--no-backfill`) diff --git a/README.md b/README.md index 116e0cc..a7a7770 100644 --- a/README.md +++ b/README.md @@ -88,9 +88,9 @@ The [`sourcerer.yml` specification](specs/sourcerer-yml.md) has the full referen ### Snapshot vs. incremental indexing (`update: `) Each source can set `update: snapshot` (the default) or `update: incremental` (branch-only). -Both modes write content that carries a `git.ref_key`, and every Agent Builder tool query and -skill resolves a commit the same way regardless of mode: -`WHERE git.ref_key == ?git_ref_key | LOOKUP JOIN sourcerer-refs ON git.ref_key`. +Every Agent Builder content tool takes the same `git_ref` param either way (a commit SHA or an +exact branch/tag name) and resolves a commit the same way regardless of mode; `git.ref_key` is +an internal storage/join detail, never something the agent constructs or passes. - **`snapshot`** (default): content is commit-addressed, exactly as before. `git.ref_key` is the commit SHA itself, so every ref (branch, tag, or pinned commit) that resolves to the same diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml index 7402b95..a2be68e 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml @@ -9,16 +9,23 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key == ?git_ref_key + // A commit SHA (snapshot content) or the exact branch/tag name (incremental + // content, which has no git.commit of its own) -- whichever this source resolves to. + AND (git.commit == ?git_ref OR git.ref == ?git_ref) AND file.path LIKE ?file_path AND line.content RLIKE ?regex - // Resolve the commit for this ref_key via the universal join -- identical for - // snapshot and incremental content. Snapshot content already carries its own - // git.commit (equal to git.ref_key), so the join's value simply overwrites it in - // place; incremental content has no git.commit of its own, so the join supplies it. + // git.ref_key is purely an internal storage/join key (never a query param): every + // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN + // resolves each row's citable commit regardless of how the row above was scoped -- + // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Optional consistency guard: assert the branch hasn't advanced since git_ref was + // resolved (only meaningful when git_ref names an incremental branch; a no-op filter + // for a commit-scoped query, since git.commit already equals git_ref there). + | WHERE git.commit LIKE ?git_commit + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -45,10 +52,15 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref_key: + git_ref: type: string - description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. optional: false + git_commit: + type: string + description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to grep (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml index 5549dd2..7d4500a 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml @@ -9,16 +9,23 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key == ?git_ref_key + // A commit SHA (snapshot content) or the exact branch/tag name (incremental + // content, which has no git.commit of its own) -- whichever this source resolves to. + AND (git.commit == ?git_ref OR git.ref == ?git_ref) AND file.path LIKE ?file_path AND MATCH(line.content.text, ?q) - // Resolve the commit for this ref_key via the universal join -- identical for - // snapshot and incremental content. Snapshot content already carries its own - // git.commit (equal to git.ref_key), so the join's value simply overwrites it in - // place; incremental content has no git.commit of its own, so the join supplies it. + // git.ref_key is purely an internal storage/join key (never a query param): every + // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN + // resolves each row's citable commit regardless of how the row above was scoped -- + // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Optional consistency guard: assert the branch hasn't advanced since git_ref was + // resolved (only meaningful when git_ref names an incremental branch; a no-op filter + // for a commit-scoped query, since git.commit already equals git_ref there). + | WHERE git.commit LIKE ?git_commit + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -45,10 +52,15 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref_key: + git_ref: type: string - description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. optional: false + git_commit: + type: string + description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to search (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml index 0de5965..2c61c4c 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml @@ -9,15 +9,22 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key == ?git_ref_key + // A commit SHA (snapshot content) or the exact branch/tag name (incremental + // content, which has no git.commit of its own) -- whichever this source resolves to. + AND (git.commit == ?git_ref OR git.ref == ?git_ref) AND file.path LIKE ?file_path - // Resolve the commit for this ref_key via the universal join -- identical for - // snapshot and incremental content. Snapshot content already carries its own - // git.commit (equal to git.ref_key), so the join's value simply overwrites it in - // place; incremental content has no git.commit of its own, so the join supplies it. + // git.ref_key is purely an internal storage/join key (never a query param): every + // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN + // resolves each row's citable commit regardless of how the row above was scoped -- + // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Optional consistency guard: assert the branch hasn't advanced since git_ref was + // resolved (only meaningful when git_ref names an incremental branch; a no-op filter + // for a commit-scoped query, since git.commit already equals git_ref there). + | WHERE git.commit LIKE ?git_commit + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -63,10 +70,15 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref_key: + git_ref: type: string - description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. optional: false + git_commit: + type: string + description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to cat (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml index 4a0994a..20ab828 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml @@ -9,15 +9,22 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key == ?git_ref_key + // A commit SHA (snapshot content) or the exact branch/tag name (incremental + // content, which has no git.commit of its own) -- whichever this source resolves to. + AND (git.commit == ?git_ref OR git.ref == ?git_ref) AND file.path LIKE ?file_path - // Resolve the commit for this ref_key via the universal join -- identical for - // snapshot and incremental content. Snapshot content already carries its own - // git.commit (equal to git.ref_key), so the join's value simply overwrites it in - // place; incremental content has no git.commit of its own, so the join supplies it. + // git.ref_key is purely an internal storage/join key (never a query param): every + // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN + // resolves each row's citable commit regardless of how the row above was scoped -- + // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Optional consistency guard: assert the branch hasn't advanced since git_ref was + // resolved (only meaningful when git_ref names an incremental branch; a no-op filter + // for a commit-scoped query, since git.commit already equals git_ref there). + | WHERE git.commit LIKE ?git_commit + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -62,10 +69,15 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref_key: + git_ref: type: string - description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. optional: false + git_commit: + type: string + description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to head (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml index 76c0558..83e9785 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml @@ -9,14 +9,21 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key == ?git_ref_key + // A commit SHA (snapshot content) or the exact branch/tag name (incremental + // content, which has no git.commit of its own) -- whichever this source resolves to. + AND (git.commit == ?git_ref OR git.ref == ?git_ref) - // Resolve the commit for this ref_key via the universal join -- identical for - // snapshot and incremental content. Snapshot content already carries its own - // git.commit (equal to git.ref_key), so the join's value simply overwrites it in - // place; incremental content has no git.commit of its own, so the join supplies it. + // git.ref_key is purely an internal storage/join key (never a query param): every + // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN + // resolves each row's citable commit regardless of how the row above was scoped -- + // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Optional consistency guard: assert the branch hasn't advanced since git_ref was + // resolved (only meaningful when git_ref names an incremental branch; a no-op filter + // for a commit-scoped query, since git.commit already equals git_ref there). + | WHERE git.commit LIKE ?git_commit + // Enforce glob depth for * and ** on file.path. // Without this, "src/*/Job.java" would match files at any depth, // not just three path segments, because a wildcard * in ES|QL @@ -97,10 +104,15 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref_key: + git_ref: type: string - description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. optional: false + git_commit: + type: string + description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + optional: true + defaultValue: "*" file_path: type: string description: File path to ls (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml index 3864cee..d6ad15f 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml @@ -9,17 +9,24 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key == ?git_ref_key + // A commit SHA (snapshot content) or the exact branch/tag name (incremental + // content, which has no git.commit of its own) -- whichever this source resolves to. + AND (git.commit == ?git_ref OR git.ref == ?git_ref) AND file.path LIKE ?file_path AND line.number >= ?line_number_start AND line.number <= ?line_number_end - // Resolve the commit for this ref_key via the universal join -- identical for - // snapshot and incremental content. Snapshot content already carries its own - // git.commit (equal to git.ref_key), so the join's value simply overwrites it in - // place; incremental content has no git.commit of its own, so the join supplies it. + // git.ref_key is purely an internal storage/join key (never a query param): every + // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN + // resolves each row's citable commit regardless of how the row above was scoped -- + // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Optional consistency guard: assert the branch hasn't advanced since git_ref was + // resolved (only meaningful when git_ref names an incremental branch; a no-op filter + // for a commit-scoped query, since git.commit already equals git_ref there). + | WHERE git.commit LIKE ?git_commit + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -65,10 +72,15 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref_key: + git_ref: type: string - description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. optional: false + git_commit: + type: string + description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to read (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml index 7249386..0daea53 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml @@ -9,15 +9,22 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key == ?git_ref_key + // A commit SHA (snapshot content) or the exact branch/tag name (incremental + // content, which has no git.commit of its own) -- whichever this source resolves to. + AND (git.commit == ?git_ref OR git.ref == ?git_ref) AND file.path LIKE ?file_path - // Resolve the commit for this ref_key via the universal join -- identical for - // snapshot and incremental content. Snapshot content already carries its own - // git.commit (equal to git.ref_key), so the join's value simply overwrites it in - // place; incremental content has no git.commit of its own, so the join supplies it. + // git.ref_key is purely an internal storage/join key (never a query param): every + // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN + // resolves each row's citable commit regardless of how the row above was scoped -- + // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Optional consistency guard: assert the branch hasn't advanced since git_ref was + // resolved (only meaningful when git_ref names an incremental branch; a no-op filter + // for a commit-scoped query, since git.commit already equals git_ref there). + | WHERE git.commit LIKE ?git_commit + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -62,10 +69,15 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref_key: + git_ref: type: string - description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. optional: false + git_commit: + type: string + description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to tail (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml index 454e161..5e3de39 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml @@ -9,14 +9,21 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key == ?git_ref_key - - // Resolve the commit for this ref_key via the universal join -- identical for - // snapshot and incremental content. Snapshot content already carries its own - // git.commit (equal to git.ref_key), so the join's value simply overwrites it in - // place; incremental content has no git.commit of its own, so the join supplies it. + // A commit SHA (snapshot content) or the exact branch/tag name (incremental + // content, which has no git.commit of its own) -- whichever this source resolves to. + AND (git.commit == ?git_ref OR git.ref == ?git_ref) + + // git.ref_key is purely an internal storage/join key (never a query param): every + // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN + // resolves each row's citable commit regardless of how the row above was scoped -- + // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Optional consistency guard: assert the branch hasn't advanced since git_ref was + // resolved (only meaningful when git_ref names an incremental branch; a no-op filter + // for a commit-scoped query, since git.commit already equals git_ref there). + | WHERE git.commit LIKE ?git_commit + // Split each file path into its segments | EVAL _segs = SPLIT(file.path, "/") | EVAL _file_segs = MV_COUNT(_segs) @@ -186,10 +193,15 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref_key: + git_ref: type: string - description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. optional: false + git_commit: + type: string + description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + optional: true + defaultValue: "*" file_path: type: string description: Directory to tree, e.g. src or src/sourcerer. The tree is rooted there and recurses; depth is controlled by L, not by this pattern. May also be a glob to filter what is listed (e.g. src/*.xml lists only XML files under src, at any depth). A single * already matches across directories, so ** is never required. diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml index 33bce5f..b7a9bbe 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml @@ -9,15 +9,22 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key == ?git_ref_key + // A commit SHA (snapshot content) or the exact branch/tag name (incremental + // content, which has no git.commit of its own) -- whichever this source resolves to. + AND (git.commit == ?git_ref OR git.ref == ?git_ref) AND file.path LIKE ?file_path - // Resolve the commit for this ref_key via the universal join -- identical for - // snapshot and incremental content. Snapshot content already carries its own - // git.commit (equal to git.ref_key), so the join's value simply overwrites it in - // place; incremental content has no git.commit of its own, so the join supplies it. + // git.ref_key is purely an internal storage/join key (never a query param): every + // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN + // resolves each row's citable commit regardless of how the row above was scoped -- + // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Optional consistency guard: assert the branch hasn't advanced since git_ref was + // resolved (only meaningful when git_ref names an incremental branch; a no-op filter + // for a commit-scoped query, since git.commit already equals git_ref there). + | WHERE git.commit LIKE ?git_commit + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -177,10 +184,15 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref_key: + git_ref: type: string - description: Exact ref_key to scope to -- a commit SHA (snapshot content) or the incremental "{host}~{org}~{repo}~{ref}" key (see sourcerer.refs.list). Required exact match, no wildcards. + description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. optional: false + git_commit: + type: string + description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to count (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml index ed7b653..e18a56f 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml @@ -20,7 +20,11 @@ configuration: // Format the response | SORT indexed_at DESC - | KEEP git.host, git.org, git.repo, git.ref, git.ref_type, git.ref_key, git.commit, git.commit_date, status, files_count, lines_count, indexed_at, indexing_started_at + // git.ref_key is not surfaced here -- it's purely an internal storage/join key (used only + // by content tools' `LOOKUP JOIN sourcerer-refs ON git.ref_key`), never something an agent + // needs to read or construct. Use git.ref (branch/tag name) or git.commit as the git_ref + // param on a content tool -- see the ref-resolution skill. + | KEEP git.host, git.org, git.repo, git.ref, git.ref_type, git.commit, git.commit_date, status, files_count, lines_count, indexed_at, indexing_started_at | LIMIT 1000000 params: git_host: diff --git a/src/sourcerer/skills/ref-resolution/SKILL.md b/src/sourcerer/skills/ref-resolution/SKILL.md index bd31ccd..aa0769b 100644 --- a/src/sourcerer/skills/ref-resolution/SKILL.md +++ b/src/sourcerer/skills/ref-resolution/SKILL.md @@ -36,10 +36,10 @@ Disambiguate based on context: - **Comparison or history** (e.g. "how has X evolved across 8.x?"): resolve to *all matching commits*. Collect every stable tag in the range; query each one separately. Label findings clearly by version. ### Comparison across versions (e.g. "8.x vs 9.x", "before and after 8.17") -Resolve each ref independently using the steps above. Run content queries against each commit, then compare results. Label each finding with its version. +Resolve each ref independently using the steps above. Run content queries against each resolved ref (see "Pinning the ref" below), then compare results. Label each finding with its version. ### Explicit ref (branch name, exact tag, commit hash) -Use as given. If it is a branch, call `refs.list` with `git_ref_type: branch` to confirm it exists and retrieve its current commit. If it is a tag, confirm and get its commit. If it is a commit hash, use it directly - optionally confirm with `git_ref_type: commit` if it may be a pinned commit rather than one reached via a branch/tag. +Use as given. If it is a branch, call `refs.list` with `git_ref_type: branch` to confirm it exists and retrieve its row. If it is a tag, confirm it the same way. If it is a commit hash, you don't need a `refs.list` call to use it in a content query (a commit hash is already a valid `git_ref` value) - optionally confirm with `git_ref_type: commit` if it may be a pinned commit rather than one reached via a branch/tag. ### Branch as of a specific date (e.g. "main as it was on 2024-03-01") When a branch was indexed with `since` (history walk), multiple snapshots of the branch exist — @@ -47,26 +47,32 @@ one per historical commit. Resolve "branch as of date D" like this: 1. Call `refs.list` with `git_ref_type: branch` and `git_ref: `. 2. From the results, filter to markers with `commit_date <= D` and `status: complete`. 3. Pick the marker with the **latest** `commit_date` among those (the branch state at the closest point on or before D). -4. Use that marker's `git.commit` for content queries. +4. Use that row for content queries as described below. If only one marker exists for the branch (tip-only indexing, no `since`), state that historical snapshots are unavailable for that branch. -## Pinning the ref_key -Every content query (`sourcerer.code.*` and `sourcerer.files.*`) takes the same single param, -`git_ref_key`, and runs the identical universal join query underneath -(`WHERE git.ref_key == ?git_ref_key | LOOKUP JOIN sourcerer-refs ON git.ref_key`) regardless of -whether the source is indexed as `snapshot` or `incremental`. Derive it once a ref is resolved: +## Pinning the ref +Every content query (`sourcerer.code.*` and `sourcerer.files.*`) takes the same single required +param, `git_ref`: an exact commit SHA, or an exact branch/tag name. The tool matches it against +whichever field the content actually carries (`git.commit` for a `snapshot` source, `git.ref` +for an `incremental` branch) and then joins to resolve the citable commit -- there is no mode +you need to reason about. -- **Snapshot** (the default; most tags and one-off branch indexes): resolve the ref to its - `git.commit` as above, then use that **commit SHA directly** as `git_ref_key` -- for snapshot - content, `git.ref_key == git.commit`. -- **Incremental** (a branch source configured with `update: incremental` in `sourcerer.yml`; - `refs.list` surfaces its join doc with `update_mode: incremental` and no separate per-commit - history): build the `git_ref_key` directly as `{host}~{org}~{repo}~{ref}` (host/org/repo - lowercased, ref case-preserved, `~`-joined) -- no commit resolution needed, since the key names - the branch itself and the join always resolves it to the CURRENT indexed commit at query time. +Once a ref is resolved above, pass the value straight through: +- Resolved to a commit (tags, one-off branch snapshots -- the common case): use that commit SHA + as `git_ref`. +- Resolved to an incremental branch (a source configured with `update: incremental` in + `sourcerer.yml`; its `refs.list` row has `status: ready`, not `complete`): use the branch name + itself as `git_ref` (e.g. `main`) -- no commit needed, the query always resolves to whatever + commit that branch is CURRENTLY at. -Use the resulting `git_ref_key` in every subsequent content call for that repo and ref, and read -the resolved `git.commit` back from the join for citations. Re-invoke this skill only when the -question introduces a new or additional ref. +`git.ref_key` is an internal storage/join detail (`LOOKUP JOIN sourcerer-refs ON git.ref_key` +inside the tool) -- it is never a param you construct or a value `refs.list` returns. + +Read the resolved `git.commit` back from the same row (or from the content query's own join) +for citations. Every content tool also accepts an optional `git_commit` param (default `"*"`): +it filters the commit the join resolves, so passing the expected commit re-asserts that an +incremental branch hasn't advanced since `git_ref` was resolved -- pass it when that matters +(e.g. a long-running investigation), otherwise leave it at the default. Re-invoke this skill +only when the question introduces a new or additional ref. diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index 1f06d56..b9fe428 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -48,20 +48,39 @@ def test_git_host_filtered_before_git_org(): ) -def test_content_tools_use_universal_ref_key_join_query(): - # INV-005: every content tool's WHERE runs the identical `git.ref_key == ?git_ref_key` - # shape with no update_mode/mode conditional, and joins sourcerer-refs on git.ref_key to - # resolve the commit for both snapshot and incremental content. +def test_content_tools_use_universal_ref_join_query(): + # INV-005: every content tool's WHERE runs the identical `(git.commit == ?git_ref OR + # git.ref == ?git_ref)` shape with no update_mode/mode conditional, and joins sourcerer-refs + # on git.ref_key (a purely internal field -- never an agent-facing param) to resolve the + # commit for both snapshot and incremental content. git_commit survives as an optional + # POST-join consistency guard (not a scoping filter -- git_ref is the scoping param). tools = _tools() for tid in _CONTENT_TOOL_IDS: query = tools[tid]["configuration"]["query"] params = tools[tid]["configuration"]["params"] assert "update_mode" not in query, f"{tid} query has an update_mode conditional" - assert "git.ref_key == ?git_ref_key" in query, f"{tid} missing the exact ref_key filter" + assert "git.commit == ?git_ref" in query, f"{tid} missing the commit-or-ref filter" + assert "git.ref == ?git_ref" in query, f"{tid} missing the commit-or-ref filter" assert "| LOOKUP JOIN sourcerer-refs ON git.ref_key" in query, f"{tid} missing the universal join" - assert "git_commit" not in params, f"{tid} still has the old git_commit param" - assert params["git_ref_key"]["optional"] is False - assert "defaultValue" not in params["git_ref_key"] + assert "git_ref_key" not in params, f"{tid} still exposes git_ref_key as a param" + assert "?git_ref_key" not in query, f"{tid} still references ?git_ref_key" + assert params["git_ref"]["optional"] is False + assert "defaultValue" not in params["git_ref"] + assert params["git_commit"]["optional"] is True + assert params["git_commit"]["defaultValue"] == "*" + # The git_commit filter must appear AFTER the join (it filters the joined value, not + # the raw content doc -- incremental content has no git.commit of its own). + assert query.index("LOOKUP JOIN sourcerer-refs") < query.index("git.commit LIKE ?git_commit") + + +def test_refs_list_does_not_surface_ref_key(): + # git.ref_key is purely an internal storage/join key -- never something an agent reads or + # constructs -- so refs.list must not surface it. + tool = _tools()["sourcerer.refs.list"] + query = tool["configuration"]["query"] + for line in query.splitlines(): + if line.strip().startswith("| KEEP"): + assert "ref_key" not in line def test_refs_list_default_status_also_surfaces_incremental_ready(): From 43212b5c1f4a4eb0e8d52130cf02cf368d5430f1 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 18 Aug 2026 23:14:31 -0600 Subject: [PATCH 04/29] Bump backing indices to v3 for incremental indexing This PR's incremental (ref-addressed) branch indexing is a big enough addition to the index schema (new update:incremental mode, ref_key join field, incremental refs join docs) to warrant its own index generation rather than folding it silently into v2. Rename every backing index constant, template file, and doc reference from sourcerer-v2-* to sourcerer-v3-*: - src/sourcerer/indices.py: FILES_INDEX_PREFIX, LINES_INDEX_PREFIX, REFS_INDEX now sourcerer-v3-*. Read aliases (sourcerer-files/-lines/-refs) are unaffected. - Index templates renamed sourcerer-v2-{files,lines,refs}.json -> sourcerer-v3-{...}.json, with matching index_patterns. - Doc/comment references across planner.py, queries.py, markers.py, schedule.py, prune/*, tests, AGENTS.md, README.md, specs/sourcerer-yml.md, sourcerer.example.yml updated to v3. - Added an "Upgrading from v2 to v3" section to AGENTS.md alongside the existing v1->v2 section, covering the index rename and the git_ref Agent Builder tool param. Existing sourcerer-v2-* installations require running `sourcerer setup` (to create the v3 templates) followed by a full re-index; the old v2-* indices can be deleted afterward. No config schema change. --- AGENTS.md | 33 +++-- sourcerer.example.yml | 2 +- specs/sourcerer-yml.md | 32 ++--- src/sourcerer/commands/index/command.py | 10 +- src/sourcerer/commands/index/markers.py | 2 +- src/sourcerer/commands/index/schedule.py | 2 +- src/sourcerer/commands/prune/execute.py | 2 +- src/sourcerer/commands/prune/report.py | 2 +- ...-v2-files.json => sourcerer-v3-files.json} | 2 +- ...-v2-lines.json => sourcerer-v3-lines.json} | 2 +- ...er-v2-refs.json => sourcerer-v3-refs.json} | 2 +- src/sourcerer/indices.py | 13 +- src/sourcerer/planner.py | 10 +- src/sourcerer/queries.py | 4 +- tests/test_index_orphans.py | 18 +-- tests/test_index_ref.py | 6 +- tests/test_indices.py | 42 +++---- tests/test_planner_orphans.py | 114 +++++++++--------- tests/test_prune_deletions.py | 16 +-- tests/test_setup.py | 24 ++-- 20 files changed, 176 insertions(+), 162 deletions(-) rename src/sourcerer/elastic/index_templates/{sourcerer-v2-files.json => sourcerer-v3-files.json} (98%) rename src/sourcerer/elastic/index_templates/{sourcerer-v2-lines.json => sourcerer-v3-lines.json} (99%) rename src/sourcerer/elastic/index_templates/{sourcerer-v2-refs.json => sourcerer-v3-refs.json} (98%) diff --git a/AGENTS.md b/AGENTS.md index 9c2d0c6..725d81e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -256,7 +256,7 @@ config files each driven by their own cron job. Run `sourcerer index --config` o cron** (e.g. every 5 minutes) and let the schedule config control the actual indexing cadence. **How the gate works**: on each invocation of `index --config`, before any ls-remote or clone -work, the command queries `sourcerer-v2-refs` to see when each source was last fully indexed +work, the command queries `sourcerer-v3-refs` to see when each source was last fully indexed (`status: complete`) and whether any ref in its scope is actively being indexed (`status: indexing`). Only sources whose schedule has fired since their last indexed run proceed to the expensive pipeline. Sources where another run is actively indexing are skipped. @@ -347,19 +347,19 @@ Content is addressed by **host + commit**, not by ref name. A file's bytes are f by `(git.host, git.org, git.repo, git.commit, file.path)`, so the same file reached via any ref collapses to a single doc (no per-ref duplication), while the same org/repo on two different git hosts stays distinct. `git.host` is a lowercase keyword, placed before `git.org` in every index -template's mappings and index sort. Backing indices are `sourcerer-v2-refs`, -`sourcerer-v2-files~{git.host}~{git.org}~{git.repo}`, and -`sourcerer-v2-lines~{git.host}~{git.org}~{git.repo}` (read via the unchanged `sourcerer-refs` / +template's mappings and index sort. Backing indices are `sourcerer-v3-refs`, +`sourcerer-v3-files~{git.host}~{git.org}~{git.repo}`, and +`sourcerer-v3-lines~{git.host}~{git.org}~{git.repo}` (read via the unchanged `sourcerer-refs` / `sourcerer-files` / `sourcerer-lines` aliases). **Index routing (`sources[i].index.level` / `index.suffix`).** A source can override the content index name: `level` (`host`/`org`/`repo` (default)/`commit`) chooses the granularity -(`sourcerer-v2-*~{host}` … `~{host}~{org}~{repo}~{commit}`), and `suffix` appends `^{suffix}` +(`sourcerer-v3-*~{host}` … `~{host}~{org}~{repo}~{commit}`), and `suffix` appends `^{suffix}` (e.g. `~{host}~{org}~{repo}^deploy`). Routing is **per-source**, so two sources of the same -`(host, org, repo)` may target different indices; the read aliases match `sourcerer-v2-files*` / -`sourcerer-v2-lines*`, so every leveled/suffixed index auto-joins them and agents are unaffected. +`(host, org, repo)` may target different indices; the read aliases match `sourcerer-v3-files*` / +`sourcerer-v3-lines*`, so every leveled/suffixed index auto-joins them and agents are unaffected. The name is built by `files_index`/`lines_index` in `src/sourcerer/indices.py`; each ref marker in -`sourcerer-v2-refs` records the source's `index_level`/`index_suffix` (semantic, not the resolved +`sourcerer-v3-refs` records the source's `index_level`/`index_suffix` (semantic, not the resolved name — so a future prefix bump stays correct). Changing a source's routing between runs triggers a **migration** (`sourcerer index` re-ingests at the new index, flips the marker, then deletes the old copy; `sourcerer prune` sweeps any crash-leftover as `orphan:stale-location`, and deletes any @@ -380,7 +380,7 @@ creates one index/shard per commit — see `specs/sourcerer-yml.md` for the cave Every content doc (file and line, both `update` modes) carries a `git.ref_key` keyword field: the bare commit SHA for `snapshot` content, or `{host}~{org}~{repo}~{ref}` for `incremental` content (see `update: ` above; `build_ref_key` in `src/sourcerer/utils.py`). A second, -distinct kind of `sourcerer-v2-refs` document -- a **refs join doc**, `_id = git.ref_key` +distinct kind of `sourcerer-v3-refs` document -- a **refs join doc**, `_id = git.ref_key` (exactly one per key) -- carries the citable `git.commit`: one per commit for snapshot content, one per branch (holding the live HEAD) for incremental content. This is a different id space from the hashed, append-only `build_ref_id` ref-name markers described above (those still drive @@ -455,4 +455,17 @@ different git hosting providers. This is a breaking change: `sourcerer-v1-*` indices can be deleted once you have re-indexed. - **Citations**: `sourcerer setup --config sourcerer.yml` reads the config's `hosts:` section and generates one citation skill per host so the agent formats links correctly for each - provider. Run `setup` again whenever you add or customize a host. \ No newline at end of file + provider. Run `setup` again whenever you add or customize a host. + +### Upgrading from v2 to v3 + +v3.0.0 adds incremental (ref-addressed) branch indexing (`update: incremental`, see above). This +is a breaking change to the backing indices, with no config schema change: + +- **Indices**: backing indices are renamed `sourcerer-v2-*` to `sourcerer-v3-*`. There is no + automatic migration - run `sourcerer setup` to create the v3 templates, then re-index every + source. The old `sourcerer-v2-*` indices can be deleted once you have re-indexed. +- **Agent Builder tools**: content tools (`sourcerer.code.*`, `sourcerer.files.*`) replace their + `git_commit` param with `git_ref` (a commit SHA or a branch/tag name); `git_commit` survives + as an optional post-join consistency guard. Run `sourcerer setup` again to push the updated + tool definitions. \ No newline at end of file diff --git a/sourcerer.example.yml b/sourcerer.example.yml index 49ccec8..01c2a8e 100644 --- a/sourcerer.example.yml +++ b/sourcerer.example.yml @@ -240,7 +240,7 @@ sources: match: deploy@{major} index: level: repo # host | org | repo (default) | commit - suffix: deploy # -> sourcerer-v2-*~github~elastic~elasticsearch^deploy + suffix: deploy # -> sourcerer-v3-*~github~elastic~elasticsearch^deploy # A second host hosting a same-named org/repo is a distinct instance: - git: diff --git a/specs/sourcerer-yml.md b/specs/sourcerer-yml.md index 3e23979..1c224a2 100644 --- a/specs/sourcerer-yml.md +++ b/specs/sourcerer-yml.md @@ -126,7 +126,7 @@ Elastic Agent Builder. URL template for repositories when running `git clone` during indexing. -The `git` fields from the `sourcerer-v2-refs` index can be referenced as +The `git` fields from the `sourcerer-v3-refs` index can be referenced as variables with curly braces (e.g. `{git.org}`, `{git.repo}`). Example: `https://github.com/{git.org}/{git.repo}.git` @@ -139,7 +139,7 @@ Example: `https://github.com/{git.org}/{git.repo}.git` URL template for citing links to a directory in a repo. Used by the citation skills in Agent Builder. -Fields from the `sourcerer-v2-files*` or `sourcerer-v2-lines*` indices can be +Fields from the `sourcerer-v3-files*` or `sourcerer-v3-lines*` indices can be referenced as variables with curly braces (e.g. `{git.org}`, `{file.directory}`). Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.directory}` @@ -152,7 +152,7 @@ Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.direct URL template for citing links to a file in a repo. Used by the citation skills in Elastic Agent Builder. -Fields from the `sourcerer-v2-files*` or `sourcerer-v2-lines*` indices can be +Fields from the `sourcerer-v3-files*` or `sourcerer-v3-lines*` indices can be referenced as variables with curly braces (e.g. `{git.org}`, `{file.path}`). Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.path}` @@ -165,7 +165,7 @@ Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.path}` URL template for citing links to a line of code in a repo. Used by the citation skills in Elastic Agent Builder. -Fields from the `sourcerer-v2-files*` or `sourcerer-v2-lines*` indices can be +Fields from the `sourcerer-v3-files*` or `sourcerer-v3-lines*` indices can be referenced as variables with curly braces (e.g. `{git.org}`, `{file.path}`, `{line.number}`). Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.path}#L{line.number}` @@ -178,7 +178,7 @@ Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.path}# URL template for citing links to a range of lines of code in a repo. Used by the citation skills in Elastic Agent Builder. -Fields from the `sourcerer-v2-files*` or `sourcerer-v2-lines*` indices can be +Fields from the `sourcerer-v3-files*` or `sourcerer-v3-lines*` indices can be referenced as variables with curly braces (e.g. `{git.org}`, `{file.path}`) as well as fields returned by the `sourcerer.code.*` and `sourcerer.files.*` tools in Elastic Agent Builder (e.g. `{line.number_start}`, `{line.number_end}`). @@ -599,7 +599,7 @@ indexing (if it doesn't also qualify for pruning). Accepts a 5-field cron expression or a duration string (see `schedules[i].schedule`). The "due" check compares against the **most recently completed** indexing run for -this source's scope `(host, org, repo, ref_type)` as recorded in `sourcerer-v2-refs`. +this source's scope `(host, org, repo, ref_type)` as recorded in `sourcerer-v3-refs`. A source is **not due** if another run has `status: indexing` for a ref in scope with `indexing_started_at` newer than 6 hours ago. This prevents redundant parallel work when the same config is invoked on a tight cron schedule. @@ -626,14 +626,14 @@ default repo-level index while its deploy tags go to a `^deploy`-suffixed index: ```yaml sources: - git: { host: github, org: elastic, repo: kibana, ref_type: tag } - match: "v{major}.{minor}.{patch}" # -> sourcerer-v2-*~github~elastic~kibana + match: "v{major}.{minor}.{patch}" # -> sourcerer-v3-*~github~elastic~kibana - git: { host: github, org: elastic, repo: kibana, ref_type: tag } match: "deploy@{major}" - index: { suffix: deploy } # -> sourcerer-v2-*~github~elastic~kibana^deploy + index: { suffix: deploy } # -> sourcerer-v3-*~github~elastic~kibana^deploy ``` Agents are unaffected by routing: they query the `sourcerer-files` / `sourcerer-lines` -read aliases, which span every `sourcerer-v2-files*` / `sourcerer-v2-lines*` index +read aliases, which span every `sourcerer-v3-files*` / `sourcerer-v3-lines*` index regardless of its level or suffix. **Changing routing on an already-indexed source (migration).** Because content @@ -666,10 +666,10 @@ Values of `level` and their effects on index names: |`level` |Index name | |----------|-------------------------------------------------------------| -|`"host"` |`sourcerer-v2-*~{git.host}` | -|`"org"` |`sourcerer-v2-*~{git.host}~{git.org}` | -|`"repo"` |`sourcerer-v2-*~{git.host}~{git.org}~{git.repo}` | -|`"commit"`|`sourcerer-v2-*~{git.host}~{git.org}~{git.repo}~{git.commit}`| +|`"host"` |`sourcerer-v3-*~{git.host}` | +|`"org"` |`sourcerer-v3-*~{git.host}~{git.org}` | +|`"repo"` |`sourcerer-v3-*~{git.host}~{git.org}~{git.repo}` | +|`"commit"`|`sourcerer-v3-*~{git.host}~{git.org}~{git.repo}~{git.commit}`| **Caveat — `"commit"`:** a commit-level index creates one physical index (and at least one shard) *per indexed commit*. On a repo/branch with many indexed commits @@ -692,12 +692,12 @@ a caret (`^`). Example index naming pattern where `sources[i].index.level` is `"repo"` and `sources[i].index.suffix` is `"deploy"`: -`sourcerer-v2-*~{git.host}~{git.org}~{git.repo}^deploy` +`sourcerer-v3-*~{git.host}~{git.org}~{git.repo}^deploy` For instance: -`sourcerer-v2-files~github~elastic~kibana^deploy` -`sourcerer-v2-lines~github~elastic~kibana^deploy` +`sourcerer-v3-files~github~elastic~kibana^deploy` +`sourcerer-v3-lines~github~elastic~kibana^deploy` - Required: No - Type: String diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index 5d86444..c861a04 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -75,27 +75,27 @@ def _load_template_mapping(name: str) -> dict | None: def _load_refs_mapping() -> dict | None: - return _load_template_mapping("sourcerer-v2-refs.json") + return _load_template_mapping("sourcerer-v3-refs.json") def _load_files_mapping() -> dict | None: - return _load_template_mapping("sourcerer-v2-files.json") + return _load_template_mapping("sourcerer-v3-files.json") def _load_lines_mapping() -> dict | None: - return _load_template_mapping("sourcerer-v2-lines.json") + return _load_template_mapping("sourcerer-v3-lines.json") def _run_uniqueness_gate(es: Elasticsearch, host: str, org: str, repo: str) -> bool: """Post-index uniqueness gate (INV-011): every distinct `git.ref_key` in this repo's - content must resolve to exactly one `sourcerer-v2-refs` join doc. Prints the offending + content must resolve to exactly one `sourcerer-v3-refs` join doc. Prints the offending ref_key(s) to stderr and returns False on any violation; True (silent) when the invariant holds.""" offending = check_ref_key_uniqueness(es, host, org, repo) if offending: click.echo( f"Error: {host}/{org}/{repo}: {len(offending)} git.ref_key value(s) missing or " - f"duplicated in sourcerer-v2-refs: {', '.join(offending)}", + f"duplicated in sourcerer-v3-refs: {', '.join(offending)}", err=True, ) return False diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index de19fce..48cf51b 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -1,7 +1,7 @@ # sourcerer/commands/index/markers.py # Refs-index idempotency: content-addressing a ref's indexed state, the guards that decide # whether a ref needs (re)indexing, and writing the completion marker. Reads use the sourcerer -# aliases; writes use sourcerer-v2-refs and the physical per-repo content indices; +# aliases; writes use sourcerer-v3-refs and the physical per-repo content indices; # broader read-only queries across the whole cluster live in sourcerer/queries.py. # Standard packages diff --git a/src/sourcerer/commands/index/schedule.py b/src/sourcerer/commands/index/schedule.py index 972b651..8f36558 100644 --- a/src/sourcerer/commands/index/schedule.py +++ b/src/sourcerer/commands/index/schedule.py @@ -1,7 +1,7 @@ # sourcerer/commands/index/schedule.py # Schedule-gating logic for `sourcerer index --config`: determines which sources are due for # indexing based on their configured schedule and the state of refs already indexed in -# sourcerer-v2-refs (last completed-at and any active in-progress indexing runs). +# sourcerer-v3-refs (last completed-at and any active in-progress indexing runs). # # The gate runs BEFORE the expensive ls-remote / clone / ingest pipeline, so a source whose # schedule hasn't fired since its last indexed run is dropped before any network I/O. diff --git a/src/sourcerer/commands/prune/execute.py b/src/sourcerer/commands/prune/execute.py index 47824c4..33abd72 100644 --- a/src/sourcerer/commands/prune/execute.py +++ b/src/sourcerer/commands/prune/execute.py @@ -173,7 +173,7 @@ def execute_orphan_deletions(es: Elasticsearch, plan: OrphanPlan) -> tuple[int, indices and Class E empty indices -- both near-instant), then the per-repo content delete_by_query (Class B -- expensive, one call per content index per repo), then the per-index stale-location delete_by_query (Class D -- the index.level/suffix migration backstop, - one call per index holding stale docs), then a single delete_by_query against sourcerer-v2-refs + one call per index holding stale docs), then a single delete_by_query against sourcerer-v3-refs covering every orphaned marker tuple across every repo (Class C -- refs is tiny, so one combined query costs one merge cycle instead of one per repo). Repo keys are (host, org, repo). Returns (indices_deleted, content_commits_dropped, marker_commits_dropped, diff --git a/src/sourcerer/commands/prune/report.py b/src/sourcerer/commands/prune/report.py index 69b349f..909d3b3 100644 --- a/src/sourcerer/commands/prune/report.py +++ b/src/sourcerer/commands/prune/report.py @@ -44,7 +44,7 @@ class _Row: without it. - orphan:content, orphan:marker -> "host/org/repo@commit" -- no ref exists for either case. - orphan:index -> the index's own name (e.g. - "sourcerer-v2-files~host~org~repo") -- not commit- or even repo-addressable, since a + "sourcerer-v3-files~host~org~repo") -- not commit- or even repo-addressable, since a host~org-level orphan index spans every repo under that host+org. - orphan:stale-location -> "@" -- location-specific, since the same commit may legitimately have content in another index; only this stale copy is deleted.""" diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v2-files.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json similarity index 98% rename from src/sourcerer/elastic/index_templates/sourcerer-v2-files.json rename to src/sourcerer/elastic/index_templates/sourcerer-v3-files.json index 5ce16ab..57b48ae 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v2-files.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json @@ -3,7 +3,7 @@ "description": "sourcerer-files" }, "index_patterns": [ - "sourcerer-v2-files*" + "sourcerer-v3-files*" ], "template": { "aliases": { diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v2-lines.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json similarity index 99% rename from src/sourcerer/elastic/index_templates/sourcerer-v2-lines.json rename to src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json index 1af54ff..c9e9719 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v2-lines.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json @@ -3,7 +3,7 @@ "description": "sourcerer-lines" }, "index_patterns": [ - "sourcerer-v2-lines*" + "sourcerer-v3-lines*" ], "template": { "aliases": { diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v2-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json similarity index 98% rename from src/sourcerer/elastic/index_templates/sourcerer-v2-refs.json rename to src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index 322a57f..8a5bb25 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v2-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -3,7 +3,7 @@ "description": "sourcerer-refs" }, "index_patterns": [ - "sourcerer-v2-refs*" + "sourcerer-v3-refs*" ], "template": { "aliases": { diff --git a/src/sourcerer/indices.py b/src/sourcerer/indices.py index 9147f2b..a24b7d7 100644 --- a/src/sourcerer/indices.py +++ b/src/sourcerer/indices.py @@ -6,11 +6,12 @@ # command's logic. # # v2 (multi-host): content index names carry a leading git.host segment, so the same org/repo on -# two different hosts lands in distinct backing indices. See sourcerer/hosts.py. +# two different hosts lands in distinct backing indices. See sourcerer/hosts.py. Bumped to v3 for +# incremental (ref-addressed) branch indexing. -FILES_INDEX_PREFIX = "sourcerer-v2-files" -LINES_INDEX_PREFIX = "sourcerer-v2-lines" -REFS_INDEX = "sourcerer-v2-refs" +FILES_INDEX_PREFIX = "sourcerer-v3-files" +LINES_INDEX_PREFIX = "sourcerer-v3-lines" +REFS_INDEX = "sourcerer-v3-refs" # Read aliases span all versioned backing indices of their respective kinds. Writes, updates, # and deletes deliberately use the physical names above so a future index version can coexist @@ -56,7 +57,7 @@ def files_index( commit: str | None = None, level: str = "repo", suffix: str | None = None, ) -> str: """Files index name for a source's `index.level`/`index.suffix`. The 3-arg call reproduces the - historical repo-level name, e.g. sourcerer-v2-files~github~elastic~elasticsearch.""" + historical repo-level name, e.g. sourcerer-v3-files~github~elastic~elasticsearch.""" return _content_index(FILES_INDEX_PREFIX, host, org, repo, commit, level, suffix) @@ -65,5 +66,5 @@ def lines_index( commit: str | None = None, level: str = "repo", suffix: str | None = None, ) -> str: """Lines index name for a source's `index.level`/`index.suffix`. The 3-arg call reproduces the - historical repo-level name, e.g. sourcerer-v2-lines~github~elastic~elasticsearch.""" + historical repo-level name, e.g. sourcerer-v3-lines~github~elastic~elasticsearch.""" return _content_index(LINES_INDEX_PREFIX, host, org, repo, commit, level, suffix) diff --git a/src/sourcerer/planner.py b/src/sourcerer/planner.py index 8fe737a..08146e4 100644 --- a/src/sourcerer/planner.py +++ b/src/sourcerer/planner.py @@ -175,15 +175,15 @@ def content_delete_set(decisions: list[Decision]) -> set[str]: # C. orphan marker -- a commit marker in refs with no content docs at all in either # content index (content manually deleted, or a whole content # index/repo/org vanished without its markers being cleaned up). -# -> delete_by_query on sourcerer-v2-refs. +# -> delete_by_query on sourcerer-v3-refs. # # Today the CLI only ever produces host~org~repo-granularity indices (see files_index/lines_index # in sourcerer/indices.py); parse_index_name and orphan_indices also recognize the host-only, # host~org, and host~org~repo~commit levels so detection keeps working if a future granularity is # introduced, even though only one level is exercised in practice right now. -_FILES_PREFIX_DEFAULT = "sourcerer-v2-files" -_LINES_PREFIX_DEFAULT = "sourcerer-v2-lines" +_FILES_PREFIX_DEFAULT = "sourcerer-v3-files" +_LINES_PREFIX_DEFAULT = "sourcerer-v3-lines" @dataclass(frozen=True) @@ -205,7 +205,7 @@ def parse_index_name( """Inverse of files_index()/lines_index() (sourcerer/indices.py), extended to also recognize the host-only, host~org, and host~org~repo~commit granularities those builders don't produce today, plus an optional trailing `^{suffix}` (index.suffix). Returns None for anything that - doesn't fit the scheme (sourcerer-v2-refs, an unrelated index, or a malformed/empty segment) so + doesn't fit the scheme (sourcerer-v3-refs, an unrelated index, or a malformed/empty segment) so callers skip it rather than risk misclassifying it as an orphan. The `^suffix` is split off BEFORE the `~` segments so it can't corrupt the last segment @@ -312,7 +312,7 @@ def orphan_markers( This single per-commit check also covers a whole repo/org's content having vanished entirely (every one of that repo's ref commits shows up as "missing"), so it subsumes what would otherwise be separate org- and repo-level refs sweeps -- one combined delete_by_query - against sourcerer-v2-refs handles all of it. `skip_repos` excludes repos already going away + against sourcerer-v3-refs handles all of it. `skip_repos` excludes repos already going away via a Class-A index DELETE (their markers are dropped as part of that, not here). Repo keys are (host, org, repo).""" out: dict[tuple[str, str, str], set[str]] = {} diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index 5e95a59..54be0c8 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -248,7 +248,7 @@ def enumerate_content_ref_keys(es: Elasticsearch, host: str, org: str, repo: str """Every distinct `git.ref_key` present in this repo's content (files + lines aliases), via a paginated composite aggregation scoped to (host, org, repo). Feeds the post-upgrade uniqueness gate (INV-011): every value this returns must resolve to exactly one - `sourcerer-v2-refs` join doc. Returns an empty set if neither alias has any matching docs.""" + `sourcerer-v3-refs` join doc. Returns an empty set if neither alias has any matching docs.""" filters = [ {"term": {"git.host": host}}, {"term": {"git.org": org}}, @@ -286,7 +286,7 @@ def enumerate_content_ref_keys(es: Elasticsearch, host: str, org: str, repo: str def check_ref_key_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> list[str]: """The post-upgrade uniqueness gate (INV-011): every distinct `git.ref_key` in this repo's - content must resolve to EXACTLY ONE `sourcerer-v2-refs` join doc. Returns the sorted list of + content must resolve to EXACTLY ONE `sourcerer-v3-refs` join doc. Returns the sorted list of offending ref_keys (missing entirely, or matched by more than one join doc) -- empty means the invariant holds. A single aggregation query counts join docs per ref_key; a key absent from the buckets has zero matches (missing).""" diff --git a/tests/test_index_orphans.py b/tests/test_index_orphans.py index c9129cb..f183ff0 100644 --- a/tests/test_index_orphans.py +++ b/tests/test_index_orphans.py @@ -43,11 +43,11 @@ class TestListSourcererIndices: def test_discovers_backing_indices_through_content_aliases(self): es = MagicMock() es.indices.get_alias.side_effect = [ - {"sourcerer-v2-files~github~acme~widgets": {}}, - {"sourcerer-v2-lines~github~acme~widgets": {}}, + {"sourcerer-v3-files~github~acme~widgets": {}}, + {"sourcerer-v3-lines~github~acme~widgets": {}}, ] names = list_sourcerer_indices(es) - assert names == ["sourcerer-v2-files~github~acme~widgets", "sourcerer-v2-lines~github~acme~widgets"] + assert names == ["sourcerer-v3-files~github~acme~widgets", "sourcerer-v3-lines~github~acme~widgets"] assert es.indices.get_alias.call_args_list[0].kwargs == {"name": FILES_ALIAS} assert es.indices.get_alias.call_args_list[1].kwargs == {"name": LINES_ALIAS} @@ -119,12 +119,12 @@ def fake_count(index): def test_returns_only_zero_doc_content_indices(self): counts = { - "sourcerer-v2-files~github~acme~widgets": 0, # empty -> returned - "sourcerer-v2-files~github~acme~widgets^deploy": 5, # non-empty -> skipped + "sourcerer-v3-files~github~acme~widgets": 0, # empty -> returned + "sourcerer-v3-files~github~acme~widgets^deploy": 5, # non-empty -> skipped } es = self._es_with_counts(counts) result = empty_content_indices(es, list(counts)) - assert result == ["sourcerer-v2-files~github~acme~widgets"] + assert result == ["sourcerer-v3-files~github~acme~widgets"] def test_non_sourcerer_index_never_considered(self): # An unrelated (even empty) index must not be counted or returned. @@ -134,11 +134,11 @@ def test_non_sourcerer_index_never_considered(self): es.count.assert_not_called() # guarded by parse_index_name before any count def test_refs_index_not_considered(self): - es = self._es_with_counts({"sourcerer-v2-refs": 0}) - assert empty_content_indices(es, ["sourcerer-v2-refs"]) == [] + es = self._es_with_counts({"sourcerer-v3-refs": 0}) + assert empty_content_indices(es, ["sourcerer-v3-refs"]) == [] es.count.assert_not_called() def test_index_that_vanished_is_skipped(self): # count raises NotFound (deleted between listing and counting) -> just skipped. es = self._es_with_counts({}) # every count -> NotFound - assert empty_content_indices(es, ["sourcerer-v2-files~github~acme~widgets"]) == [] + assert empty_content_indices(es, ["sourcerer-v3-files~github~acme~widgets"]) == [] diff --git a/tests/test_index_ref.py b/tests/test_index_ref.py index 143ee77..a33984b 100644 --- a/tests/test_index_ref.py +++ b/tests/test_index_ref.py @@ -66,7 +66,7 @@ def test_reuse_writes_marker_counts_not_index_repo(self): # unchanged), so the reuse probe targets exactly where this commit's content lives. mock_cp.assert_called_once_with( es, "github", "elastic", "myrepo", FULL_SHA, - at_index="sourcerer-v2-files~github~elastic~myrepo", + at_index="sourcerer-v3-files~github~elastic~myrepo", ) mock_index.assert_not_called() mock_wim.assert_not_called() # indexing marker only written for fresh ingest @@ -266,8 +266,8 @@ def track_delete(es_, host, org, repo, sha, index_names): i for i, c in enumerate(call_order) if isinstance(c, tuple) ) deleted_indices = mock_del.call_args.args[5] - assert "sourcerer-v2-files~github~elastic~myrepo" in deleted_indices - assert "sourcerer-v2-lines~github~elastic~myrepo" in deleted_indices + assert "sourcerer-v3-files~github~elastic~myrepo" in deleted_indices + assert "sourcerer-v3-lines~github~elastic~myrepo" in deleted_indices def test_unchanged_routing_does_not_delete(self): """Same routing as recorded -> normal ingest, no migration delete.""" diff --git a/tests/test_indices.py b/tests/test_indices.py index aeea7cd..ea63831 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -1,5 +1,5 @@ """Unit tests for the index-name builders in sourcerer.indices, including a round-trip -check against planner.parse_index_name (their documented inverse). v2 names carry a leading +check against planner.parse_index_name (their documented inverse). v3 names carry a leading git.host segment.""" # App packages @@ -15,54 +15,54 @@ def test_read_aliases(self): "sourcerer-refs", ) - def test_refs_index_is_v2(self): - assert REFS_INDEX == "sourcerer-v2-refs" + def test_refs_index_is_v3(self): + assert REFS_INDEX == "sourcerer-v3-refs" def test_files_index_shape(self): assert files_index("github", "elastic", "elasticsearch") == \ - "sourcerer-v2-files~github~elastic~elasticsearch" + "sourcerer-v3-files~github~elastic~elasticsearch" def test_lines_index_shape(self): assert lines_index("github", "elastic", "elasticsearch") == \ - "sourcerer-v2-lines~github~elastic~elasticsearch" + "sourcerer-v3-lines~github~elastic~elasticsearch" def test_host_org_repo_lowercased(self): assert files_index("GitHub", "Elastic", "ElasticSearch") == \ - "sourcerer-v2-files~github~elastic~elasticsearch" + "sourcerer-v3-files~github~elastic~elasticsearch" assert lines_index("MyGitea", "ACME", "Widgets") == \ - "sourcerer-v2-lines~mygitea~acme~widgets" + "sourcerer-v3-lines~mygitea~acme~widgets" def test_same_org_repo_distinct_hosts_differ(self): assert files_index("github", "acme", "w") != files_index("gitlab", "acme", "w") def test_level_host(self): - assert files_index("github", "elastic", "kibana", level="host") == "sourcerer-v2-files~github" + assert files_index("github", "elastic", "kibana", level="host") == "sourcerer-v3-files~github" def test_level_org(self): - assert files_index("github", "Elastic", "Kibana", level="org") == "sourcerer-v2-files~github~elastic" + assert files_index("github", "Elastic", "Kibana", level="org") == "sourcerer-v3-files~github~elastic" def test_level_commit_requires_commit(self): assert lines_index("github", "elastic", "kibana", commit="ABC123", level="commit") == \ - "sourcerer-v2-lines~github~elastic~kibana~abc123" + "sourcerer-v3-lines~github~elastic~kibana~abc123" import pytest with pytest.raises(ValueError): files_index("github", "elastic", "kibana", level="commit") # no commit def test_suffix_appended_with_caret(self): assert files_index("github", "elastic", "kibana", suffix="deploy") == \ - "sourcerer-v2-files~github~elastic~kibana^deploy" + "sourcerer-v3-files~github~elastic~kibana^deploy" def test_suffix_lowercased(self): assert files_index("github", "elastic", "kibana", suffix="Deploy") == \ - "sourcerer-v2-files~github~elastic~kibana^deploy" + "sourcerer-v3-files~github~elastic~kibana^deploy" def test_empty_suffix_is_no_suffix(self): assert files_index("github", "elastic", "kibana", suffix="") == \ - "sourcerer-v2-files~github~elastic~kibana" + "sourcerer-v3-files~github~elastic~kibana" def test_level_and_suffix_combine(self): assert files_index("github", "elastic", "kibana", commit="abc", level="commit", suffix="deploy") == \ - "sourcerer-v2-files~github~elastic~kibana~abc^deploy" + "sourcerer-v3-files~github~elastic~kibana~abc^deploy" class TestRoundTripWithParseIndexName: @@ -80,30 +80,30 @@ def test_refs_index_is_not_parsed(self): assert parse_index_name(REFS_INDEX) is None def test_host_only_granularity(self): - parsed = parse_index_name("sourcerer-v2-files~github") + parsed = parse_index_name("sourcerer-v3-files~github") assert (parsed.host, parsed.org, parsed.repo, parsed.commit) == ("github", None, None, None) def test_host_org_repo_commit_granularity(self): - parsed = parse_index_name("sourcerer-v2-files~github~acme~w~abc123") + parsed = parse_index_name("sourcerer-v3-files~github~acme~w~abc123") assert (parsed.host, parsed.org, parsed.repo, parsed.commit) == ("github", "acme", "w", "abc123") def test_too_many_segments_rejected(self): - assert parse_index_name("sourcerer-v2-files~a~b~c~d~e") is None + assert parse_index_name("sourcerer-v3-files~a~b~c~d~e") is None def test_empty_segment_rejected(self): - assert parse_index_name("sourcerer-v2-files~github~~repo") is None + assert parse_index_name("sourcerer-v3-files~github~~repo") is None def test_suffix_parsed_and_identity_suffix_blind(self): - parsed = parse_index_name("sourcerer-v2-files~github~acme~widgets^deploy") + parsed = parse_index_name("sourcerer-v3-files~github~acme~widgets^deploy") assert (parsed.host, parsed.org, parsed.repo, parsed.commit, parsed.suffix) == \ ("github", "acme", "widgets", None, "deploy") def test_commit_level_with_suffix(self): - parsed = parse_index_name("sourcerer-v2-lines~github~acme~w~abc123^deploy") + parsed = parse_index_name("sourcerer-v3-lines~github~acme~w~abc123^deploy") assert (parsed.repo, parsed.commit, parsed.suffix) == ("w", "abc123", "deploy") def test_trailing_caret_is_malformed(self): - assert parse_index_name("sourcerer-v2-files~github~acme~widgets^") is None + assert parse_index_name("sourcerer-v3-files~github~acme~widgets^") is None def test_round_trip_invariant_all_levels_and_suffix(self): """The builder <-> parser contract: parse_index_name(files_index(...)) recovers the inputs diff --git a/tests/test_planner_orphans.py b/tests/test_planner_orphans.py index 8b21268..78f4446 100644 --- a/tests/test_planner_orphans.py +++ b/tests/test_planner_orphans.py @@ -1,6 +1,6 @@ """Unit tests for the pure orphan-detection helpers in sourcerer.planner. No Elasticsearch here -- every case is expressed as plain index-name lists and (host, org, repo, commit) tuple -sets. v2 index names carry a leading host segment; repo tuples are keyed (host, org, repo).""" +sets. v3 index names carry a leading host segment; repo tuples are keyed (host, org, repo).""" # App packages from sourcerer.planner import ( @@ -15,29 +15,29 @@ class TestParseIndexName: def test_host_org(self): - assert parse_index_name("sourcerer-v2-lines~github~acme~widgets") == ParsedIndex( + assert parse_index_name("sourcerer-v3-lines~github~acme~widgets") == ParsedIndex( kind="lines", host="github", org="acme", repo="widgets", commit=None, - name="sourcerer-v2-lines~github~acme~widgets", + name="sourcerer-v3-lines~github~acme~widgets", ) def test_host_org_repo_commit(self): - parsed = parse_index_name("sourcerer-v2-files~github~acme~widgets~deadbeef") + parsed = parse_index_name("sourcerer-v3-files~github~acme~widgets~deadbeef") assert parsed == ParsedIndex( kind="files", host="github", org="acme", repo="widgets", commit="deadbeef", - name="sourcerer-v2-files~github~acme~widgets~deadbeef", + name="sourcerer-v3-files~github~acme~widgets~deadbeef", ) def test_refs_index_is_not_a_files_or_lines_index(self): - assert parse_index_name("sourcerer-v2-refs") is None + assert parse_index_name("sourcerer-v3-refs") is None def test_unrelated_index_returns_none(self): assert parse_index_name("kibana_sample_data_ecommerce") is None def test_too_many_segments_returns_none(self): - assert parse_index_name("sourcerer-v2-files~a~b~c~d~e") is None + assert parse_index_name("sourcerer-v3-files~a~b~c~d~e") is None def test_empty_segment_returns_none(self): - assert parse_index_name("sourcerer-v2-files~github~acme~~deadbeef") is None + assert parse_index_name("sourcerer-v3-files~github~acme~~deadbeef") is None def test_custom_prefixes(self): parsed = parse_index_name("myprefix-files~github~acme~widgets", files_prefix="myprefix-files") @@ -46,12 +46,12 @@ def test_custom_prefixes(self): class TestOrphanIndices: def test_host_org_repo_index_with_no_ref_repo_is_orphaned(self): - names = ["sourcerer-v2-files~github~acme~widgets"] + names = ["sourcerer-v3-files~github~acme~widgets"] result = orphan_indices(names, ref_orgs=set(), ref_repos=set(), ref_commits=set()) - assert result == ["sourcerer-v2-files~github~acme~widgets"] + assert result == ["sourcerer-v3-files~github~acme~widgets"] def test_index_with_ref_repo_is_not_orphaned(self): - names = ["sourcerer-v2-files~github~acme~widgets", "sourcerer-v2-lines~github~acme~widgets"] + names = ["sourcerer-v3-files~github~acme~widgets", "sourcerer-v3-lines~github~acme~widgets"] result = orphan_indices( names, ref_orgs={("github", "acme")}, ref_repos={("github", "acme", "widgets")}, ref_commits=set() ) @@ -61,33 +61,33 @@ def test_same_repo_distinct_hosts_judged_independently(self): # github/acme/widgets is in refs; gitlab/acme/widgets is not -> only the gitlab index # is an orphan. names = [ - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-files~gitlab~acme~widgets", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~gitlab~acme~widgets", ] result = orphan_indices( names, ref_orgs={("github", "acme")}, ref_repos={("github", "acme", "widgets")}, ref_commits=set() ) - assert result == ["sourcerer-v2-files~gitlab~acme~widgets"] + assert result == ["sourcerer-v3-files~gitlab~acme~widgets"] def test_org_level_orphan_subsumes_repo_and_commit_level_of_the_same_kind(self): names = [ - "sourcerer-v2-files~github~acme", - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-files~github~acme~widgets~deadbeef", + "sourcerer-v3-files~github~acme", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~github~acme~widgets~deadbeef", ] result = orphan_indices(names, ref_orgs=set(), ref_repos=set(), ref_commits=set()) - assert result == ["sourcerer-v2-files~github~acme"] + assert result == ["sourcerer-v3-files~github~acme"] def test_repo_level_orphan_subsumes_commit_level(self): names = [ - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-files~github~acme~widgets~deadbeef", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~github~acme~widgets~deadbeef", ] result = orphan_indices(names, ref_orgs={("github", "acme")}, ref_repos=set(), ref_commits=set()) - assert result == ["sourcerer-v2-files~github~acme~widgets"] + assert result == ["sourcerer-v3-files~github~acme~widgets"] def test_commit_level_not_orphaned_when_ref_commit_present(self): - names = ["sourcerer-v2-files~github~acme~widgets~deadbeef"] + names = ["sourcerer-v3-files~github~acme~widgets~deadbeef"] result = orphan_indices( names, ref_orgs={("github", "acme")}, @@ -98,15 +98,15 @@ def test_commit_level_not_orphaned_when_ref_commit_present(self): def test_subsumption_does_not_cross_files_and_lines(self): names = [ - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-lines~github~acme~widgets~aaa", - "sourcerer-v2-lines~github~acme~widgets~bbb", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-lines~github~acme~widgets~aaa", + "sourcerer-v3-lines~github~acme~widgets~bbb", ] result = orphan_indices(names, ref_orgs=set(), ref_repos=set(), ref_commits=set()) assert set(result) == set(names) def test_unparseable_names_are_ignored(self): - names = ["sourcerer-v2-refs", "some-other-index"] + names = ["sourcerer-v3-refs", "some-other-index"] assert orphan_indices(names, ref_orgs=set(), ref_repos=set(), ref_commits=set()) == [] @@ -138,7 +138,7 @@ def test_skip_repos_excludes_class_a_repos(self): class TestPlanOrphans: def test_no_orphans(self): - names = ["sourcerer-v2-files~github~acme~widgets", "sourcerer-v2-lines~github~acme~widgets"] + names = ["sourcerer-v3-files~github~acme~widgets", "sourcerer-v3-lines~github~acme~widgets"] ref_tuples = {("github", "acme", "widgets", "aaa")} content_tuples = {("github", "acme", "widgets", "aaa")} plan = plan_orphans(names, ref_tuples, content_tuples) @@ -147,16 +147,16 @@ def test_no_orphans(self): assert plan.orphan_marker_commits == {} def test_class_a_subsumes_class_b_for_the_same_repo(self): - names = ["sourcerer-v2-files~github~acme~widgets"] + names = ["sourcerer-v3-files~github~acme~widgets"] ref_tuples: set[tuple[str, str, str, str]] = set() content_tuples = {("github", "acme", "widgets", "bbb")} plan = plan_orphans(names, ref_tuples, content_tuples) - assert plan.orphan_index_names == ["sourcerer-v2-files~github~acme~widgets"] + assert plan.orphan_index_names == ["sourcerer-v3-files~github~acme~widgets"] assert plan.orphan_content == {} assert plan.orphan_marker_commits == {} def test_class_b_and_c_fire_independently_when_index_present(self): - names = ["sourcerer-v2-files~github~acme~widgets"] + names = ["sourcerer-v3-files~github~acme~widgets"] ref_tuples = {("github", "acme", "widgets", "aaa"), ("github", "acme", "widgets", "bbb")} content_tuples = {("github", "acme", "widgets", "aaa"), ("github", "acme", "widgets", "ccc")} plan = plan_orphans(names, ref_tuples, content_tuples) @@ -171,13 +171,13 @@ def test_same_repo_two_hosts_do_not_cross_contaminate(self): # must be judged independently: github's marker orphan must not be cancelled by gitlab's # content just because org/repo match. names = [ - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-files~gitlab~acme~widgets", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~gitlab~acme~widgets", ] ref_tuples = {("github", "acme", "widgets", "aaa")} content_tuples = {("gitlab", "acme", "widgets", "bbb")} plan = plan_orphans(names, ref_tuples, content_tuples) - assert plan.orphan_index_names == ["sourcerer-v2-files~gitlab~acme~widgets"] + assert plan.orphan_index_names == ["sourcerer-v3-files~gitlab~acme~widgets"] assert plan.orphan_content == {} # gitlab content subsumed by the Class-A index DELETE assert plan.orphan_marker_commits == {("github", "acme", "widgets"): {"aaa"}} @@ -190,53 +190,53 @@ def test_content_at_unintended_index_is_stale(self): from sourcerer.planner import orphan_stale_content ct = ("github", "acme", "widgets", "abc") content_by_index = { - "sourcerer-v2-files~github~acme~widgets": {ct}, # old copy, left by a migration - "sourcerer-v2-files~github~acme~widgets^deploy": {ct}, # new (intended) copy + "sourcerer-v3-files~github~acme~widgets": {ct}, # old copy, left by a migration + "sourcerer-v3-files~github~acme~widgets^deploy": {ct}, # new (intended) copy } - intended = {ct: {"sourcerer-v2-files~github~acme~widgets^deploy"}} + intended = {ct: {"sourcerer-v3-files~github~acme~widgets^deploy"}} stale = orphan_stale_content(content_by_index, intended, skip_indices=set()) - assert stale == {"sourcerer-v2-files~github~acme~widgets": {"abc"}} + assert stale == {"sourcerer-v3-files~github~acme~widgets": {"abc"}} def test_intended_index_not_flagged(self): from sourcerer.planner import orphan_stale_content ct = ("github", "acme", "widgets", "abc") - content_by_index = {"sourcerer-v2-files~github~acme~widgets^deploy": {ct}} - intended = {ct: {"sourcerer-v2-files~github~acme~widgets^deploy"}} + content_by_index = {"sourcerer-v3-files~github~acme~widgets^deploy": {ct}} + intended = {ct: {"sourcerer-v3-files~github~acme~widgets^deploy"}} assert orphan_stale_content(content_by_index, intended, set()) == {} def test_commit_without_marker_is_not_class_d(self): """No marker at all -> Class B territory (orphan_content), not stale-location.""" from sourcerer.planner import orphan_stale_content ct = ("github", "acme", "widgets", "abc") - content_by_index = {"sourcerer-v2-files~github~acme~widgets": {ct}} + content_by_index = {"sourcerer-v3-files~github~acme~widgets": {ct}} assert orphan_stale_content(content_by_index, {}, set()) == {} def test_skip_indices_excluded(self): from sourcerer.planner import orphan_stale_content ct = ("github", "acme", "widgets", "abc") - content_by_index = {"sourcerer-v2-files~github~acme~widgets": {ct}} - intended = {ct: {"sourcerer-v2-files~github~acme~widgets^deploy"}} + content_by_index = {"sourcerer-v3-files~github~acme~widgets": {ct}} + intended = {ct: {"sourcerer-v3-files~github~acme~widgets^deploy"}} # index is already going away via a Class-A whole-index DELETE assert orphan_stale_content(content_by_index, intended, - {"sourcerer-v2-files~github~acme~widgets"}) == {} + {"sourcerer-v3-files~github~acme~widgets"}) == {} def test_plan_orphans_wires_class_d(self): ct = ("github", "acme", "widgets", "abc") names = [ - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-files~github~acme~widgets^deploy", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~github~acme~widgets^deploy", ] ref_tuples = {ct} content_tuples = {ct} content_by_index = { - "sourcerer-v2-files~github~acme~widgets": {ct}, - "sourcerer-v2-files~github~acme~widgets^deploy": {ct}, + "sourcerer-v3-files~github~acme~widgets": {ct}, + "sourcerer-v3-files~github~acme~widgets^deploy": {ct}, } - intended = {ct: {"sourcerer-v2-files~github~acme~widgets^deploy"}} + intended = {ct: {"sourcerer-v3-files~github~acme~widgets^deploy"}} plan = plan_orphans(names, ref_tuples, content_tuples, content_by_index_commit=content_by_index, intended_index_by_commit=intended) - assert plan.orphan_stale == {"sourcerer-v2-files~github~acme~widgets": {"abc"}} + assert plan.orphan_stale == {"sourcerer-v3-files~github~acme~widgets": {"abc"}} class TestEmptyIndexSweep: @@ -246,25 +246,25 @@ class TestEmptyIndexSweep: def test_empty_index_included_even_when_identity_has_markers(self): ct = ("github", "acme", "widgets", "abc") names = [ - "sourcerer-v2-files~github~acme~widgets^a", # drained by a suffix a->b migration - "sourcerer-v2-files~github~acme~widgets^b", # now holds the content + "sourcerer-v3-files~github~acme~widgets^a", # drained by a suffix a->b migration + "sourcerer-v3-files~github~acme~widgets^b", # now holds the content ] # Identity (github, acme, widgets) still has markers (they point at ^b), so ^a is NOT a # Class-A orphan -- but it's empty, so Class E must catch it. plan = plan_orphans( names, ref_commit_tuples={ct}, content_commit_tuples={ct}, - empty_index_names=["sourcerer-v2-files~github~acme~widgets^a"], + empty_index_names=["sourcerer-v3-files~github~acme~widgets^a"], ) - assert "sourcerer-v2-files~github~acme~widgets^a" in plan.empty_index_names + assert "sourcerer-v3-files~github~acme~widgets^a" in plan.empty_index_names # ^a is not a Class-A orphan (identity is backed by refs). - assert "sourcerer-v2-files~github~acme~widgets^a" not in plan.orphan_index_names + assert "sourcerer-v3-files~github~acme~widgets^a" not in plan.orphan_index_names def test_empty_index_deduped_against_class_a(self): # An index that is BOTH empty AND identity-orphaned appears only under Class A (one DELETE). - names = ["sourcerer-v2-files~github~ghostorg~gone"] + names = ["sourcerer-v3-files~github~ghostorg~gone"] plan = plan_orphans( names, ref_commit_tuples=set(), content_commit_tuples=set(), - empty_index_names=["sourcerer-v2-files~github~ghostorg~gone"], + empty_index_names=["sourcerer-v3-files~github~ghostorg~gone"], ) - assert "sourcerer-v2-files~github~ghostorg~gone" in plan.orphan_index_names + assert "sourcerer-v3-files~github~ghostorg~gone" in plan.orphan_index_names assert plan.empty_index_names == [] diff --git a/tests/test_prune_deletions.py b/tests/test_prune_deletions.py index 84421ac..ebd463b 100644 --- a/tests/test_prune_deletions.py +++ b/tests/test_prune_deletions.py @@ -2,7 +2,7 @@ delete_index and execute_orphan_deletions (the orphan sweep), and execute_deletions (retention). Every ES call is mocked -- these assert the shape of the requests (wildcard-free deletes, query filters, single combined refs query), not against a real cluster. Repo tuples -are keyed (host, org, repo); index names are v2 and carry a leading host segment.""" +are keyed (host, org, repo); index names are v3 and carry a leading host segment.""" # Standard packages from unittest.mock import MagicMock @@ -25,26 +25,26 @@ def _not_found() -> NotFoundError: class TestDeleteIndex: def test_deletes_by_exact_name_no_wildcard(self): es = MagicMock() - assert delete_index(es, "sourcerer-v2-files~github~acme~widgets") is True - es.indices.delete.assert_called_once_with(index="sourcerer-v2-files~github~acme~widgets") + assert delete_index(es, "sourcerer-v3-files~github~acme~widgets") is True + es.indices.delete.assert_called_once_with(index="sourcerer-v3-files~github~acme~widgets") def test_missing_index_returns_false_not_raise(self): es = MagicMock() es.indices.delete.side_effect = _not_found() - assert delete_index(es, "sourcerer-v2-files~github~acme~widgets") is False + assert delete_index(es, "sourcerer-v3-files~github~acme~widgets") is False class TestExecuteOrphanDeletions: def test_deletes_indices_first(self): es = MagicMock() plan = OrphanPlan( - orphan_index_names=["sourcerer-v2-files~github~ghostorg"], + orphan_index_names=["sourcerer-v3-files~github~ghostorg"], orphan_content={}, orphan_marker_commits={}, ) indices_deleted, content_dropped, markers_dropped, stale_dropped, empty_deleted = execute_orphan_deletions(es, plan) assert (indices_deleted, content_dropped, markers_dropped, stale_dropped, empty_deleted) == (1, 0, 0, 0, 0) - es.indices.delete.assert_called_once_with(index="sourcerer-v2-files~github~ghostorg") + es.indices.delete.assert_called_once_with(index="sourcerer-v3-files~github~ghostorg") es.delete_by_query.assert_not_called() def test_content_delete_by_query_targets_both_content_indices_with_terms_and_host_filter(self): @@ -98,12 +98,12 @@ def test_empty_indices_deleted_whole(self): es = MagicMock() plan = OrphanPlan( orphan_index_names=[], orphan_content={}, orphan_marker_commits={}, - empty_index_names=["sourcerer-v2-files~github~acme~widgets^olddeploy"], + empty_index_names=["sourcerer-v3-files~github~acme~widgets^olddeploy"], ) _, _, _, _, empty_deleted = execute_orphan_deletions(es, plan) assert empty_deleted == 1 es.indices.delete.assert_called_once_with( - index="sourcerer-v2-files~github~acme~widgets^olddeploy") + index="sourcerer-v3-files~github~acme~widgets^olddeploy") es.delete_by_query.assert_not_called() diff --git a/tests/test_setup.py b/tests/test_setup.py index e34a81d..845a9b0 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -45,48 +45,48 @@ def test_multiple_categories_returned_as_set(self): class TestLoadIndexTemplates: def test_loads_template_and_applies_its_alias_to_existing_indices(self, tmp_path): - (tmp_path / "sourcerer-v2-files.json").write_text(json.dumps({ - "index_patterns": ["sourcerer-v2-files*"], + (tmp_path / "sourcerer-v3-files.json").write_text(json.dumps({ + "index_patterns": ["sourcerer-v3-files*"], "template": {"aliases": {"sourcerer-files": {}}}, })) es = MagicMock() loaded = load_index_templates(es, tmp_path) - assert loaded == ["sourcerer-v2-files"] + assert loaded == ["sourcerer-v3-files"] es.indices.put_index_template.assert_called_once_with( - name="sourcerer-v2-files", - index_patterns=["sourcerer-v2-files*"], + name="sourcerer-v3-files", + index_patterns=["sourcerer-v3-files*"], template={"aliases": {"sourcerer-files": {}}}, _meta=None, ) es.indices.update_aliases.assert_called_once_with(actions=[{ "add": { "alias": "sourcerer-files", - "index": "sourcerer-v2-files*", + "index": "sourcerer-v3-files*", }, }]) def test_ignores_missing_indices_when_applying_template_alias(self, tmp_path): - (tmp_path / "sourcerer-v2-files.json").write_text(json.dumps({ - "index_patterns": ["sourcerer-v2-files*"], + (tmp_path / "sourcerer-v3-files.json").write_text(json.dumps({ + "index_patterns": ["sourcerer-v3-files*"], "template": {"aliases": {"sourcerer-files": {}}}, })) es = MagicMock() es.indices.update_aliases.side_effect = _not_found() - assert load_index_templates(es, tmp_path) == ["sourcerer-v2-files"] + assert load_index_templates(es, tmp_path) == ["sourcerer-v3-files"] assert es.indices.method_calls == [ call.put_index_template( - name="sourcerer-v2-files", - index_patterns=["sourcerer-v2-files*"], + name="sourcerer-v3-files", + index_patterns=["sourcerer-v3-files*"], template={"aliases": {"sourcerer-files": {}}}, _meta=None, ), call.update_aliases(actions=[{ "add": { "alias": "sourcerer-files", - "index": "sourcerer-v2-files*", + "index": "sourcerer-v3-files*", }, }]), ] From 15e032dc77c4f7129b193472594d85793d8ab904 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 18 Aug 2026 23:14:37 -0600 Subject: [PATCH 05/29] Release v3.0.0 --- .claude-plugin/marketplace.json | 2 +- README.md | 10 +++++----- pyproject.toml | 2 +- uv.lock | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index d498d24..0a87578 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ "plugins": [ { "name": "sourcerer", - "version": "2.4.5", + "version": "3.0.0", "description": "Search indexed git repositories and generate responses with citations.", "source": "./", "strict": false, diff --git a/README.md b/README.md index a7a7770..1c371c6 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Make sure you have [uv](https://docs.astral.sh/uv/) and [git](https://git-scm.co 1. Install the `sourcerer` CLI: ```sh - uv tool install "git+https://github.com/elastic/sourcerer.git@v2.4.5" + uv tool install "git+https://github.com/elastic/sourcerer.git@v3.0.0" ``` 2. Add connection details. Create a `.env` in your working directory, then fill it in: ```sh @@ -346,10 +346,10 @@ claude plugin marketplace remove elastic-sourcerer ## Upgrades -To upgrade, reinstall from the desired release tag, replacing `v2.4.5` with the release you want: +To upgrade, reinstall from the desired release tag, replacing `v3.0.0` with the release you want: ```sh -uv tool install --reinstall "git+https://github.com/elastic/sourcerer.git@v2.4.5" +uv tool install --reinstall "git+https://github.com/elastic/sourcerer.git@v3.0.0" ``` Git tag installations remain pinned to that release. `uv tool upgrade sourcerer` does not automatically discover a newer GitHub tag. @@ -436,7 +436,7 @@ uv run pytest tests/ #### Prepare a release ```sh -./scripts/release.sh prepare v2.4.5 +./scripts/release.sh prepare v3.0.0 ``` `prepare` bumps the version numbers in `pyproject.toml`, `uv.lock`, @@ -450,7 +450,7 @@ Then from an up-to-date `main` with no tracked changes, publish the tag to make an official release: ```sh -./scripts/release.sh publish v2.4.5 +./scripts/release.sh publish v3.0.0 ``` `publish` verifies that all version files are consistent, `main` matches diff --git a/pyproject.toml b/pyproject.toml index db3960c..5da3049 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sourcerer" -version = "2.4.5" +version = "3.0.0" description = "Index and search source code in Elasticsearch." requires-python = ">=3.10" dependencies = [ diff --git a/uv.lock b/uv.lock index 77b7263..22b6d77 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -1667,7 +1667,7 @@ wheels = [ [[package]] name = "sourcerer" -version = "2.4.5" +version = "3.0.0" source = { editable = "." } dependencies = [ { name = "click" }, From ee4d6c250badddbd8fde2133bfea69eab6cb4e3b Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Fri, 14 Aug 2026 15:42:45 -0400 Subject: [PATCH 06/29] Update sourcerer-v2-refs to map new string fields as keywords to prevent breaking changes when new fields are added to the mappings --- .../elastic/index_templates/sourcerer-v3-refs.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index 8a5bb25..13bc66a 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -30,6 +30,16 @@ } }, "mappings": { + "dynamic_templates": [ + { + "strings_as_keyword": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ], "properties": { "git": { "properties": { From d7d73d7f0c8a53db16d65d3dc75efe53e7620b04 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Mon, 17 Aug 2026 14:48:55 -0400 Subject: [PATCH 07/29] Update v2 -> v3 --- .gitignore | 3 ++- README.md | 1 + src/sourcerer/commands/index/markers.py | 2 +- src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 1c76771..9235135 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ vendor/ # Secrets & local config .env +sourcerer.yml # macOS -.DS_Store +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 1c371c6..22b5661 100644 --- a/README.md +++ b/README.md @@ -413,6 +413,7 @@ the repo root. uv reads `pyproject.toml`, provisions a matching Python, and sync dependencies into an isolated `./.venv` (gitignored) on first run: ```sh +uv sync --extra dev uv run sourcerer help uv run sourcerer setup uv run sourcerer index elastic/elasticsearch -b main diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index 48cf51b..b083c98 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -458,7 +458,7 @@ def write_ref_marker( # # index_level/index_suffix record this source's index.* routing (semantic, not the resolved # index name). The physical files/lines index is reconstructed on demand from git.host/org/ - # repo/commit + these two fields via indices.files_index/lines_index, so a v2->v3 prefix bump + # repo/commit + these two fields via indices.files_index/lines_index, so a v3->v4 prefix bump # stays correct and prune/migration can find (and clean up) exactly where content lives. # Legacy markers written before this feature omit both; readers fall back to the "repo"/None # defaults, which reconstruct to the historical repo-level name where that content actually is. diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index 13bc66a..244af4f 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -96,6 +96,9 @@ }, "index_suffix": { "type": "keyword" + }, + "update_mode": { + "type": "keyword" } } } From cdb737fb197400cd47b832792ff402fe677c8497 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Mon, 17 Aug 2026 15:28:02 -0400 Subject: [PATCH 08/29] Set indexing status to 'complete' instead of 'ready' for consistency with snapshot indexing. Show the number of files and lines actually indexed in the progress bar for incremental indexing. --- AGENTS.md | 13 ++++++++++++- src/sourcerer/commands/index/command.py | 10 +++++----- src/sourcerer/commands/index/markers.py | 6 +++--- .../agent_builder_tools/sourcerer.refs.list.yml | 8 ++------ src/sourcerer/progress.py | 6 ++++-- src/sourcerer/skills/ref-resolution/SKILL.md | 2 +- tests/test_agent_builder_tools.py | 10 +++++----- tests/test_incremental_index.py | 4 ++-- tests/test_markers.py | 2 +- 9 files changed, 35 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 725d81e..1c3d569 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,7 +73,7 @@ lives only on its refs join doc (`_id = git.ref_key`). A HEAD advance runs `git reindexes the paths git reports changed -- add/modify/delete/rename/copy -- instead of reindexing the whole tree. A missing diff base (force-push, GC'd, or the first index) rebuilds the whole branch namespace. The refs join doc publishes `status: indexing` before any content -change and `status: ready` (with the new commit) only after the deletes/indexes/refresh all +change and `status: complete` (with the new commit) only after the deletes/indexes/refresh all succeed, so a crash mid-update leaves the prior commit and content in place. ```yaml @@ -386,6 +386,17 @@ one per branch (holding the live HEAD) for incremental content. This is a differ from the hashed, append-only `build_ref_id` ref-name markers described above (those still drive `since`/retention history and are untouched by this). +#### `status` field values + +Every `sourcerer-v3-refs` document — ref-name markers (keyed by `build_ref_id`) and refs join +docs (keyed by `ref_key`) alike — carries a `status` field drawn from a shared two-value +vocabulary, so the scheduler and `sourcerer.refs.list` can query both families uniformly: + +| Value | Meaning | +|---|---| +| `indexing` | A run is mid-flight. `indexing_started_at` is set; `indexed_at` is absent/null. Present on both ref-name markers (written by `write_indexing_marker` just before snapshot ingest) and incremental join docs (written by `write_incremental_indexing` just before incremental ingest). A stale `indexing` doc whose `indexing_started_at` is older than the retry window (default 6 h) marks a crashed run and is treated as due for re-indexing. | +| `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot ref-name markers), `write_snapshot_join_doc` (snapshot join docs), and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | + Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) therefore runs the same query shape regardless of mode, with no `update_mode` conditional -- and `git.ref_key` is NEVER an agent-facing param, only the internal join field: diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index c861a04..a47bd80 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -299,7 +299,7 @@ def index_incremental_branch_in_dir( new commit, deleting only the paths git reports removed/changed and (re)indexing only the paths git reports added/changed (INV-008 -- scoped by the exact `ref_key`, never a whole namespace sweep). - The refs join doc is published `indexing` before any mutation and `ready` only after the + The refs join doc is published `indexing` before any mutation and `complete` only after the content deletes/indexes and a refresh all succeed (INV-006); a raised exception instead records `write_incremental_failed` and leaves the completed pointer untouched, then re-raises so the caller's per-unit error handling reports it. @@ -318,7 +318,7 @@ def index_incremental_branch_in_dir( old_sha = None if force else (prior.get("git", {}).get("commit") if prior else None) if old_sha == new_sha and not force: - reporter.finish(unit, "skipped") + reporter.finish(unit, "no-changes") return level = unit.index_level @@ -336,7 +336,7 @@ def index_incremental_branch_in_dir( if full_rebuild: delete_incremental_branch(es, host, org, repo, branch, index_level=level, index_suffix=suffix) reporter.set_total_files(unit, count_tracked_files(repo_dir)) - index_incremental_paths( + indexed_files, indexed_lines = index_incremental_paths( es, host, org, repo, repo_dir, branch, None, on_progress=lambda f, l: reporter.update_counts(unit, f, l), index_level=level, index_suffix=suffix, @@ -345,7 +345,7 @@ def index_incremental_branch_in_dir( delete_incremental_paths(es, host, org, repo, branch, plan.delete_paths, index_level=level, index_suffix=suffix) reporter.set_total_files(unit, len(plan.index_paths)) - index_incremental_paths( + indexed_files, indexed_lines = index_incremental_paths( es, host, org, repo, repo_dir, branch, plan.index_paths, on_progress=lambda f, l: reporter.update_counts(unit, f, l), index_level=level, index_suffix=suffix, @@ -365,7 +365,7 @@ def index_incremental_branch_in_dir( write_incremental_failed(es, host, org, repo, branch, completed_commit=old_sha, target_commit=new_sha, error=str(e), prior=prior) raise - reporter.finish(unit, "indexed", files_count, lines_count) + reporter.finish(unit, "indexed", indexed_files, indexed_lines) def index_one( diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index b083c98..99f10d8 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -539,7 +539,7 @@ def pre_clone_skip( # --- refs join docs (git.ref_key), keyed by `_id = ref_key` ------------------------------- # One document per `ref_key` (INV-004): snapshot content's join doc lives at `_id = `; # an incremental branch's single join doc lives at `_id = {host}~{org}~{repo}~{ref}` and its -# `git.commit` is the branch's live HEAD, advanced only by a two-phase indexing -> ready +# `git.commit` is the branch's live HEAD, advanced only by a two-phase indexing -> complete # publication (INV-006). These are a DISTINCT id space from `build_ref_id`'s hashed, append-only # ref-name markers above (untouched -- they still drive `since`/retention history); a join doc's # `_id` is a plain, unhashed `ref_key` string, which a `build_ref_id` hash can never collide with. @@ -678,12 +678,12 @@ def write_incremental_ready( lines_count: int, refresh: bool = True, ) -> None: - """Publish `status: ready` at the NEW completed commit, clearing `target_commit` and any + """Publish `status: complete` at the NEW completed commit, clearing `target_commit` and any prior failure fields. This is the pointer-advancing publication boundary (INV-006): callers must delete+index+refresh the content indices FIRST, then call this.""" doc = _build_incremental_join_doc( host, org, repo, ref, - status="ready", + status="complete", commit=commit, target_commit=None, commit_date_iso=commit_date_iso, diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml index e18a56f..2cf5ebd 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml @@ -12,11 +12,7 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - // An incremental branch's join doc is never "complete" (only "ready"/"indexing" -- - // see markers.write_incremental_ready/write_incremental_indexing), so the default - // ?status == "complete" must also surface "ready" docs, or the default (no-arg) call - // this skill documents would silently omit every incremental branch from the results. - AND (status LIKE ?status OR (?status == "complete" AND status == "ready")) + AND status LIKE ?status // Format the response | SORT indexed_at DESC @@ -59,6 +55,6 @@ configuration: defaultValue: "*" status: type: string - description: Filter by ref status. "complete" = fully indexed (default; also includes incremental branches, whose join doc status is "ready" rather than "complete"); "indexing" = currently being indexed; "*" = all statuses. + description: Filter by ref status. "complete" = fully indexed (default); "indexing" = currently being indexed; "*" = all statuses. optional: true defaultValue: "complete" diff --git a/src/sourcerer/progress.py b/src/sourcerer/progress.py index 88ee5ee..40d8d2d 100644 --- a/src/sourcerer/progress.py +++ b/src/sourcerer/progress.py @@ -40,7 +40,7 @@ class Unit: `ref` may be None until a default branch is resolved. `kind` is one of branch|tag|commit|default. `status` is set once on completion to one of - indexed|skipped|tagged|recorded|error. + indexed|skipped|no-changes|tagged|recorded|error. """ host: str @@ -188,6 +188,8 @@ def _completion_text(self, unit: Unit) -> str: return f"✓ {unit.label} - tagged existing content ({counts})" if unit.status == "recorded": return f"✓ {unit.label} - content already indexed, recorded ref ({counts})" + if unit.status == "no-changes": + return f"• {unit.label} - no changes, skipped" if unit.status == "skipped": return f"• {unit.label} - already indexed, skipped" if unit.status == "error": @@ -207,7 +209,7 @@ def _summary_text(self) -> str: files = sum(u.files for u in self.units) lines = sum(u.lines for u in self.units) order = [("indexed", "indexed"), ("tagged", "tagged"), ("recorded", "recorded"), - ("skipped", "skipped"), ("error", "failed")] + ("skipped", "skipped"), ("no-changes", "no changes"), ("error", "failed")] parts = [f"{by[k]} {label}" for k, label in order if by.get(k)] body = ", ".join(parts) or "nothing to do" return f"Done in {format_elapsed(time.monotonic() - self.start_time)} - {body}; {files:,} files, {lines:,} lines" diff --git a/src/sourcerer/skills/ref-resolution/SKILL.md b/src/sourcerer/skills/ref-resolution/SKILL.md index aa0769b..18db145 100644 --- a/src/sourcerer/skills/ref-resolution/SKILL.md +++ b/src/sourcerer/skills/ref-resolution/SKILL.md @@ -63,7 +63,7 @@ Once a ref is resolved above, pass the value straight through: - Resolved to a commit (tags, one-off branch snapshots -- the common case): use that commit SHA as `git_ref`. - Resolved to an incremental branch (a source configured with `update: incremental` in - `sourcerer.yml`; its `refs.list` row has `status: ready`, not `complete`): use the branch name + `sourcerer.yml`; its `refs.list` row has `status: complete`, same as a snapshot ref): use the branch name itself as `git_ref` (e.g. `main`) -- no commit needed, the query always resolves to whatever commit that branch is CURRENTLY at. diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index b9fe428..fd8e929 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -83,13 +83,13 @@ def test_refs_list_does_not_surface_ref_key(): assert "ref_key" not in line -def test_refs_list_default_status_also_surfaces_incremental_ready(): - # Incremental join docs are never status:"complete" (only "ready"/"indexing" -- see - # markers.write_incremental_ready/write_incremental_indexing), so the default (no-arg) - # call must not silently omit every incremental branch. +def test_refs_list_default_status_surfaces_all_refs(): + # Incremental join docs now use status:"complete" (same as snapshot), so the default + # ?status == "complete" correctly surfaces all indexed refs without a special-case. tool = _tools()["sourcerer.refs.list"] query = tool["configuration"]["query"] - assert 'status == "ready"' in query + assert 'status == "ready"' not in query # special-case removed; no longer needed + assert "status LIKE ?status" in query assert tool["configuration"]["params"]["status"]["defaultValue"] == "complete" diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py index 92d2a77..ee37b3c 100644 --- a/tests/test_incremental_index.py +++ b/tests/test_incremental_index.py @@ -1,6 +1,6 @@ """Tests for the incremental (ref-addressed) branch orchestration in sourcerer.commands.index.command.index_incremental_branch_in_dir: the two-phase -indexing -> ready publication, full rebuild vs delta update selection, and failure handling. +indexing -> complete publication, full rebuild vs delta update selection, and failure handling. Every ES call and every git/documents side effect is mocked/patched -- these are orchestration tests, not an end-to-end index run (see specs/incremental-indexing.md Task 16 for that).""" @@ -116,7 +116,7 @@ def test_no_change_skips_entirely(self): mocks["index_incremental_paths"].assert_not_called() mocks["write_incremental_indexing"].assert_not_called() mocks["write_incremental_ready"].assert_not_called() - assert unit.status == "skipped" + assert unit.status == "no-changes" finally: _stop(patchers) diff --git a/tests/test_markers.py b/tests/test_markers.py index efc77c6..7110e78 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -471,7 +471,7 @@ def test_incremental_marker_advances_commit_and_clears_target_and_error(self): commit_date_iso="2026-02-02T00:00:00+00:00", files_count=5, lines_count=99) doc = _indexed_doc(es) - assert doc["status"] == "ready" + assert doc["status"] == "complete" assert doc["git"]["commit"] == NEW # advances only after a successful run (INV-006) assert doc["git"]["target_commit"] is None assert doc["error"] is None and doc["failed_at"] is None From 871b6f38ad651c519379e7fe035e1fd45d06db28 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Tue, 18 Aug 2026 08:25:33 -0400 Subject: [PATCH 09/29] Remove update_mode field from files and lines indices, which were written to but never read or used in joins --- AGENTS.md | 8 ++--- src/sourcerer/cli.py | 2 +- src/sourcerer/commands/index/documents.py | 4 --- src/sourcerer/commands/index/markers.py | 35 +++++++++---------- .../index_templates/sourcerer-v3-files.json | 4 --- .../index_templates/sourcerer-v3-lines.json | 4 --- tests/test_backfill.py | 4 +-- tests/test_documents.py | 11 +----- 8 files changed, 24 insertions(+), 48 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1c3d569..a7a310a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -398,8 +398,8 @@ vocabulary, so the scheduler and `sourcerer.refs.list` can query both families u | `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot ref-name markers), `write_snapshot_join_doc` (snapshot join docs), and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) therefore runs the -same query shape regardless of mode, with no `update_mode` conditional -- and `git.ref_key` is -NEVER an agent-facing param, only the internal join field: +same query shape regardless of mode -- and `git.ref_key` is NEVER an agent-facing param, only +the internal join field: ```esql FROM sourcerer-lines @@ -423,8 +423,8 @@ incremental content (which has none) gets it from the join. ### Upgrade backfill (`--no-backfill`) `sourcerer index` runs a one-time, idempotent upgrade backfill by default on every invocation: -an `_update_by_query` stamps `git.ref_key = git.commit` + `update_mode: snapshot` onto -pre-existing snapshot content that predates this feature, the refs index's mapping is +an `_update_by_query` stamps `git.ref_key = git.commit` onto pre-existing snapshot content +that predates this feature, the refs index's mapping is re-applied to the existing physical index (a template change alone only affects indices created afterward), and a snapshot refs join doc is created for every already-indexed commit that lacks one. Pass `--no-backfill` to skip it. Safe to run every time: a repeat run touches diff --git a/src/sourcerer/cli.py b/src/sourcerer/cli.py index 74237da..6e53f7a 100755 --- a/src/sourcerer/cli.py +++ b/src/sourcerer/cli.py @@ -272,7 +272,7 @@ def setup(url, api_key, username, password, kb_url, config_path, include_experim "--no-backfill", is_flag=True, default=False, - help="Skip the one-time upgrade backfill that stamps git.ref_key/update_mode onto " + help="Skip the one-time upgrade backfill that stamps git.ref_key onto " "pre-existing snapshot content and migrates the refs index (default: run it, idempotently, " "on every invocation).", ) diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index 42fee80..9901250 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -95,7 +95,6 @@ def build_file_doc( # commit itself is the stable join key (see build_ref_key for the incremental shape). "ref_key": commit_sha, }, - "update_mode": "snapshot", "file": file_fields, } # Content identity is (host, org, repo, commit, path): the same blob reached via any ref @@ -144,7 +143,6 @@ def iter_line_docs( "commit": commit_sha, "ref_key": commit_sha, }, - "update_mode": "snapshot", "file": file_fields, } for line_num, line_content in enumerate(content.splitlines(), start=1): @@ -208,7 +206,6 @@ def build_incremental_file_doc( "ref_type": "branch", "ref_key": build_ref_key(host, org, repo, ref), }, - "update_mode": "incremental", "file": file_fields, } _id = make_doc_id(host, org, repo, "branch", ref, rel_path) @@ -255,7 +252,6 @@ def iter_incremental_line_docs( "ref_type": "branch", "ref_key": build_ref_key(host, org, repo, ref), }, - "update_mode": "incremental", "file": file_fields, } for line_num, line_content in enumerate(content.splitlines(), start=1): diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index 99f10d8..a434db1 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -846,16 +846,16 @@ def refresh_incremental_content( # --- one-time upgrade backfill (default-on; --no-backfill opts out) ----------------------- -# Stamps `git.ref_key`/`update_mode` onto pre-existing snapshot content that predates this -# feature, migrates the refs index mapping, and creates the missing `_id = commit` join docs -# (INV-009/INV-010). Safe to run on every `index` invocation: both the content update and the -# join-doc creation are no-ops the second time. +# Stamps `git.ref_key` onto pre-existing snapshot content that predates this feature, migrates +# the refs index mapping, and creates the missing `_id = commit` join docs (INV-009/INV-010). +# Safe to run on every `index` invocation: both the content update and the join-doc creation +# are no-ops the second time. def backfill_snapshot_ref_keys(es: Elasticsearch, host: str, org: str, repo: str) -> int: - """Idempotent `_update_by_query` stamping `git.ref_key = git.commit` + `update_mode: - "snapshot"` onto this repo's content docs that lack `git.ref_key` (pre-upgrade data). - Returns the total number of docs updated across the files and lines aliases; 0 on a repeat - run (INV-009) since the `must_not: exists` filter then matches nothing.""" + """Idempotent `_update_by_query` stamping `git.ref_key = git.commit` onto this repo's + content docs that lack `git.ref_key` (pre-upgrade data). Returns the total number of docs + updated across the files and lines aliases; 0 on a repeat run (INV-009) since the + `must_not: exists` filter then matches nothing.""" query = { "bool": { "filter": [ @@ -867,10 +867,7 @@ def backfill_snapshot_ref_keys(es: Elasticsearch, host: str, org: str, repo: str } } script = { - "source": ( - "ctx._source.git.ref_key = ctx._source.git.commit; " - "ctx._source.update_mode = 'snapshot';" - ), + "source": "ctx._source.git.ref_key = ctx._source.git.commit;", "lang": "painless", } total = 0 @@ -988,10 +985,10 @@ def apply_refs_index_mapping(es: Elasticsearch, mapping: dict) -> None: def apply_content_index_mapping(es: Elasticsearch, files_mapping: dict, lines_mapping: dict) -> None: """Apply the updated files/lines template mappings to every EXISTING physical content index behind the read aliases. This must run BEFORE `backfill_snapshot_ref_keys` writes - `git.ref_key`/`update_mode` onto pre-existing content: an index created before this feature - has no explicit mapping for those fields, so the first `_update_by_query` write would - otherwise fall back to ES's dynamic string mapping (`text`, no fielddata) instead of the - `keyword` type the template defines -- silently breaking every later `git.ref_key` + `git.ref_key` onto pre-existing content: an index created before this feature has no + explicit mapping for that field, so the first `_update_by_query` write would otherwise + fall back to ES's dynamic string mapping (`text`, no fielddata) instead of the `keyword` + type the template defines -- silently breaking every later `git.ref_key` aggregation/sort/exact-match query. `put_mapping` against an alias updates every backing index it resolves to. A no-op if neither alias has any backing index yet.""" for alias, mapping in ((FILES_ALIAS, files_mapping), (LINES_ALIAS, lines_mapping)): @@ -1006,9 +1003,9 @@ def backfill_repo( files_mapping: dict | None = None, lines_mapping: dict | None = None, ) -> dict: """Run the full one-time upgrade for one repo: apply the updated content/refs index - mappings (once, if given -- must happen BEFORE the content update so the new fields land - typed correctly rather than dynamically guessed), stamp `ref_key`/`update_mode` onto - pre-existing snapshot content (idempotent), and create a join doc for every already-indexed + mappings (once, if given -- must happen BEFORE the content update so the new field lands + typed correctly rather than dynamically guessed), stamp `ref_key` onto pre-existing + snapshot content (idempotent), and create a join doc for every already-indexed commit that lacks one. Returns a small summary dict for reporting; every field is 0 on a repeat run (INV-009).""" if files_mapping is not None and lines_mapping is not None: diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json index 57b48ae..1bb91e2 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json @@ -68,10 +68,6 @@ } } }, - "update_mode": { - "type": "keyword", - "normalizer": "lowercase" - }, "file": { "properties": { "path": { diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json index c9e9719..1ae9e33 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json @@ -110,10 +110,6 @@ } } }, - "update_mode": { - "type": "keyword", - "normalizer": "lowercase" - }, "file": { "properties": { "path": { diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 33d1190..e55e364 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -1,6 +1,6 @@ """Tests for the one-time upgrade backfill in sourcerer.commands.index.markers: stamping -git.ref_key/update_mode onto pre-existing snapshot content, migrating the refs index mapping, -and creating missing snapshot join docs. Every ES call is mocked (INV-009/INV-010).""" +git.ref_key onto pre-existing snapshot content, migrating the refs index mapping, and +creating missing snapshot join docs. Every ES call is mocked (INV-009/INV-010).""" # Standard packages from unittest.mock import MagicMock diff --git a/tests/test_documents.py b/tests/test_documents.py index fd96476..75e841b 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -80,7 +80,6 @@ def test_git_fields(self, tmp_path): _id, doc = build_file_doc("github", "acme", "widgets", "deadbeef", "a.txt", p) assert doc["git"] == {"host": "github", "org": "acme", "repo": "widgets", "commit": "deadbeef", "ref_key": "deadbeef"} - assert doc["update_mode"] == "snapshot" def test_host_changes_id(self, tmp_path): p = tmp_path / "a.txt" @@ -131,11 +130,10 @@ def test_broken_symlink_has_target_path_but_no_target_size(self, tmp_path): class TestIterLineDocs: - def test_snapshot_ref_key_and_update_mode(self): + def test_snapshot_ref_key(self): docs = list(iter_line_docs("github", "acme", "widgets", "deadbeef", "a.txt", "one")) _id, doc = docs[0] assert doc["git"]["ref_key"] == "deadbeef" - assert doc["update_mode"] == "snapshot" def test_line_numbering_starts_at_one(self): docs = list(iter_line_docs("github", "acme", "widgets", "deadbeef", "a.txt", "one\ntwo\nthree")) @@ -192,12 +190,6 @@ def test_no_commit_field(self, tmp_path): _id, doc = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) assert "commit" not in doc["git"] - def test_update_mode_incremental(self, tmp_path): - p = tmp_path / "a.txt" - p.write_text("hello") - _id, doc = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) - assert doc["update_mode"] == "incremental" - def test_id_stable_across_commits(self, tmp_path): # The whole point of ref-addressing: the id does not depend on the commit, only the # ref, so a modified file's doc overwrites in place rather than minting a new id. @@ -220,7 +212,6 @@ def test_line_docs_ref_key_and_no_commit(self): for _id, d in docs: assert d["git"]["ref_key"] == "github~acme~widgets~main" assert "commit" not in d["git"] - assert d["update_mode"] == "incremental" def test_worker_ctx_routes_to_incremental_builders(self, tmp_path): (tmp_path / "a.txt").write_text("one\ntwo\n") From cfd2dd44f16709403c03f733d2020602aebca30e Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Tue, 18 Aug 2026 08:45:08 -0400 Subject: [PATCH 10/29] Fix snapshot indexing false-failing the uniqueness gate by refreshing the refs join doc write so it's visible before the INV-011 gate runs --- src/sourcerer/commands/index/command.py | 8 +++++++- tests/test_markers.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index a47bd80..5c2c65c 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -263,7 +263,13 @@ def index_ref_in_dir( # Every snapshot unit -- whether freshly indexed or reusing a sibling's already-indexed # content -- must have its `_id = commit` refs join doc so the universal join query resolves # a commit for this content regardless of which ref reached it (INV-004). - write_snapshot_join_doc(es, host, org, repo, ref_type, ref_for_id, commit_sha, commit_date_iso) + # refresh=True so the post-index uniqueness gate (_run_uniqueness_gate, INV-011) sees this join + # doc immediately instead of racing the refs index's default (~1s) refresh interval: the bulk + # context manager refreshes the CONTENT indices on exit but not refs, so an unrefreshed write + # here would make the gate read this ref_key's content but miss its join doc and false-fail + # "missing". Mirrors write_incremental_ready (refresh=True) and backfill_refs_join_docs. + write_snapshot_join_doc(es, host, org, repo, ref_type, ref_for_id, commit_sha, commit_date_iso, + refresh=True) if migrating: # Reconstruct the OLD index name from the prior marker's routing and drop this commit's # stale copy there. Commit-safety (another surviving ref sharing the commit) is respected diff --git a/tests/test_markers.py b/tests/test_markers.py index 7110e78..2a4838c 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -424,6 +424,20 @@ def test_join_doc_idempotent_rewrite_same_id(self): write_snapshot_join_doc(second, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None) assert first.index.call_args.kwargs["id"] == second.index.call_args.kwargs["id"] + def test_default_write_does_not_refresh(self): + es = MagicMock() + write_snapshot_join_doc(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None) + assert es.index.call_args.kwargs["refresh"] is False + + def test_refresh_true_is_propagated(self): + # The snapshot indexing path (command.index_one) passes refresh=True so the post-index + # uniqueness gate (INV-011) doesn't race the refs index's async refresh and false-fail + # "git.ref_key missing". Guards that the write actually threads the flag to es.index. + es = MagicMock() + write_snapshot_join_doc(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + refresh=True) + assert es.index.call_args.kwargs["refresh"] is True + class TestIncrementalRefKeyIdentity: def test_id_is_ref_key_not_a_hash(self): From 7af4d46921a98b95721695b4f97a58f71be2fa94 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Tue, 18 Aug 2026 13:25:29 -0400 Subject: [PATCH 11/29] Replace the git_ref and git_commit params into one optional git_commit_ish param (resolved via a sourcerer-refs subquery) in the agent tools, and guard content reads with post-join status=='complete' --- AGENTS.md | 47 ++++++++---- README.md | 7 +- .../sourcerer.code.grep.yml | 35 +++++---- .../sourcerer.code.search.yml | 35 +++++---- .../sourcerer.files.cat.yml | 35 +++++---- .../sourcerer.files.head.yml | 35 +++++---- .../sourcerer.files.ls.yml | 45 +++++++----- .../sourcerer.files.read_lines.yml | 35 +++++---- .../sourcerer.files.tail.yml | 35 +++++---- .../sourcerer.files.tree.yml | 35 +++++---- .../sourcerer.files.wc.yml | 35 +++++---- .../sourcerer.repos.search.yml | 31 ++++++-- src/sourcerer/skills/ref-resolution/SKILL.md | 33 +++++---- tests/test_agent_builder_tools.py | 72 +++++++++++++++---- 14 files changed, 346 insertions(+), 169 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a7a310a..77c0a04 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -403,23 +403,46 @@ the internal join field: ```esql FROM sourcerer-lines -| WHERE ... AND (git.commit == ?git_ref OR git.ref == ?git_ref) +| WHERE ... AND git.ref_key IN ( + // Resolve git_commit_ish against the small sourcerer-refs table once, then do a cheap + // membership check on the large content index (instead of a per-row LIKE/OR wildcard match). + FROM sourcerer-refs + | WHERE ... AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) | LOOKUP JOIN sourcerer-refs ON git.ref_key -| WHERE git.commit LIKE ?git_commit +| WHERE status == "complete" ``` -`git_ref` is a required, exact-match param (no wildcards) -- resolve a ref first (see -`src/sourcerer/skills/ref-resolution/SKILL.md`), then pass through whatever it resolved to: a -snapshot ref's commit SHA, or an incremental branch's plain name (e.g. `main`) -- no construction, -no `ref_key` involved. The tool matches `git_ref` against whichever field the row actually -carries (`git.commit` for snapshot, `git.ref` for incremental), so the same param and the same -query shape work for both without the caller knowing which mode it is. `git_commit` is optional -(default `"*"`) and filters the commit the join resolves -- it lets a caller assert the branch it -resolved `git_ref` against hasn't since advanced; a no-op for a commit-scoped query, since -`git.commit` already equals `git_ref` there. The join adds/overwrites `git.commit` on every row, -so snapshot content (which already carries its own, identical `git.commit`) is unaffected and +`git_commit_ish` is the single scoping param and supports `*`/`?` wildcards (the filter uses +`LIKE`). It is optional (default `"*"`, matching every indexed ref), but for a normal content +question resolve a ref first (see `src/sourcerer/skills/ref-resolution/SKILL.md`) and pass through +whatever it resolved to: a snapshot ref's commit SHA, or an incremental branch's plain name (e.g. +`main`) -- no construction, no `ref_key` involved. Leaving it at `"*"` matches content across all +refs at once; because every content tool carries `git.commit` through to output (and any +aggregation groups `BY git.commit`), unpinned results stay attributable per commit rather than +being blended -- but a version-specific answer should still pin a ref. The subquery matches `git_commit_ish` +against whichever field the ref actually carries (`git.commit` for snapshot, `git.ref` for +incremental) and collapses it to a set of `git.ref_key` values; the outer query scopes content by +membership in that set, so the same param and the same query shape work for both modes without the +caller knowing which one it is. The join then adds/overwrites `git.commit` on every row, so +snapshot content (which already carries its own, identical `git.commit`) is unaffected and incremental content (which has none) gets it from the join. +The post-join `| WHERE status == "complete"` is an automatic consistency guard (no param): it +serves content only from a ref whose latest index is complete, excluding the torn/partial-read +window while an incremental branch is mid-reindex -- during a HEAD advance the branch's refs join +doc is `status: indexing` and its content is being mutated in place, so a query joining to it +would otherwise read a half-applied mix of the old and new commits. It is a no-op for snapshot +content (whose join doc is always `status: complete`). Trade-off: a *failed* incremental run +leaves the join doc at `status: indexing` with the prior commit's content still fully consistent; +the guard hides that content until the next successful run republishes `status: complete`. Note +this guard is about intra-update consistency, not inter-query staleness: because incremental +content overwrites in place, only a branch's current HEAD is ever indexed, so a query always +returns whatever commit the branch is at *now* -- to detect that a branch advanced since an +earlier resolution, re-check `sourcerer.refs.list`. + ### Upgrade backfill (`--no-backfill`) `sourcerer index` runs a one-time, idempotent upgrade backfill by default on every invocation: diff --git a/README.md b/README.md index 22b5661..7f14d33 100644 --- a/README.md +++ b/README.md @@ -88,9 +88,10 @@ The [`sourcerer.yml` specification](specs/sourcerer-yml.md) has the full referen ### Snapshot vs. incremental indexing (`update: `) Each source can set `update: snapshot` (the default) or `update: incremental` (branch-only). -Every Agent Builder content tool takes the same `git_ref` param either way (a commit SHA or an -exact branch/tag name) and resolves a commit the same way regardless of mode; `git.ref_key` is -an internal storage/join detail, never something the agent constructs or passes. +Every Agent Builder content tool takes the same `git_commit_ish` param either way (a commit SHA or +a branch/tag name, `*`/`?` wildcards supported) and resolves a commit the same way regardless of +mode; `git.ref_key` is an internal storage/join detail, never something the agent constructs or +passes. - **`snapshot`** (default): content is commit-addressed, exactly as before. `git.ref_key` is the commit SHA itself, so every ref (branch, tag, or pinned commit) that resolves to the same diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml index a2be68e..ef4106a 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml @@ -9,9 +9,20 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - // A commit SHA (snapshot content) or the exact branch/tag name (incremental - // content, which has no git.commit of its own) -- whichever this source resolves to. - AND (git.commit == ?git_ref OR git.ref == ?git_ref) + AND git.ref_key IN ( + // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than + // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. + // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values + // once; the outer query then does a cheap membership check instead of a wildcard match on + // every row. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) AND file.path LIKE ?file_path AND line.content RLIKE ?regex @@ -21,10 +32,12 @@ configuration: // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key - // Optional consistency guard: assert the branch hasn't advanced since git_ref was - // resolved (only meaningful when git_ref names an incremental branch; a no-op filter - // for a commit-scoped query, since git.commit already equals git_ref there). - | WHERE git.commit LIKE ?git_commit + // Consistency guard: only serve content from a ref whose latest index is complete. This + // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs + // join doc is status:"indexing" until the new commit's content is fully published). A no-op + // for snapshot content, whose join doc is always status:"complete". `status` lives only on + // refs docs, so after the join it unambiguously means the joined ref's status. + | WHERE status == "complete" // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") @@ -52,13 +65,9 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref: + git_commit_ish: type: string - description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. - optional: false - git_commit: - type: string - description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + description: Filter by git commit(s) or ref(s) (supports * wildcards) optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml index 7d4500a..c348992 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml @@ -9,9 +9,20 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - // A commit SHA (snapshot content) or the exact branch/tag name (incremental - // content, which has no git.commit of its own) -- whichever this source resolves to. - AND (git.commit == ?git_ref OR git.ref == ?git_ref) + AND git.ref_key IN ( + // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than + // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. + // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values + // once; the outer query then does a cheap membership check instead of a wildcard match on + // every row. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) AND file.path LIKE ?file_path AND MATCH(line.content.text, ?q) @@ -21,10 +32,12 @@ configuration: // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key - // Optional consistency guard: assert the branch hasn't advanced since git_ref was - // resolved (only meaningful when git_ref names an incremental branch; a no-op filter - // for a commit-scoped query, since git.commit already equals git_ref there). - | WHERE git.commit LIKE ?git_commit + // Consistency guard: only serve content from a ref whose latest index is complete. This + // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs + // join doc is status:"indexing" until the new commit's content is fully published). A no-op + // for snapshot content, whose join doc is always status:"complete". `status` lives only on + // refs docs, so after the join it unambiguously means the joined ref's status. + | WHERE status == "complete" // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") @@ -52,13 +65,9 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref: + git_commit_ish: type: string - description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. - optional: false - git_commit: - type: string - description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + description: Filter by git commit(s) or ref(s) (supports * wildcards) optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml index 2c61c4c..1784ca8 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml @@ -9,9 +9,20 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - // A commit SHA (snapshot content) or the exact branch/tag name (incremental - // content, which has no git.commit of its own) -- whichever this source resolves to. - AND (git.commit == ?git_ref OR git.ref == ?git_ref) + AND git.ref_key IN ( + // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than + // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. + // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values + // once; the outer query then does a cheap membership check instead of a wildcard match on + // every row. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) AND file.path LIKE ?file_path // git.ref_key is purely an internal storage/join key (never a query param): every @@ -20,10 +31,12 @@ configuration: // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key - // Optional consistency guard: assert the branch hasn't advanced since git_ref was - // resolved (only meaningful when git_ref names an incremental branch; a no-op filter - // for a commit-scoped query, since git.commit already equals git_ref there). - | WHERE git.commit LIKE ?git_commit + // Consistency guard: only serve content from a ref whose latest index is complete. This + // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs + // join doc is status:"indexing" until the new commit's content is fully published). A no-op + // for snapshot content, whose join doc is always status:"complete". `status` lives only on + // refs docs, so after the join it unambiguously means the joined ref's status. + | WHERE status == "complete" // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") @@ -70,13 +83,9 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref: + git_commit_ish: type: string - description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. - optional: false - git_commit: - type: string - description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + description: Filter by git commit(s) or ref(s) (supports * wildcards) optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml index 20ab828..7cec93d 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml @@ -9,9 +9,20 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - // A commit SHA (snapshot content) or the exact branch/tag name (incremental - // content, which has no git.commit of its own) -- whichever this source resolves to. - AND (git.commit == ?git_ref OR git.ref == ?git_ref) + AND git.ref_key IN ( + // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than + // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. + // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values + // once; the outer query then does a cheap membership check instead of a wildcard match on + // every row. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) AND file.path LIKE ?file_path // git.ref_key is purely an internal storage/join key (never a query param): every @@ -20,10 +31,12 @@ configuration: // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key - // Optional consistency guard: assert the branch hasn't advanced since git_ref was - // resolved (only meaningful when git_ref names an incremental branch; a no-op filter - // for a commit-scoped query, since git.commit already equals git_ref there). - | WHERE git.commit LIKE ?git_commit + // Consistency guard: only serve content from a ref whose latest index is complete. This + // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs + // join doc is status:"indexing" until the new commit's content is fully published). A no-op + // for snapshot content, whose join doc is always status:"complete". `status` lives only on + // refs docs, so after the join it unambiguously means the joined ref's status. + | WHERE status == "complete" // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") @@ -69,13 +82,9 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref: + git_commit_ish: type: string - description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. - optional: false - git_commit: - type: string - description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + description: Filter by git commit(s) or ref(s) (supports * wildcards) optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml index 83e9785..9933f39 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml @@ -9,9 +9,20 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - // A commit SHA (snapshot content) or the exact branch/tag name (incremental - // content, which has no git.commit of its own) -- whichever this source resolves to. - AND (git.commit == ?git_ref OR git.ref == ?git_ref) + AND git.ref_key IN ( + // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than + // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. + // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values + // once; the outer query then does a cheap membership check instead of a wildcard match on + // every row. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) // git.ref_key is purely an internal storage/join key (never a query param): every // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN @@ -19,10 +30,12 @@ configuration: // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key - // Optional consistency guard: assert the branch hasn't advanced since git_ref was - // resolved (only meaningful when git_ref names an incremental branch; a no-op filter - // for a commit-scoped query, since git.commit already equals git_ref there). - | WHERE git.commit LIKE ?git_commit + // Consistency guard: only serve content from a ref whose latest index is complete. This + // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs + // join doc is status:"indexing" until the new commit's content is fully published). A no-op + // for snapshot content, whose join doc is always status:"complete". `status` lives only on + // refs docs, so after the join it unambiguously means the joined ref's status. + | WHERE status == "complete" // Enforce glob depth for * and ** on file.path. // Without this, "src/*/Job.java" would match files at any depth, @@ -77,7 +90,11 @@ configuration: ) // Collapse to one row per unique entry - this turns raw paths into an ls listing. - | STATS files = COUNT(*), bytes = SUM(file.size) BY name, is_dir + // Group by the commit too: when git_commit_ish matches more than one ref (e.g. a + // wildcard or the default "*"), each ref must get its own listing rather than having + // its file counts/bytes summed together into a meaningless cross-ref total. + | STATS files = COUNT(*), bytes = SUM(file.size) + BY git.host, git.org, git.repo, git.commit, name, is_dir // Append a trailing "/" to directory entries so they read like ls output. | EVAL name = CASE(is_dir, CONCAT(name, "/"), name) @@ -85,8 +102,8 @@ configuration: // Format the response. Each row is a single entry name, the cheapest row // in this toolset, so limit defaults to a high ceiling that mainly exists // to opt out of ES|QL's implicit row cap rather than to trim real usage. - | SORT is_dir DESC, name - | KEEP name + | SORT git.host, git.org, git.repo, git.commit, is_dir DESC, name + | KEEP git.host, git.org, git.repo, git.commit, name | LIMIT ?limit params: git_host: @@ -104,13 +121,9 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref: + git_commit_ish: type: string - description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. - optional: false - git_commit: - type: string - description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + description: Filter by git commit(s) or ref(s) (supports * wildcards) optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml index d6ad15f..3eb7798 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml @@ -9,9 +9,20 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - // A commit SHA (snapshot content) or the exact branch/tag name (incremental - // content, which has no git.commit of its own) -- whichever this source resolves to. - AND (git.commit == ?git_ref OR git.ref == ?git_ref) + AND git.ref_key IN ( + // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than + // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. + // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values + // once; the outer query then does a cheap membership check instead of a wildcard match on + // every row. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) AND file.path LIKE ?file_path AND line.number >= ?line_number_start AND line.number <= ?line_number_end @@ -22,10 +33,12 @@ configuration: // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key - // Optional consistency guard: assert the branch hasn't advanced since git_ref was - // resolved (only meaningful when git_ref names an incremental branch; a no-op filter - // for a commit-scoped query, since git.commit already equals git_ref there). - | WHERE git.commit LIKE ?git_commit + // Consistency guard: only serve content from a ref whose latest index is complete. This + // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs + // join doc is status:"indexing" until the new commit's content is fully published). A no-op + // for snapshot content, whose join doc is always status:"complete". `status` lives only on + // refs docs, so after the join it unambiguously means the joined ref's status. + | WHERE status == "complete" // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") @@ -72,13 +85,9 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref: + git_commit_ish: type: string - description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. - optional: false - git_commit: - type: string - description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + description: Filter by git commit(s) or ref(s) (supports * wildcards) optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml index 0daea53..260a3ef 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml @@ -9,9 +9,20 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - // A commit SHA (snapshot content) or the exact branch/tag name (incremental - // content, which has no git.commit of its own) -- whichever this source resolves to. - AND (git.commit == ?git_ref OR git.ref == ?git_ref) + AND git.ref_key IN ( + // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than + // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. + // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values + // once; the outer query then does a cheap membership check instead of a wildcard match on + // every row. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) AND file.path LIKE ?file_path // git.ref_key is purely an internal storage/join key (never a query param): every @@ -20,10 +31,12 @@ configuration: // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key - // Optional consistency guard: assert the branch hasn't advanced since git_ref was - // resolved (only meaningful when git_ref names an incremental branch; a no-op filter - // for a commit-scoped query, since git.commit already equals git_ref there). - | WHERE git.commit LIKE ?git_commit + // Consistency guard: only serve content from a ref whose latest index is complete. This + // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs + // join doc is status:"indexing" until the new commit's content is fully published). A no-op + // for snapshot content, whose join doc is always status:"complete". `status` lives only on + // refs docs, so after the join it unambiguously means the joined ref's status. + | WHERE status == "complete" // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") @@ -69,13 +82,9 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref: + git_commit_ish: type: string - description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. - optional: false - git_commit: - type: string - description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + description: Filter by git commit(s) or ref(s) (supports * wildcards) optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml index 5e3de39..43f8e87 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml @@ -9,9 +9,20 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - // A commit SHA (snapshot content) or the exact branch/tag name (incremental - // content, which has no git.commit of its own) -- whichever this source resolves to. - AND (git.commit == ?git_ref OR git.ref == ?git_ref) + AND git.ref_key IN ( + // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than + // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. + // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values + // once; the outer query then does a cheap membership check instead of a wildcard match on + // every row. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) // git.ref_key is purely an internal storage/join key (never a query param): every // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN @@ -19,10 +30,12 @@ configuration: // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key - // Optional consistency guard: assert the branch hasn't advanced since git_ref was - // resolved (only meaningful when git_ref names an incremental branch; a no-op filter - // for a commit-scoped query, since git.commit already equals git_ref there). - | WHERE git.commit LIKE ?git_commit + // Consistency guard: only serve content from a ref whose latest index is complete. This + // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs + // join doc is status:"indexing" until the new commit's content is fully published). A no-op + // for snapshot content, whose join doc is always status:"complete". `status` lives only on + // refs docs, so after the join it unambiguously means the joined ref's status. + | WHERE status == "complete" // Split each file path into its segments | EVAL _segs = SPLIT(file.path, "/") @@ -193,13 +206,9 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref: + git_commit_ish: type: string - description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. - optional: false - git_commit: - type: string - description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + description: Filter by git commit(s) or ref(s) (supports * wildcards) optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml index b7a9bbe..a16913b 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml @@ -9,9 +9,20 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - // A commit SHA (snapshot content) or the exact branch/tag name (incremental - // content, which has no git.commit of its own) -- whichever this source resolves to. - AND (git.commit == ?git_ref OR git.ref == ?git_ref) + AND git.ref_key IN ( + // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than + // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. + // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values + // once; the outer query then does a cheap membership check instead of a wildcard match on + // every row. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) AND file.path LIKE ?file_path // git.ref_key is purely an internal storage/join key (never a query param): every @@ -20,10 +31,12 @@ configuration: // by commit (snapshot) or by ref name (incremental). | LOOKUP JOIN sourcerer-refs ON git.ref_key - // Optional consistency guard: assert the branch hasn't advanced since git_ref was - // resolved (only meaningful when git_ref names an incremental branch; a no-op filter - // for a commit-scoped query, since git.commit already equals git_ref there). - | WHERE git.commit LIKE ?git_commit + // Consistency guard: only serve content from a ref whose latest index is complete. This + // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs + // join doc is status:"indexing" until the new commit's content is fully published). A no-op + // for snapshot content, whose join doc is always status:"complete". `status` lives only on + // refs docs, so after the join it unambiguously means the joined ref's status. + | WHERE status == "complete" // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") @@ -184,13 +197,9 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_ref: + git_commit_ish: type: string - description: Exact commit SHA, or exact branch/tag name for an incremental source (no wildcards). Resolve via sourcerer.refs.list first. - optional: false - git_commit: - type: string - description: Optional consistency guard (supports * wildcards) against the commit resolved by the LOOKUP JOIN -- e.g. assert an incremental branch hasn't advanced since git_ref was resolved. Omit (default "*") to skip the check. + description: Filter by git commit(s) or ref(s) (supports * wildcards) optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml index 356f14e..7ec216a 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml @@ -9,7 +9,20 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND git.ref_key IN ( + // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than + // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. + // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values + // once; the outer query then does a cheap membership check instead of a wildcard match on + // every row. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND status == "complete" + AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) + | KEEP git.ref_key + ) AND file.path LIKE ?file_path AND MATCH(line.content.text, ?q) @@ -19,16 +32,22 @@ configuration: | EVAL _file_segs = MV_COUNT(SPLIT(file.path, "/")) | WHERE _fp_is_recursive OR _file_segs == _fp_segs - // Collapse matching lines to one row per repo + commit + // Collapse matching lines to one row per repo + ref. Grouping by git.ref_key + // rather than git.commit matters for two reasons: incremental (branch-tracked) + // rows have no git.commit of their own, so grouping by commit would collapse + // every branch in a repo into one bucket; and two refs can point at the same + // commit (e.g. a release tag and the branch it was cut from), which grouping + // by commit would wrongly conflate into a single ref. ref_key is already + // present on every sourcerer-lines row, so no join is needed to get it. | STATS _commit_file_count_distinct = COUNT_DISTINCT(file.path), _commit_score_sum = SUM(_score) - BY git.host, git.org, git.repo, git.commit + BY git.host, git.org, git.repo, git.ref_key // Normalize by distinct files so breadth beats duplication | EVAL _commit_score_density = _commit_score_sum / SQRT(TO_DOUBLE(_commit_file_count_distinct)) - // Collapse commits to one row per repo + // Collapse refs to one row per repo | STATS _repo_ref_count = COUNT(*), _repo_score_sum = SUM(_commit_score_density) @@ -57,9 +76,9 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit: + git_commit_ish: type: string - description: Filter by git commit(s) (supports * wildcards) + description: Filter by git commit(s) or ref(s) (supports * wildcards) optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/skills/ref-resolution/SKILL.md b/src/sourcerer/skills/ref-resolution/SKILL.md index 18db145..969ef3e 100644 --- a/src/sourcerer/skills/ref-resolution/SKILL.md +++ b/src/sourcerer/skills/ref-resolution/SKILL.md @@ -39,7 +39,7 @@ Disambiguate based on context: Resolve each ref independently using the steps above. Run content queries against each resolved ref (see "Pinning the ref" below), then compare results. Label each finding with its version. ### Explicit ref (branch name, exact tag, commit hash) -Use as given. If it is a branch, call `refs.list` with `git_ref_type: branch` to confirm it exists and retrieve its row. If it is a tag, confirm it the same way. If it is a commit hash, you don't need a `refs.list` call to use it in a content query (a commit hash is already a valid `git_ref` value) - optionally confirm with `git_ref_type: commit` if it may be a pinned commit rather than one reached via a branch/tag. +Use as given. If it is a branch, call `refs.list` with `git_ref_type: branch` to confirm it exists and retrieve its row. If it is a tag, confirm it the same way. If it is a commit hash, you don't need a `refs.list` call to use it in a content query (a commit hash is already a valid `git_commit_ish` value) - optionally confirm with `git_ref_type: commit` if it may be a pinned commit rather than one reached via a branch/tag. ### Branch as of a specific date (e.g. "main as it was on 2024-03-01") When a branch was indexed with `since` (history walk), multiple snapshots of the branch exist — @@ -53,26 +53,31 @@ If only one marker exists for the branch (tip-only indexing, no `since`), state snapshots are unavailable for that branch. ## Pinning the ref -Every content query (`sourcerer.code.*` and `sourcerer.files.*`) takes the same single required -param, `git_ref`: an exact commit SHA, or an exact branch/tag name. The tool matches it against -whichever field the content actually carries (`git.commit` for a `snapshot` source, `git.ref` -for an `incremental` branch) and then joins to resolve the citable commit -- there is no mode -you need to reason about. +Every content query (`sourcerer.code.*` and `sourcerer.files.*`) takes the same scoping param, +`git_commit_ish`: a commit SHA, or a branch/tag name (`*` and `?` wildcards are supported). The +tool matches it against whichever field the content actually carries (`git.commit` for a +`snapshot` source, `git.ref` for an `incremental` branch) and then joins to resolve the citable +commit -- there is no mode you need to reason about, and no separate commit param to supply. + +The param is optional (default `"*"`, matching every indexed ref), but for a version-specific +question you should still resolve and pin a ref: an unpinned `"*"` query returns matches from +*every* ref at once, which blends versions. Results stay attributable (each row carries its +`git.commit`), so `"*"` is fine for "does this symbol exist anywhere" style questions -- just not +for "how does X behave in 8.17". Once a ref is resolved above, pass the value straight through: - Resolved to a commit (tags, one-off branch snapshots -- the common case): use that commit SHA - as `git_ref`. + as `git_commit_ish`. - Resolved to an incremental branch (a source configured with `update: incremental` in `sourcerer.yml`; its `refs.list` row has `status: complete`, same as a snapshot ref): use the branch name - itself as `git_ref` (e.g. `main`) -- no commit needed, the query always resolves to whatever + itself as `git_commit_ish` (e.g. `main`) -- no commit needed, the query always resolves to whatever commit that branch is CURRENTLY at. `git.ref_key` is an internal storage/join detail (`LOOKUP JOIN sourcerer-refs ON git.ref_key` inside the tool) -- it is never a param you construct or a value `refs.list` returns. -Read the resolved `git.commit` back from the same row (or from the content query's own join) -for citations. Every content tool also accepts an optional `git_commit` param (default `"*"`): -it filters the commit the join resolves, so passing the expected commit re-asserts that an -incremental branch hasn't advanced since `git_ref` was resolved -- pass it when that matters -(e.g. a long-running investigation), otherwise leave it at the default. Re-invoke this skill -only when the question introduces a new or additional ref. +Read the resolved `git.commit` back from each result row (the content query's own join supplies +it) for citations. Because incremental content overwrites in place, a branch query always returns +its current HEAD -- if you need to confirm a branch hasn't advanced since you resolved it (e.g. a +long-running investigation), re-check `sourcerer.refs.list` for its current commit. Re-invoke this +skill only when the question introduces a new or additional ref. diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index fd8e929..141fcb0 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -4,6 +4,7 @@ # Standard packages import importlib.resources as resources +import re # Third-party packages import pytest @@ -49,28 +50,42 @@ def test_git_host_filtered_before_git_org(): def test_content_tools_use_universal_ref_join_query(): - # INV-005: every content tool's WHERE runs the identical `(git.commit == ?git_ref OR - # git.ref == ?git_ref)` shape with no update_mode/mode conditional, and joins sourcerer-refs - # on git.ref_key (a purely internal field -- never an agent-facing param) to resolve the - # commit for both snapshot and incremental content. git_commit survives as an optional - # POST-join consistency guard (not a scoping filter -- git_ref is the scoping param). + # INV-005: every content tool scopes to a set of git.ref_key values resolved from the small + # sourcerer-refs table via a subquery (`git.ref_key IN (FROM sourcerer-refs | WHERE ... (git.commit + # LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) | KEEP git.ref_key)`) rather than a + # per-row LIKE/OR wildcard match on the (far larger) content index, with no update_mode/mode + # conditional. It then joins sourcerer-refs on git.ref_key (a purely internal field -- never an + # agent-facing param) to resolve the commit for both snapshot and incremental content. + # git_commit_ish is the single scoping param (LIKE, so it supports wildcards); there is no + # separate git_commit param. A post-join `status == "complete"` guard excludes torn/partial + # reads while an incremental branch is mid-reindex (a no-op for always-complete snapshot content). tools = _tools() for tid in _CONTENT_TOOL_IDS: query = tools[tid]["configuration"]["query"] params = tools[tid]["configuration"]["params"] assert "update_mode" not in query, f"{tid} query has an update_mode conditional" - assert "git.commit == ?git_ref" in query, f"{tid} missing the commit-or-ref filter" - assert "git.ref == ?git_ref" in query, f"{tid} missing the commit-or-ref filter" + # The commit-or-ref match resolves inside the sourcerer-refs subquery, keyed by git.ref_key. + assert "git.ref_key IN (" in query, f"{tid} missing the ref_key subquery scope" + assert "git.commit LIKE ?git_commit_ish" in query, f"{tid} missing the commit-or-ref filter" + assert "git.ref LIKE ?git_commit_ish" in query, f"{tid} missing the commit-or-ref filter" assert "| LOOKUP JOIN sourcerer-refs ON git.ref_key" in query, f"{tid} missing the universal join" assert "git_ref_key" not in params, f"{tid} still exposes git_ref_key as a param" assert "?git_ref_key" not in query, f"{tid} still references ?git_ref_key" - assert params["git_ref"]["optional"] is False - assert "defaultValue" not in params["git_ref"] - assert params["git_commit"]["optional"] is True - assert params["git_commit"]["defaultValue"] == "*" - # The git_commit filter must appear AFTER the join (it filters the joined value, not - # the raw content doc -- incremental content has no git.commit of its own). - assert query.index("LOOKUP JOIN sourcerer-refs") < query.index("git.commit LIKE ?git_commit") + # git_commit_ish is optional with a "*" default: an unpinned query matches all indexed + # refs. Every content tool keeps this safe by carrying git.commit through to output (and, + # where it aggregates, grouping BY git.commit) so multi-ref matches stay attributable and + # are never summed across refs. + assert params["git_commit_ish"]["optional"] is True + assert params["git_commit_ish"]["defaultValue"] == "*" + # The old standalone git_commit guard param is gone: git_commit_ish is the sole scoping + # param, and the consistency guard is now the automatic, no-param `status == "complete"`. + # (Match on word boundary so ?git_commit_ish / git_commit_ish don't false-positive.) + assert not re.search(r"\bgit_commit\b", "\n".join(params)), f"{tid} still exposes git_commit as a param" + assert not re.search(r"\?git_commit\b", query), f"{tid} still references ?git_commit" + # The status guard must appear AFTER the join (status lives only on the refs doc the join + # brings in, never on the raw content doc). + assert '| WHERE status == "complete"' in query, f"{tid} missing the post-join status guard" + assert query.index("LOOKUP JOIN sourcerer-refs") < query.index('WHERE status == "complete"') def test_refs_list_does_not_surface_ref_key(): @@ -104,6 +119,35 @@ def test_output_keeps_git_host(): assert stripped.index("git.host") < stripped.index("git.org") +def test_content_tool_aggregation_is_ref_scoped(): + # git_commit_ish defaults to "*", so a content query can match more than one ref at once. + # That is only safe if aggregation never blends refs: every STATS in a content tool must carry + # git.commit in its BY grouping key, so per-ref counts/bytes/line-blobs stay separate rather + # than being summed or interleaved across commits. Guards the files.ls-style regression where a + # `BY name` grouping silently summed file counts across every matching ref. + tools = _tools() + for tid in _CONTENT_TOOL_IDS: + query = tools[tid]["configuration"]["query"] + # Walk each STATS block: from a "| STATS" line through its trailing "BY ..." clause(s). + lines = query.splitlines() + for i, line in enumerate(lines): + if not line.strip().startswith("| STATS"): + continue + # Collect this STATS block's text up to the next pipe command. + block = [line] + for nxt in lines[i + 1:]: + if nxt.strip().startswith("|"): + break + block.append(nxt) + block_text = "\n".join(block) + assert " BY " in block_text, f"{tid} has a STATS with no BY grouping" + by_clause = block_text.split(" BY ", 1)[1] + assert "git.commit" in by_clause, ( + f"{tid} STATS groups without git.commit -- would blend refs when git_commit_ish " + f"matches more than one ref" + ) + + # --------------------------------------------------------------------------- # strip_esql_comments tests # --------------------------------------------------------------------------- From 8e89fd504a5d81f53baeb25293f8f3707fad335b Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Tue, 18 Aug 2026 16:04:59 -0400 Subject: [PATCH 12/29] Remove unused ref_type field from files and lines indices --- src/sourcerer/commands/index/documents.py | 2 -- src/sourcerer/elastic/index_templates/sourcerer-v3-files.json | 4 ---- src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json | 4 ---- 3 files changed, 10 deletions(-) diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index 9901250..805e5a9 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -203,7 +203,6 @@ def build_incremental_file_doc( "org": org, "repo": repo, "ref": ref, - "ref_type": "branch", "ref_key": build_ref_key(host, org, repo, ref), }, "file": file_fields, @@ -249,7 +248,6 @@ def iter_incremental_line_docs( "org": org, "repo": repo, "ref": ref, - "ref_type": "branch", "ref_key": build_ref_key(host, org, repo, ref), }, "file": file_fields, diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json index 1bb91e2..17cb50e 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json @@ -61,10 +61,6 @@ }, "ref": { "type": "keyword" - }, - "ref_type": { - "type": "keyword", - "normalizer": "lowercase" } } }, diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json index 1ae9e33..2ef42f5 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json @@ -103,10 +103,6 @@ }, "ref": { "type": "keyword" - }, - "ref_type": { - "type": "keyword", - "normalizer": "lowercase" } } }, From 4039f839e162105a8d85836c1e4ac39af2104a90 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Wed, 19 Aug 2026 10:26:25 -0400 Subject: [PATCH 13/29] Fold snapshot join doc into ref-name marker so each snapshot source writes one refs doc --- AGENTS.md | 31 ++-- src/sourcerer/commands/index/command.py | 20 +-- src/sourcerer/commands/index/markers.py | 189 +++++++++++++++--------- tests/test_backfill.py | 125 ++++++++++++---- tests/test_markers.py | 53 ++++--- 5 files changed, 275 insertions(+), 143 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 77c0a04..17407e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -379,23 +379,30 @@ creates one index/shard per commit — see `specs/sourcerer-yml.md` for the cave Every content doc (file and line, both `update` modes) carries a `git.ref_key` keyword field: the bare commit SHA for `snapshot` content, or `{host}~{org}~{repo}~{ref}` for `incremental` -content (see `update: ` above; `build_ref_key` in `src/sourcerer/utils.py`). A second, -distinct kind of `sourcerer-v3-refs` document -- a **refs join doc**, `_id = git.ref_key` -(exactly one per key) -- carries the citable `git.commit`: one per commit for snapshot content, -one per branch (holding the live HEAD) for incremental content. This is a different id space -from the hashed, append-only `build_ref_id` ref-name markers described above (those still drive -`since`/retention history and are untouched by this). +content (see `update: ` above; `build_ref_key` in `src/sourcerer/utils.py`). The refs doc +that carries `git.commit` for the join is: + +- **Snapshot:** the `build_ref_id`-keyed **ref-name marker** itself. `write_ref_marker` sets + `git.ref_key = commit_sha` on the marker (one doc per snapshot source). There is no separate + shadow join doc for snapshot content. +- **Incremental:** a dedicated refs join doc at `_id = git.ref_key = {host}~{org}~{repo}~{ref}`, + holding the live HEAD commit and advanced two-phase (INV-006). One doc per branch. + +INV-004: exactly one `sourcerer-v3-refs` doc per `git.ref_key` value. For snapshot content this +is the marker (one per ref+commit); for incremental it is the branch's join doc. Because ES|QL +`LOOKUP JOIN` fans out on duplicate right-side keys, having >1 doc with the same `git.ref_key` +would multiply content rows — the uniqueness gate (`_run_uniqueness_gate`, INV-011) guards this. #### `status` field values -Every `sourcerer-v3-refs` document — ref-name markers (keyed by `build_ref_id`) and refs join -docs (keyed by `ref_key`) alike — carries a `status` field drawn from a shared two-value -vocabulary, so the scheduler and `sourcerer.refs.list` can query both families uniformly: +Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental join docs alike +— carries a `status` field drawn from a shared two-value vocabulary, so the scheduler and +`sourcerer.refs.list` can query both families uniformly: | Value | Meaning | |---|---| -| `indexing` | A run is mid-flight. `indexing_started_at` is set; `indexed_at` is absent/null. Present on both ref-name markers (written by `write_indexing_marker` just before snapshot ingest) and incremental join docs (written by `write_incremental_indexing` just before incremental ingest). A stale `indexing` doc whose `indexing_started_at` is older than the retry window (default 6 h) marks a crashed run and is treated as due for re-indexing. | -| `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot ref-name markers), `write_snapshot_join_doc` (snapshot join docs), and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | +| `indexing` | A run is mid-flight. `indexing_started_at` is set; `indexed_at` is absent/null. Present on snapshot ref-name markers (written by `write_indexing_marker` just before ingest) and incremental join docs (written by `write_incremental_indexing`). A stale `indexing` doc whose `indexing_started_at` is older than the retry window (default 6 h) marks a crashed run and is treated as due for re-indexing. | +| `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot markers) and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) therefore runs the same query shape regardless of mode -- and `git.ref_key` is NEVER an agent-facing param, only @@ -435,7 +442,7 @@ serves content only from a ref whose latest index is complete, excluding the tor window while an incremental branch is mid-reindex -- during a HEAD advance the branch's refs join doc is `status: indexing` and its content is being mutated in place, so a query joining to it would otherwise read a half-applied mix of the old and new commits. It is a no-op for snapshot -content (whose join doc is always `status: complete`). Trade-off: a *failed* incremental run +content (snapshot markers are always `status: complete`). Trade-off: a *failed* incremental run leaves the join doc at `status: indexing` with the prior commit's content still fully consistent; the guard hides that content until the next successful run republishes `status: complete`. Note this guard is about intra-update consistency, not inter-query staleness: because incremental diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index 5c2c65c..d69d2b1 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -53,7 +53,7 @@ fully_indexed_counts, markers_status_by_id, _needs_index, pre_clone_skip, read_incremental_ref, recorded_routing, refresh_incremental_content, should_index, write_incremental_failed, write_incremental_indexing, write_incremental_ready, - write_indexing_marker, write_ref_marker, write_snapshot_join_doc, + write_indexing_marker, write_ref_marker, ) from .report import dry_run_config from .schedule import filter_config_by_schedule @@ -258,18 +258,14 @@ def index_ref_in_dir( # write-new -> FLIP MARKER -> delete-old: the marker now points at the new location before any # old copy is deleted, so a crash between here and the delete below leaves stale (not missing) # data that the prune stale-location sweep reclaims. + # refresh=True so the post-index uniqueness gate (_run_uniqueness_gate, INV-011) sees the + # git.ref_key carrier immediately instead of racing the refs index's default (~1s) refresh + # interval: the bulk context manager refreshes the CONTENT indices on exit but not refs, so an + # unrefreshed write here would make the gate read this ref_key's content but miss its carrier + # and false-fail "missing". Mirrors write_incremental_ready (refresh=True). write_ref_marker(es, host, org, repo, ref_type, ref_for_id, commit_sha, commit_date_iso, - files_count, lines_count, index_level=level, index_suffix=suffix) - # Every snapshot unit -- whether freshly indexed or reusing a sibling's already-indexed - # content -- must have its `_id = commit` refs join doc so the universal join query resolves - # a commit for this content regardless of which ref reached it (INV-004). - # refresh=True so the post-index uniqueness gate (_run_uniqueness_gate, INV-011) sees this join - # doc immediately instead of racing the refs index's default (~1s) refresh interval: the bulk - # context manager refreshes the CONTENT indices on exit but not refs, so an unrefreshed write - # here would make the gate read this ref_key's content but miss its join doc and false-fail - # "missing". Mirrors write_incremental_ready (refresh=True) and backfill_refs_join_docs. - write_snapshot_join_doc(es, host, org, repo, ref_type, ref_for_id, commit_sha, commit_date_iso, - refresh=True) + files_count, lines_count, index_level=level, index_suffix=suffix, + refresh=True) if migrating: # Reconstruct the OLD index name from the prior marker's routing and drop this commit's # stale copy there. Commit-safety (another surviving ref sharing the commit) is respected diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index a434db1..fa072f9 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -450,6 +450,7 @@ def write_ref_marker( lines_count: int, index_level: str = "repo", index_suffix: str | None = None, + refresh: bool = False, ) -> None: # (ref, ref_type) replaces the old git.branch/git.tag fields: those were write-only and # fully reconstructable as `git.ref filtered by git.ref_type`. git.tag was an array that @@ -462,9 +463,14 @@ def write_ref_marker( # stays correct and prune/migration can find (and clean up) exactly where content lives. # Legacy markers written before this feature omit both; readers fall back to the "repo"/None # defaults, which reconstruct to the historical repo-level name where that content actually is. + # + # git.ref_key = commit_sha folds the snapshot join doc into this single marker: it lets the + # content tools' LOOKUP JOIN sourcerer-refs ON git.ref_key resolve git.commit without a + # separate _id=commit shadow doc. One snapshot source → one refs doc (INV-004). ref_id = build_ref_id(host, org, repo, ref_type, ref, commit_sha) doc = { "git": { + "ref_key": commit_sha, "host": host, "org": org, "repo": repo, @@ -480,7 +486,7 @@ def write_ref_marker( "index_level": index_level, "index_suffix": index_suffix, } - es.index(index=REFS_INDEX, id=ref_id, document=doc) + es.index(index=REFS_INDEX, id=ref_id, document=doc, refresh=refresh) def pre_clone_skip( @@ -536,50 +542,21 @@ def pre_clone_skip( return False, ref_for_id, remote_sha -# --- refs join docs (git.ref_key), keyed by `_id = ref_key` ------------------------------- -# One document per `ref_key` (INV-004): snapshot content's join doc lives at `_id = `; -# an incremental branch's single join doc lives at `_id = {host}~{org}~{repo}~{ref}` and its -# `git.commit` is the branch's live HEAD, advanced only by a two-phase indexing -> complete -# publication (INV-006). These are a DISTINCT id space from `build_ref_id`'s hashed, append-only -# ref-name markers above (untouched -- they still drive `since`/retention history); a join doc's -# `_id` is a plain, unhashed `ref_key` string, which a `build_ref_id` hash can never collide with. +# --- incremental refs join docs (git.ref_key), keyed by `_id = ref_key` ------------------- +# One document per incremental branch (INV-004): the branch's single join doc lives at +# `_id = {host}~{org}~{repo}~{ref}` and its `git.commit` is the branch's live HEAD, advanced +# only by a two-phase indexing -> complete publication (INV-006). This is a DISTINCT id space +# from `build_ref_id`'s hashed, append-only ref-name markers above; a join doc's `_id` is a +# plain, unhashed `ref_key` string, which a `build_ref_id` hash can never collide with. +# +# Snapshot mode no longer writes a separate join doc: `write_ref_marker` now carries +# `git.ref_key = commit_sha` directly on the hashed marker (one doc per snapshot source, +# INV-004). Legacy `_id = commit` snapshot join docs written before this change are cleaned +# up by `backfill_refs_join_docs` / the migration `backfill_repo` step. ERROR_MAX_LEN = 2000 # bound stored failure text so a giant git/ES error can't bloat the doc -def write_snapshot_join_doc( - es: Elasticsearch, - host: str, - org: str, - repo: str, - ref_type: str, - ref: str, - commit_sha: str, - commit_date_iso: str | None, - refresh: bool = False, -) -> None: - """Write (or idempotently re-write) the snapshot refs join doc: `_id = commit_sha`, - `git.ref_key = commit_sha`, `git.commit = commit_sha` (the commit is its own citable - identity). `ref`/`ref_type` record the ref that produced this write (informational only -- - multiple refs resolving to the same commit all write the same doc, so re-writes are a - no-op change).""" - doc = { - "git": { - "ref_key": commit_sha, - "host": host, - "org": org, - "repo": repo, - "ref": ref, - "ref_type": ref_type, - "commit": commit_sha, - "commit_date": commit_date_iso, - }, - "update_mode": "snapshot", - "status": "complete", - } - es.index(index=REFS_INDEX, id=commit_sha, document=doc, refresh=refresh) - - def read_incremental_ref(es: Elasticsearch, host: str, org: str, repo: str, ref: str) -> dict | None: """The branch's incremental join doc `_source`, or None if never indexed. A real-time GET (by `_id = ref_key`), so it reflects the last write even without a refresh.""" @@ -907,34 +884,43 @@ def distinct_commits_for_repo(es: Elasticsearch, host: str, org: str, repo: str) return {b["key"] for b in resp["aggregations"]["commits"]["buckets"]} -def commits_with_join_doc(es: Elasticsearch, commits: set[str]) -> set[str]: - """The subset of `commits` that already have a `_id = commit` refs join doc. A batched - ids lookup, the join-doc analogue of `commits_with_content`.""" +def commits_with_ref_key_carrier(es: Elasticsearch, commits: set[str]) -> set[str]: + """The subset of `commits` for which a refs doc already carries `git.ref_key == commit` + (i.e. has a snapshot ref_key carrier). Used by `backfill_refs_join_docs` to skip commits + whose marker already carries the ref_key from a normal index run, so the backfill is a + no-op on up-to-date repos (INV-009). A terms query on `git.ref_key` covers both the + hashed-marker carrier (new) and any legacy `_id = commit` shadow docs (old).""" if not commits: return set() try: resp = es.search( - index=REFS_ALIAS, size=len(commits), query={"ids": {"values": sorted(commits)}}, - source_includes=[], + index=REFS_ALIAS, size=0, + query={"terms": {"git.ref_key": sorted(commits)}}, + aggs={"carriers": {"terms": {"field": "git.ref_key", "size": len(commits)}}}, ) except NotFoundError: return set() - return {hit["_id"] for hit in resp["hits"]["hits"]} + return {b["key"] for b in resp["aggregations"]["carriers"]["buckets"]} + + +# Keep the old name as an alias so external callers (if any) and tests can migrate gradually. +commits_with_join_doc = commits_with_ref_key_carrier def backfill_refs_join_docs(es: Elasticsearch, host: str, org: str, repo: str) -> int: - """Ensure a snapshot `_id = commit` join doc exists for every distinct content commit in - this repo (INV-010). For each missing commit, an existing `build_ref_id` marker (if any) - supplies the ref/ref_type/commit_date it was originally indexed under; falls back to - generic values if none is found (the content is authoritative either way -- the join doc's - ref/ref_type are informational). Returns the number of join docs created; 0 on a repeat run - (INV-009).""" + """Ensure every distinct content commit for this repo has a refs doc carrying + `git.ref_key = commit` (INV-010). For each commit lacking a carrier, stamps `git.ref_key` + onto its existing complete `build_ref_id` marker (the normal post-collapse shape) by + partially updating that doc. Falls back to writing a minimal `_id = commit` carrier doc if + no marker is found (orphan content). Returns the number of carriers stamped/created; 0 on a + repeat run (INV-009).""" commits = distinct_commits_for_repo(es, host, org, repo) if not commits: return 0 - missing = commits - commits_with_join_doc(es, commits) - created = 0 + missing = commits - commits_with_ref_key_carrier(es, commits) + stamped = 0 for commit_sha in missing: + # Look for the existing complete marker for this commit (build_ref_id key space). query = { "bool": { "filter": [ @@ -943,6 +929,9 @@ def backfill_refs_join_docs(es: Elasticsearch, host: str, org: str, repo: str) - {"term": {"git.repo": repo}}, {"term": {"git.commit": commit_sha}}, {"term": {"status": "complete"}}, + # Only hashed markers (no ref_key yet); legacy shadow docs (update_mode: + # "snapshot") are already carriers and handled by commits_with_ref_key_carrier. + {"bool": {"must_not": {"exists": {"field": "git.ref_key"}}}}, ] } } @@ -952,22 +941,52 @@ def backfill_refs_join_docs(es: Elasticsearch, host: str, org: str, repo: str) - resp = {"hits": {"hits": []}} hits = resp["hits"]["hits"] if hits: - src_git = hits[0]["_source"].get("git", {}) + # Stamp git.ref_key onto the existing marker in place. We use es.update with a + # partial doc rather than a full re-index to avoid touching the counts/timestamps. + marker_id = hits[0]["_id"] + es.update(index=REFS_INDEX, id=marker_id, doc={"git": {"ref_key": commit_sha}}) + else: + # No marker exists (orphan content): write a minimal carrier doc so the gate passes. + src_git = {} + try: + # Try to derive informational ref/ref_type from any refs doc for this commit. + any_resp = es.search( + index=REFS_ALIAS, size=1, + query={"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.commit": commit_sha}}, + ]}}, + ) + if any_resp["hits"]["hits"]: + src_git = any_resp["hits"]["hits"][0]["_source"].get("git", {}) + except NotFoundError: + pass ref = src_git.get("ref") or commit_sha ref_type = src_git.get("ref_type") or "commit" commit_date_iso = src_git.get("commit_date") - else: - ref, ref_type, commit_date_iso = commit_sha, "commit", None - write_snapshot_join_doc(es, host, org, repo, ref_type, ref, commit_sha, commit_date_iso) - created += 1 - if created: - # Refresh so a uniqueness gate run immediately afterward (see command._run_uniqueness_gate) - # sees every join doc just created rather than racing the refs index's refresh interval. + es.index( + index=REFS_INDEX, id=commit_sha, + document={ + "git": { + "ref_key": commit_sha, + "host": host, "org": org, "repo": repo, + "ref": ref, "ref_type": ref_type, + "commit": commit_sha, "commit_date": commit_date_iso, + }, + "status": "complete", + }, + ) + stamped += 1 + if stamped: + # Refresh so a uniqueness gate run immediately afterward sees every carrier just written + # rather than racing the refs index's default refresh interval. try: es.indices.refresh(index=REFS_INDEX) except NotFoundError: pass - return created + return stamped def apply_refs_index_mapping(es: Elasticsearch, mapping: dict) -> None: @@ -1005,16 +1024,48 @@ def backfill_repo( """Run the full one-time upgrade for one repo: apply the updated content/refs index mappings (once, if given -- must happen BEFORE the content update so the new field lands typed correctly rather than dynamically guessed), stamp `ref_key` onto pre-existing - snapshot content (idempotent), and create a join doc for every already-indexed - commit that lacks one. Returns a small summary dict for reporting; every field is 0 on a - repeat run (INV-009).""" + snapshot content (idempotent), stamp `git.ref_key` onto existing snapshot markers that + predate the one-doc-per-source change, and delete any now-redundant legacy `_id = commit` + shadow join docs (those written by the old write_snapshot_join_doc path). Returns a small + summary dict for reporting; every field is 0 on a repeat run (INV-009).""" if files_mapping is not None and lines_mapping is not None: apply_content_index_mapping(es, files_mapping, lines_mapping) if refs_mapping is not None: apply_refs_index_mapping(es, refs_mapping) updated = backfill_snapshot_ref_keys(es, host, org, repo) - created = backfill_refs_join_docs(es, host, org, repo) - return {"content_updated": updated, "join_docs_created": created} + # Stamp git.ref_key onto existing markers that lack it; this also covers repos that were + # indexed before the one-doc-per-source change where markers had no ref_key. + stamped = backfill_refs_join_docs(es, host, org, repo) + # Delete legacy standalone `_id = commit` shadow docs (update_mode: "snapshot", written by + # the old write_snapshot_join_doc). Ordered AFTER the backfill stamp + implicit refresh so + # every commit always has at least one carrier (the marker) before the shadow is removed. + deleted = _delete_legacy_snapshot_join_docs(es, host, org, repo) + return {"content_updated": updated, "carriers_stamped": stamped, "shadow_docs_deleted": deleted} + + +def _delete_legacy_snapshot_join_docs(es: Elasticsearch, host: str, org: str, repo: str) -> int: + """Delete legacy `_id = commit` snapshot join docs written by the old write_snapshot_join_doc + path (distinguishable by update_mode == "snapshot"). These are now superseded by the + git.ref_key field on the hashed ref-name markers. Safe to call only AFTER backfill_refs_join_docs + has stamped ref_key onto all markers (so no commit loses its carrier). Returns 0 if none exist.""" + query = { + "bool": { + "filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"update_mode": "snapshot"}}, + ] + } + } + try: + resp = es.delete_by_query( + index=REFS_INDEX, query=query, + wait_for_completion=True, conflicts="proceed", refresh=True, + ) + return resp.get("deleted", 0) + except NotFoundError: + return 0 def resolve_head(es: Elasticsearch, host: str, org: str, repo: str, ref_type: str, ref: str) -> dict | None: diff --git a/tests/test_backfill.py b/tests/test_backfill.py index e55e364..50a84c6 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -1,9 +1,10 @@ """Tests for the one-time upgrade backfill in sourcerer.commands.index.markers: stamping -git.ref_key onto pre-existing snapshot content, migrating the refs index mapping, and -creating missing snapshot join docs. Every ES call is mocked (INV-009/INV-010).""" +git.ref_key onto pre-existing snapshot content, stamping ref_key onto existing markers that +predate the one-doc-per-source change, and deleting legacy shadow join docs. +Every ES call is mocked (INV-009/INV-010).""" # Standard packages -from unittest.mock import MagicMock +from unittest.mock import MagicMock, call # Third-party packages from elastic_transport import ApiResponseMeta, HttpHeaders @@ -17,6 +18,7 @@ backfill_repo, backfill_snapshot_ref_keys, commits_with_join_doc, + commits_with_ref_key_carrier, distinct_commits_for_repo, ) from sourcerer.indices import FILES_ALIAS, LINES_ALIAS, REFS_INDEX @@ -70,67 +72,110 @@ def test_missing_index_returns_empty_set(self): assert distinct_commits_for_repo(es, "github", "acme", "widgets") == set() -class TestCommitsWithJoinDoc: +class TestCommitsWithRefKeyCarrier: + """commits_with_ref_key_carrier (and its alias commits_with_join_doc) returns the subset + of commits for which a refs doc carries git.ref_key == commit.""" + def test_empty_input_short_circuits(self): es = MagicMock() - assert commits_with_join_doc(es, set()) == set() + assert commits_with_ref_key_carrier(es, set()) == set() es.search.assert_not_called() - def test_returns_hit_ids(self): + def test_returns_commits_from_agg_buckets(self): es = MagicMock() - es.search.return_value = {"hits": {"hits": [{"_id": "aaa"}]}} - assert commits_with_join_doc(es, {"aaa", "bbb"}) == {"aaa"} + es.search.return_value = {"aggregations": {"carriers": {"buckets": [{"key": "aaa"}]}}} + assert commits_with_ref_key_carrier(es, {"aaa", "bbb"}) == {"aaa"} + + def test_missing_index_returns_empty_set(self): + es = MagicMock() + es.search.side_effect = _not_found() + assert commits_with_ref_key_carrier(es, {"aaa"}) == set() + + def test_alias_commits_with_join_doc_is_the_same_function(self): + # commits_with_join_doc kept as alias for backward compatibility. + assert commits_with_join_doc is commits_with_ref_key_carrier class TestBackfillRefsJoinDocs: - def test_creates_join_doc_for_missing_commit(self): + def test_stamps_ref_key_onto_existing_marker(self): + # Normal case: a complete marker exists for the commit; backfill stamps ref_key on it. + es = MagicMock() + marker_id = "deadbeef" * 4 # any string + es.search.side_effect = [ + # distinct_commits_for_repo + {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, + # commits_with_ref_key_carrier -- no carrier yet + {"aggregations": {"carriers": {"buckets": []}}}, + # per-missing-commit: find the existing marker (no ref_key yet) + {"hits": {"hits": [{"_id": marker_id, "_source": {"git": {"ref": "v1.0", "ref_type": "tag"}}}]}}, + ] + stamped = backfill_refs_join_docs(es, "github", "acme", "widgets") + assert stamped == 1 + # Must use es.update (partial doc), not es.index + es.update.assert_called_once() + assert es.update.call_args.kwargs["id"] == marker_id + assert es.update.call_args.kwargs["doc"] == {"git": {"ref_key": FULL_SHA}} + es.index.assert_not_called() + + def test_falls_back_to_index_for_orphan_content(self): + # No marker found for the commit: write a minimal _id=commit carrier doc. es = MagicMock() es.search.side_effect = [ # distinct_commits_for_repo {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, - # commits_with_join_doc -- no existing join docs + # commits_with_ref_key_carrier -- no carrier + {"aggregations": {"carriers": {"buckets": []}}}, + # per-missing-commit marker search -- no marker {"hits": {"hits": []}}, - # per-missing-commit lookup for a build_ref_id marker (none found) + # fallback: any refs doc for this commit (for informational fields) {"hits": {"hits": []}}, ] - created = backfill_refs_join_docs(es, "github", "acme", "widgets") - assert created == 1 - assert es.index.call_args.kwargs["id"] == FULL_SHA - assert es.index.call_args.kwargs["document"]["git"]["ref_key"] == FULL_SHA - - def test_second_run_creates_nothing(self): - # INV-009/INV-010: every commit already has a join doc -> no-op. + stamped = backfill_refs_join_docs(es, "github", "acme", "widgets") + assert stamped == 1 + es.update.assert_not_called() + # Falls back to writing an _id=commit carrier + es.index.assert_called_once() + call_kwargs = es.index.call_args.kwargs + assert call_kwargs["id"] == FULL_SHA + assert call_kwargs["document"]["git"]["ref_key"] == FULL_SHA + assert call_kwargs["document"]["git"]["commit"] == FULL_SHA + + def test_second_run_stamps_nothing(self): + # INV-009/INV-010: every commit already has a carrier -> no-op. es = MagicMock() es.search.side_effect = [ {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, - {"hits": {"hits": [{"_id": FULL_SHA}]}}, + {"aggregations": {"carriers": {"buckets": [{"key": FULL_SHA}]}}}, ] assert backfill_refs_join_docs(es, "github", "acme", "widgets") == 0 + es.update.assert_not_called() es.index.assert_not_called() def test_no_commits_short_circuits(self): es = MagicMock() es.search.return_value = {"aggregations": {"commits": {"buckets": []}}} assert backfill_refs_join_docs(es, "github", "acme", "widgets") == 0 + es.update.assert_not_called() es.index.assert_not_called() - def test_refreshes_refs_index_when_docs_created(self): - # A uniqueness-gate run immediately afterward must see the just-created join doc + def test_refreshes_refs_index_when_carriers_stamped(self): + # A uniqueness-gate run immediately afterward must see the just-stamped carrier # rather than racing the refs index's refresh interval. es = MagicMock() + marker_id = "aabbccdd" * 4 es.search.side_effect = [ {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, - {"hits": {"hits": []}}, - {"hits": {"hits": []}}, + {"aggregations": {"carriers": {"buckets": []}}}, + {"hits": {"hits": [{"_id": marker_id, "_source": {"git": {"ref": "v1.0", "ref_type": "tag"}}}]}}, ] backfill_refs_join_docs(es, "github", "acme", "widgets") assert es.indices.refresh.call_args.kwargs["index"] == REFS_INDEX - def test_no_refresh_when_nothing_created(self): + def test_no_refresh_when_nothing_stamped(self): es = MagicMock() es.search.side_effect = [ {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, - {"hits": {"hits": [{"_id": FULL_SHA}]}}, + {"aggregations": {"carriers": {"buckets": [{"key": FULL_SHA}]}}}, ] backfill_refs_join_docs(es, "github", "acme", "widgets") es.indices.refresh.assert_not_called() @@ -169,9 +214,35 @@ class TestBackfillRepo: def test_second_run_is_fully_idempotent(self): es = MagicMock() es.update_by_query.return_value = {"updated": 0} - es.search.return_value = {"hits": {"hits": []}, "aggregations": {"commits": {"buckets": []}}} + # All search calls return empty (no commits -> backfill no-ops; delete_by_query finds nothing) + es.search.return_value = {"aggregations": {"commits": {"buckets": []}}} + es.delete_by_query.return_value = {"deleted": 0} summary = backfill_repo( es, "github", "acme", "widgets", refs_mapping={"properties": {}}, files_mapping={"properties": {}}, lines_mapping={"properties": {}}, ) - assert summary == {"content_updated": 0, "join_docs_created": 0} + assert summary == {"content_updated": 0, "carriers_stamped": 0, "shadow_docs_deleted": 0} + + def test_summary_keys(self): + # Confirm the returned dict uses the new key names. + es = MagicMock() + es.update_by_query.return_value = {"updated": 0} + es.search.return_value = {"aggregations": {"commits": {"buckets": []}}} + es.delete_by_query.return_value = {"deleted": 0} + summary = backfill_repo(es, "github", "acme", "widgets") + assert set(summary.keys()) == {"content_updated", "carriers_stamped", "shadow_docs_deleted"} + + def test_deletes_legacy_shadow_docs(self): + # Migration: after stamping carriers, delete legacy update_mode:snapshot shadow docs. + es = MagicMock() + es.update_by_query.return_value = {"updated": 0} + es.search.return_value = {"aggregations": {"commits": {"buckets": []}}} + es.delete_by_query.return_value = {"deleted": 3} + summary = backfill_repo(es, "github", "acme", "widgets") + assert summary["shadow_docs_deleted"] == 3 + # delete_by_query must target the physical REFS_INDEX (not the alias -- only writes go there) + assert es.delete_by_query.call_args.kwargs["index"] == REFS_INDEX + # Must scope to update_mode: "snapshot" (the legacy shadow doc shape) + filters = es.delete_by_query.call_args.kwargs["query"]["bool"]["filter"] + assert {"term": {"git.host": "github"}} in filters + assert {"term": {"update_mode": "snapshot"}} in filters diff --git a/tests/test_markers.py b/tests/test_markers.py index 2a4838c..aa692c0 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -29,7 +29,6 @@ write_incremental_indexing, write_incremental_ready, write_ref_marker, - write_snapshot_join_doc, ) from sourcerer.indices import FILES_ALIAS, REFS_ALIAS, REFS_INDEX from sourcerer.utils import build_ref_key @@ -404,40 +403,48 @@ def _indexed_doc(es): return es.index.call_args.kwargs["document"] -class TestSnapshotJoinDoc: - def test_join_doc_id_and_ref_key_are_the_commit(self): +class TestWriteRefMarker: + """write_ref_marker is the single refs doc per snapshot source; it carries git.ref_key so + the LOOKUP JOIN resolves git.commit without a separate shadow join doc (INV-004).""" + + def test_marker_carries_ref_key_equal_to_commit(self): es = MagicMock() - write_snapshot_join_doc(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None) - call = es.index.call_args.kwargs - assert call["id"] == OLD - assert call["index"] == REFS_INDEX - doc = call["document"] + write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + files_count=10, lines_count=200) + doc = es.index.call_args.kwargs["document"] assert doc["git"]["ref_key"] == OLD assert doc["git"]["commit"] == OLD - assert doc["update_mode"] == "snapshot" - assert doc["status"] == "complete" - def test_join_doc_idempotent_rewrite_same_id(self): - first = MagicMock() - second = MagicMock() - write_snapshot_join_doc(first, "github", "acme", "widgets", "branch", "main", OLD, None) - write_snapshot_join_doc(second, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None) - assert first.index.call_args.kwargs["id"] == second.index.call_args.kwargs["id"] + def test_marker_id_is_hashed_not_the_commit(self): + # _id is build_ref_id (BLAKE2b hash) -- one per (ref, commit), NOT the bare commit SHA. + es = MagicMock() + write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + files_count=10, lines_count=200) + call = es.index.call_args.kwargs + assert call["index"] == REFS_INDEX + assert call["id"] != OLD # hashed, not the bare commit + assert call["id"] == build_ref_id("github", "acme", "widgets", "tag", "v1.0.0", OLD) def test_default_write_does_not_refresh(self): es = MagicMock() - write_snapshot_join_doc(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None) - assert es.index.call_args.kwargs["refresh"] is False + write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + files_count=1, lines_count=1) + assert es.index.call_args.kwargs.get("refresh") is False def test_refresh_true_is_propagated(self): - # The snapshot indexing path (command.index_one) passes refresh=True so the post-index - # uniqueness gate (INV-011) doesn't race the refs index's async refresh and false-fail - # "git.ref_key missing". Guards that the write actually threads the flag to es.index. + # command.py passes refresh=True so the post-index uniqueness gate (INV-011) sees the + # ref_key carrier immediately rather than racing the refs index's async refresh. es = MagicMock() - write_snapshot_join_doc(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, - refresh=True) + write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + files_count=1, lines_count=1, refresh=True) assert es.index.call_args.kwargs["refresh"] is True + def test_marker_status_complete(self): + es = MagicMock() + write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + files_count=5, lines_count=100) + assert es.index.call_args.kwargs["document"]["status"] == "complete" + class TestIncrementalRefKeyIdentity: def test_id_is_ref_key_not_a_hash(self): From 83d91fef1208590e2b81dbef0e8e1dc3df320976 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Thu, 20 Aug 2026 07:43:59 -0400 Subject: [PATCH 14/29] Refactor indices and queries to no longer require a ref_key to join on (serendipitously improving search speed, too). Make incremental indexing compatible with sources[i].index.level and sources[i].index.suffix. --- AGENTS.md | 170 +++++---- src/sourcerer/cli.py | 14 +- src/sourcerer/commands/index/command.py | 103 ++---- src/sourcerer/commands/index/documents.py | 19 +- src/sourcerer/commands/index/markers.py | 345 +++++------------- src/sourcerer/commands/index/selection.py | 31 +- src/sourcerer/commands/prune/command.py | 15 +- src/sourcerer/commands/prune/execute.py | 64 +++- src/sourcerer/config.py | 2 +- .../elastic/agent_builder_tools/README.md | 75 +++- .../sourcerer.code.grep.yml | 95 +++-- .../sourcerer.code.search.yml | 95 +++-- .../sourcerer.files.cat.yml | 95 +++-- .../sourcerer.files.head.yml | 92 +++-- .../sourcerer.files.ls.yml | 97 +++-- .../sourcerer.files.read_lines.yml | 95 +++-- .../sourcerer.files.tail.yml | 95 +++-- .../sourcerer.files.tree.yml | 95 +++-- .../sourcerer.files.wc.yml | 95 +++-- .../sourcerer.refs.list.yml | 6 +- .../sourcerer.repos.search.yml | 84 +++-- .../index_templates/sourcerer-v3-files.json | 3 - .../index_templates/sourcerer-v3-lines.json | 3 - .../index_templates/sourcerer-v3-refs.json | 3 - src/sourcerer/queries.py | 132 +++++-- src/sourcerer/skills/ref-resolution/SKILL.md | 7 +- src/sourcerer/utils.py | 14 +- tests/test_agent_builder_tools.py | 77 ++-- tests/test_backfill.py | 265 ++++---------- tests/test_cli_index.py | 44 +-- tests/test_documents.py | 25 +- tests/test_markers.py | 66 +++- tests/test_uniqueness_gate.py | 185 +++++++--- 33 files changed, 1518 insertions(+), 1088 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 17407e2..ea2f634 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,14 +61,11 @@ config's `sources:` is a YAML list, one entry per (host, org, repo, ref_type). S #### `update: ` (snapshot vs. incremental) -`snapshot` (default): content is commit-addressed, as always -- `git.ref_key` on every content -doc equals its `git.commit`, and a HEAD advance on a branch indexes a whole new snapshot under -the new commit. +`snapshot` (default): content is commit-addressed. A HEAD advance on a branch +indexes a whole new snapshot under the new commit. `incremental` (branch-only; rejects `since`/`retain` -- there is no per-commit history for either -to apply to): content is ref-addressed instead. `git.ref_key` is -`{host}~{org}~{repo}~{ref}` and carries no `git.commit` of its own; the branch's live commit -lives only on its refs join doc (`_id = git.ref_key`). A HEAD advance runs `git diff +to apply to): content is ref-addressed instead. A HEAD advance runs `git diff --name-status` between the previously-completed commit and the new tip and only deletes/ reindexes the paths git reports changed -- add/modify/delete/rename/copy -- instead of reindexing the whole tree. A missing diff base (force-push, GC'd, or the first index) rebuilds @@ -375,90 +372,113 @@ creates one index/shard per commit — see `specs/sourcerer-yml.md` for the cave resolve it to a commit via the refs index (the `sourcerer.refs.list` tool), then filter content by `git.host` + `git.commit`. -### `git.ref_key` and the universal join query +### Universal join query -Every content doc (file and line, both `update` modes) carries a `git.ref_key` keyword field: -the bare commit SHA for `snapshot` content, or `{host}~{org}~{repo}~{ref}` for `incremental` -content (see `update: ` above; `build_ref_key` in `src/sourcerer/utils.py`). The refs doc -that carries `git.commit` for the join is: +Content docs come in two disjoint shapes depending on how they were indexed: -- **Snapshot:** the `build_ref_id`-keyed **ref-name marker** itself. `write_ref_marker` sets - `git.ref_key = commit_sha` on the marker (one doc per snapshot source). There is no separate - shadow join doc for snapshot content. -- **Incremental:** a dedicated refs join doc at `_id = git.ref_key = {host}~{org}~{repo}~{ref}`, - holding the live HEAD commit and advanced two-phase (INV-006). One doc per branch. +- **Snapshot** (`update: snapshot`): content docs carry `git.commit` and no `git.ref`. The ref-name + marker in `sourcerer-v3-refs` (keyed by `build_ref_id`, one per snapshot source) carries the commit + and status. `git.commit` on the content row is already the answer; no join is needed to resolve it. +- **Incremental** (`update: incremental`): content docs carry `git.ref` and no `git.commit`. A + dedicated refs join doc at `_id = build_ref_key(host,org,repo,ref)` (one per branch) holds the + live HEAD commit and is advanced two-phase (INV-006). The join resolves `git.commit` from this doc. -INV-004: exactly one `sourcerer-v3-refs` doc per `git.ref_key` value. For snapshot content this -is the marker (one per ref+commit); for incremental it is the branch's join doc. Because ES|QL -`LOOKUP JOIN` fans out on duplicate right-side keys, having >1 doc with the same `git.ref_key` -would multiply content rows — the uniqueness gate (`_run_uniqueness_gate`, INV-011) guards this. +Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) uses +the same shape that handles both modes without fan-out: + +```esql +FROM sourcerer-lines +| WHERE git.host LIKE ?git_host AND ... + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small sourcerer-refs + // index first (content docs carry no git.ref_type); the two membership sets handle + // both content-doc shapes in one pass. + (git.commit IS NOT NULL AND git.commit IN ( + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host AND ... AND status == "complete" + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host AND ... AND status == "complete" + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref + )) + ) +// Branch by content-doc shape: snapshot rows already carry git.commit and status was +// pre-confirmed by the membership subquery above (status=="complete"), so the snapshot +// arm needs no join -- it just asserts status to match the incremental arm's column. +// Incremental rows carry only git.ref; the join resolves the ref's current status from +// its join doc. Safety of the incremental join (one doc per (host,org,repo,ref)) is +// enforced by the "one update mode owns a ref name" invariant at index time. +| FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) +| WHERE status == "complete" +``` + +**Snapshot arm**: no join needed. The commit already lives on the content row and was pre-confirmed +`complete` by the membership subquery; `EVAL status = "complete"` asserts the column so it matches +the incremental arm's shape. Critically, the snapshot arm never touches `sourcerer-refs`, so two +complete markers sharing the same commit (branch + same-named tag) do NOT fan out — they just produce +one row each in the pre-FORK membership filter, which deduplicates naturally. + +**Incremental arm**: joins `sourcerer-refs ON (git.host, git.org, git.repo, git.ref)`. This join is +safe (no fan-out) because there is always exactly one incremental join doc per `(host,org,repo,ref)`: +all three incremental writers use `_id = build_ref_key(...)` (overwrite-in-place), the runtime +mode-conflict guard in `selection.py` prevents two selectors of different modes from claiming the same +ref name simultaneously, and the flip-status switchover marks any old snapshot marker `"stale"` BEFORE +the incremental join doc is published as `"complete"` — so the two-complete-docs window never opens. + +**Scoping params** (`git_commit`, `git_ref`, `git_ref_type`) are all optional (default `"*"`) and +support `*`/`?` wildcards (filters use `LIKE`). For a normal content question, resolve a ref first +(see `src/sourcerer/skills/ref-resolution/SKILL.md`), then pass the result through the appropriate +param: a commit SHA goes to `git_commit`; a branch or tag name goes to `git_ref` (optionally narrow +further with `git_ref_type: branch` or `git_ref_type: tag`). Leaving all three at `"*"` matches +content across all refs at once; because every content tool carries `git.commit` through to output +(and aggregations group `BY git.commit`), unpinned results stay attributable per commit rather than +being blended — but a version-specific answer should still pin a ref. + +**The post-FORK `| WHERE status == "complete"`** is an automatic consistency guard (no param): it +serves content only from a ref whose latest index is complete. For incremental content this excludes +the torn/partial-read window while a branch is mid-reindex — during a HEAD advance the branch's refs +join doc is `status: indexing` and its content is being mutated in place. For snapshot content this +is a no-op (status is already `"complete"` from the `EVAL` above). Trade-off: a *failed* incremental +run leaves the join doc at `status: indexing` with the prior commit's content still fully consistent; +the guard hides that content until the next successful run republishes `status: complete`. #### `status` field values Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental join docs alike -— carries a `status` field drawn from a shared two-value vocabulary, so the scheduler and -`sourcerer.refs.list` can query both families uniformly: +— carries a `status` field drawn from a three-value vocabulary: | Value | Meaning | |---|---| | `indexing` | A run is mid-flight. `indexing_started_at` is set; `indexed_at` is absent/null. Present on snapshot ref-name markers (written by `write_indexing_marker` just before ingest) and incremental join docs (written by `write_incremental_indexing`). A stale `indexing` doc whose `indexing_started_at` is older than the retry window (default 6 h) marks a crashed run and is treated as due for re-indexing. | | `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot markers) and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | +| `stale` | A snapshot marker superseded by a mode switch to incremental. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). Stale markers are invisible to all content tools (all gate on `status == "complete"`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | -Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) therefore runs the -same query shape regardless of mode -- and `git.ref_key` is NEVER an agent-facing param, only -the internal join field: +#### Uniqueness gate (INV-011 backstop) -```esql -FROM sourcerer-lines -| WHERE ... AND git.ref_key IN ( - // Resolve git_commit_ish against the small sourcerer-refs table once, then do a cheap - // membership check on the large content index (instead of a per-row LIKE/OR wildcard match). - FROM sourcerer-refs - | WHERE ... AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key - ) -| LOOKUP JOIN sourcerer-refs ON git.ref_key -| WHERE status == "complete" -``` +`_run_uniqueness_gate` (`commands/index/command.py`) runs after each index pass and calls +`check_join_uniqueness` (`queries.py`) to verify: + +- **Snapshot** (git.commit IS NOT NULL in content): each distinct commit must have ≥1 complete refs + doc (presence check — multi-ref-per-commit is legal). +- **Incremental** (git.ref IS NOT NULL in content): each distinct ref must have **exactly one** + incremental join doc with `update_mode == "incremental"` (anti-fan-out guard for the surviving join). -`git_commit_ish` is the single scoping param and supports `*`/`?` wildcards (the filter uses -`LIKE`). It is optional (default `"*"`, matching every indexed ref), but for a normal content -question resolve a ref first (see `src/sourcerer/skills/ref-resolution/SKILL.md`) and pass through -whatever it resolved to: a snapshot ref's commit SHA, or an incremental branch's plain name (e.g. -`main`) -- no construction, no `ref_key` involved. Leaving it at `"*"` matches content across all -refs at once; because every content tool carries `git.commit` through to output (and any -aggregation groups `BY git.commit`), unpinned results stay attributable per commit rather than -being blended -- but a version-specific answer should still pin a ref. The subquery matches `git_commit_ish` -against whichever field the ref actually carries (`git.commit` for snapshot, `git.ref` for -incremental) and collapses it to a set of `git.ref_key` values; the outer query scopes content by -membership in that set, so the same param and the same query shape work for both modes without the -caller knowing which one it is. The join then adds/overwrites `git.commit` on every row, so -snapshot content (which already carries its own, identical `git.commit`) is unaffected and -incremental content (which has none) gets it from the join. - -The post-join `| WHERE status == "complete"` is an automatic consistency guard (no param): it -serves content only from a ref whose latest index is complete, excluding the torn/partial-read -window while an incremental branch is mid-reindex -- during a HEAD advance the branch's refs join -doc is `status: indexing` and its content is being mutated in place, so a query joining to it -would otherwise read a half-applied mix of the old and new commits. It is a no-op for snapshot -content (snapshot markers are always `status: complete`). Trade-off: a *failed* incremental run -leaves the join doc at `status: indexing` with the prior commit's content still fully consistent; -the guard hides that content until the next successful run republishes `status: complete`. Note -this guard is about intra-update consistency, not inter-query staleness: because incremental -content overwrites in place, only a branch's current HEAD is ever indexed, so a query always -returns whatever commit the branch is at *now* -- to detect that a branch advanced since an -earlier resolution, re-check `sourcerer.refs.list`. - -### Upgrade backfill (`--no-backfill`) - -`sourcerer index` runs a one-time, idempotent upgrade backfill by default on every invocation: -an `_update_by_query` stamps `git.ref_key = git.commit` onto pre-existing snapshot content -that predates this feature, the refs index's mapping is -re-applied to the existing physical index (a template change alone only affects indices -created afterward), and a snapshot refs join doc is created for every already-indexed commit -that lacks one. Pass `--no-backfill` to skip it. Safe to run every time: a repeat run touches -nothing (see `backfill_repo` in `src/sourcerer/commands/index/markers.py`). +The gate is non-fatal (logs a warning, does not block): with the flip-status switchover in place, +violations should only occur if a stale-flip was skipped or crashed mid-way; the next prune run +reclaims the stale marker and resolves the violation automatically. ## Releases diff --git a/src/sourcerer/cli.py b/src/sourcerer/cli.py index 6e53f7a..db4fb4a 100755 --- a/src/sourcerer/cli.py +++ b/src/sourcerer/cli.py @@ -268,18 +268,10 @@ def setup(url, api_key, username, password, kb_url, config_path, include_experim "run (skip re-indexing it); older markers are treated as stuck and re-indexed. Also " "drives the schedule gate's stuck-run detection. Duration like 30m, 1h, 6h, 1d. Default 1h.", ) -@click.option( - "--no-backfill", - is_flag=True, - default=False, - help="Skip the one-time upgrade backfill that stamps git.ref_key onto " - "pre-existing snapshot content and migrates the refs index (default: run it, idempotently, " - "on every invocation).", -) @env_option @insecure_option @auth_options -def index(repo_spec, branch, tag, commit, config_path, force, quiet, cache_dir, ephemeral, prune, dry_run, retry_window, no_backfill, url, api_key, username, password, insecure): +def index(repo_spec, branch, tag, commit, config_path, force, quiet, cache_dir, ephemeral, prune, dry_run, retry_window, url, api_key, username, password, insecure): """Index a remote GitHub repo's git-tracked files into Elasticsearch. Provide a REPO_SPEC ('//') for a single repo, or --config to index multiple @@ -292,7 +284,7 @@ def index(repo_spec, branch, tag, commit, config_path, force, quiet, cache_dir, if config_path: if repo_spec or branch or tag or commit: raise click.UsageError("--config cannot be combined with REPO_SPEC or -b/-t/-c") - index_cmd.run_config(config_path, url, api_key, username, password, force, quiet, cache_dir, ephemeral, prune, dry_run, retry_window=retry_window, insecure=insecure, no_backfill=no_backfill) + index_cmd.run_config(config_path, url, api_key, username, password, force, quiet, cache_dir, ephemeral, prune, dry_run, retry_window=retry_window, insecure=insecure) else: if prune: raise click.UsageError("--prune requires --config (there is no retention policy for a single ref)") @@ -300,7 +292,7 @@ def index(repo_spec, branch, tag, commit, config_path, force, quiet, cache_dir, raise click.UsageError("--dry-run requires --config") if not repo_spec: raise click.UsageError("provide a REPO_SPEC ('//') or --config") - index_cmd.run(repo_spec, branch, tag, commit, url, api_key, username, password, force, quiet, cache_dir, ephemeral, retry_window=retry_window, insecure=insecure, no_backfill=no_backfill) + index_cmd.run(repo_spec, branch, tag, commit, url, api_key, username, password, force, quiet, cache_dir, ephemeral, retry_window=retry_window, insecure=insecure) @cli.command() diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index d69d2b1..e5194c5 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -11,8 +11,6 @@ # Standard packages import datetime -import json -import pathlib import subprocess import sys import threading @@ -28,7 +26,7 @@ from ...planner import Marker, plan_repo from ...progress import ProgressReporter, Unit, make_reporter from ...indices import files_index, lines_index -from ...queries import check_ref_key_uniqueness +from ...queries import check_join_uniqueness from ...utils import ES_ERRORS, make_client from ..prune import command as prune_cmd from ..prune.execute import delete_commit_from_indices @@ -48,11 +46,11 @@ _rev_info, ) from .markers import ( - backfill_repo, build_ref_id, commits_with_content, content_present, + build_ref_id, commits_with_content, content_present, count_incremental_branch_docs, delete_incremental_branch, delete_incremental_paths, - fully_indexed_counts, markers_status_by_id, _needs_index, pre_clone_skip, - read_incremental_ref, recorded_routing, refresh_incremental_content, should_index, - write_incremental_failed, write_incremental_indexing, write_incremental_ready, + fully_indexed_counts, mark_snapshot_markers_stale, markers_status_by_id, _needs_index, + pre_clone_skip, read_incremental_ref, recorded_routing, refresh_incremental_content, + should_index, write_incremental_failed, write_incremental_indexing, write_incremental_ready, write_indexing_marker, write_ref_marker, ) from .report import dry_run_config @@ -61,41 +59,18 @@ from .selection import _effective_since_floor, _load_config, _resolve_entry -# The index template files, reused by the upgrade backfill to migrate the mapping of EXISTING -# physical indices (a put_index_template change alone only affects indices created afterward). -_INDEX_TEMPLATES_DIR = pathlib.Path(__file__).resolve().parents[2] / "elastic" / "index_templates" - - -def _load_template_mapping(name: str) -> dict | None: - try: - body = json.loads((_INDEX_TEMPLATES_DIR / name).read_text()) - except OSError: - return None - return body.get("template", {}).get("mappings") - - -def _load_refs_mapping() -> dict | None: - return _load_template_mapping("sourcerer-v3-refs.json") - - -def _load_files_mapping() -> dict | None: - return _load_template_mapping("sourcerer-v3-files.json") - - -def _load_lines_mapping() -> dict | None: - return _load_template_mapping("sourcerer-v3-lines.json") - - def _run_uniqueness_gate(es: Elasticsearch, host: str, org: str, repo: str) -> bool: - """Post-index uniqueness gate (INV-011): every distinct `git.ref_key` in this repo's - content must resolve to exactly one `sourcerer-v3-refs` join doc. Prints the offending - ref_key(s) to stderr and returns False on any violation; True (silent) when the invariant - holds.""" - offending = check_ref_key_uniqueness(es, host, org, repo) + """Post-index join-uniqueness gate (INV-011 backstop): every distinct content commit/ref in + this repo must resolve to a complete refs join doc. For snapshot content (git.commit IS NOT + NULL) each commit must have at least one complete refs doc; for incremental (git.ref IS NOT + NULL) each ref must have exactly one incremental join doc. Prints offenders to stderr and + returns False; True (silent) when the invariant holds.""" + offending = check_join_uniqueness(es, host, org, repo) if offending: click.echo( - f"Error: {host}/{org}/{repo}: {len(offending)} git.ref_key value(s) missing or " - f"duplicated in sourcerer-v3-refs: {', '.join(offending)}", + f"Warning: {host}/{org}/{repo}: {len(offending)} content key(s) with join-doc " + f"mismatch in sourcerer-v3-refs (run prune to clean stale markers): " + f"{', '.join(offending)}", err=True, ) return False @@ -258,14 +233,8 @@ def index_ref_in_dir( # write-new -> FLIP MARKER -> delete-old: the marker now points at the new location before any # old copy is deleted, so a crash between here and the delete below leaves stale (not missing) # data that the prune stale-location sweep reclaims. - # refresh=True so the post-index uniqueness gate (_run_uniqueness_gate, INV-011) sees the - # git.ref_key carrier immediately instead of racing the refs index's default (~1s) refresh - # interval: the bulk context manager refreshes the CONTENT indices on exit but not refs, so an - # unrefreshed write here would make the gate read this ref_key's content but miss its carrier - # and false-fail "missing". Mirrors write_incremental_ready (refresh=True). write_ref_marker(es, host, org, repo, ref_type, ref_for_id, commit_sha, commit_date_iso, - files_count, lines_count, index_level=level, index_suffix=suffix, - refresh=True) + files_count, lines_count, index_level=level, index_suffix=suffix) if migrating: # Reconstruct the OLD index name from the prior marker's routing and drop this commit's # stale copy there. Commit-safety (another surviving ref sharing the commit) is respected @@ -299,8 +268,8 @@ def index_incremental_branch_in_dir( the whole branch namespace, then index every currently-tracked path, or - does a delta update: `git diff --name-status` (via `plan_changes`) between the prior and new commit, deleting only the paths git reports removed/changed and (re)indexing only the - paths git reports added/changed (INV-008 -- scoped by the exact `ref_key`, never a whole - namespace sweep). + paths git reports added/changed (INV-008 -- scoped by the exact (host,org,repo,ref) tuple, + never a whole namespace sweep). The refs join doc is published `indexing` before any mutation and `complete` only after the content deletes/indexes and a refresh all succeed (INV-006); a raised exception instead records `write_incremental_failed` and leaves the completed pointer untouched, then @@ -328,7 +297,8 @@ def index_incremental_branch_in_dir( reporter.set_stage(unit, "indexing") write_incremental_indexing(es, host, org, repo, branch, completed_commit=old_sha, - target_commit=new_sha, prior=prior) + target_commit=new_sha, prior=prior, + index_level=level, index_suffix=suffix) try: full_rebuild = old_sha is None or force if not full_rebuild: @@ -357,15 +327,23 @@ def index_incremental_branch_in_dir( files_count, lines_count = count_incremental_branch_docs( es, host, org, repo, branch, index_level=level, index_suffix=suffix, ) + # Mode-switch: flip any complete snapshot markers for this (host,org,repo,ref) to + # "stale" BEFORE publishing the incremental join doc as "complete". This ensures the + # two-complete-docs fan-out window (one snapshot + one incremental marker both matching + # LOOKUP JOIN ON git.ref) never opens. Stale content is reclaimed by prune. + mark_snapshot_markers_stale(es, host, org, repo, branch) write_incremental_ready(es, host, org, repo, branch, new_sha, commit_date_iso, - files_count, lines_count) + files_count, lines_count, + index_level=level, index_suffix=suffix) except KeyboardInterrupt: write_incremental_failed(es, host, org, repo, branch, completed_commit=old_sha, - target_commit=new_sha, error="interrupted", prior=prior) + target_commit=new_sha, error="interrupted", prior=prior, + index_level=level, index_suffix=suffix) raise except Exception as e: write_incremental_failed(es, host, org, repo, branch, completed_commit=old_sha, - target_commit=new_sha, error=str(e), prior=prior) + target_commit=new_sha, error=str(e), prior=prior, + index_level=level, index_suffix=suffix) raise reporter.finish(unit, "indexed", indexed_files, indexed_lines) @@ -444,7 +422,6 @@ def run( ephemeral: bool = False, retry_window: datetime.timedelta | None = None, insecure: bool = False, - no_backfill: bool = False, ) -> None: parts = repo_spec.split("/", 2) if len(parts) != 3 or not all(parts): @@ -468,12 +445,6 @@ def run( es = make_client(url, api_key, username, password, insecure=insecure) cache_root = None if ephemeral else resolve_cache_root(cache_dir) - if not no_backfill: - backfill_repo( - es, host, org, repo, refs_mapping=_load_refs_mapping(), - files_mapping=_load_files_mapping(), lines_mapping=_load_lines_mapping(), - ) - kind = "branch" if branch else "tag" if tag else "commit" if commit else "default" unit = Unit(host=host, org=org, repo=repo, ref=branch or tag or commit, kind=kind) reporter = make_reporter(quiet) @@ -516,7 +487,6 @@ def run_config( dry_run: bool = False, retry_window: datetime.timedelta | None = None, insecure: bool = False, - no_backfill: bool = False, ) -> None: """ Index every (repo, ref) the config selects. First list the remote branches and tags for @@ -542,19 +512,6 @@ def run_config( es = make_client(url, api_key, username, password, insecure=insecure) cache_root = None if ephemeral else resolve_cache_root(cache_dir) - # One-time upgrade backfill (default-on; --no-backfill opts out; skipped on --dry-run, - # which promises no ES writes). Runs once per distinct (host, org, repo) in the config, - # before the schedule gate, so it applies regardless of which sources are due this tick. - if not no_backfill and not dry_run: - refs_mapping = _load_refs_mapping() - files_mapping = _load_files_mapping() - lines_mapping = _load_lines_mapping() - for repo_cfg in config.repos: - backfill_repo( - es, repo_cfg.host, repo_cfg.org, repo_cfg.repo, refs_mapping=refs_mapping, - files_mapping=files_mapping, lines_mapping=lines_mapping, - ) - # Schedule gate: determine which sources are due for indexing based on their configured # schedule and the refs index's record of when they were last indexed. Sources with no # schedule (or schedule "* * * * *") are always due; others are skipped until their next diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index 805e5a9..cd2f71b 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -18,7 +18,7 @@ # App packages from ...indices import files_index, lines_index -from ...utils import build_ref_key, make_doc_id +from ...utils import make_doc_id from .git import get_symlink_paths, iter_tracked_files from .runtime import _aborted, _tuning @@ -91,9 +91,6 @@ def build_file_doc( "org": org, "repo": repo, "commit": commit_sha, - # Snapshot ref_key is the bare commit -- the content is addressed by commit, so the - # commit itself is the stable join key (see build_ref_key for the incremental shape). - "ref_key": commit_sha, }, "file": file_fields, } @@ -141,7 +138,6 @@ def iter_line_docs( "org": org, "repo": repo, "commit": commit_sha, - "ref_key": commit_sha, }, "file": file_fields, } @@ -163,10 +159,9 @@ def build_incremental_file_doc( target_path: str | None = None, target_size: int | None = None, ) -> tuple[str, dict]: - """Ref-addressed (incremental) file doc: no `git.commit`; `git.ref_key` is the tilde-joined - `build_ref_key(host, org, repo, ref)` and `_id` is stable across commits (derived from the - branch name, not the commit), so a modified file's doc overwrites in place on the next - HEAD advance rather than minting a new id.""" + """Ref-addressed (incremental) file doc: carries `git.ref` but no `git.commit`; `_id` is + stable across commits (derived from the branch name, not the commit), so a modified file's + doc overwrites in place on the next HEAD advance rather than minting a new id.""" p = pathlib.PurePosixPath(rel_path) directory = "" if str(p.parent) == "." else str(p.parent) extension = p.suffix.lstrip(".") or None @@ -203,7 +198,6 @@ def build_incremental_file_doc( "org": org, "repo": repo, "ref": ref, - "ref_key": build_ref_key(host, org, repo, ref), }, "file": file_fields, } @@ -248,7 +242,6 @@ def iter_incremental_line_docs( "org": org, "repo": repo, "ref": ref, - "ref_key": build_ref_key(host, org, repo, ref), }, "file": file_fields, } @@ -492,8 +485,8 @@ def index_incremental_paths( diff base is unavailable). A given `rel_paths` list indexes only those paths -- the delta indexer's changed/added set (see `commands/index/git.py:plan_changes`), which is what makes an incremental HEAD advance only touch the files git reports changed. Deletions for removed - paths are the caller's responsibility (see `markers.delete_by_ref_key`) since they need no - doc generation. Mirrors `index_repo`'s worker-pool ingest loop. + paths are the caller's responsibility (see `markers.delete_incremental_paths`) since they + need no doc generation. Mirrors `index_repo`'s worker-pool ingest loop. """ files_count = 0 lines_count = 0 diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index fa072f9..85d91e0 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -427,6 +427,7 @@ def write_indexing_marker( "commit": commit_sha, "commit_date": commit_date_iso, }, + "update_mode": "snapshot", "status": "indexing", "indexing_started_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "files_count": 0, @@ -463,14 +464,9 @@ def write_ref_marker( # stays correct and prune/migration can find (and clean up) exactly where content lives. # Legacy markers written before this feature omit both; readers fall back to the "repo"/None # defaults, which reconstruct to the historical repo-level name where that content actually is. - # - # git.ref_key = commit_sha folds the snapshot join doc into this single marker: it lets the - # content tools' LOOKUP JOIN sourcerer-refs ON git.ref_key resolve git.commit without a - # separate _id=commit shadow doc. One snapshot source → one refs doc (INV-004). ref_id = build_ref_id(host, org, repo, ref_type, ref, commit_sha) doc = { "git": { - "ref_key": commit_sha, "host": host, "org": org, "repo": repo, @@ -479,6 +475,7 @@ def write_ref_marker( "commit": commit_sha, "commit_date": commit_date_iso, }, + "update_mode": "snapshot", "status": "complete", "files_count": files_count, "lines_count": lines_count, @@ -542,17 +539,15 @@ def pre_clone_skip( return False, ref_for_id, remote_sha -# --- incremental refs join docs (git.ref_key), keyed by `_id = ref_key` ------------------- +# --- incremental refs join docs, keyed by `_id = build_ref_key(...)` ---------------------- # One document per incremental branch (INV-004): the branch's single join doc lives at -# `_id = {host}~{org}~{repo}~{ref}` and its `git.commit` is the branch's live HEAD, advanced +# `_id = {host}~{org}~{repo}~{ref}` (constructed by build_ref_key, a plain tilde-joined +# string -- not a stored field) and its `git.commit` is the branch's live HEAD, advanced # only by a two-phase indexing -> complete publication (INV-006). This is a DISTINCT id space # from `build_ref_id`'s hashed, append-only ref-name markers above; a join doc's `_id` is a -# plain, unhashed `ref_key` string, which a `build_ref_id` hash can never collide with. -# -# Snapshot mode no longer writes a separate join doc: `write_ref_marker` now carries -# `git.ref_key = commit_sha` directly on the hashed marker (one doc per snapshot source, -# INV-004). Legacy `_id = commit` snapshot join docs written before this change are cleaned -# up by `backfill_refs_join_docs` / the migration `backfill_repo` step. +# plain, unhashed build_ref_key() string, which a `build_ref_id` hash can never collide with. +# build_ref_key is still used as the `_id` constructor even though git.ref_key is no longer +# a stored field -- the id itself remains the stable overwrite key for each branch. ERROR_MAX_LEN = 2000 # bound stored failure text so a giant git/ES error can't bloat the doc @@ -586,10 +581,11 @@ def _build_incremental_join_doc( indexing_started_at: str | None = None, failed_at: str | None = None, error: str | None = None, + index_level: str = "repo", + index_suffix: str | None = None, ) -> dict: return { "git": { - "ref_key": build_ref_key(host, org, repo, ref), "host": host, "org": org, "repo": repo, @@ -607,6 +603,8 @@ def _build_incremental_join_doc( "indexing_started_at": indexing_started_at, "failed_at": failed_at, "error": error[:ERROR_MAX_LEN] if error else None, + "index_level": index_level, + "index_suffix": index_suffix, } @@ -620,6 +618,8 @@ def write_incremental_indexing( target_commit: str, prior: dict | None = None, refresh: bool = False, + index_level: str = "repo", + index_suffix: str | None = None, ) -> None: """Publish `status: indexing`: the completed pointer (`git.commit`) stays at the LAST completed SHA (or None on a first index) while `git.target_commit` advertises the candidate @@ -639,6 +639,8 @@ def write_incremental_indexing( indexing_started_at=_now_iso(), failed_at=prior.get("failed_at"), error=prior.get("error"), + index_level=index_level, + index_suffix=index_suffix, ) es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref), document=doc, refresh=refresh) @@ -654,6 +656,8 @@ def write_incremental_ready( files_count: int, lines_count: int, refresh: bool = True, + index_level: str = "repo", + index_suffix: str | None = None, ) -> None: """Publish `status: complete` at the NEW completed commit, clearing `target_commit` and any prior failure fields. This is the pointer-advancing publication boundary (INV-006): callers @@ -670,6 +674,8 @@ def write_incremental_ready( indexing_started_at=None, failed_at=None, error=None, + index_level=index_level, + index_suffix=index_suffix, ) es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref), document=doc, refresh=refresh) @@ -685,6 +691,8 @@ def write_incremental_failed( error: str, prior: dict | None = None, refresh: bool = False, + index_level: str = "repo", + index_suffix: str | None = None, ) -> None: """Record a failed update WITHOUT advancing the completed pointer: status stays `indexing`, `git.commit` remains the last completed SHA, and a bounded `error`/`failed_at` are stored for @@ -703,6 +711,8 @@ def write_incremental_failed( indexing_started_at=prior.get("indexing_started_at") or _now_iso(), failed_at=_now_iso(), error=error, + index_level=index_level, + index_suffix=index_suffix, ) es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref), document=doc, refresh=refresh) @@ -738,17 +748,19 @@ def delete_incremental_paths( refresh: bool = False, ) -> None: """Synchronously delete the file and line docs for `paths` on this exact branch. Scoped by - the exact `git.ref_key` (a single keyword term, so one branch's docs can never bleed into - another's -- INV-008) plus a `file.path` terms filter, never a wildcard. A no-op for an - empty path set.""" + the exact (git.host, git.org, git.repo, git.ref) 4-term filter (INV-008: one branch's docs + can never bleed into another's) plus a `file.path` terms filter, never a wildcard. A no-op + for an empty path set.""" paths = list(paths) if not paths: return - ref_key = build_ref_key(host, org, repo, ref) query = { "bool": { "filter": [ - {"term": {"git.ref_key": ref_key}}, + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.ref": ref}}, {"terms": {"file.path": paths}}, ] } @@ -771,10 +783,14 @@ def delete_incremental_branch( refresh: bool = False, ) -> None: """Delete EVERY incremental content doc for this branch (full namespace), scoped by the - exact `git.ref_key` (INV-008). Used for the initial index and the missing-diff-base rebuild - (INV-007).""" - ref_key = build_ref_key(host, org, repo, ref) - query = {"bool": {"filter": [{"term": {"git.ref_key": ref_key}}]}} + exact (git.host, git.org, git.repo, git.ref) 4-term filter (INV-008). Used for the initial + index and the missing-diff-base rebuild (INV-007).""" + query = {"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.ref": ref}}, + ]}} for index in ( files_index(host, org, repo, None, index_level, index_suffix), lines_index(host, org, repo, None, index_level, index_suffix), @@ -787,11 +803,15 @@ def count_incremental_branch_docs( index_level: str = "repo", index_suffix: str | None = None, ) -> tuple[int, int]: """Authoritative (files, lines) totals for a branch's current incremental view, counted by - exact `git.ref_key`. Call AFTER refreshing the content indices so counts reflect the just- - applied deletes and indexes -- these become the ready marker's files_count/lines_count. - Returns 0 for an index that does not exist yet.""" - ref_key = build_ref_key(host, org, repo, ref) - query = {"bool": {"filter": [{"term": {"git.ref_key": ref_key}}]}} + exact (git.host, git.org, git.repo, git.ref). Call AFTER refreshing the content indices so + counts reflect the just-applied deletes and indexes -- these become the ready marker's + files_count/lines_count. Returns 0 for an index that does not exist yet.""" + query = {"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.ref": ref}}, + ]}} def _count(index: str) -> int: try: @@ -822,179 +842,10 @@ def refresh_incremental_content( ) -# --- one-time upgrade backfill (default-on; --no-backfill opts out) ----------------------- -# Stamps `git.ref_key` onto pre-existing snapshot content that predates this feature, migrates -# the refs index mapping, and creates the missing `_id = commit` join docs (INV-009/INV-010). -# Safe to run on every `index` invocation: both the content update and the join-doc creation -# are no-ops the second time. - -def backfill_snapshot_ref_keys(es: Elasticsearch, host: str, org: str, repo: str) -> int: - """Idempotent `_update_by_query` stamping `git.ref_key = git.commit` onto this repo's - content docs that lack `git.ref_key` (pre-upgrade data). Returns the total number of docs - updated across the files and lines aliases; 0 on a repeat run (INV-009) since the - `must_not: exists` filter then matches nothing.""" - query = { - "bool": { - "filter": [ - {"term": {"git.host": host}}, - {"term": {"git.org": org}}, - {"term": {"git.repo": repo}}, - ], - "must_not": [{"exists": {"field": "git.ref_key"}}], - } - } - script = { - "source": "ctx._source.git.ref_key = ctx._source.git.commit;", - "lang": "painless", - } - total = 0 - for index in (FILES_ALIAS, LINES_ALIAS): - try: - resp = es.update_by_query( - index=index, query=query, script=script, - wait_for_completion=True, conflicts="proceed", refresh=True, - ignore_unavailable=True, allow_no_indices=True, - ) - total += int(resp.get("updated", 0)) - except NotFoundError: - pass - return total - - -def distinct_commits_for_repo(es: Elasticsearch, host: str, org: str, repo: str) -> set[str]: - """Every distinct `git.commit` present in this repo's content, via a terms aggregation. - Used by the backfill to find every already-indexed snapshot commit that needs a refs join - doc (INV-010). Returns an empty set when the files index doesn't exist yet.""" - query = { - "bool": { - "filter": [ - {"term": {"git.host": host}}, - {"term": {"git.org": org}}, - {"term": {"git.repo": repo}}, - ] - } - } - try: - resp = es.search( - index=FILES_ALIAS, size=0, query=query, - aggs={"commits": {"terms": {"field": "git.commit", "size": 10000}}}, - ) - except NotFoundError: - return set() - return {b["key"] for b in resp["aggregations"]["commits"]["buckets"]} - - -def commits_with_ref_key_carrier(es: Elasticsearch, commits: set[str]) -> set[str]: - """The subset of `commits` for which a refs doc already carries `git.ref_key == commit` - (i.e. has a snapshot ref_key carrier). Used by `backfill_refs_join_docs` to skip commits - whose marker already carries the ref_key from a normal index run, so the backfill is a - no-op on up-to-date repos (INV-009). A terms query on `git.ref_key` covers both the - hashed-marker carrier (new) and any legacy `_id = commit` shadow docs (old).""" - if not commits: - return set() - try: - resp = es.search( - index=REFS_ALIAS, size=0, - query={"terms": {"git.ref_key": sorted(commits)}}, - aggs={"carriers": {"terms": {"field": "git.ref_key", "size": len(commits)}}}, - ) - except NotFoundError: - return set() - return {b["key"] for b in resp["aggregations"]["carriers"]["buckets"]} - - -# Keep the old name as an alias so external callers (if any) and tests can migrate gradually. -commits_with_join_doc = commits_with_ref_key_carrier - - -def backfill_refs_join_docs(es: Elasticsearch, host: str, org: str, repo: str) -> int: - """Ensure every distinct content commit for this repo has a refs doc carrying - `git.ref_key = commit` (INV-010). For each commit lacking a carrier, stamps `git.ref_key` - onto its existing complete `build_ref_id` marker (the normal post-collapse shape) by - partially updating that doc. Falls back to writing a minimal `_id = commit` carrier doc if - no marker is found (orphan content). Returns the number of carriers stamped/created; 0 on a - repeat run (INV-009).""" - commits = distinct_commits_for_repo(es, host, org, repo) - if not commits: - return 0 - missing = commits - commits_with_ref_key_carrier(es, commits) - stamped = 0 - for commit_sha in missing: - # Look for the existing complete marker for this commit (build_ref_id key space). - query = { - "bool": { - "filter": [ - {"term": {"git.host": host}}, - {"term": {"git.org": org}}, - {"term": {"git.repo": repo}}, - {"term": {"git.commit": commit_sha}}, - {"term": {"status": "complete"}}, - # Only hashed markers (no ref_key yet); legacy shadow docs (update_mode: - # "snapshot") are already carriers and handled by commits_with_ref_key_carrier. - {"bool": {"must_not": {"exists": {"field": "git.ref_key"}}}}, - ] - } - } - try: - resp = es.search(index=REFS_ALIAS, size=1, query=query) - except NotFoundError: - resp = {"hits": {"hits": []}} - hits = resp["hits"]["hits"] - if hits: - # Stamp git.ref_key onto the existing marker in place. We use es.update with a - # partial doc rather than a full re-index to avoid touching the counts/timestamps. - marker_id = hits[0]["_id"] - es.update(index=REFS_INDEX, id=marker_id, doc={"git": {"ref_key": commit_sha}}) - else: - # No marker exists (orphan content): write a minimal carrier doc so the gate passes. - src_git = {} - try: - # Try to derive informational ref/ref_type from any refs doc for this commit. - any_resp = es.search( - index=REFS_ALIAS, size=1, - query={"bool": {"filter": [ - {"term": {"git.host": host}}, - {"term": {"git.org": org}}, - {"term": {"git.repo": repo}}, - {"term": {"git.commit": commit_sha}}, - ]}}, - ) - if any_resp["hits"]["hits"]: - src_git = any_resp["hits"]["hits"][0]["_source"].get("git", {}) - except NotFoundError: - pass - ref = src_git.get("ref") or commit_sha - ref_type = src_git.get("ref_type") or "commit" - commit_date_iso = src_git.get("commit_date") - es.index( - index=REFS_INDEX, id=commit_sha, - document={ - "git": { - "ref_key": commit_sha, - "host": host, "org": org, "repo": repo, - "ref": ref, "ref_type": ref_type, - "commit": commit_sha, "commit_date": commit_date_iso, - }, - "status": "complete", - }, - ) - stamped += 1 - if stamped: - # Refresh so a uniqueness gate run immediately afterward sees every carrier just written - # rather than racing the refs index's default refresh interval. - try: - es.indices.refresh(index=REFS_INDEX) - except NotFoundError: - pass - return stamped - - def apply_refs_index_mapping(es: Elasticsearch, mapping: dict) -> None: """Apply an updated mapping to the physical REFS_INDEX. A `put_index_template` change alone - (see `setup`) only affects indices created AFTER the change -- an existing repo's refs index - predates the `git.ref_key` field and needs its mapping updated explicitly so the field is - typed as intended rather than dynamically guessed on first write. A no-op if the index - doesn't exist yet.""" + (see `setup`) only affects indices created AFTER the change -- an existing refs index may + need its mapping updated explicitly for new fields. A no-op if the index doesn't exist.""" try: es.indices.put_mapping(index=REFS_INDEX, properties=mapping.get("properties", {})) except NotFoundError: @@ -1003,13 +854,8 @@ def apply_refs_index_mapping(es: Elasticsearch, mapping: dict) -> None: def apply_content_index_mapping(es: Elasticsearch, files_mapping: dict, lines_mapping: dict) -> None: """Apply the updated files/lines template mappings to every EXISTING physical content index - behind the read aliases. This must run BEFORE `backfill_snapshot_ref_keys` writes - `git.ref_key` onto pre-existing content: an index created before this feature has no - explicit mapping for that field, so the first `_update_by_query` write would otherwise - fall back to ES's dynamic string mapping (`text`, no fielddata) instead of the `keyword` - type the template defines -- silently breaking every later `git.ref_key` - aggregation/sort/exact-match query. `put_mapping` against an alias updates every backing - index it resolves to. A no-op if neither alias has any backing index yet.""" + behind the read aliases. `put_mapping` against an alias updates every backing index it + resolves to. A no-op if neither alias has any backing index yet.""" for alias, mapping in ((FILES_ALIAS, files_mapping), (LINES_ALIAS, lines_mapping)): try: es.indices.put_mapping(index=alias, properties=mapping.get("properties", {})) @@ -1017,55 +863,48 @@ def apply_content_index_mapping(es: Elasticsearch, files_mapping: dict, lines_ma pass -def backfill_repo( - es: Elasticsearch, host: str, org: str, repo: str, refs_mapping: dict | None = None, - files_mapping: dict | None = None, lines_mapping: dict | None = None, -) -> dict: - """Run the full one-time upgrade for one repo: apply the updated content/refs index - mappings (once, if given -- must happen BEFORE the content update so the new field lands - typed correctly rather than dynamically guessed), stamp `ref_key` onto pre-existing - snapshot content (idempotent), stamp `git.ref_key` onto existing snapshot markers that - predate the one-doc-per-source change, and delete any now-redundant legacy `_id = commit` - shadow join docs (those written by the old write_snapshot_join_doc path). Returns a small - summary dict for reporting; every field is 0 on a repeat run (INV-009).""" - if files_mapping is not None and lines_mapping is not None: - apply_content_index_mapping(es, files_mapping, lines_mapping) - if refs_mapping is not None: - apply_refs_index_mapping(es, refs_mapping) - updated = backfill_snapshot_ref_keys(es, host, org, repo) - # Stamp git.ref_key onto existing markers that lack it; this also covers repos that were - # indexed before the one-doc-per-source change where markers had no ref_key. - stamped = backfill_refs_join_docs(es, host, org, repo) - # Delete legacy standalone `_id = commit` shadow docs (update_mode: "snapshot", written by - # the old write_snapshot_join_doc). Ordered AFTER the backfill stamp + implicit refresh so - # every commit always has at least one carrier (the marker) before the shadow is removed. - deleted = _delete_legacy_snapshot_join_docs(es, host, org, repo) - return {"content_updated": updated, "carriers_stamped": stamped, "shadow_docs_deleted": deleted} - - -def _delete_legacy_snapshot_join_docs(es: Elasticsearch, host: str, org: str, repo: str) -> int: - """Delete legacy `_id = commit` snapshot join docs written by the old write_snapshot_join_doc - path (distinguishable by update_mode == "snapshot"). These are now superseded by the - git.ref_key field on the hashed ref-name markers. Safe to call only AFTER backfill_refs_join_docs - has stamped ref_key onto all markers (so no commit loses its carrier). Returns 0 if none exist.""" - query = { - "bool": { - "filter": [ - {"term": {"git.host": host}}, - {"term": {"git.org": org}}, - {"term": {"git.repo": repo}}, - {"term": {"update_mode": "snapshot"}}, - ] - } - } +def stale_snapshot_markers_for_ref( + es: Elasticsearch, host: str, org: str, repo: str, ref: str, +) -> list[dict]: + """Return any complete snapshot ref-name markers (update_mode: "snapshot", status: "complete") + for (host, org, repo, ref). Used by the incremental index path to detect and mark stale snapshot + markers left behind by a mode switch. The must_not form is kept as a fallback for legacy markers + written before update_mode was added to snapshot markers.""" + query = {"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.ref": ref}}, + {"term": {"status": "complete"}}, + # Exclude incremental join docs; matches update_mode="snapshot" and any legacy snapshot + # markers that predate the update_mode field. + {"bool": {"must_not": {"term": {"update_mode": "incremental"}}}}, + ]}} try: - resp = es.delete_by_query( - index=REFS_INDEX, query=query, - wait_for_completion=True, conflicts="proceed", refresh=True, - ) - return resp.get("deleted", 0) + resp = es.search(index=REFS_ALIAS, size=100, query=query, source_includes=["git.commit"]) except NotFoundError: - return 0 + return [] + return resp["hits"]["hits"] + + +def mark_snapshot_markers_stale( + es: Elasticsearch, host: str, org: str, repo: str, ref: str, +) -> int: + """Flip any complete snapshot markers for this (host, org, repo, ref) to status:"stale", + making them invisible to all content tools without deleting them immediately. Content + reclamation is deferred to the prune command's stale-marker step. Returns the count of + markers flipped. + + ORDER: callers must call this BEFORE publishing the incremental join doc as "complete", + so the two-complete-docs fan-out window (one snapshot + one incremental, both reachable by + the LOOKUP JOIN ON git.ref) never opens.""" + markers = stale_snapshot_markers_for_ref(es, host, org, repo, ref) + for hit in markers: + try: + es.update(index=REFS_INDEX, id=hit["_id"], doc={"status": "stale"}) + except NotFoundError: + pass # already gone; not an error + return len(markers) def resolve_head(es: Elasticsearch, host: str, org: str, repo: str, ref_type: str, ref: str) -> dict | None: diff --git a/src/sourcerer/commands/index/selection.py b/src/sourcerer/commands/index/selection.py index b04f998..4e7c585 100644 --- a/src/sourcerer/commands/index/selection.py +++ b/src/sourcerer/commands/index/selection.py @@ -33,6 +33,12 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: # Phase 2's pre-clone skip check. fetched: dict[str, dict[str, str] | None] = {} seen: set[tuple[str, str]] = set() + # Maps (ref_type, name) -> update mode for the winning selector, so we can detect when a + # second selector of a DIFFERENT mode also claims the same ref. Mixed-mode refs are unsafe: + # the incremental LOOKUP JOIN ON git.ref requires exactly one refs doc per (host,org,repo,ref), + # but a snapshot marker and an incremental join doc would both be present (fan-out). + seen_mode: dict[tuple[str, str], str] = {} + mode_conflicts: list[tuple[str, str, str, str]] = [] # (ref_type, name, mode_a, mode_b) units: list[Unit] = [] for sel in cfg.selectors: rt = sel.ref_type @@ -44,6 +50,7 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: if (rt, prefix) in seen: continue seen.add((rt, prefix)) + seen_mode[(rt, prefix)] = sel.update units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=prefix, kind=rt, index_level=sel.index_level, index_suffix=sel.index_suffix, update=sel.update, @@ -56,20 +63,38 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: continue # ls-remote failed for this ref type, skip floor = sel.since_version_floor() # version-based `since: {ref}`, name-only for name in sorted(ref_map): - if (rt, name) in seen: - continue v = sel.matches(rt, name) if v is None: continue if floor is not None and v.components < floor: continue # below the since version floor - seen.add((rt, name)) + key = (rt, name) + if key in seen: + # Already claimed by an earlier selector: check for a mode conflict. + prior_mode = seen_mode[key] + if prior_mode != sel.update: + mode_conflicts.append((rt, name, prior_mode, sel.update)) + continue + seen.add(key) + seen_mode[key] = sel.update units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=name, kind=rt, remote_sha=ref_map[name], index_level=sel.index_level, index_suffix=sel.index_suffix, update=sel.update, )) + if mode_conflicts: + conflicts_str = ", ".join( + f"{rt}/{name} ({mode_a} vs {mode_b})" + for rt, name, mode_a, mode_b in mode_conflicts + ) + click.echo( + f"Warning: {cfg.org}/{cfg.repo}: selectors claim the same ref(s) with different " + f"update modes -- skipping all units for this repo to avoid fan-out: {conflicts_str}", + err=True, + ) + return [] + failed_kinds = sorted(k for k, v in fetched.items() if v is None) if failed_kinds: click.echo( diff --git a/src/sourcerer/commands/prune/command.py b/src/sourcerer/commands/prune/command.py index 33aea4b..4cf9dfa 100644 --- a/src/sourcerer/commands/prune/command.py +++ b/src/sourcerer/commands/prune/command.py @@ -22,7 +22,8 @@ from ...planner import Decision, Marker, plan_repo from ...queries import content_indices_for_commit, fetch_markers, resolve_content_commit from ...utils import ES_ERRORS, make_client -from .execute import delete_commit_content, execute_deletions, execute_orphan_deletions, plan_orphans_now +from .execute import (delete_commit_content, execute_deletions, execute_orphan_deletions, + execute_stale_marker_deletions, plan_orphans_now) from .report import _Row, _orphan_rows, _print, _ref_rows, _retention_rows @@ -102,6 +103,7 @@ def run(config_path=None, url=None, api_key=None, username=None, password=None, total_orphan_indices = total_orphan_content = total_orphan_markers = 0 total_orphan_stale = total_empty_indices = 0 + total_stale_markers = total_stale_commits = 0 if not dry_run: for cfg, decisions in repo_decisions: if any(d.action == "delete" for d in decisions): @@ -113,6 +115,16 @@ def run(config_path=None, url=None, api_key=None, username=None, password=None, failures[0] += 1 click.echo(f"{cfg.host}/{cfg.org}/{cfg.repo}: error deleting: {e}", err=True) + # Reclaim stale snapshot markers from mode-switches (snapshot → incremental). These are + # markers flipped to status="stale" by index_incremental_branch_in_dir before the new + # incremental join doc was published, so they are never visible to content tools but do + # hold snapshot content that may no longer be needed. + try: + total_stale_markers, total_stale_commits = execute_stale_marker_deletions(es) + except ES_ERRORS as e: + failures[0] += 1 + click.echo(f"error reclaiming stale markers: {e}", err=True) + if orphan_plan is not None: (total_orphan_indices, total_orphan_content, total_orphan_markers, total_orphan_stale, total_empty_indices) = _apply_orphan_plan(es, orphan_plan, failures) @@ -122,6 +134,7 @@ def run(config_path=None, url=None, api_key=None, username=None, password=None, elif not quiet: click.echo( f"Pruned {total_markers} marker(s) and {total_commits} commit(s) of content; " + f"reclaimed {total_stale_markers} stale marker(s) and {total_stale_commits} stale commit(s); " f"removed {total_orphan_indices} orphaned index(es), " f"{total_orphan_content} orphaned content commit(s), " f"{total_orphan_markers} orphaned marker commit(s), " diff --git a/src/sourcerer/commands/prune/execute.py b/src/sourcerer/commands/prune/execute.py index 33abd72..a337c1f 100644 --- a/src/sourcerer/commands/prune/execute.py +++ b/src/sourcerer/commands/prune/execute.py @@ -14,8 +14,9 @@ from ...indices import REFS_INDEX, files_index, lines_index from ...planner import OrphanPlan, content_delete_set, plan_orphans from ...queries import ( - empty_content_indices, enumerate_ref_tuples, gather_content_by_index, - gather_content_commit_tuples, gather_intended_index_by_commit, list_sourcerer_indices, + empty_content_indices, enumerate_ref_tuples, fetch_complete_commits_for_repo, + fetch_stale_markers, gather_content_by_index, gather_content_commit_tuples, + gather_intended_index_by_commit, list_sourcerer_indices, ) @@ -146,6 +147,65 @@ def delete_index(es: Elasticsearch, name: str) -> bool: return False +def execute_stale_marker_deletions(es: Elasticsearch) -> tuple[int, int]: + """Reclaim snapshot content that was superseded by a mode switch to incremental indexing. + + The flip-status switchover in `index_incremental_branch_in_dir` marks old snapshot ref-name + markers as status="stale" BEFORE publishing the incremental join doc as "complete", so the + two-complete-docs fan-out window never opens. Stale markers carry git.commit (so their content + can be reclaimed) but are invisible to all content tools (which gate on status=="complete"). + + This function: + 1. Enumerates all status="stale" snapshot markers cluster-wide. + 2. Groups them by (host, org, repo) so the commit-safety guard can be applied per-repo. + 3. For each stale marker, drops its snapshot content ONLY if the commit is not referenced + by any surviving complete marker in the same repo (the same commit-safety guard used by + execute_deletions via content_delete_set). A commit shared with another snapshot ref is + left in place. + 4. Deletes the stale marker doc from sourcerer-v3-refs. + + Returns (stale_markers_deleted, stale_commits_content_dropped). + + Crash-safety: if a crash occurs between step 3 and step 4, the stale marker doc remains and + this function is idempotent -- it will reattempt the same cleanup on the next prune run. + Unreachable content left by a crashed step 3 is also reclaimed by the orphan sweep.""" + stale_hits = fetch_stale_markers(es) + if not stale_hits: + return (0, 0) + + # Group stale markers by repo for the per-repo commit-safety check. + from collections import defaultdict + by_repo: dict[tuple[str, str, str], list[dict]] = defaultdict(list) + for hit in stale_hits: + g = hit["_source"].get("git", {}) + host = g.get("host", "") + org = g.get("org", "") + repo = g.get("repo", "") + if host and org and repo: + by_repo[(host, org, repo)].append(hit) + + markers_deleted = 0 + commits_dropped = 0 + for (host, org, repo), hits in by_repo.items(): + # Fetch commits currently protected by any complete marker in this repo. + protected_commits = fetch_complete_commits_for_repo(es, host, org, repo) + for hit in hits: + commit = hit["_source"].get("git", {}).get("commit") + if commit and commit not in protected_commits: + # Safe to delete this commit's content. + delete_commit_content(es, host, org, repo, commit) + commits_dropped += 1 + # Delete the stale marker doc regardless (the content is either dropped or still + # protected by another marker, so the stale marker itself is never useful again). + try: + es.delete(index=REFS_INDEX, id=hit["_id"]) + except NotFoundError: + pass # already gone -- race with another prune run + markers_deleted += 1 + + return (markers_deleted, commits_dropped) + + def plan_orphans_now(es: Elasticsearch) -> OrphanPlan: """Take one read-only snapshot of the cluster (index names, ref tuples, content tuples) via the read helpers in sourcerer/queries.py, and compute the full orphan plan from it via diff --git a/src/sourcerer/config.py b/src/sourcerer/config.py index a2a9184..6c888ca 100644 --- a/src/sourcerer/config.py +++ b/src/sourcerer/config.py @@ -548,7 +548,7 @@ def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: if raw.get("index") is not None: index_level, index_suffix = _parse_index(raw["index"], ctx) if update == "incremental" and index_level == "commit": - # Incremental content carries no git.commit of its own (see build_ref_key), so a + # Incremental content is ref-addressed (no git.commit on content docs), so a # commit-level index name -- which requires a commit sha -- can never be built for it. raise ValueError(f"{ctx}: 'update: incremental' cannot be combined with 'index.level: commit'") diff --git a/src/sourcerer/elastic/agent_builder_tools/README.md b/src/sourcerer/elastic/agent_builder_tools/README.md index 2cec784..8fd3518 100644 --- a/src/sourcerer/elastic/agent_builder_tools/README.md +++ b/src/sourcerer/elastic/agent_builder_tools/README.md @@ -13,9 +13,70 @@ Query snippet: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) + ) // other filters +// Branch by content-doc shape: +// 1. Content for commit snapshots already carry git.commit, and status +// was confirmed by the membership subquery above (status=="complete"), +// so the snapshot arm needs no join; it just asserts status to match +// the incremental arm's column. +// 2. Content for incremental refs carry only git.ref. The join resolves +// the ref's current status from its join doc. This join assumes at +// most one doc per (host,org,repo,ref). That assumption requires a +// "one update mode owns a ref name" invariant (no repo may have both +// a snapshot and incremental source targeting the same ref name), +// which must be enforced separately (e.g. at config-validation time); +// nothing in this query itself enforces it. +| FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + +// Consistency guard: Only retrieve from a ref whose indexing is complete. +// Excludes torn/partial-read windows when an incrementally indexed ref is +// in the middle of an update. This is a no-op for the commit snapshot arm, +// whose status is always "complete" from the EVAL above. +| WHERE status == "complete" + // rest of query ``` @@ -43,6 +104,16 @@ params: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" ``` ## Glob matching `file.path` @@ -57,7 +128,7 @@ Query snippet: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + // filter by git_commit, git_ref, and git_ref_type AND file.path LIKE ?file_path // other filters diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml index ef4106a..a940aa5 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml @@ -9,34 +9,69 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key IN ( - // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than - // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. - // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values - // once; the outer query then does a cheap membership check instead of a wildcard match on - // every row. - FROM sourcerer-refs - | WHERE git.host LIKE ?git_host - AND git.org LIKE ?git_org - AND git.repo LIKE ?git_repo - AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) ) AND file.path LIKE ?file_path AND line.content RLIKE ?regex - // git.ref_key is purely an internal storage/join key (never a query param): every - // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN - // resolves each row's citable commit regardless of how the row above was scoped -- - // by commit (snapshot) or by ref name (incremental). - | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Branch by content-doc shape: + // 1. Content for commit snapshots already carry git.commit, and status + // was confirmed by the membership subquery above (status=="complete"), + // so the snapshot arm needs no join; it just asserts status to match + // the incremental arm's column. + // 2. Content for incremental refs carry only git.ref. The join resolves + // the ref's current status from its join doc. This join assumes at + // most one doc per (host,org,repo,ref). That assumption requires a + // "one update mode owns a ref name" invariant (no repo may have both + // a snapshot and incremental source targeting the same ref name), + // which must be enforced separately (e.g. at config-validation time); + // nothing in this query itself enforces it. + | FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: only serve content from a ref whose latest index is complete. This - // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs - // join doc is status:"indexing" until the new commit's content is fully published). A no-op - // for snapshot content, whose join doc is always status:"complete". `status` lives only on - // refs docs, so after the join it unambiguously means the joined ref's status. + // Consistency guard: Only retrieve from a ref whose indexing is complete. + // Excludes torn/partial-read windows when an incrementally indexed ref is + // in the middle of an update. This is a no-op for the commit snapshot arm, + // whose status is always "complete" from the EVAL above. | WHERE status == "complete" // Glob match (* and **) on file.path @@ -65,9 +100,19 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit_ish: + git_commit: type: string - description: Filter by git commit(s) or ref(s) (supports * wildcards) + description: Filter by git commit(s) (supports * wildcards) + optional: true + defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml index c348992..02a6548 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml @@ -9,34 +9,69 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key IN ( - // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than - // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. - // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values - // once; the outer query then does a cheap membership check instead of a wildcard match on - // every row. - FROM sourcerer-refs - | WHERE git.host LIKE ?git_host - AND git.org LIKE ?git_org - AND git.repo LIKE ?git_repo - AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) ) AND file.path LIKE ?file_path AND MATCH(line.content.text, ?q) - // git.ref_key is purely an internal storage/join key (never a query param): every - // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN - // resolves each row's citable commit regardless of how the row above was scoped -- - // by commit (snapshot) or by ref name (incremental). - | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Branch by content-doc shape: + // 1. Content for commit snapshots already carry git.commit, and status + // was confirmed by the membership subquery above (status=="complete"), + // so the snapshot arm needs no join; it just asserts status to match + // the incremental arm's column. + // 2. Content for incremental refs carry only git.ref. The join resolves + // the ref's current status from its join doc. This join assumes at + // most one doc per (host,org,repo,ref). That assumption requires a + // "one update mode owns a ref name" invariant (no repo may have both + // a snapshot and incremental source targeting the same ref name), + // which must be enforced separately (e.g. at config-validation time); + // nothing in this query itself enforces it. + | FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: only serve content from a ref whose latest index is complete. This - // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs - // join doc is status:"indexing" until the new commit's content is fully published). A no-op - // for snapshot content, whose join doc is always status:"complete". `status` lives only on - // refs docs, so after the join it unambiguously means the joined ref's status. + // Consistency guard: Only retrieve from a ref whose indexing is complete. + // Excludes torn/partial-read windows when an incrementally indexed ref is + // in the middle of an update. This is a no-op for the commit snapshot arm, + // whose status is always "complete" from the EVAL above. | WHERE status == "complete" // Glob match (* and **) on file.path @@ -65,9 +100,19 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit_ish: + git_commit: type: string - description: Filter by git commit(s) or ref(s) (supports * wildcards) + description: Filter by git commit(s) (supports * wildcards) + optional: true + defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml index 1784ca8..3e261d7 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml @@ -9,33 +9,68 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key IN ( - // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than - // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. - // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values - // once; the outer query then does a cheap membership check instead of a wildcard match on - // every row. - FROM sourcerer-refs - | WHERE git.host LIKE ?git_host - AND git.org LIKE ?git_org - AND git.repo LIKE ?git_repo - AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) ) AND file.path LIKE ?file_path - // git.ref_key is purely an internal storage/join key (never a query param): every - // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN - // resolves each row's citable commit regardless of how the row above was scoped -- - // by commit (snapshot) or by ref name (incremental). - | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Branch by content-doc shape: + // 1. Content for commit snapshots already carry git.commit, and status + // was confirmed by the membership subquery above (status=="complete"), + // so the snapshot arm needs no join; it just asserts status to match + // the incremental arm's column. + // 2. Content for incremental refs carry only git.ref. The join resolves + // the ref's current status from its join doc. This join assumes at + // most one doc per (host,org,repo,ref). That assumption requires a + // "one update mode owns a ref name" invariant (no repo may have both + // a snapshot and incremental source targeting the same ref name), + // which must be enforced separately (e.g. at config-validation time); + // nothing in this query itself enforces it. + | FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: only serve content from a ref whose latest index is complete. This - // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs - // join doc is status:"indexing" until the new commit's content is fully published). A no-op - // for snapshot content, whose join doc is always status:"complete". `status` lives only on - // refs docs, so after the join it unambiguously means the joined ref's status. + // Consistency guard: Only retrieve from a ref whose indexing is complete. + // Excludes torn/partial-read windows when an incrementally indexed ref is + // in the middle of an update. This is a no-op for the commit snapshot arm, + // whose status is always "complete" from the EVAL above. | WHERE status == "complete" // Glob match (* and **) on file.path @@ -83,9 +118,19 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit_ish: + git_commit: type: string - description: Filter by git commit(s) or ref(s) (supports * wildcards) + description: Filter by git commit(s) (supports * wildcards) + optional: true + defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml index 7cec93d..9c4e6ab 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml @@ -9,33 +9,65 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key IN ( - // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than - // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. - // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values - // once; the outer query then does a cheap membership check instead of a wildcard match on - // every row. - FROM sourcerer-refs - | WHERE git.host LIKE ?git_host - AND git.org LIKE ?git_org - AND git.repo LIKE ?git_repo - AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) ) AND file.path LIKE ?file_path - // git.ref_key is purely an internal storage/join key (never a query param): every - // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN - // resolves each row's citable commit regardless of how the row above was scoped -- - // by commit (snapshot) or by ref name (incremental). - | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Branch by content-doc shape: + // 1. Content for commit snapshots already carry git.commit, and status + // was confirmed by the membership subquery above (status=="complete"), + // so the snapshot arm needs no join; it just asserts status to match + // the incremental arm's column. + // 2. Content for incremental refs carry only git.ref. The join resolves + // the ref's current status from its join doc. Safety of the incremental + // join (one doc per (host,org,repo,ref)) is enforced by the "one update + // mode owns a ref name" invariant at index time. + | FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: only serve content from a ref whose latest index is complete. This - // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs - // join doc is status:"indexing" until the new commit's content is fully published). A no-op - // for snapshot content, whose join doc is always status:"complete". `status` lives only on - // refs docs, so after the join it unambiguously means the joined ref's status. + // Consistency guard: Only retrieve from a ref whose indexing is complete. + // Excludes torn/partial-read windows when an incrementally indexed ref is + // in the middle of an update. This is a no-op for the commit snapshot arm, + // whose status is always "complete" from the EVAL above. | WHERE status == "complete" // Glob match (* and **) on file.path @@ -82,9 +114,19 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit_ish: + git_commit: type: string - description: Filter by git commit(s) or ref(s) (supports * wildcards) + description: Filter by git commit(s) (supports * wildcards) + optional: true + defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml index 9933f39..d761ac1 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml @@ -9,32 +9,67 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key IN ( - // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than - // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. - // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values - // once; the outer query then does a cheap membership check instead of a wildcard match on - // every row. - FROM sourcerer-refs - | WHERE git.host LIKE ?git_host - AND git.org LIKE ?git_org - AND git.repo LIKE ?git_repo - AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) ) - // git.ref_key is purely an internal storage/join key (never a query param): every - // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN - // resolves each row's citable commit regardless of how the row above was scoped -- - // by commit (snapshot) or by ref name (incremental). - | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Branch by content-doc shape: + // 1. Content for commit snapshots already carry git.commit, and status + // was confirmed by the membership subquery above (status=="complete"), + // so the snapshot arm needs no join; it just asserts status to match + // the incremental arm's column. + // 2. Content for incremental refs carry only git.ref. The join resolves + // the ref's current status from its join doc. This join assumes at + // most one doc per (host,org,repo,ref). That assumption requires a + // "one update mode owns a ref name" invariant (no repo may have both + // a snapshot and incremental source targeting the same ref name), + // which must be enforced separately (e.g. at config-validation time); + // nothing in this query itself enforces it. + | FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: only serve content from a ref whose latest index is complete. This - // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs - // join doc is status:"indexing" until the new commit's content is fully published). A no-op - // for snapshot content, whose join doc is always status:"complete". `status` lives only on - // refs docs, so after the join it unambiguously means the joined ref's status. + // Consistency guard: Only retrieve from a ref whose indexing is complete. + // Excludes torn/partial-read windows when an incrementally indexed ref is + // in the middle of an update. This is a no-op for the commit snapshot arm, + // whose status is always "complete" from the EVAL above. | WHERE status == "complete" // Enforce glob depth for * and ** on file.path. @@ -90,7 +125,7 @@ configuration: ) // Collapse to one row per unique entry - this turns raw paths into an ls listing. - // Group by the commit too: when git_commit_ish matches more than one ref (e.g. a + // Group by the commit too: when git_commit/git_ref/git_ref_type match more than one ref (e.g. a // wildcard or the default "*"), each ref must get its own listing rather than having // its file counts/bytes summed together into a meaningless cross-ref total. | STATS files = COUNT(*), bytes = SUM(file.size) @@ -121,9 +156,19 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit_ish: + git_commit: type: string - description: Filter by git commit(s) or ref(s) (supports * wildcards) + description: Filter by git commit(s) (supports * wildcards) + optional: true + defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml index 3eb7798..a9fd956 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml @@ -9,35 +9,70 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key IN ( - // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than - // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. - // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values - // once; the outer query then does a cheap membership check instead of a wildcard match on - // every row. - FROM sourcerer-refs - | WHERE git.host LIKE ?git_host - AND git.org LIKE ?git_org - AND git.repo LIKE ?git_repo - AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) ) AND file.path LIKE ?file_path AND line.number >= ?line_number_start AND line.number <= ?line_number_end - // git.ref_key is purely an internal storage/join key (never a query param): every - // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN - // resolves each row's citable commit regardless of how the row above was scoped -- - // by commit (snapshot) or by ref name (incremental). - | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Branch by content-doc shape: + // 1. Content for commit snapshots already carry git.commit, and status + // was confirmed by the membership subquery above (status=="complete"), + // so the snapshot arm needs no join; it just asserts status to match + // the incremental arm's column. + // 2. Content for incremental refs carry only git.ref. The join resolves + // the ref's current status from its join doc. This join assumes at + // most one doc per (host,org,repo,ref). That assumption requires a + // "one update mode owns a ref name" invariant (no repo may have both + // a snapshot and incremental source targeting the same ref name), + // which must be enforced separately (e.g. at config-validation time); + // nothing in this query itself enforces it. + | FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: only serve content from a ref whose latest index is complete. This - // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs - // join doc is status:"indexing" until the new commit's content is fully published). A no-op - // for snapshot content, whose join doc is always status:"complete". `status` lives only on - // refs docs, so after the join it unambiguously means the joined ref's status. + // Consistency guard: Only retrieve from a ref whose indexing is complete. + // Excludes torn/partial-read windows when an incrementally indexed ref is + // in the middle of an update. This is a no-op for the commit snapshot arm, + // whose status is always "complete" from the EVAL above. | WHERE status == "complete" // Glob match (* and **) on file.path @@ -85,9 +120,19 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit_ish: + git_commit: type: string - description: Filter by git commit(s) or ref(s) (supports * wildcards) + description: Filter by git commit(s) (supports * wildcards) + optional: true + defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml index 260a3ef..28df24c 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml @@ -9,33 +9,68 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key IN ( - // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than - // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. - // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values - // once; the outer query then does a cheap membership check instead of a wildcard match on - // every row. - FROM sourcerer-refs - | WHERE git.host LIKE ?git_host - AND git.org LIKE ?git_org - AND git.repo LIKE ?git_repo - AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) ) AND file.path LIKE ?file_path - // git.ref_key is purely an internal storage/join key (never a query param): every - // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN - // resolves each row's citable commit regardless of how the row above was scoped -- - // by commit (snapshot) or by ref name (incremental). - | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Branch by content-doc shape: + // 1. Content for commit snapshots already carry git.commit, and status + // was confirmed by the membership subquery above (status=="complete"), + // so the snapshot arm needs no join; it just asserts status to match + // the incremental arm's column. + // 2. Content for incremental refs carry only git.ref. The join resolves + // the ref's current status from its join doc. This join assumes at + // most one doc per (host,org,repo,ref). That assumption requires a + // "one update mode owns a ref name" invariant (no repo may have both + // a snapshot and incremental source targeting the same ref name), + // which must be enforced separately (e.g. at config-validation time); + // nothing in this query itself enforces it. + | FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: only serve content from a ref whose latest index is complete. This - // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs - // join doc is status:"indexing" until the new commit's content is fully published). A no-op - // for snapshot content, whose join doc is always status:"complete". `status` lives only on - // refs docs, so after the join it unambiguously means the joined ref's status. + // Consistency guard: Only retrieve from a ref whose indexing is complete. + // Excludes torn/partial-read windows when an incrementally indexed ref is + // in the middle of an update. This is a no-op for the commit snapshot arm, + // whose status is always "complete" from the EVAL above. | WHERE status == "complete" // Glob match (* and **) on file.path @@ -82,9 +117,19 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit_ish: + git_commit: type: string - description: Filter by git commit(s) or ref(s) (supports * wildcards) + description: Filter by git commit(s) (supports * wildcards) + optional: true + defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml index 43f8e87..5135d71 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml @@ -9,32 +9,67 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key IN ( - // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than - // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. - // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values - // once; the outer query then does a cheap membership check instead of a wildcard match on - // every row. - FROM sourcerer-refs - | WHERE git.host LIKE ?git_host - AND git.org LIKE ?git_org - AND git.repo LIKE ?git_repo - AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) ) - // git.ref_key is purely an internal storage/join key (never a query param): every - // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN - // resolves each row's citable commit regardless of how the row above was scoped -- - // by commit (snapshot) or by ref name (incremental). - | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Branch by content-doc shape: + // 1. Content for commit snapshots already carry git.commit, and status + // was confirmed by the membership subquery above (status=="complete"), + // so the snapshot arm needs no join; it just asserts status to match + // the incremental arm's column. + // 2. Content for incremental refs carry only git.ref. The join resolves + // the ref's current status from its join doc. This join assumes at + // most one doc per (host,org,repo,ref). That assumption requires a + // "one update mode owns a ref name" invariant (no repo may have both + // a snapshot and incremental source targeting the same ref name), + // which must be enforced separately (e.g. at config-validation time); + // nothing in this query itself enforces it. + | FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: only serve content from a ref whose latest index is complete. This - // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs - // join doc is status:"indexing" until the new commit's content is fully published). A no-op - // for snapshot content, whose join doc is always status:"complete". `status` lives only on - // refs docs, so after the join it unambiguously means the joined ref's status. + // Consistency guard: Only retrieve from a ref whose indexing is complete. + // Excludes torn/partial-read windows when an incrementally indexed ref is + // in the middle of an update. This is a no-op for the commit snapshot arm, + // whose status is always "complete" from the EVAL above. | WHERE status == "complete" // Split each file path into its segments @@ -206,9 +241,19 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit_ish: + git_commit: type: string - description: Filter by git commit(s) or ref(s) (supports * wildcards) + description: Filter by git commit(s) (supports * wildcards) + optional: true + defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml index a16913b..bfeee6f 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml @@ -9,33 +9,68 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key IN ( - // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than - // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. - // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values - // once; the outer query then does a cheap membership check instead of a wildcard match on - // every row. - FROM sourcerer-refs - | WHERE git.host LIKE ?git_host - AND git.org LIKE ?git_org - AND git.repo LIKE ?git_repo - AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) ) AND file.path LIKE ?file_path - // git.ref_key is purely an internal storage/join key (never a query param): every - // content doc and its refs join doc share the same ref_key, so this LOOKUP JOIN - // resolves each row's citable commit regardless of how the row above was scoped -- - // by commit (snapshot) or by ref name (incremental). - | LOOKUP JOIN sourcerer-refs ON git.ref_key + // Branch by content-doc shape: + // 1. Content for commit snapshots already carry git.commit, and status + // was confirmed by the membership subquery above (status=="complete"), + // so the snapshot arm needs no join; it just asserts status to match + // the incremental arm's column. + // 2. Content for incremental refs carry only git.ref. The join resolves + // the ref's current status from its join doc. This join assumes at + // most one doc per (host,org,repo,ref). That assumption requires a + // "one update mode owns a ref name" invariant (no repo may have both + // a snapshot and incremental source targeting the same ref name), + // which must be enforced separately (e.g. at config-validation time); + // nothing in this query itself enforces it. + | FORK + ( WHERE git.commit IS NOT NULL + | EVAL status = "complete" ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: only serve content from a ref whose latest index is complete. This - // excludes the torn/partial-read window while an incremental branch is mid-reindex (its refs - // join doc is status:"indexing" until the new commit's content is fully published). A no-op - // for snapshot content, whose join doc is always status:"complete". `status` lives only on - // refs docs, so after the join it unambiguously means the joined ref's status. + // Consistency guard: Only retrieve from a ref whose indexing is complete. + // Excludes torn/partial-read windows when an incrementally indexed ref is + // in the middle of an update. This is a no-op for the commit snapshot arm, + // whose status is always "complete" from the EVAL above. | WHERE status == "complete" // Glob match (* and **) on file.path @@ -197,9 +232,19 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit_ish: + git_commit: type: string - description: Filter by git commit(s) or ref(s) (supports * wildcards) + description: Filter by git commit(s) (supports * wildcards) + optional: true + defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml index 2cf5ebd..750957a 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml @@ -16,10 +16,6 @@ configuration: // Format the response | SORT indexed_at DESC - // git.ref_key is not surfaced here -- it's purely an internal storage/join key (used only - // by content tools' `LOOKUP JOIN sourcerer-refs ON git.ref_key`), never something an agent - // needs to read or construct. Use git.ref (branch/tag name) or git.commit as the git_ref - // param on a content tool -- see the ref-resolution skill. | KEEP git.host, git.org, git.repo, git.ref, git.ref_type, git.commit, git.commit_date, status, files_count, lines_count, indexed_at, indexing_started_at | LIMIT 1000000 params: @@ -57,4 +53,4 @@ configuration: type: string description: Filter by ref status. "complete" = fully indexed (default); "indexing" = currently being indexed; "*" = all statuses. optional: true - defaultValue: "complete" + defaultValue: "complete" \ No newline at end of file diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml index 7ec216a..68acbd7 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml @@ -9,19 +9,43 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.ref_key IN ( - // Resolve ?git_commit_ish against the small sourcerer-refs lookup table first, rather than - // evaluating the LIKE/OR wildcard match per line against this (far larger) content index. - // The subquery narrows host/org/repo/status/commit-ish down to a set of git.ref_key values - // once; the outer query then does a cheap membership check instead of a wildcard match on - // every row. - FROM sourcerer-refs - | WHERE git.host LIKE ?git_host - AND git.org LIKE ?git_org - AND git.repo LIKE ?git_repo - AND status == "complete" - AND (git.commit LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) - | KEEP git.ref_key + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + // Includes a consistency guard by only retrieving content from refs + // whose indexing is complete. + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + AND status == "complete" + | KEEP git.ref + )) ) AND file.path LIKE ?file_path AND MATCH(line.content.text, ?q) @@ -32,17 +56,21 @@ configuration: | EVAL _file_segs = MV_COUNT(SPLIT(file.path, "/")) | WHERE _fp_is_recursive OR _file_segs == _fp_segs - // Collapse matching lines to one row per repo + ref. Grouping by git.ref_key - // rather than git.commit matters for two reasons: incremental (branch-tracked) - // rows have no git.commit of their own, so grouping by commit would collapse - // every branch in a repo into one bucket; and two refs can point at the same - // commit (e.g. a release tag and the branch it was cut from), which grouping - // by commit would wrongly conflate into a single ref. ref_key is already - // present on every sourcerer-lines row, so no join is needed to get it. + // Collapse matching lines to one row per (repo, ref) with COALESCE(git.ref, git.commit). + // git.ref is checked first because it's the only field that distinguishes + // two incremental branches that happen to currently point at the same commit + // (this query never joins, so an incremental row's git.commit stays NULL + // throughout. There's nothing for the COALESCE to prefer it over anyway). + // For snapshot rows git.ref is NULL, so this falls through to git.commit, + // which is already a correct per-commit identity since snapshot content is + // naturally deduplicated by commit. Same two reasons as before: incremental + // rows have no commit of their own to group by, and two refs sharing a + // commit must not collapse into one bucket. + | EVAL _ref_ish = COALESCE(git.ref, git.commit) | STATS _commit_file_count_distinct = COUNT_DISTINCT(file.path), _commit_score_sum = SUM(_score) - BY git.host, git.org, git.repo, git.ref_key + BY git.host, git.org, git.repo, _ref_ish // Normalize by distinct files so breadth beats duplication | EVAL _commit_score_density = _commit_score_sum / SQRT(TO_DOUBLE(_commit_file_count_distinct)) @@ -76,9 +104,19 @@ configuration: description: Filter by git repo(s) (supports * wildcards) optional: true defaultValue: "*" - git_commit_ish: + git_commit: type: string - description: Filter by git commit(s) or ref(s) (supports * wildcards) + description: Filter by git commit(s) (supports * wildcards) + optional: true + defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") optional: true defaultValue: "*" file_path: diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json index 17cb50e..eac08e3 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json @@ -56,9 +56,6 @@ "type": "keyword", "normalizer": "lowercase" }, - "ref_key": { - "type": "keyword" - }, "ref": { "type": "keyword" } diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json index 2ef42f5..04bafee 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json @@ -98,9 +98,6 @@ "type": "keyword", "normalizer": "lowercase" }, - "ref_key": { - "type": "keyword" - }, "ref": { "type": "keyword" } diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index 244af4f..a430c34 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -69,9 +69,6 @@ }, "commit_date": { "type": "date" - }, - "ref_key": { - "type": "keyword" } } }, diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index 54be0c8..619f38c 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -65,6 +65,44 @@ def fetch_markers( return out +def fetch_stale_markers(es: Elasticsearch) -> list[dict]: + """Return all refs docs with status="stale" across all repos. Each element is the raw + Elasticsearch hit dict (keys: _id, _source). Called by the prune command to reclaim snapshot + content that was switched to incremental mode (the flip-status switchover writes "stale" + markers before publishing the incremental join doc as "complete", so these never re-appear in + any content-tool query, which all gate on status=="complete").""" + try: + hits = [] + for hit in scan(es, index=REFS_ALIAS, + query={"query": {"term": {"status": "stale"}}}, + preserve_order=False): + hits.append(hit) + return hits + except NotFoundError: + return [] + + +def fetch_complete_commits_for_repo(es: Elasticsearch, host: str, org: str, repo: str) -> set[str]: + """Return the set of git.commit values referenced by any complete (non-stale) marker in this + repo. Used by the stale-marker reclamation step to apply the commit-safety guard: a commit + still held by a surviving complete marker must not have its content deleted.""" + try: + resp = es.search( + index=REFS_ALIAS, size=0, + query={"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"status": "complete"}}, + {"exists": {"field": "git.commit"}}, + ]}}, + aggs={"commits": {"terms": {"field": "git.commit", "size": 10000}}}, + ) + return {b["key"] for b in resp["aggregations"]["commits"]["buckets"]} + except NotFoundError: + return set() + + # --- Orphan sweep: ES-facing read helpers -------------------------------------------------- # The detection logic itself (orphan_indices/orphan_content_commits/orphan_markers/ # plan_orphans) is pure and lives in planner.py; these are the thin, mockable, READ-ONLY @@ -244,15 +282,16 @@ def content_indices_for_commit( return sorted(names) -def enumerate_content_ref_keys(es: Elasticsearch, host: str, org: str, repo: str) -> set[str]: - """Every distinct `git.ref_key` present in this repo's content (files + lines aliases), via - a paginated composite aggregation scoped to (host, org, repo). Feeds the post-upgrade - uniqueness gate (INV-011): every value this returns must resolve to exactly one - `sourcerer-v3-refs` join doc. Returns an empty set if neither alias has any matching docs.""" +def _enumerate_content_field( + es: Elasticsearch, host: str, org: str, repo: str, field: str, +) -> set[str]: + """Every distinct value of `field` (git.commit or git.ref) in this repo's content, + restricted to docs where that field IS NOT NULL, via paginated composite aggregation.""" filters = [ {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, + {"exists": {"field": field}}, ] out: set[str] = set() for index in (FILES_ALIAS, LINES_ALIAS): @@ -260,7 +299,7 @@ def enumerate_content_ref_keys(es: Elasticsearch, host: str, org: str, repo: str while True: composite: dict = { "size": _COMPOSITE_PAGE_SIZE, - "sources": [{"ref_key": {"terms": {"field": "git.ref_key"}}}], + "sources": [{"val": {"terms": {"field": field}}}], } if after is not None: composite["after"] = after @@ -277,32 +316,73 @@ def enumerate_content_ref_keys(es: Elasticsearch, host: str, org: str, repo: str if not buckets: break for b in buckets: - out.add(b["key"]["ref_key"]) + out.add(b["key"]["val"]) after = agg.get("after_key") if after is None: break return out -def check_ref_key_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> list[str]: - """The post-upgrade uniqueness gate (INV-011): every distinct `git.ref_key` in this repo's - content must resolve to EXACTLY ONE `sourcerer-v3-refs` join doc. Returns the sorted list of - offending ref_keys (missing entirely, or matched by more than one join doc) -- empty means - the invariant holds. A single aggregation query counts join docs per ref_key; a key absent - from the buckets has zero matches (missing).""" - ref_keys = enumerate_content_ref_keys(es, host, org, repo) - if not ref_keys: - return [] - try: - resp = es.search( - index=REFS_ALIAS, size=0, - query={"terms": {"git.ref_key": sorted(ref_keys)}}, - aggs={"keys": {"terms": {"field": "git.ref_key", "size": len(ref_keys)}}}, - ) - counts = {b["key"]: b["doc_count"] for b in resp["aggregations"]["keys"]["buckets"]} - except NotFoundError: - counts = {} - return sorted(key for key in ref_keys if counts.get(key, 0) != 1) +def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> list[str]: + """Join-uniqueness gate (INV-011 backstop): verifies every content key maps to a correct + refs join doc. Split by content shape (no `update_mode` on content docs since d77726a): + + - Snapshot (git.commit IS NOT NULL): each commit must resolve to ≥1 complete refs doc + (presence check -- multi-ref-per-commit is legal; the snapshot FORK arm no longer joins + so the uniqueness requirement there is already removed). + - Incremental (git.ref IS NOT NULL): each ref must resolve to EXACTLY ONE refs doc with + `update_mode == "incremental"` -- the anti-fan-out invariant for the surviving join. + + Returns the sorted list of offending keys (commits/refs that fail their respective check); + an empty list means the invariant holds.""" + offending: list[str] = [] + + # --- snapshot: each commit must have ≥1 complete refs doc --- + commits = _enumerate_content_field(es, host, org, repo, "git.commit") + if commits: + try: + resp = es.search( + index=REFS_ALIAS, size=0, + query={"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"terms": {"git.commit": sorted(commits)}}, + {"term": {"status": "complete"}}, + ]}}, + aggs={"commits": {"terms": {"field": "git.commit", "size": len(commits)}}}, + ) + found_commits = {b["key"] for b in resp["aggregations"]["commits"]["buckets"]} + except NotFoundError: + found_commits = set() + offending.extend(sorted(commits - found_commits)) + + # --- incremental: each ref must have EXACTLY ONE incremental join doc --- + refs = _enumerate_content_field(es, host, org, repo, "git.ref") + if refs: + try: + resp = es.search( + index=REFS_ALIAS, size=0, + query={"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"terms": {"git.ref": sorted(refs)}}, + {"term": {"update_mode": "incremental"}}, + ]}}, + aggs={"refs": {"terms": {"field": "git.ref", "size": len(refs)}}}, + ) + ref_counts = {b["key"]: b["doc_count"] for b in resp["aggregations"]["refs"]["buckets"]} + except NotFoundError: + ref_counts = {} + offending.extend(sorted(ref for ref in refs if ref_counts.get(ref, 0) != 1)) + + return sorted(offending) + + +# Legacy alias: used by tests and any external callers referencing the old name. +# Prefer check_join_uniqueness for new code. +check_ref_key_uniqueness = check_join_uniqueness def resolve_content_commit( diff --git a/src/sourcerer/skills/ref-resolution/SKILL.md b/src/sourcerer/skills/ref-resolution/SKILL.md index 969ef3e..971076c 100644 --- a/src/sourcerer/skills/ref-resolution/SKILL.md +++ b/src/sourcerer/skills/ref-resolution/SKILL.md @@ -73,11 +73,12 @@ Once a ref is resolved above, pass the value straight through: itself as `git_commit_ish` (e.g. `main`) -- no commit needed, the query always resolves to whatever commit that branch is CURRENTLY at. -`git.ref_key` is an internal storage/join detail (`LOOKUP JOIN sourcerer-refs ON git.ref_key` -inside the tool) -- it is never a param you construct or a value `refs.list` returns. +Internally, each tool resolves the citable commit via a FORK that branches on content-doc shape: +a `LOOKUP JOIN` on (`git.host`, `git.org`, `git.repo`, `git.commit`) for snapshot-shaped rows, or +on (`git.host`, `git.org`, `git.repo`, `git.ref`) for incremental-shaped rows. Read the resolved `git.commit` back from each result row (the content query's own join supplies it) for citations. Because incremental content overwrites in place, a branch query always returns its current HEAD -- if you need to confirm a branch hasn't advanced since you resolved it (e.g. a long-running investigation), re-check `sourcerer.refs.list` for its current commit. Re-invoke this -skill only when the question introduces a new or additional ref. +skill only when the question introduces a new or additional ref. \ No newline at end of file diff --git a/src/sourcerer/utils.py b/src/sourcerer/utils.py index b979d72..1b06c85 100644 --- a/src/sourcerer/utils.py +++ b/src/sourcerer/utils.py @@ -18,14 +18,16 @@ def build_ref_key(host: str, org: str, repo: str, ref: str) -> str: - """Deterministic incremental ref_key: `{host}~{org}~{repo}~{ref}` (host/org/repo lowercased, - ref case-preserved). + """Deterministic `_id` string for incremental join docs: `{host}~{org}~{repo}~{ref}` + (host/org/repo lowercased, ref case-preserved). + + Used exclusively as the Elasticsearch `_id` for the incremental refs join doc. Not a stored + field -- `git.ref_key` was removed from all index mappings and content builders. The string + is opaque to queries; the join uses `(git.host, git.org, git.repo, git.ref)` natively. `~` is safe as a delimiter because it is illegal in git ref names (see - `git check-ref-format`) and is already the index-name segment delimiter used for - host/org/repo elsewhere (see `indices.py`), so it cannot collide with any of the joined - values. Snapshot content instead uses the bare commit SHA as its `ref_key` -- this helper - is only for the incremental (ref-addressed) shape. + `git check-ref-format`) and matches the index-name segment delimiter used for + host/org/repo elsewhere (see `indices.py`), so it cannot collide with any joined value. """ return "~".join((host.lower(), org.lower(), repo.lower(), ref)) diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index 141fcb0..1e8ff21 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -50,41 +50,48 @@ def test_git_host_filtered_before_git_org(): def test_content_tools_use_universal_ref_join_query(): - # INV-005: every content tool scopes to a set of git.ref_key values resolved from the small - # sourcerer-refs table via a subquery (`git.ref_key IN (FROM sourcerer-refs | WHERE ... (git.commit - # LIKE ?git_commit_ish OR git.ref LIKE ?git_commit_ish) | KEEP git.ref_key)`) rather than a - # per-row LIKE/OR wildcard match on the (far larger) content index, with no update_mode/mode - # conditional. It then joins sourcerer-refs on git.ref_key (a purely internal field -- never an - # agent-facing param) to resolve the commit for both snapshot and incremental content. - # git_commit_ish is the single scoping param (LIKE, so it supports wildcards); there is no - # separate git_commit param. A post-join `status == "complete"` guard excludes torn/partial - # reads while an incremental branch is mid-reindex (a no-op for always-complete snapshot content). + # Every content tool uses a two-OR'd-IN subquery to scope rows to matching + # refs (git.commit OR git.ref), then a FORK to handle the two content shapes + # separately without fan-out: + # - Snapshot arm (git.commit IS NOT NULL): EVAL status = "complete" -- no join needed; the + # commit already lives on the content row, and status was pre-confirmed by the subquery. + # - Incremental arm (git.ref IS NOT NULL AND git.commit IS NULL): LOOKUP JOIN sourcerer-refs + # ON (host,org,repo,ref) to resolve status from the incremental join doc. + # Ref scoping uses three separate params: git_commit, git_ref, git_ref_type. tools = _tools() for tid in _CONTENT_TOOL_IDS: query = tools[tid]["configuration"]["query"] params = tools[tid]["configuration"]["params"] assert "update_mode" not in query, f"{tid} query has an update_mode conditional" - # The commit-or-ref match resolves inside the sourcerer-refs subquery, keyed by git.ref_key. - assert "git.ref_key IN (" in query, f"{tid} missing the ref_key subquery scope" - assert "git.commit LIKE ?git_commit_ish" in query, f"{tid} missing the commit-or-ref filter" - assert "git.ref LIKE ?git_commit_ish" in query, f"{tid} missing the commit-or-ref filter" - assert "| LOOKUP JOIN sourcerer-refs ON git.ref_key" in query, f"{tid} missing the universal join" + # git.ref_key must not be used as a field or join key (comments may reference it by name) + assert "git.ref_key" not in query, f"{tid} still uses git.ref_key as a field" + assert "ON git.ref_key" not in query, f"{tid} still joins on git.ref_key" + # The membership subquery uses two OR'd IN paths (one for snapshot commits, one for + # incremental refs), scoped by git_commit, git_ref, and git_ref_type params. + assert "git.commit LIKE ?git_commit" in query, f"{tid} missing git.commit LIKE ?git_commit" + assert "git.ref LIKE ?git_ref" in query, f"{tid} missing git.ref LIKE ?git_ref" + assert "git.ref_type LIKE ?git_ref_type" in query, f"{tid} missing git.ref_type LIKE ?git_ref_type" + # Snapshot arm: no join; asserts status = "complete" inline. + assert "git.commit IS NOT NULL" in query, f"{tid} missing snapshot FORK arm (git.commit IS NOT NULL)" + assert 'EVAL status = "complete"' in query, f"{tid} missing EVAL status = \"complete\" in snapshot arm" + # Incremental arm: join on the 4-tuple (no ref_key). + assert "git.ref IS NOT NULL" in query, f"{tid} missing incremental FORK arm (git.ref IS NOT NULL)" + assert "LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref" in query, \ + f"{tid} missing the incremental join on (host,org,repo,ref)" + # No ref_key param or join shape. assert "git_ref_key" not in params, f"{tid} still exposes git_ref_key as a param" assert "?git_ref_key" not in query, f"{tid} still references ?git_ref_key" - # git_commit_ish is optional with a "*" default: an unpinned query matches all indexed - # refs. Every content tool keeps this safe by carrying git.commit through to output (and, - # where it aggregates, grouping BY git.commit) so multi-ref matches stay attributable and - # are never summed across refs. - assert params["git_commit_ish"]["optional"] is True - assert params["git_commit_ish"]["defaultValue"] == "*" - # The old standalone git_commit guard param is gone: git_commit_ish is the sole scoping - # param, and the consistency guard is now the automatic, no-param `status == "complete"`. - # (Match on word boundary so ?git_commit_ish / git_commit_ish don't false-positive.) - assert not re.search(r"\bgit_commit\b", "\n".join(params)), f"{tid} still exposes git_commit as a param" - assert not re.search(r"\?git_commit\b", query), f"{tid} still references ?git_commit" - # The status guard must appear AFTER the join (status lives only on the refs doc the join - # brings in, never on the raw content doc). - assert '| WHERE status == "complete"' in query, f"{tid} missing the post-join status guard" + # git_commit, git_ref, git_ref_type are all optional with a "*" default. + for p in ("git_commit", "git_ref", "git_ref_type"): + assert p in params, f"{tid} missing param {p}" + assert params[p]["optional"] is True, f"{tid} param {p} is not optional" + assert params[p]["defaultValue"] == "*", f"{tid} param {p} defaultValue != '*'" + # No collapsed git_commit_ish param. + assert "git_commit_ish" not in params, f"{tid} still exposes git_commit_ish as a param" + assert "?git_commit_ish" not in query, f"{tid} still references ?git_commit_ish" + # The post-FORK status guard must appear after the join (defense-in-depth; free no-op for + # snapshot arm since status is already "complete" from the EVAL). + assert '| WHERE status == "complete"' in query, f"{tid} missing the post-FORK status guard" assert query.index("LOOKUP JOIN sourcerer-refs") < query.index('WHERE status == "complete"') @@ -120,11 +127,11 @@ def test_output_keeps_git_host(): def test_content_tool_aggregation_is_ref_scoped(): - # git_commit_ish defaults to "*", so a content query can match more than one ref at once. - # That is only safe if aggregation never blends refs: every STATS in a content tool must carry - # git.commit in its BY grouping key, so per-ref counts/bytes/line-blobs stay separate rather - # than being summed or interleaved across commits. Guards the files.ls-style regression where a - # `BY name` grouping silently summed file counts across every matching ref. + # git_commit/git_ref/git_ref_type all default to "*", so a content query can match more than + # one ref at once. That is only safe if aggregation never blends refs: every STATS in a content + # tool must carry git.commit in its BY grouping key, so per-ref counts/bytes/line-blobs stay + # separate rather than being summed or interleaved across commits. Guards the files.ls-style + # regression where a `BY name` grouping silently summed file counts across every matching ref. tools = _tools() for tid in _CONTENT_TOOL_IDS: query = tools[tid]["configuration"]["query"] @@ -143,8 +150,8 @@ def test_content_tool_aggregation_is_ref_scoped(): assert " BY " in block_text, f"{tid} has a STATS with no BY grouping" by_clause = block_text.split(" BY ", 1)[1] assert "git.commit" in by_clause, ( - f"{tid} STATS groups without git.commit -- would blend refs when git_commit_ish " - f"matches more than one ref" + f"{tid} STATS groups without git.commit -- would blend refs when git_commit/git_ref " + f"params match more than one ref" ) diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 50a84c6..99f699a 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -1,7 +1,10 @@ -"""Tests for the one-time upgrade backfill in sourcerer.commands.index.markers: stamping -git.ref_key onto pre-existing snapshot content, stamping ref_key onto existing markers that -predate the one-doc-per-source change, and deleting legacy shadow join docs. -Every ES call is mocked (INV-009/INV-010).""" +"""Tests for mapping helpers and the stale-marker switchover in markers.py. + +The ref_key backfill subsystem (backfill_snapshot_ref_keys, backfill_refs_join_docs, +backfill_repo, etc.) has been removed along with git.ref_key. This file tests the +surviving apply_*_index_mapping helpers and the new mark_snapshot_markers_stale helper +that handles mode-switch from snapshot to incremental. +""" # Standard packages from unittest.mock import MagicMock, call @@ -14,180 +17,24 @@ from sourcerer.commands.index.markers import ( apply_content_index_mapping, apply_refs_index_mapping, - backfill_refs_join_docs, - backfill_repo, - backfill_snapshot_ref_keys, - commits_with_join_doc, - commits_with_ref_key_carrier, - distinct_commits_for_repo, + mark_snapshot_markers_stale, + stale_snapshot_markers_for_ref, ) from sourcerer.indices import FILES_ALIAS, LINES_ALIAS, REFS_INDEX -FULL_SHA = "cfefb3b2378ccbadefa7c8f4f9e21b3a1d2e5f60" - def _not_found() -> NotFoundError: meta = ApiResponseMeta(status=404, http_version="1.1", headers=HttpHeaders({}), duration=0.0, node=None) return NotFoundError("index_not_found_exception", meta, None) -class TestBackfillSnapshotRefKeys: - def test_scoped_to_repo_and_missing_ref_key(self): - es = MagicMock() - es.update_by_query.return_value = {"updated": 3} - total = backfill_snapshot_ref_keys(es, "github", "acme", "widgets") - assert total == 6 # 3 (files) + 3 (lines) - assert es.update_by_query.call_count == 2 - indices = {c.kwargs["index"] for c in es.update_by_query.call_args_list} - assert indices == {FILES_ALIAS, LINES_ALIAS} - for call in es.update_by_query.call_args_list: - query = call.kwargs["query"] - assert {"term": {"git.host": "github"}} in query["bool"]["filter"] - assert {"exists": {"field": "git.ref_key"}} in query["bool"]["must_not"] - - def test_second_run_is_a_no_op(self): - # Idempotency (INV-009): once every doc has ref_key, the must_not:exists filter - # matches nothing, so a repeat run updates 0 docs. - es = MagicMock() - es.update_by_query.return_value = {"updated": 0} - assert backfill_snapshot_ref_keys(es, "github", "acme", "widgets") == 0 - - def test_missing_index_is_ignored(self): - es = MagicMock() - es.update_by_query.side_effect = _not_found() - assert backfill_snapshot_ref_keys(es, "github", "acme", "widgets") == 0 - - -class TestDistinctCommitsForRepo: - def test_returns_bucket_keys(self): - es = MagicMock() - es.search.return_value = {"aggregations": {"commits": {"buckets": [ - {"key": "aaa"}, {"key": "bbb"}, - ]}}} - assert distinct_commits_for_repo(es, "github", "acme", "widgets") == {"aaa", "bbb"} - - def test_missing_index_returns_empty_set(self): - es = MagicMock() - es.search.side_effect = _not_found() - assert distinct_commits_for_repo(es, "github", "acme", "widgets") == set() - - -class TestCommitsWithRefKeyCarrier: - """commits_with_ref_key_carrier (and its alias commits_with_join_doc) returns the subset - of commits for which a refs doc carries git.ref_key == commit.""" - - def test_empty_input_short_circuits(self): - es = MagicMock() - assert commits_with_ref_key_carrier(es, set()) == set() - es.search.assert_not_called() - - def test_returns_commits_from_agg_buckets(self): - es = MagicMock() - es.search.return_value = {"aggregations": {"carriers": {"buckets": [{"key": "aaa"}]}}} - assert commits_with_ref_key_carrier(es, {"aaa", "bbb"}) == {"aaa"} - - def test_missing_index_returns_empty_set(self): - es = MagicMock() - es.search.side_effect = _not_found() - assert commits_with_ref_key_carrier(es, {"aaa"}) == set() - - def test_alias_commits_with_join_doc_is_the_same_function(self): - # commits_with_join_doc kept as alias for backward compatibility. - assert commits_with_join_doc is commits_with_ref_key_carrier - - -class TestBackfillRefsJoinDocs: - def test_stamps_ref_key_onto_existing_marker(self): - # Normal case: a complete marker exists for the commit; backfill stamps ref_key on it. - es = MagicMock() - marker_id = "deadbeef" * 4 # any string - es.search.side_effect = [ - # distinct_commits_for_repo - {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, - # commits_with_ref_key_carrier -- no carrier yet - {"aggregations": {"carriers": {"buckets": []}}}, - # per-missing-commit: find the existing marker (no ref_key yet) - {"hits": {"hits": [{"_id": marker_id, "_source": {"git": {"ref": "v1.0", "ref_type": "tag"}}}]}}, - ] - stamped = backfill_refs_join_docs(es, "github", "acme", "widgets") - assert stamped == 1 - # Must use es.update (partial doc), not es.index - es.update.assert_called_once() - assert es.update.call_args.kwargs["id"] == marker_id - assert es.update.call_args.kwargs["doc"] == {"git": {"ref_key": FULL_SHA}} - es.index.assert_not_called() - - def test_falls_back_to_index_for_orphan_content(self): - # No marker found for the commit: write a minimal _id=commit carrier doc. - es = MagicMock() - es.search.side_effect = [ - # distinct_commits_for_repo - {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, - # commits_with_ref_key_carrier -- no carrier - {"aggregations": {"carriers": {"buckets": []}}}, - # per-missing-commit marker search -- no marker - {"hits": {"hits": []}}, - # fallback: any refs doc for this commit (for informational fields) - {"hits": {"hits": []}}, - ] - stamped = backfill_refs_join_docs(es, "github", "acme", "widgets") - assert stamped == 1 - es.update.assert_not_called() - # Falls back to writing an _id=commit carrier - es.index.assert_called_once() - call_kwargs = es.index.call_args.kwargs - assert call_kwargs["id"] == FULL_SHA - assert call_kwargs["document"]["git"]["ref_key"] == FULL_SHA - assert call_kwargs["document"]["git"]["commit"] == FULL_SHA - - def test_second_run_stamps_nothing(self): - # INV-009/INV-010: every commit already has a carrier -> no-op. - es = MagicMock() - es.search.side_effect = [ - {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, - {"aggregations": {"carriers": {"buckets": [{"key": FULL_SHA}]}}}, - ] - assert backfill_refs_join_docs(es, "github", "acme", "widgets") == 0 - es.update.assert_not_called() - es.index.assert_not_called() - - def test_no_commits_short_circuits(self): - es = MagicMock() - es.search.return_value = {"aggregations": {"commits": {"buckets": []}}} - assert backfill_refs_join_docs(es, "github", "acme", "widgets") == 0 - es.update.assert_not_called() - es.index.assert_not_called() - - def test_refreshes_refs_index_when_carriers_stamped(self): - # A uniqueness-gate run immediately afterward must see the just-stamped carrier - # rather than racing the refs index's refresh interval. - es = MagicMock() - marker_id = "aabbccdd" * 4 - es.search.side_effect = [ - {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, - {"aggregations": {"carriers": {"buckets": []}}}, - {"hits": {"hits": [{"_id": marker_id, "_source": {"git": {"ref": "v1.0", "ref_type": "tag"}}}]}}, - ] - backfill_refs_join_docs(es, "github", "acme", "widgets") - assert es.indices.refresh.call_args.kwargs["index"] == REFS_INDEX - - def test_no_refresh_when_nothing_stamped(self): - es = MagicMock() - es.search.side_effect = [ - {"aggregations": {"commits": {"buckets": [{"key": FULL_SHA}]}}}, - {"aggregations": {"carriers": {"buckets": [{"key": FULL_SHA}]}}}, - ] - backfill_refs_join_docs(es, "github", "acme", "widgets") - es.indices.refresh.assert_not_called() - - class TestApplyContentIndexMapping: def test_puts_mapping_on_both_aliases(self): es = MagicMock() apply_content_index_mapping( es, - {"properties": {"git": {"properties": {"ref_key": {"type": "keyword"}}}}}, - {"properties": {"git": {"properties": {"ref_key": {"type": "keyword"}}}}}, + {"properties": {"git": {"properties": {"ref": {"type": "keyword"}}}}}, + {"properties": {"git": {"properties": {"ref": {"type": "keyword"}}}}}, ) indices = {c.kwargs["index"] for c in es.indices.put_mapping.call_args_list} assert indices == {FILES_ALIAS, LINES_ALIAS} @@ -201,7 +48,7 @@ def test_missing_index_is_ignored(self): class TestApplyRefsIndexMapping: def test_puts_mapping_on_refs_index(self): es = MagicMock() - apply_refs_index_mapping(es, {"properties": {"git": {"properties": {"ref_key": {"type": "keyword"}}}}}) + apply_refs_index_mapping(es, {"properties": {"status": {"type": "keyword"}}}) assert es.indices.put_mapping.call_args.kwargs["index"] == REFS_INDEX def test_missing_index_is_ignored(self): @@ -210,39 +57,63 @@ def test_missing_index_is_ignored(self): apply_refs_index_mapping(es, {"properties": {}}) # no raise -class TestBackfillRepo: - def test_second_run_is_fully_idempotent(self): +class TestStaleSnapshotMarkersForRef: + def test_returns_complete_non_incremental_markers(self): + """Returns complete markers that are NOT update_mode=incremental (i.e. snapshot markers).""" es = MagicMock() - es.update_by_query.return_value = {"updated": 0} - # All search calls return empty (no commits -> backfill no-ops; delete_by_query finds nothing) - es.search.return_value = {"aggregations": {"commits": {"buckets": []}}} - es.delete_by_query.return_value = {"deleted": 0} - summary = backfill_repo( - es, "github", "acme", "widgets", refs_mapping={"properties": {}}, - files_mapping={"properties": {}}, lines_mapping={"properties": {}}, - ) - assert summary == {"content_updated": 0, "carriers_stamped": 0, "shadow_docs_deleted": 0} + es.search.return_value = {"hits": {"hits": [ + {"_id": "abc123", "_source": {"git": {"commit": "deadbeef"}}}, + ]}} + hits = stale_snapshot_markers_for_ref(es, "github", "acme", "widgets", "main") + assert len(hits) == 1 + assert hits[0]["_id"] == "abc123" - def test_summary_keys(self): - # Confirm the returned dict uses the new key names. + def test_missing_index_returns_empty(self): es = MagicMock() - es.update_by_query.return_value = {"updated": 0} - es.search.return_value = {"aggregations": {"commits": {"buckets": []}}} - es.delete_by_query.return_value = {"deleted": 0} - summary = backfill_repo(es, "github", "acme", "widgets") - assert set(summary.keys()) == {"content_updated", "carriers_stamped", "shadow_docs_deleted"} - - def test_deletes_legacy_shadow_docs(self): - # Migration: after stamping carriers, delete legacy update_mode:snapshot shadow docs. + es.search.side_effect = _not_found() + assert stale_snapshot_markers_for_ref(es, "github", "acme", "widgets", "main") == [] + + def test_query_scopes_to_host_org_repo_ref(self): + es = MagicMock() + es.search.return_value = {"hits": {"hits": []}} + stale_snapshot_markers_for_ref(es, "github", "acme", "widgets", "main") + query = es.search.call_args.kwargs["query"] + filt = query["bool"]["filter"] + assert {"term": {"git.host": "github"}} in filt + assert {"term": {"git.org": "acme"}} in filt + assert {"term": {"git.repo": "widgets"}} in filt + assert {"term": {"git.ref": "main"}} in filt + assert {"term": {"status": "complete"}} in filt + + +class TestMarkSnapshotMarkersStale: + def test_flips_each_marker_to_stale(self): + """mark_snapshot_markers_stale flips every found marker to status:'stale'.""" + es = MagicMock() + es.search.return_value = {"hits": {"hits": [ + {"_id": "marker1", "_source": {"git": {"commit": "aaa"}}}, + {"_id": "marker2", "_source": {"git": {"commit": "aaa"}}}, + ]}} + count = mark_snapshot_markers_stale(es, "github", "acme", "widgets", "main") + assert count == 2 + assert es.update.call_count == 2 + ids_updated = {c.kwargs["id"] for c in es.update.call_args_list} + assert ids_updated == {"marker1", "marker2"} + for c in es.update.call_args_list: + assert c.kwargs["doc"] == {"status": "stale"} + + def test_no_markers_returns_zero(self): + es = MagicMock() + es.search.return_value = {"hits": {"hits": []}} + assert mark_snapshot_markers_stale(es, "github", "acme", "widgets", "main") == 0 + es.update.assert_not_called() + + def test_update_not_found_is_ignored(self): + """A marker that disappears between the search and the update is not an error.""" es = MagicMock() - es.update_by_query.return_value = {"updated": 0} - es.search.return_value = {"aggregations": {"commits": {"buckets": []}}} - es.delete_by_query.return_value = {"deleted": 3} - summary = backfill_repo(es, "github", "acme", "widgets") - assert summary["shadow_docs_deleted"] == 3 - # delete_by_query must target the physical REFS_INDEX (not the alias -- only writes go there) - assert es.delete_by_query.call_args.kwargs["index"] == REFS_INDEX - # Must scope to update_mode: "snapshot" (the legacy shadow doc shape) - filters = es.delete_by_query.call_args.kwargs["query"]["bool"]["filter"] - assert {"term": {"git.host": "github"}} in filters - assert {"term": {"update_mode": "snapshot"}} in filters + es.search.return_value = {"hits": {"hits": [ + {"_id": "gone", "_source": {"git": {"commit": "aaa"}}}, + ]}} + es.update.side_effect = _not_found() + count = mark_snapshot_markers_stale(es, "github", "acme", "widgets", "main") + assert count == 1 # 1 found, even if the update race-lost diff --git a/tests/test_cli_index.py b/tests/test_cli_index.py index a37f3d7..d80fe58 100644 --- a/tests/test_cli_index.py +++ b/tests/test_cli_index.py @@ -68,7 +68,7 @@ def test_insecure_env_var_true_resolves_to_true(self): def fake_run(repo_spec, branch, tag, commit, url, api_key, username, password, force=False, quiet=False, cache_dir=None, ephemeral=False, - retry_window=None, insecure=False, no_backfill=False): + retry_window=None, insecure=False): captured["insecure"] = insecure with patch("sourcerer.commands.index.command.run", side_effect=fake_run): @@ -86,7 +86,7 @@ def test_insecure_env_var_absent_resolves_to_false(self): def fake_run(repo_spec, branch, tag, commit, url, api_key, username, password, force=False, quiet=False, cache_dir=None, ephemeral=False, - retry_window=None, insecure=False, no_backfill=False): + retry_window=None, insecure=False): captured["insecure"] = insecure with patch("sourcerer.commands.index.command.run", side_effect=fake_run): @@ -96,43 +96,3 @@ def fake_run(repo_spec, branch, tag, commit, url, api_key, username, password, ], env={}, catch_exceptions=False) assert captured.get("insecure") is False - - -class TestNoBackfillOption: - def test_help_shows_no_backfill(self): - runner = CliRunner() - result = runner.invoke(index, ["--help"]) - assert result.exit_code == 0 - assert "--no-backfill" in result.output - - def test_no_backfill_flag_forwarded_to_run(self): - runner = CliRunner() - captured = {} - - def fake_run(repo_spec, branch, tag, commit, url, api_key, username, password, - force=False, quiet=False, cache_dir=None, ephemeral=False, - retry_window=None, insecure=False, no_backfill=False): - captured["no_backfill"] = no_backfill - - with patch("sourcerer.commands.index.command.run", side_effect=fake_run): - runner.invoke(index, [ - "--url", "http://es:9200", "--no-backfill", "github/org/repo", - ], catch_exceptions=False) - - assert captured.get("no_backfill") is True - - def test_default_is_backfill_enabled(self): - runner = CliRunner() - captured = {} - - def fake_run(repo_spec, branch, tag, commit, url, api_key, username, password, - force=False, quiet=False, cache_dir=None, ephemeral=False, - retry_window=None, insecure=False, no_backfill=False): - captured["no_backfill"] = no_backfill - - with patch("sourcerer.commands.index.command.run", side_effect=fake_run): - runner.invoke(index, [ - "--url", "http://es:9200", "github/org/repo", - ], catch_exceptions=False) - - assert captured.get("no_backfill") is False diff --git a/tests/test_documents.py b/tests/test_documents.py index 75e841b..22cf9eb 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -78,8 +78,8 @@ def test_git_fields(self, tmp_path): p = tmp_path / "a.txt" p.write_text("hello") _id, doc = build_file_doc("github", "acme", "widgets", "deadbeef", "a.txt", p) - assert doc["git"] == {"host": "github", "org": "acme", "repo": "widgets", "commit": "deadbeef", - "ref_key": "deadbeef"} + assert doc["git"] == {"host": "github", "org": "acme", "repo": "widgets", "commit": "deadbeef"} + assert "ref_key" not in doc["git"] def test_host_changes_id(self, tmp_path): p = tmp_path / "a.txt" @@ -130,10 +130,12 @@ def test_broken_symlink_has_target_path_but_no_target_size(self, tmp_path): class TestIterLineDocs: - def test_snapshot_ref_key(self): + def test_snapshot_no_ref_key(self): + # Snapshot line docs carry git.commit but no git.ref_key (field removed). docs = list(iter_line_docs("github", "acme", "widgets", "deadbeef", "a.txt", "one")) _id, doc = docs[0] - assert doc["git"]["ref_key"] == "deadbeef" + assert doc["git"]["commit"] == "deadbeef" + assert "ref_key" not in doc["git"] def test_line_numbering_starts_at_one(self): docs = list(iter_line_docs("github", "acme", "widgets", "deadbeef", "a.txt", "one\ntwo\nthree")) @@ -178,11 +180,13 @@ def test_no_optional_fields_when_omitted(self): class TestIncrementalDocs: - def test_ref_key_is_tilde_joined(self, tmp_path): + def test_ref_field_set_no_ref_key(self, tmp_path): + # Incremental docs carry git.ref (the branch name) but no git.ref_key (field removed). p = tmp_path / "a.txt" p.write_text("hello") _id, doc = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) - assert doc["git"]["ref_key"] == "github~acme~widgets~main" + assert doc["git"]["ref"] == "main" + assert "ref_key" not in doc["git"] def test_no_commit_field(self, tmp_path): p = tmp_path / "a.txt" @@ -207,18 +211,21 @@ def test_id_differs_from_snapshot_id(self, tmp_path): incr_id, _ = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) assert snap_id != incr_id - def test_line_docs_ref_key_and_no_commit(self): + def test_line_docs_ref_and_no_commit_no_ref_key(self): + # Incremental line docs carry git.ref, no git.commit, no git.ref_key. docs = list(iter_incremental_line_docs("github", "acme", "widgets", "main", "a.txt", "one\ntwo")) for _id, d in docs: - assert d["git"]["ref_key"] == "github~acme~widgets~main" + assert d["git"]["ref"] == "main" assert "commit" not in d["git"] + assert "ref_key" not in d["git"] def test_worker_ctx_routes_to_incremental_builders(self, tmp_path): (tmp_path / "a.txt").write_text("one\ntwo\n") _set_worker_ctx_incremental("github", "acme", "widgets", "main", tmp_path) actions = _build_one_file_actions("a.txt") - assert actions[0]["_source"]["git"]["ref_key"] == "github~acme~widgets~main" + assert actions[0]["_source"]["git"]["ref"] == "main" assert "commit" not in actions[0]["_source"]["git"] + assert "ref_key" not in actions[0]["_source"]["git"] class TestFileAttributes: diff --git a/tests/test_markers.py b/tests/test_markers.py index aa692c0..e1de463 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -404,16 +404,17 @@ def _indexed_doc(es): class TestWriteRefMarker: - """write_ref_marker is the single refs doc per snapshot source; it carries git.ref_key so - the LOOKUP JOIN resolves git.commit without a separate shadow join doc (INV-004).""" + """write_ref_marker is the single refs doc per snapshot source (INV-004). git.ref_key has + been removed; snapshot refs are identified by (host, org, repo, ref, commit) on the marker.""" - def test_marker_carries_ref_key_equal_to_commit(self): + def test_marker_carries_commit_no_ref_key(self): es = MagicMock() write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, files_count=10, lines_count=200) doc = es.index.call_args.kwargs["document"] - assert doc["git"]["ref_key"] == OLD assert doc["git"]["commit"] == OLD + assert "ref_key" not in doc["git"] + assert doc["update_mode"] == "snapshot" def test_marker_id_is_hashed_not_the_commit(self): # _id is build_ref_id (BLAKE2b hash) -- one per (ref, commit), NOT the bare commit SHA. @@ -432,8 +433,8 @@ def test_default_write_does_not_refresh(self): assert es.index.call_args.kwargs.get("refresh") is False def test_refresh_true_is_propagated(self): - # command.py passes refresh=True so the post-index uniqueness gate (INV-011) sees the - # ref_key carrier immediately rather than racing the refs index's async refresh. + # write_ref_marker accepts refresh=True for callers that need the doc visible before + # the next gate (INV-011) runs. es = MagicMock() write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, files_count=1, lines_count=1, refresh=True) @@ -484,6 +485,21 @@ def test_incremental_marker_carries_prior_counts(self): assert doc["files_count"] == 12 and doc["lines_count"] == 340 assert doc["git"]["commit_date"] == "2026-01-01T00:00:00+00:00" + def test_incremental_indexing_carries_routing(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "main", OLD, NEW, + index_level="commit", index_suffix="s1") + doc = _indexed_doc(es) + assert doc["index_level"] == "commit" + assert doc["index_suffix"] == "s1" + + def test_incremental_indexing_default_routing(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "main", OLD, NEW) + doc = _indexed_doc(es) + assert doc["index_level"] == "repo" + assert doc["index_suffix"] is None + class TestWriteIncrementalReady: def test_incremental_marker_advances_commit_and_clears_target_and_error(self): @@ -499,6 +515,15 @@ def test_incremental_marker_advances_commit_and_clears_target_and_error(self): assert doc["files_count"] == 5 and doc["lines_count"] == 99 assert es.index.call_args.kwargs["refresh"] is True # publication boundary + def test_incremental_ready_carries_routing(self): + es = MagicMock() + write_incremental_ready(es, "github", "acme", "widgets", "main", commit=NEW, + commit_date_iso=None, files_count=1, lines_count=1, + index_level="commit", index_suffix="s1") + doc = _indexed_doc(es) + assert doc["index_level"] == "commit" + assert doc["index_suffix"] == "s1" + class TestWriteIncrementalFailed: def test_incremental_marker_keeps_status_indexing_and_retains_old_pointer(self): @@ -517,6 +542,14 @@ def test_incremental_marker_error_text_is_bounded(self): write_incremental_failed(es, "github", "acme", "widgets", "main", OLD, NEW, error="x" * 5000) assert len(_indexed_doc(es)["error"]) == ERROR_MAX_LEN + def test_incremental_failed_carries_routing(self): + es = MagicMock() + write_incremental_failed(es, "github", "acme", "widgets", "main", OLD, NEW, error="boom", + index_level="commit", index_suffix="s1") + doc = _indexed_doc(es) + assert doc["index_level"] == "commit" + assert doc["index_suffix"] == "s1" + class TestReadIncrementalRef: def test_returns_source(self): @@ -545,10 +578,12 @@ def test_scoped_to_exact_ref_key_and_paths(self): assert es.delete_by_query.call_count == 2 # files + lines indices for call in es.delete_by_query.call_args_list: query = call.kwargs["query"] - assert {"term": {"git.ref_key": build_ref_key("github", "acme", "widgets", "main")}} in ( - query["bool"]["filter"] - ) - assert {"terms": {"file.path": ["a.txt", "b.txt"]}} in query["bool"]["filter"] + filt = query["bool"]["filter"] + assert {"term": {"git.host": "github"}} in filt + assert {"term": {"git.org": "acme"}} in filt + assert {"term": {"git.repo": "widgets"}} in filt + assert {"term": {"git.ref": "main"}} in filt + assert {"terms": {"file.path": ["a.txt", "b.txt"]}} in filt def test_missing_index_is_ignored(self): es = MagicMock() @@ -563,13 +598,16 @@ def test_scoped_to_exact_ref_key_only(self): assert es.delete_by_query.call_count == 2 for call in es.delete_by_query.call_args_list: query = call.kwargs["query"] - assert query["bool"]["filter"] == [ - {"term": {"git.ref_key": build_ref_key("github", "acme", "widgets", "main")}} - ] + filt = query["bool"]["filter"] + assert {"term": {"git.host": "github"}} in filt + assert {"term": {"git.org": "acme"}} in filt + assert {"term": {"git.repo": "widgets"}} in filt + assert {"term": {"git.ref": "main"}} in filt + assert not any("ref_key" in str(f) for f in filt) def test_isolated_from_another_branch(self): # Two incremental branches indexed; deleting one's docs must never scope to the other's - # ref_key (INV-008) -- asserted here at the query-construction level. + # (host,org,repo,ref) quadruple (INV-008) -- asserted here at the query-construction level. es_a = MagicMock() es_b = MagicMock() delete_incremental_branch(es_a, "github", "acme", "widgets", "main") diff --git a/tests/test_uniqueness_gate.py b/tests/test_uniqueness_gate.py index aee9db4..73ea52e 100644 --- a/tests/test_uniqueness_gate.py +++ b/tests/test_uniqueness_gate.py @@ -1,5 +1,10 @@ -"""Tests for the post-upgrade uniqueness gate: sourcerer.queries.check_ref_key_uniqueness -(INV-011) and its command.py wiring in _run_uniqueness_gate. Every ES call is mocked.""" +"""Tests for the post-index join-uniqueness gate: sourcerer.queries.check_join_uniqueness +(INV-011) and its command.py wiring in _run_uniqueness_gate. Every ES call is mocked. + +The gate is split by content shape (no update_mode on content docs since d77726a): + - Snapshot (git.commit IS NOT NULL): each commit must have ≥1 complete refs doc. + - Incremental (git.ref IS NOT NULL): each ref must have EXACTLY ONE incremental join doc. +""" # Standard packages from unittest.mock import MagicMock @@ -10,7 +15,7 @@ # App packages from sourcerer.commands.index.command import _run_uniqueness_gate -from sourcerer.queries import check_ref_key_uniqueness, enumerate_content_ref_keys +from sourcerer.queries import check_join_uniqueness def _not_found() -> NotFoundError: @@ -18,76 +23,138 @@ def _not_found() -> NotFoundError: return NotFoundError("index_not_found_exception", meta, None) -class TestEnumerateContentRefKeys: - def test_collects_keys_across_both_aliases(self): - es = MagicMock() - es.search.side_effect = [ - {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "aaa"}}]}}}, # files - {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "bbb"}}]}}}, # lines - ] - assert enumerate_content_ref_keys(es, "github", "acme", "widgets") == {"aaa", "bbb"} +def _composite_resp(values: list[str]) -> dict: + """Build a composite agg response with the given values.""" + return {"aggregations": {"keys": {"buckets": [{"key": {"val": v}} for v in values]}}} - def test_missing_index_contributes_nothing(self): - es = MagicMock() - es.search.side_effect = _not_found() - assert enumerate_content_ref_keys(es, "github", "acme", "widgets") == set() +def _terms_resp(counts: dict[str, int]) -> dict: + """Build a terms agg response mapping key -> doc_count.""" + return {"aggregations": { + "commits": {"buckets": [{"key": k, "doc_count": v} for k, v in counts.items()]}, + "refs": {"buckets": [{"key": k, "doc_count": v} for k, v in counts.items()]}, + }} -class TestCheckRefKeyUniqueness: - def test_clean_repo_returns_empty(self): - es = MagicMock() - es.search.side_effect = [ - # enumerate_content_ref_keys: files then lines - {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "aaa"}}]}}}, - {"aggregations": {"keys": {"buckets": []}}}, - # uniqueness count query - {"aggregations": {"keys": {"buckets": [{"key": "aaa", "doc_count": 1}]}}}, - ] - assert check_ref_key_uniqueness(es, "github", "acme", "widgets") == [] - - def test_missing_join_doc_is_offending(self): + +class TestCheckJoinUniqueness: + """Tests for check_join_uniqueness: the combined snapshot + incremental gate.""" + + def _make_es(self, snapshot_commits=(), snapshot_found=(), incremental_refs=(), incremental_counts=None): + """Build a mock ES with side_effects matching the exact call order of check_join_uniqueness: + + 1. _enumerate_content_field(git.commit): 1 search per index (FILES, LINES) + 2. if commits non-empty → snapshot presence check (1 search on sourcerer-refs) + 3. _enumerate_content_field(git.ref): 1 search per index (FILES, LINES) + 4. if refs non-empty → incremental uniqueness check (1 search on sourcerer-refs) + + The composite agg loop breaks on the first empty page (no after_key returned), so exactly + one search per index per field enumeration. + """ es = MagicMock() - es.search.side_effect = [ - {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "aaa"}}]}}}, - {"aggregations": {"keys": {"buckets": []}}}, - {"aggregations": {"keys": {"buckets": []}}}, # no join doc at all - ] - assert check_ref_key_uniqueness(es, "github", "acme", "widgets") == ["aaa"] - - def test_duplicate_join_doc_is_offending(self): + side_effects = [] + # (1) enumerate git.commit: FILES then LINES + side_effects.append(_composite_resp(list(snapshot_commits))) # FILES git.commit + side_effects.append(_composite_resp(list(snapshot_commits))) # LINES git.commit + # (2) snapshot join-doc presence check (only if commits found) + if snapshot_commits: + found = {c: 1 for c in snapshot_found} + side_effects.append({"aggregations": {"commits": {"buckets": [ + {"key": k, "doc_count": v} for k, v in found.items() + ]}}}) + # (3) enumerate git.ref: FILES then LINES + side_effects.append(_composite_resp(list(incremental_refs))) # FILES git.ref + side_effects.append(_composite_resp(list(incremental_refs))) # LINES git.ref + # (4) incremental uniqueness check (only if refs found) + if incremental_refs: + counts = incremental_counts or {} + side_effects.append({"aggregations": {"refs": {"buckets": [ + {"key": k, "doc_count": v} for k, v in counts.items() + ]}}}) + es.search.side_effect = side_effects + return es + + def test_clean_repo_no_content(self): + """No content at all → gate passes.""" es = MagicMock() - es.search.side_effect = [ - {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "aaa"}}]}}}, - {"aggregations": {"keys": {"buckets": []}}}, - {"aggregations": {"keys": {"buckets": [{"key": "aaa", "doc_count": 2}]}}}, - ] - assert check_ref_key_uniqueness(es, "github", "acme", "widgets") == ["aaa"] - - def test_no_content_short_circuits(self): + es.search.return_value = {"aggregations": {"keys": {"buckets": []}}} + assert check_join_uniqueness(es, "github", "acme", "widgets") == [] + + def test_clean_snapshot_all_present(self): + """Snapshot commits all have a complete refs doc → clean.""" + es = self._make_es( + snapshot_commits=["aaa"], + snapshot_found=["aaa"], + ) + assert check_join_uniqueness(es, "github", "acme", "widgets") == [] + + def test_snapshot_missing_refs_doc_is_offending(self): + """A snapshot commit with no complete refs doc is reported.""" + es = self._make_es( + snapshot_commits=["aaa"], + snapshot_found=[], # no complete refs doc found + ) + result = check_join_uniqueness(es, "github", "acme", "widgets") + assert "aaa" in result + + def test_clean_incremental_exactly_one_join_doc(self): + """Incremental ref with exactly one join doc → clean.""" + es = self._make_es( + incremental_refs=["main"], + incremental_counts={"main": 1}, + ) + assert check_join_uniqueness(es, "github", "acme", "widgets") == [] + + def test_incremental_missing_join_doc_is_offending(self): + """An incremental ref with no join doc is reported.""" + es = self._make_es( + incremental_refs=["main"], + incremental_counts={}, # zero docs found + ) + result = check_join_uniqueness(es, "github", "acme", "widgets") + assert "main" in result + + def test_incremental_duplicate_join_doc_is_offending(self): + """An incremental ref with more than one join doc (e.g. stale snapshot marker) is reported.""" + es = self._make_es( + incremental_refs=["main"], + incremental_counts={"main": 2}, # two docs — fan-out! + ) + result = check_join_uniqueness(es, "github", "acme", "widgets") + assert "main" in result + + def test_missing_index_contributes_nothing(self): es = MagicMock() - es.search.side_effect = [ - {"aggregations": {"keys": {"buckets": []}}}, - {"aggregations": {"keys": {"buckets": []}}}, - ] - assert check_ref_key_uniqueness(es, "github", "acme", "widgets") == [] + es.search.side_effect = _not_found() + assert check_join_uniqueness(es, "github", "acme", "widgets") == [] class TestRunUniquenessGate: def test_passes_silently_when_clean(self): es = MagicMock() - es.search.side_effect = [ - {"aggregations": {"keys": {"buckets": []}}}, - {"aggregations": {"keys": {"buckets": []}}}, - ] + # No content: all composite aggs return empty + es.search.return_value = {"aggregations": {"keys": {"buckets": []}}} assert _run_uniqueness_gate(es, "github", "acme", "widgets") is True - def test_fails_and_reports_on_violation(self, capsys): + def test_fails_and_reports_on_snapshot_violation(self, capsys): es = MagicMock() - es.search.side_effect = [ - {"aggregations": {"keys": {"buckets": [{"key": {"ref_key": "aaa"}}]}}}, - {"aggregations": {"keys": {"buckets": []}}}, - {"aggregations": {"keys": {"buckets": []}}}, - ] + # Snapshot commit "aaa" exists in content but has no complete refs doc. + # sources structure: [{"val": {"terms": {"field": "git.commit"}}}] + def side_effect(*args, **kwargs): + aggs = kwargs.get("aggs", {}) + if "keys" in aggs and aggs["keys"].get("composite"): + sources = aggs["keys"]["composite"].get("sources", []) + # Each source is {"": {"terms": {"field": ""}}} + field = None + if sources: + src = sources[0] + for alias_val in src.values(): + field = alias_val.get("terms", {}).get("field") + if field == "git.commit": + return {"aggregations": {"keys": {"buckets": [{"key": {"val": "aaa"}}]}}} + return {"aggregations": {"keys": {"buckets": []}}} + # terms agg for join doc presence (snapshot or incremental) + return {"aggregations": {"commits": {"buckets": []}, "refs": {"buckets": []}}} + es.search.side_effect = side_effect assert _run_uniqueness_gate(es, "github", "acme", "widgets") is False captured = capsys.readouterr() assert "aaa" in captured.err From ad4ab74918ca681b38bfb41c5757eba787e7efa8 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Thu, 20 Aug 2026 08:09:20 -0400 Subject: [PATCH 15/29] Add incremental index.level/suffix migration: backfill into new routing, delete old copy, extend prune with Class D-I backstop --- src/sourcerer/commands/index/command.py | 34 ++++++--- src/sourcerer/commands/prune/execute.py | 32 ++++++++- src/sourcerer/planner.py | 63 +++++++++++++++- src/sourcerer/queries.py | 89 +++++++++++++++++++++++ tests/test_incremental_index.py | 96 +++++++++++++++++++++++++ tests/test_planner_orphans.py | 74 +++++++++++++++++++ 6 files changed, 376 insertions(+), 12 deletions(-) diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index e5194c5..dc2a08f 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -48,10 +48,10 @@ from .markers import ( build_ref_id, commits_with_content, content_present, count_incremental_branch_docs, delete_incremental_branch, delete_incremental_paths, - fully_indexed_counts, mark_snapshot_markers_stale, markers_status_by_id, _needs_index, - pre_clone_skip, read_incremental_ref, recorded_routing, refresh_incremental_content, - should_index, write_incremental_failed, write_incremental_indexing, write_incremental_ready, - write_indexing_marker, write_ref_marker, + fully_indexed_counts, mark_snapshot_markers_stale, marker_routing, markers_status_by_id, + _needs_index, pre_clone_skip, read_incremental_ref, recorded_routing, + refresh_incremental_content, should_index, write_incremental_failed, + write_incremental_indexing, write_incremental_ready, write_indexing_marker, write_ref_marker, ) from .report import dry_run_config from .schedule import filter_config_by_schedule @@ -288,19 +288,25 @@ def index_incremental_branch_in_dir( prior = read_incremental_ref(es, host, org, repo, branch) old_sha = None if force else (prior.get("git", {}).get("commit") if prior else None) - if old_sha == new_sha and not force: - reporter.finish(unit, "no-changes") - return - level = unit.index_level suffix = unit.index_suffix + # Detect a routing (index.level / index.suffix) change from the prior completed run. When the + # routing changes we must migrate even if the commit hasn't advanced -- so routing_changed + # bypasses the no-changes early-return and forces a full rebuild into the new index (below). + old_routing = marker_routing(prior) if prior else None + routing_changed = old_routing is not None and old_routing != (level, suffix) + + if old_sha == new_sha and not force and not routing_changed: + reporter.finish(unit, "no-changes") + return + reporter.set_stage(unit, "indexing") write_incremental_indexing(es, host, org, repo, branch, completed_commit=old_sha, target_commit=new_sha, prior=prior, index_level=level, index_suffix=suffix) try: - full_rebuild = old_sha is None or force + full_rebuild = old_sha is None or force or routing_changed if not full_rebuild: plan = plan_changes(repo_dir, old_sha, new_sha) full_rebuild = plan.base_missing @@ -335,6 +341,16 @@ def index_incremental_branch_in_dir( write_incremental_ready(es, host, org, repo, branch, new_sha, commit_date_iso, files_count, lines_count, index_level=level, index_suffix=suffix) + # Migration cleanup (write-new -> flip join doc -> delete-old): now that the join doc is + # complete and points at the new routing, delete this branch's docs from the old physical + # index. Scoped to the exact (host,org,repo,ref) 4-term filter so a sibling source that + # still lives in the old index is never touched. A crash between the ready write above + # and this delete leaves stale-location incremental docs in the old index; prune's + # incremental stale-location sweep (Class D-I) reclaims them. + if routing_changed: + old_level, old_suffix = old_routing + delete_incremental_branch(es, host, org, repo, branch, + index_level=old_level, index_suffix=old_suffix) except KeyboardInterrupt: write_incremental_failed(es, host, org, repo, branch, completed_commit=old_sha, target_commit=new_sha, error="interrupted", prior=prior, diff --git a/src/sourcerer/commands/prune/execute.py b/src/sourcerer/commands/prune/execute.py index a337c1f..7c9ef89 100644 --- a/src/sourcerer/commands/prune/execute.py +++ b/src/sourcerer/commands/prune/execute.py @@ -16,6 +16,7 @@ from ...queries import ( empty_content_indices, enumerate_ref_tuples, fetch_complete_commits_for_repo, fetch_stale_markers, gather_content_by_index, gather_content_commit_tuples, + gather_incremental_content_by_index, gather_intended_incremental_index_by_ref, gather_intended_index_by_commit, list_sourcerer_indices, ) @@ -219,13 +220,19 @@ def plan_orphans_now(es: Elasticsearch) -> OrphanPlan: # detection (the index.level/suffix migration backstop). content_by_index = gather_content_by_index(es, index_names) intended_by_commit = gather_intended_index_by_commit(es) + # Class D-I: incremental (ref-addressed, commit-less) stale-location detection -- the + # incremental mirror of Class D, since the commit-keyed sweep can't see incremental docs. + incremental_content_by_index = gather_incremental_content_by_index(es, index_names) + intended_incremental_by_ref = gather_intended_incremental_index_by_ref(es) # Class E: content indices already drained to zero docs (a fully-pruned repo, or a suffix # a->b migration that emptied ~repo^a while its identity still has markers at ~repo^b). empty = empty_content_indices(es, index_names) return plan_orphans(index_names, ref_tuples, content_tuples, content_by_index_commit=content_by_index, intended_index_by_commit=intended_by_commit, - empty_index_names=empty) + empty_index_names=empty, + incremental_content_by_index=incremental_content_by_index, + intended_incremental_index_by_ref=intended_incremental_by_ref) def execute_orphan_deletions(es: Elasticsearch, plan: OrphanPlan) -> tuple[int, int, int, int, int]: @@ -291,6 +298,29 @@ def execute_orphan_deletions(es: Elasticsearch, plan: OrphanPlan) -> tuple[int, except NotFoundError: pass + # Class D-I: stale-location incremental content (ref-addressed, no git.commit). Mirrors Class D + # but keyed on (host, org, repo, ref) tuples -- the commit-keyed filter above cannot match + # incremental docs whose git.commit is absent. + for index_name, ref_tuples in plan.orphan_stale_incremental.items(): + stale_dropped += len(ref_tuples) + for (host, org, repo, ref) in ref_tuples: + try: + es.delete_by_query( + index=index_name, + query={"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.ref": ref}}, + ]}}, + conflicts="proceed", + refresh=False, + scroll_size=5000, + wait_for_completion=False, + ) + except NotFoundError: + pass + markers_dropped = sum(len(commits) for commits in plan.orphan_marker_commits.values()) if plan.orphan_marker_commits: should = [ diff --git a/src/sourcerer/planner.py b/src/sourcerer/planner.py index 08146e4..d0662ce 100644 --- a/src/sourcerer/planner.py +++ b/src/sourcerer/planner.py @@ -360,6 +360,40 @@ def orphan_stale_content( return out +def orphan_stale_incremental_content( + incremental_content_by_index: dict[str, set[tuple[str, str, str, str]]], + intended_incremental_index_by_ref: dict[tuple[str, str, str, str], set[str]], + skip_indices: set[str], +) -> dict[str, set[tuple[str, str, str, str]]]: + """Class D-I orphans: incremental content docs sitting in a physical index that the branch's + join doc no longer intends. This is the incremental migration backstop -- an index.level/suffix + change re-homes a branch's content to a new index and flips its join doc there; if a crash + happens before the old copy is deleted, the old-location docs survive with no join doc + referencing that location. + + `incremental_content_by_index` maps a physical index name -> the set of (host, org, repo, ref) + tuples with incremental content docs in it. `intended_incremental_index_by_ref` maps a ref + tuple -> the set of index names its join doc intends (reconstructed from index_level/index_suffix + with commit=None). `skip_indices` excludes indices already going away via a Class-A whole-index + DELETE. + + Returns {index_name -> set of (host, org, repo, ref) tuples to delete-by-query from that index}. + A ref with NO join doc at all is not flagged here -- that is a different category; this class is + specifically 'has a join doc, but content lives somewhere the join doc doesn't intend'.""" + out: dict[str, set[tuple[str, str, str, str]]] = {} + for index_name, ref_tuples in incremental_content_by_index.items(): + if index_name in skip_indices: + continue + for ref_tuple in ref_tuples: + intended = intended_incremental_index_by_ref.get(ref_tuple) + # No join doc for this ref at all -> different sweep; leave it alone. + if not intended: + continue + if index_name not in intended: + out.setdefault(index_name, set()).add(ref_tuple) + return out + + @dataclass class OrphanPlan: orphan_index_names: list[str] # Class A -> DELETE {index} @@ -374,6 +408,12 @@ class OrphanPlan: # identity still has markers at ~repo^b). Disjoint from orphan_index_names (Class A). Defaults # empty so callers/tests without empty-index data don't need to supply it. empty_index_names: list[str] = field(default_factory=list) + # Class D-I -> delete_by_query per index for incremental (ref-addressed, commit-less) content + # sitting in a physical index its branch's join doc no longer intends. The incremental mirror + # of Class D, since the commit-keyed Class-D sweep cannot see incremental docs. Value is + # {index_name -> set of (host, org, repo, ref) tuples to reclaim from that index}. Defaults + # empty so callers/tests without incremental location data don't need to supply it. + orphan_stale_incremental: dict[str, set[tuple[str, str, str, str]]] = field(default_factory=dict) def plan_orphans( @@ -383,6 +423,8 @@ def plan_orphans( content_by_index_commit: dict[str, set[tuple[str, str, str, str]]] | None = None, intended_index_by_commit: dict[tuple[str, str, str, str], set[str]] | None = None, empty_index_names: list[str] | None = None, + incremental_content_by_index: dict[str, set[tuple[str, str, str, str]]] | None = None, + intended_incremental_index_by_ref: dict[tuple[str, str, str, str], set[str]] | None = None, ) -> OrphanPlan: """Combine the orphan classes into one plan from cheap snapshots: the physical index names, the distinct (host, org, repo, commit) tuples in refs, and the distinct (host, org, repo, commit) @@ -392,7 +434,13 @@ def plan_orphans( `content_by_index_commit` (index name -> content commit tuples in it) and `intended_index_by_commit` (commit tuple -> index names its markers intend) enable Class-D stale-location detection (the index.level/suffix migration backstop). When omitted, Class D is - empty -- back-compat for callers that don't supply per-index location data.""" + empty -- back-compat for callers that don't supply per-index location data. + + `incremental_content_by_index` (index name -> incremental ref tuples in it) and + `intended_incremental_index_by_ref` (ref tuple -> index names its join doc intends) enable + Class-D-I stale-location detection for incremental (ref-addressed, commit-less) content -- the + incremental mirror of Class D since the commit-keyed sweep can't see incremental docs. When + omitted, Class D-I is empty -- back-compat for callers without incremental location data.""" ref_orgs = {(host, org) for host, org, _, _ in ref_commit_tuples} ref_repos = {(host, org, repo) for host, org, repo, _ in ref_commit_tuples} @@ -422,8 +470,19 @@ def plan_orphans( content_by_index_commit, intended_index_by_commit, skip_indices=orphaned_names, ) + # Class D-I: stale-location incremental content (ref-addressed, no git.commit). Mirrors Class D + # but keyed on (host, org, repo, ref) tuples from the branch's join doc. Also skip Class-A + # indices since they'll be deleted whole. + orphan_stale_incremental: dict[str, set[tuple[str, str, str, str]]] = {} + if incremental_content_by_index is not None and intended_incremental_index_by_ref is not None: + orphan_stale_incremental = orphan_stale_incremental_content( + incremental_content_by_index, intended_incremental_index_by_ref, + skip_indices=orphaned_names, + ) + # Class E: empty content indices. Exclude any already slated for a Class-A DELETE (an index # that is both empty AND identity-orphaned only needs to be deleted once). empty = [n for n in (empty_index_names or []) if n not in orphaned_names] - return OrphanPlan(orphan_index_names, orphan_content, orphan_marker_commits, orphan_stale, empty) + return OrphanPlan(orphan_index_names, orphan_content, orphan_marker_commits, orphan_stale, + empty, orphan_stale_incremental) diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index 619f38c..6f9115a 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -252,6 +252,95 @@ def gather_intended_index_by_commit( return out +def gather_intended_incremental_index_by_ref( + es: Elasticsearch, +) -> dict[tuple[str, str, str, str], set[str]]: + """For every (host, org, repo, ref) with an incremental join doc, the set of physical content + index names that join doc intends -- reconstructed from its index_level/index_suffix via + files_index/lines_index with commit=None (incremental content is ref-addressed, not + commit-addressed). + + Feeds the incremental stale-location sweep (Class D-I): content for a branch sitting in an + index NOT in this set is stale from a crashed migration and should be reclaimed. Filters to + update_mode=="incremental" docs only, so snapshot markers (which always have git.commit) are + not double-counted. Returns {} if the refs index doesn't exist.""" + out: dict[tuple[str, str, str, str], set[str]] = {} + body = {"query": {"term": {"update_mode": "incremental"}}} + src_fields = ["git.host", "git.org", "git.repo", "git.ref", "index_level", "index_suffix"] + try: + for hit in scan(es, index=REFS_ALIAS, query=body, _source=src_fields, preserve_order=False): + src = hit["_source"] + g = src.get("git", {}) + host, org, repo, ref = g.get("host"), g.get("org"), g.get("repo"), g.get("ref") + if not (host and org and repo and ref): + continue + level = src.get("index_level") or "repo" + suffix = src.get("index_suffix") or None + key = (host, org, repo, ref) + intended = out.setdefault(key, set()) + intended.add(files_index(host, org, repo, None, level, suffix)) + intended.add(lines_index(host, org, repo, None, level, suffix)) + except NotFoundError: + return {} + return out + + +def gather_incremental_content_by_index( + es: Elasticsearch, index_names: list[str], +) -> dict[str, set[tuple[str, str, str, str]]]: + """Per physical index, the distinct (host, org, repo, ref) tuples with incremental content + docs in it (docs that have git.ref and a null/absent git.commit). + + Feeds the incremental stale-location sweep (Class D-I) in planner.orphan_stale_incremental_content: + to decide a doc is stale we must know WHICH physical index holds it AND which ref it belongs to, + so this enumerates each backing index by name via a composite aggregation over git.ref. + Empty/missing indices contribute nothing.""" + out: dict[str, set[tuple[str, str, str, str]]] = {} + for name in index_names: + tuples = _composite_incremental_ref_tuples(es, name) + if tuples: + out[name] = tuples + return out + + +def _composite_incremental_ref_tuples( + es: Elasticsearch, index: str, +) -> set[tuple[str, str, str, str]]: + """Distinct (host, org, repo, ref) tuples from incremental content docs (git.ref present, + git.commit absent) in `index`. Returns empty set if the index doesn't exist.""" + out: set[tuple[str, str, str, str]] = set() + after: dict | None = None + while True: + composite: dict = { + "size": _COMPOSITE_PAGE_SIZE, + "sources": [ + {"host": {"terms": {"field": "git.host"}}}, + {"org": {"terms": {"field": "git.org"}}}, + {"repo": {"terms": {"field": "git.repo"}}}, + {"ref": {"terms": {"field": "git.ref"}}}, + ], + } + if after is not None: + composite["after"] = after + # Filter to docs that have git.ref but no git.commit (incremental content). + query = {"bool": {"filter": [{"exists": {"field": "git.ref"}}], + "must_not": [{"exists": {"field": "git.commit"}}]}} + try: + resp = es.search(index=index, size=0, query=query, + aggs={"tuples": {"composite": composite}}) + except NotFoundError: + return out + agg = resp["aggregations"]["tuples"] + buckets = agg["buckets"] + if not buckets: + return out + for b in buckets: + out.add((b["key"]["host"], b["key"]["org"], b["key"]["repo"], b["key"]["ref"])) + after = agg.get("after_key") + if after is None: + return out + + _FULL_SHA_LEN = 40 _MIN_PREFIX_LEN = 7 diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py index ee37b3c..8cbabe9 100644 --- a/tests/test_incremental_index.py +++ b/tests/test_incremental_index.py @@ -143,3 +143,99 @@ def test_failed_run_does_not_advance_commit(self): assert mocks["write_incremental_failed"].call_args.kwargs["completed_commit"] == OLD finally: _stop(patchers) + + +class TestIncrementalIndexRoutingMigration: + """When index.level or index.suffix changes on an already-indexed branch the run must: + 1. NOT return no-changes even when the commit hasn't advanced. + 2. Do a full rebuild into the NEW routing. + 3. After the ready marker flips, delete the old-routing copy (write-new -> flip -> delete-old). + """ + + def _prior_at_routing(self, level="repo", suffix=None): + """A completed incremental join doc recorded at the given routing.""" + doc = {"git": {"commit": NEW}, "index_level": level} + if suffix is not None: + doc["index_suffix"] = suffix + return doc + + def test_suffix_change_forces_full_rebuild_and_old_copy_delete(self): + """repo -> repo^deploy: full rebuild at new routing, then delete at old routing.""" + prior = self._prior_at_routing(level="repo", suffix=None) + patchers, mocks = _patch_common(prior=prior) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + update="incremental", index_level="repo", index_suffix="deploy") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + # Full rebuild path: delete_incremental_branch called at new routing, full tree indexed. + assert mocks["delete_incremental_branch"].call_count == 2, ( + "Expected 2 calls to delete_incremental_branch: one for new routing (rebuild), " + "one for old routing (migration cleanup)" + ) + call_kwargs_list = [c.kwargs for c in mocks["delete_incremental_branch"].call_args_list] + # First call: full rebuild at new (repo^deploy) routing. + assert call_kwargs_list[0].get("index_level") == "repo" + assert call_kwargs_list[0].get("index_suffix") == "deploy" + # Second call: delete old (repo, no suffix) routing. + assert call_kwargs_list[1].get("index_level") == "repo" + assert call_kwargs_list[1].get("index_suffix") is None + # Ready marker was published. + mocks["write_incremental_ready"].assert_called_once() + finally: + _stop(patchers) + + def test_level_change_forces_full_rebuild_and_old_copy_delete(self): + """repo -> org level: full rebuild at org routing, then delete at repo routing.""" + prior = self._prior_at_routing(level="repo", suffix=None) + patchers, mocks = _patch_common(prior=prior) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + update="incremental", index_level="org", index_suffix=None) + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + assert mocks["delete_incremental_branch"].call_count == 2 + call_kwargs_list = [c.kwargs for c in mocks["delete_incremental_branch"].call_args_list] + assert call_kwargs_list[0].get("index_level") == "org" + assert call_kwargs_list[1].get("index_level") == "repo" + mocks["write_incremental_ready"].assert_called_once() + finally: + _stop(patchers) + + def test_no_changes_with_routing_change_still_migrates(self): + """Commit unchanged but routing changed: must NOT return no-changes; must migrate.""" + # prior already at NEW sha, but at old routing + prior = self._prior_at_routing(level="repo", suffix=None) + prior["git"]["commit"] = NEW # same commit as what resolve_commit returns + patchers, mocks = _patch_common(prior=prior) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + update="incremental", index_level="repo", index_suffix="v2") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + # Must NOT skip even though old_sha == new_sha. + mocks["write_incremental_indexing"].assert_called_once() + mocks["write_incremental_ready"].assert_called_once() + # Old routing must be cleaned up. + assert mocks["delete_incremental_branch"].call_count == 2 + finally: + _stop(patchers) + + def test_same_routing_no_old_copy_delete(self): + """When routing is unchanged a delta run must not call delete_incremental_branch at all.""" + prior = self._prior_at_routing(level="repo", suffix=None) + plan = ChangePlan(delete_paths=[], index_paths=["changed.txt"]) + patchers, mocks = _patch_common(prior=prior, plan=plan) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + update="incremental", index_level="repo", index_suffix=None) + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + # Delta run: no full rebuild (delete_incremental_branch not called), no extra delete. + mocks["delete_incremental_branch"].assert_not_called() + finally: + _stop(patchers) diff --git a/tests/test_planner_orphans.py b/tests/test_planner_orphans.py index 78f4446..22a2f12 100644 --- a/tests/test_planner_orphans.py +++ b/tests/test_planner_orphans.py @@ -239,6 +239,80 @@ def test_plan_orphans_wires_class_d(self): assert plan.orphan_stale == {"sourcerer-v3-files~github~acme~widgets": {"abc"}} +class TestOrphanStaleIncrementalContent: + """Class D-I: incremental (ref-addressed, commit-less) content in an index that the branch's + join doc no longer intends. Mirrors Class D but keyed on (host, org, repo, ref) tuples.""" + + def test_incremental_content_at_unintended_index_is_stale(self): + from sourcerer.planner import orphan_stale_incremental_content + rt = ("github", "acme", "widgets", "main") + content_by_index = { + "sourcerer-v3-files~github~acme~widgets": {rt}, # old copy from a suffix migration + "sourcerer-v3-files~github~acme~widgets^deploy": {rt}, # new (intended) copy + } + intended = {rt: {"sourcerer-v3-files~github~acme~widgets^deploy"}} + stale = orphan_stale_incremental_content(content_by_index, intended, skip_indices=set()) + assert stale == {"sourcerer-v3-files~github~acme~widgets": {rt}} + + def test_intended_index_not_flagged(self): + from sourcerer.planner import orphan_stale_incremental_content + rt = ("github", "acme", "widgets", "main") + content_by_index = {"sourcerer-v3-files~github~acme~widgets^deploy": {rt}} + intended = {rt: {"sourcerer-v3-files~github~acme~widgets^deploy"}} + assert orphan_stale_incremental_content(content_by_index, intended, set()) == {} + + def test_ref_without_join_doc_is_not_class_di(self): + """No join doc for this ref -> different sweep, not stale-location.""" + from sourcerer.planner import orphan_stale_incremental_content + rt = ("github", "acme", "widgets", "main") + content_by_index = {"sourcerer-v3-files~github~acme~widgets": {rt}} + assert orphan_stale_incremental_content(content_by_index, {}, set()) == {} + + def test_skip_indices_excluded(self): + from sourcerer.planner import orphan_stale_incremental_content + rt = ("github", "acme", "widgets", "main") + content_by_index = {"sourcerer-v3-files~github~acme~widgets": {rt}} + intended = {rt: {"sourcerer-v3-files~github~acme~widgets^deploy"}} + # Index is already going away via a Class-A whole-index DELETE. + assert orphan_stale_incremental_content( + content_by_index, intended, {"sourcerer-v3-files~github~acme~widgets"} + ) == {} + + def test_plan_orphans_wires_class_di(self): + """plan_orphans exposes orphan_stale_incremental when location data is supplied.""" + rt = ("github", "acme", "widgets", "main") + ct = ("github", "acme", "widgets", "abc") + names = [ + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~github~acme~widgets^deploy", + ] + # One snapshot commit ref present so Class A/B/C don't fire on the identity. + ref_tuples = {ct} + content_tuples = {ct} + incremental_content_by_index = { + "sourcerer-v3-files~github~acme~widgets": {rt}, # stale copy + "sourcerer-v3-files~github~acme~widgets^deploy": {rt}, # intended + } + intended_incremental_by_ref = {rt: {"sourcerer-v3-files~github~acme~widgets^deploy"}} + plan = plan_orphans( + names, ref_tuples, content_tuples, + incremental_content_by_index=incremental_content_by_index, + intended_incremental_index_by_ref=intended_incremental_by_ref, + ) + assert plan.orphan_stale_incremental == { + "sourcerer-v3-files~github~acme~widgets": {rt}, + } + + def test_plan_orphans_class_di_empty_when_not_supplied(self): + """Back-compat: callers that omit incremental location data get an empty Class D-I.""" + ct = ("github", "acme", "widgets", "abc") + plan = plan_orphans( + ["sourcerer-v3-files~github~acme~widgets"], + {ct}, {ct}, + ) + assert plan.orphan_stale_incremental == {} + + class TestEmptyIndexSweep: """Class E: an empty content index is deleted even when its git identity still has markers (the suffix a->b migration case), and is de-duped against Class-A orphans.""" From f9f05f01df17ea1d78d6ec8b23809fac9fcabbd6 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Thu, 20 Aug 2026 08:23:55 -0400 Subject: [PATCH 16/29] Don't normalize git.ref to lowercase because those are case sensitive in git --- src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index a430c34..a45cd8a 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -56,8 +56,7 @@ "normalizer": "lowercase" }, "ref": { - "type": "keyword", - "normalizer": "lowercase" + "type": "keyword" }, "ref_type": { "type": "keyword", From cd25b28552636090a814fe78d4b1d45ca0cc7755 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Thu, 20 Aug 2026 09:02:39 -0400 Subject: [PATCH 17/29] Use the name sources[i].index.strategy instead of sources[i].update_mode, which better describes the behaviors of both 'snapshot' and 'incremental' since snapshots generally aren't updated. --- AGENTS.md | 26 ++++--- README.md | 39 +++++----- sourcerer.example.yml | 16 ++-- specs/sourcerer-yml.md | 13 ++++ src/sourcerer/commands/index/command.py | 6 +- src/sourcerer/commands/index/documents.py | 8 +- src/sourcerer/commands/index/markers.py | 15 ++-- src/sourcerer/commands/index/selection.py | 39 +++++----- src/sourcerer/config.py | 78 +++++++++++-------- .../index_templates/sourcerer-v3-refs.json | 2 +- src/sourcerer/progress.py | 8 +- src/sourcerer/queries.py | 10 +-- tests/test_agent_builder_tools.py | 2 +- tests/test_backfill.py | 3 +- tests/test_config.py | 43 +++++----- tests/test_documents.py | 4 +- tests/test_incremental_index.py | 18 ++--- tests/test_markers.py | 4 +- tests/test_uniqueness_gate.py | 2 +- 19 files changed, 184 insertions(+), 152 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ea2f634..b4d473c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,9 +57,9 @@ config's `sources:` is a YAML list, one entry per (host, org, repo, ref_type). S | `match` | yes | For `branch`/`tag`: pattern string or list of patterns matched against ref names (version DSL + glob), a ref matches if any pattern hits. For `commit`: a commit SHA/prefix string or list of them (see below). | | `since` | no | Index-side inclusion floor: the earliest commit to start indexing from. See below. Not valid for `git.ref_type: commit`. | | `retain` | no | Retention policy (see below). Omit to keep forever. For `git.ref_type: commit`, only `age` is valid. | -| `update` | no | `snapshot` (default) or `incremental` (branch-only). See below. | +| `index.strategy` | no | `snapshot` (default) or `incremental` (branch-only). See below. | -#### `update: ` (snapshot vs. incremental) +#### `index.strategy` (snapshot vs. incremental) `snapshot` (default): content is commit-addressed. A HEAD advance on a branch indexes a whole new snapshot under the new commit. @@ -80,7 +80,8 @@ succeed, so a crash mid-update leaves the prior commit and content in place. repo: serverless-gitops ref_type: branch match: main - update: incremental + index: + strategy: incremental ``` #### `git.ref_type: commit` (pinning an explicit commit) @@ -376,15 +377,15 @@ creates one index/shard per commit — see `specs/sourcerer-yml.md` for the cave Content docs come in two disjoint shapes depending on how they were indexed: -- **Snapshot** (`update: snapshot`): content docs carry `git.commit` and no `git.ref`. The ref-name +- **Snapshot** (`index.strategy: snapshot`): content docs carry `git.commit` and no `git.ref`. The ref-name marker in `sourcerer-v3-refs` (keyed by `build_ref_id`, one per snapshot source) carries the commit and status. `git.commit` on the content row is already the answer; no join is needed to resolve it. -- **Incremental** (`update: incremental`): content docs carry `git.ref` and no `git.commit`. A +- **Incremental** (`index.strategy: incremental`): content docs carry `git.ref` and no `git.commit`. A dedicated refs join doc at `_id = build_ref_key(host,org,repo,ref)` (one per branch) holds the live HEAD commit and is advanced two-phase (INV-006). The join resolves `git.commit` from this doc. Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) uses -the same shape that handles both modes without fan-out: +the same shape that handles both index strategies without fan-out: ```esql FROM sourcerer-lines @@ -416,7 +417,7 @@ FROM sourcerer-lines // arm needs no join -- it just asserts status to match the incremental arm's column. // Incremental rows carry only git.ref; the join resolves the ref's current status from // its join doc. Safety of the incremental join (one doc per (host,org,repo,ref)) is -// enforced by the "one update mode owns a ref name" invariant at index time. +// enforced by the "one index strategy owns a ref name" invariant at index time. | FORK ( WHERE git.commit IS NOT NULL | EVAL status = "complete" ) @@ -434,9 +435,10 @@ one row each in the pre-FORK membership filter, which deduplicates naturally. **Incremental arm**: joins `sourcerer-refs ON (git.host, git.org, git.repo, git.ref)`. This join is safe (no fan-out) because there is always exactly one incremental join doc per `(host,org,repo,ref)`: all three incremental writers use `_id = build_ref_key(...)` (overwrite-in-place), the runtime -mode-conflict guard in `selection.py` prevents two selectors of different modes from claiming the same -ref name simultaneously, and the flip-status switchover marks any old snapshot marker `"stale"` BEFORE -the incremental join doc is published as `"complete"` — so the two-complete-docs window never opens. +strategy-conflict guard in `selection.py` prevents two selectors of different index strategies from +claiming the same ref name simultaneously, and the flip-status switchover marks any old snapshot +marker `"stale"` BEFORE the incremental join doc is published as `"complete"` — so the +two-complete-docs window never opens. **Scoping params** (`git_commit`, `git_ref`, `git_ref_type`) are all optional (default `"*"`) and support `*`/`?` wildcards (filters use `LIKE`). For a normal content question, resolve a ref first @@ -464,7 +466,7 @@ Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental |---|---| | `indexing` | A run is mid-flight. `indexing_started_at` is set; `indexed_at` is absent/null. Present on snapshot ref-name markers (written by `write_indexing_marker` just before ingest) and incremental join docs (written by `write_incremental_indexing`). A stale `indexing` doc whose `indexing_started_at` is older than the retry window (default 6 h) marks a crashed run and is treated as due for re-indexing. | | `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot markers) and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | -| `stale` | A snapshot marker superseded by a mode switch to incremental. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). Stale markers are invisible to all content tools (all gate on `status == "complete"`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | +| `stale` | A snapshot marker superseded by an index strategy switch to incremental. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). Stale markers are invisible to all content tools (all gate on `status == "complete"`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | #### Uniqueness gate (INV-011 backstop) @@ -474,7 +476,7 @@ Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental - **Snapshot** (git.commit IS NOT NULL in content): each distinct commit must have ≥1 complete refs doc (presence check — multi-ref-per-commit is legal). - **Incremental** (git.ref IS NOT NULL in content): each distinct ref must have **exactly one** - incremental join doc with `update_mode == "incremental"` (anti-fan-out guard for the surviving join). + incremental join doc with `index_strategy == "incremental"` (anti-fan-out guard for the surviving join). The gate is non-fatal (logs a warning, does not block): with the flip-status switchover in place, violations should only occur if a stale-flip was skipped or crashed mid-way; the next prune run diff --git a/README.md b/README.md index 7f14d33..aba18c2 100644 --- a/README.md +++ b/README.md @@ -85,31 +85,30 @@ Make sure you have [uv](https://docs.astral.sh/uv/) and [git](https://git-scm.co The [`sourcerer.yml` specification](specs/sourcerer-yml.md) has the full reference of fields supported by the configuration file. -### Snapshot vs. incremental indexing (`update: `) - -Each source can set `update: snapshot` (the default) or `update: incremental` (branch-only). -Every Agent Builder content tool takes the same `git_commit_ish` param either way (a commit SHA or -a branch/tag name, `*`/`?` wildcards supported) and resolves a commit the same way regardless of -mode; `git.ref_key` is an internal storage/join detail, never something the agent constructs or -passes. - -- **`snapshot`** (default): content is commit-addressed, exactly as before. `git.ref_key` is the - commit SHA itself, so every ref (branch, tag, or pinned commit) that resolves to the same - commit collapses to one snapshot. A moving branch's HEAD advance indexes a brand-new snapshot - under the new commit. -- **`incremental`** (branch-only): content is ref-addressed instead. `git.ref_key` is - `{host}~{org}~{repo}~{ref}` and content carries no `git.commit` of its own -- the branch's - current commit lives only on its refs join doc, resolved via the join above. A HEAD advance - re-indexes only the files `git diff --name-status` reports changed (add/modify/delete/rename), - not the whole tree, so staying current on a fast-moving branch (e.g. GitOps/IaC repos that - deploy off `main`) is cheap. `since` and `retain` don't apply to an incremental source (there is - no per-commit history to filter or retain) and are rejected if given. +### Snapshot vs. incremental indexing (`index.strategy`) + +Each source can set `index.strategy: snapshot` (the default) or `index.strategy: incremental` +(branch-only). Every Agent Builder content tool takes the same `git_commit_ish` param either way +(a commit SHA or a branch/tag name, `*`/`?` wildcards supported) and resolves a commit the same +way regardless of strategy. + +- **`snapshot`** (default): content is commit-addressed. Every ref (branch, tag, or pinned commit) + that resolves to the same commit collapses to one snapshot. A moving branch's HEAD advance indexes + a brand-new snapshot under the new commit. +- **`incremental`** (branch-only): content is ref-addressed instead. Content docs carry `git.ref` + but no `git.commit` of their own — the branch's current commit lives only on its refs join doc, + resolved at query time via a LOOKUP JOIN. A HEAD advance re-indexes only the files + `git diff --name-status` reports changed (add/modify/delete/rename), not the whole tree, so + staying current on a fast-moving branch (e.g. GitOps/IaC repos that deploy off `main`) is cheap. + `since` and `retain` don't apply to an incremental source (there is no per-commit history to + filter or retain) and are rejected if given. ```yaml sources: - git: { host: "github", org: "elastic", repo: "serverless-gitops", ref_type: "branch" } match: "main" - update: incremental + index: + strategy: incremental ``` Upgrading from a pre-`ref_key` install is automatic and invisible: every `index` run backfills diff --git a/sourcerer.example.yml b/sourcerer.example.yml index 01c2a8e..88a7580 100644 --- a/sourcerer.example.yml +++ b/sourcerer.example.yml @@ -184,20 +184,20 @@ sources: retain: count: 5 -# Incremental (ref-addressed) update mode -- branch-only. Instead of a new commit-addressed -# snapshot on every HEAD advance, content is keyed by git.ref_key = "{host}~{org}~{repo}~{ref}" -# and stays in place: a HEAD advance re-indexes only the files `git diff` reports changed -# (a delta update), rather than the whole tree. Good for a fast-moving branch that deploys off -# main, where staying current matters more than retaining per-commit history. `since` and -# `retain` are not meaningful here (there is no per-commit history to filter/retain -- see -# specs/incremental-indexing.md) and are rejected if given. +# Incremental index strategy -- branch-only. Instead of a new commit-addressed snapshot on +# every HEAD advance, content is keyed by git.ref and stays in place: a HEAD advance +# re-indexes only the files `git diff` reports changed (a delta update), rather than the whole +# tree. Good for a fast-moving branch that deploys off main, where staying current matters +# more than retaining per-commit history. `since` and `retain` are not meaningful here (there +# is no per-commit history to filter/retain) and are rejected if given. - git: host: github org: elastic repo: serverless-gitops ref_type: branch match: main - update: incremental # default: snapshot + index: + strategy: incremental # default: snapshot # Feature/fix branches as of a week ago; keep the newest commit, prune > 1 month. - git: diff --git a/specs/sourcerer-yml.md b/specs/sourcerer-yml.md index 1c224a2..c2a6570 100644 --- a/specs/sourcerer-yml.md +++ b/specs/sourcerer-yml.md @@ -50,6 +50,7 @@ performs its `setup`, `index`, and `prune` commands. |`sources[i].index` |Object |No || |`sources[i].index.level` |String |No || |`sources[i].index.suffix` |String |No || +|`sources[i].index.strategy` |String |No || Notes: - Fields can be expressed either in nested format or flat dotted format. @@ -707,6 +708,18 @@ For instance: - Cannot contain uppercase characters or whitespace characters - An empty string (`""`) is treated as omitted (`null`) +### `sources[i].index.strategy` + +Defines whether to index the content of each matching ref as an immutable commit +snapshot (`"snapshot"`) or maintain a single ref-addressed view that is updated +incrementally as the HEAD moves (`"incremental"`). + +- Required: No +- Type: String +- Default: `"snapshot"` +- Validation: + - Must be one of: `"snapshot"`, `"incremental"` + ## Example Here are the full example contents of sourcerer.yml that will replace repos.yml diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index dc2a08f..3b63129 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -278,7 +278,7 @@ def index_incremental_branch_in_dir( if reporter is None: reporter = ProgressReporter() if unit is None: - unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", update="incremental") + unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", index_strategy="incremental") reporter.set_stage(unit, "checkout") checkout_branch(repo_dir, branch) @@ -626,8 +626,8 @@ def process_group(item: tuple[tuple[str, str, str], list[Unit]]) -> None: # reuse -- each is a standalone two-phase delta update against its own prior state # (see index_incremental_branch_in_dir). Processed here, before the snapshot-only # `group` continues below with incremental units filtered out. - incremental_units = [u for u in group if u.update == "incremental"] - group = [u for u in group if u.update != "incremental"] + incremental_units = [u for u in group if u.index_strategy == "incremental"] + group = [u for u in group if u.index_strategy != "incremental"] for unit in incremental_units: reporter.start(unit) if incremental_units: diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index cd2f71b..0a7107e 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -268,7 +268,7 @@ def _init_worker( _WORKER_CTX.update( host=host, org=org, repo=repo, commit_sha=commit_sha, repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, - mode="snapshot", + strategy="snapshot", ) @@ -277,12 +277,12 @@ def _init_worker_incremental( index_level: str = "repo", index_suffix: str | None = None, ) -> None: """Same as `_init_worker`, but for the incremental (ref-addressed) path: `ref` replaces - `commit_sha` and `mode` routes `_build_one_file_actions` to the incremental doc builders.""" + `commit_sha` and `strategy` routes `_build_one_file_actions` to the incremental doc builders.""" signal.signal(signal.SIGINT, signal.SIG_IGN) _WORKER_CTX.update( host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, - mode="incremental", + strategy="incremental", ) @@ -303,7 +303,7 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: text. Runs in a worker process (see _init_worker for the shared context). Mirrors the old inline generator -- a binary file or one that can't be read yields only its file doc.""" ctx = _WORKER_CTX - incremental = ctx.get("mode", "snapshot") == "incremental" + incremental = ctx.get("strategy", "snapshot") == "incremental" host, org, repo = ctx["host"], ctx["org"], ctx["repo"] commit_sha = ctx.get("ref") if incremental else ctx["commit_sha"] level, suffix = ctx.get("index_level", "repo"), ctx.get("index_suffix") diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index 85d91e0..093c947 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -427,7 +427,7 @@ def write_indexing_marker( "commit": commit_sha, "commit_date": commit_date_iso, }, - "update_mode": "snapshot", + "index_strategy": "snapshot", "status": "indexing", "indexing_started_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "files_count": 0, @@ -475,7 +475,7 @@ def write_ref_marker( "commit": commit_sha, "commit_date": commit_date_iso, }, - "update_mode": "snapshot", + "index_strategy": "snapshot", "status": "complete", "files_count": files_count, "lines_count": lines_count, @@ -595,7 +595,7 @@ def _build_incremental_join_doc( "target_commit": target_commit, "commit_date": commit_date_iso, }, - "update_mode": "incremental", + "index_strategy": "incremental", "status": status, "files_count": files_count, "lines_count": lines_count, @@ -866,19 +866,16 @@ def apply_content_index_mapping(es: Elasticsearch, files_mapping: dict, lines_ma def stale_snapshot_markers_for_ref( es: Elasticsearch, host: str, org: str, repo: str, ref: str, ) -> list[dict]: - """Return any complete snapshot ref-name markers (update_mode: "snapshot", status: "complete") + """Return any complete snapshot ref-name markers (index_strategy: "snapshot", status: "complete") for (host, org, repo, ref). Used by the incremental index path to detect and mark stale snapshot - markers left behind by a mode switch. The must_not form is kept as a fallback for legacy markers - written before update_mode was added to snapshot markers.""" + markers left behind by an index strategy switch from snapshot to incremental.""" query = {"bool": {"filter": [ {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"term": {"git.ref": ref}}, {"term": {"status": "complete"}}, - # Exclude incremental join docs; matches update_mode="snapshot" and any legacy snapshot - # markers that predate the update_mode field. - {"bool": {"must_not": {"term": {"update_mode": "incremental"}}}}, + {"term": {"index_strategy": "snapshot"}}, ]}} try: resp = es.search(index=REFS_ALIAS, size=100, query=query, source_includes=["git.commit"]) diff --git a/src/sourcerer/commands/index/selection.py b/src/sourcerer/commands/index/selection.py index 4e7c585..ecb4b94 100644 --- a/src/sourcerer/commands/index/selection.py +++ b/src/sourcerer/commands/index/selection.py @@ -33,12 +33,13 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: # Phase 2's pre-clone skip check. fetched: dict[str, dict[str, str] | None] = {} seen: set[tuple[str, str]] = set() - # Maps (ref_type, name) -> update mode for the winning selector, so we can detect when a - # second selector of a DIFFERENT mode also claims the same ref. Mixed-mode refs are unsafe: - # the incremental LOOKUP JOIN ON git.ref requires exactly one refs doc per (host,org,repo,ref), - # but a snapshot marker and an incremental join doc would both be present (fan-out). - seen_mode: dict[tuple[str, str], str] = {} - mode_conflicts: list[tuple[str, str, str, str]] = [] # (ref_type, name, mode_a, mode_b) + # Maps (ref_type, name) -> index strategy for the winning selector, so we can detect when a + # second selector of a DIFFERENT strategy also claims the same ref. Mixed-strategy refs are + # unsafe: the incremental LOOKUP JOIN ON git.ref requires exactly one refs doc per + # (host,org,repo,ref), but a snapshot marker and an incremental join doc would both be present + # (fan-out). + seen_strategy: dict[tuple[str, str], str] = {} + strategy_conflicts: list[tuple[str, str, str, str]] = [] # (ref_type, name, strategy_a, strategy_b) units: list[Unit] = [] for sel in cfg.selectors: rt = sel.ref_type @@ -50,10 +51,11 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: if (rt, prefix) in seen: continue seen.add((rt, prefix)) - seen_mode[(rt, prefix)] = sel.update + seen_strategy[(rt, prefix)] = sel.index_strategy units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=prefix, kind=rt, - index_level=sel.index_level, index_suffix=sel.index_suffix, update=sel.update, + index_level=sel.index_level, index_suffix=sel.index_suffix, + index_strategy=sel.index_strategy, )) continue if rt not in fetched: @@ -70,27 +72,28 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: continue # below the since version floor key = (rt, name) if key in seen: - # Already claimed by an earlier selector: check for a mode conflict. - prior_mode = seen_mode[key] - if prior_mode != sel.update: - mode_conflicts.append((rt, name, prior_mode, sel.update)) + # Already claimed by an earlier selector: check for a strategy conflict. + prior_strategy = seen_strategy[key] + if prior_strategy != sel.index_strategy: + strategy_conflicts.append((rt, name, prior_strategy, sel.index_strategy)) continue seen.add(key) - seen_mode[key] = sel.update + seen_strategy[key] = sel.index_strategy units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=name, kind=rt, remote_sha=ref_map[name], - index_level=sel.index_level, index_suffix=sel.index_suffix, update=sel.update, + index_level=sel.index_level, index_suffix=sel.index_suffix, + index_strategy=sel.index_strategy, )) - if mode_conflicts: + if strategy_conflicts: conflicts_str = ", ".join( - f"{rt}/{name} ({mode_a} vs {mode_b})" - for rt, name, mode_a, mode_b in mode_conflicts + f"{rt}/{name} ({strategy_a} vs {strategy_b})" + for rt, name, strategy_a, strategy_b in strategy_conflicts ) click.echo( f"Warning: {cfg.org}/{cfg.repo}: selectors claim the same ref(s) with different " - f"update modes -- skipping all units for this repo to avoid fan-out: {conflicts_str}", + f"index strategies -- skipping all units for this repo to avoid fan-out: {conflicts_str}", err=True, ) return [] diff --git a/src/sourcerer/config.py b/src/sourcerer/config.py index 6c888ca..a56bc09 100644 --- a/src/sourcerer/config.py +++ b/src/sourcerer/config.py @@ -282,14 +282,13 @@ class Selector: retain: Retain | None levels: tuple[str, ...] = () # numeric levels shared by the versioned match patterns schedule: Schedule | None = None # per-source schedule override (sources[i].schedule) - # sources[i].index routing (see specs/sourcerer-yml.md): which physical files/lines index this - # source's content docs land in. Per-source, so two sources sharing a (host, org, repo) may - # route to different indices (e.g. kibana release tags -> ~repo, deploy tags -> ~repo^deploy). + # sources[i].index routing + strategy (see specs/sourcerer-yml.md): which physical + # files/lines index this source's content docs land in, and whether indexing is + # commit-addressed snapshots or ref-addressed incremental deltas. Per-source, so two + # sources sharing a (host, org, repo) may route differently. index_level: str = "repo" # "host" | "org" | "repo" | "commit" index_suffix: str | None = None # appended as ^{suffix}; None == no suffix - # sources[i].update: "snapshot" (default, commit-addressed content) or "incremental" - # (ref-addressed content, branch-only -- see specs/incremental-indexing.md). - update: str = "snapshot" + index_strategy: str = "snapshot" # "snapshot" (default) or "incremental" (branch-only) def matches(self, ref_type: str, ref: str) -> Version | None: if self.ref_type != ref_type: @@ -411,22 +410,25 @@ def _parse_commit_match(raw: dict, ctx: str) -> list[str]: _GIT_KEYS = {"host", "org", "repo", "ref_type"} _INDEX_LEVELS = ("host", "org", "repo", "commit") -_UPDATE_MODES = ("snapshot", "incremental") +_INDEX_STRATEGIES = ("snapshot", "incremental") # A suffix goes into a physical index name after a `^`, so it must be safe as an index-name # segment: the same characters forbidden in a host id, plus the `^` we use as the suffix delimiter. _FORBIDDEN_SUFFIX_CHARS = _FORBIDDEN_HOST_CHARS | {"^"} -def _parse_index(raw: dict, ctx: str) -> tuple[str, str | None]: - """Validate a source's `index:` block and return (level, suffix). `level` defaults to "repo"; - `suffix` defaults to None. An empty-string suffix is treated as omitted (per the spec). The - suffix charset mirrors the host-id rules (lowercase, no whitespace, no index-name-forbidden - chars) plus a ban on the `^` delimiter itself.""" +def _parse_index(raw: dict, ctx: str) -> tuple[str, str | None, str]: + """Validate a source's `index:` block and return (level, suffix, strategy). + + `level` defaults to "repo"; `suffix` defaults to None; `strategy` defaults to "snapshot". + An empty-string suffix is treated as omitted (per the spec). The suffix charset mirrors the + host-id rules (lowercase, no whitespace, no index-name-forbidden chars) plus a ban on the `^` + delimiter itself. Strategy "incremental" is rejected with index.level "commit" here (within- + block check) because incremental content is ref-addressed and cannot form a commit-keyed name.""" if not isinstance(raw, dict): - raise ValueError(f"{ctx} index: must be a mapping with 'level' and/or 'suffix'") - unknown = set(raw) - {"level", "suffix"} + raise ValueError(f"{ctx} index: must be a mapping with 'level', 'suffix', and/or 'strategy'") + unknown = set(raw) - {"level", "suffix", "strategy"} if unknown: - raise ValueError(f"{ctx} index: unknown keys {sorted(unknown)} (use 'level', 'suffix')") + raise ValueError(f"{ctx} index: unknown keys {sorted(unknown)} (use 'level', 'suffix', 'strategy')") level = "repo" if raw.get("level") is not None: @@ -448,7 +450,19 @@ def _parse_index(raw: dict, ctx: str) -> tuple[str, str | None]: if any(c.isspace() for c in s): raise ValueError(f"{ctx} index.suffix: {s!r} must not contain whitespace") suffix = s - return level, suffix + + strategy = "snapshot" + if raw.get("strategy") is not None: + strategy = raw["strategy"] + if strategy not in _INDEX_STRATEGIES: + raise ValueError(f"{ctx} index.strategy: must be one of {list(_INDEX_STRATEGIES)} " + f"(got {strategy!r})") + if strategy == "incremental" and level == "commit": + # Incremental content is ref-addressed (no git.commit on content docs), so a + # commit-level index name -- which requires a commit sha -- can never be built for it. + raise ValueError(f"{ctx} index.strategy: 'incremental' cannot be combined with " + f"'index.level: commit'") + return level, suffix, strategy def _parse_git_scope(raw: dict, ctx: str) -> tuple[str, str, str, str]: @@ -479,25 +493,28 @@ def _parse_git_scope(raw: dict, ctx: str) -> tuple[str, str, str, str]: def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: """Parse one `sources[i]` entry into (host, org, repo, Selector). The ref_type comes from the `git` block; `match`/`since`/`retain` are top-level siblings.""" - unknown = set(raw) - {"git", "match", "since", "retain", "schedule", "index", "update"} + unknown = set(raw) - {"git", "match", "since", "retain", "schedule", "index"} if unknown: raise ValueError(f"{ctx}: unknown keys {sorted(unknown)}") host, org, repo, ref_type = _parse_git_scope(raw, ctx) - update = raw.get("update", "snapshot") - if update not in _UPDATE_MODES: - raise ValueError(f"{ctx} update: must be one of {list(_UPDATE_MODES)} (got {update!r})") - if update == "incremental": + # Parse the index: block early so that index_strategy is available for the incremental + # constraints below (strategy and level are both validated inside _parse_index). + index_level, index_suffix, index_strategy = "repo", None, "snapshot" + if raw.get("index") is not None: + index_level, index_suffix, index_strategy = _parse_index(raw["index"], ctx) + + if index_strategy == "incremental": if ref_type != "branch": - raise ValueError(f"{ctx} update: 'incremental' is only valid for git.ref_type: branch " - f"(got ref_type {ref_type!r})") + raise ValueError(f"{ctx} index.strategy: 'incremental' is only valid for " + f"git.ref_type: branch (got ref_type {ref_type!r})") # An incremental branch maintains a single mutable ref-addressed view with no per-commit # history for retention to trim and no inclusion floor to apply -- both since and retain # are meaningless here (see specs/incremental-indexing.md). if raw.get("since") is not None: - raise ValueError(f"{ctx}: 'update: incremental' cannot be combined with 'since'") + raise ValueError(f"{ctx}: 'index.strategy: incremental' cannot be combined with 'since'") if raw.get("retain") is not None: - raise ValueError(f"{ctx}: 'update: incremental' cannot be combined with 'retain'") + raise ValueError(f"{ctx}: 'index.strategy: incremental' cannot be combined with 'retain'") if ref_type == "commit": # A pinned commit has no enumerable name to pattern-match against (see selection.py), @@ -544,17 +561,10 @@ def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: except ValueError as e: raise ValueError(f"{ctx} schedule: {e}") from e - index_level, index_suffix = "repo", None - if raw.get("index") is not None: - index_level, index_suffix = _parse_index(raw["index"], ctx) - if update == "incremental" and index_level == "commit": - # Incremental content is ref-addressed (no git.commit on content docs), so a - # commit-level index name -- which requires a commit sha -- can never be built for it. - raise ValueError(f"{ctx}: 'update: incremental' cannot be combined with 'index.level: commit'") - selector = Selector(ref_type=ref_type, raw_patterns=patterns, compiled=compiled, since=since, retain=retain, levels=levels, schedule=schedule, - index_level=index_level, index_suffix=index_suffix, update=update) + index_level=index_level, index_suffix=index_suffix, + index_strategy=index_strategy) return host, org, repo, selector diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index a45cd8a..bf85f7d 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -93,7 +93,7 @@ "index_suffix": { "type": "keyword" }, - "update_mode": { + "index_strategy": { "type": "keyword" } } diff --git a/src/sourcerer/progress.py b/src/sourcerer/progress.py index 40d8d2d..f08d45e 100644 --- a/src/sourcerer/progress.py +++ b/src/sourcerer/progress.py @@ -67,10 +67,10 @@ class Unit: # unit's content docs are written to; defaults reproduce the historical repo-level name. index_level: str = "repo" index_suffix: str | None = None - # sources[i].update carried from the selector that emitted this unit: "snapshot" (default, - # commit-addressed) or "incremental" (ref-addressed, branch-only). Routes the unit to the - # incremental delta-index path instead of the snapshot pre-clone/skip/retention flow. - update: str = "snapshot" + # sources[i].index.strategy carried from the selector that emitted this unit: "snapshot" + # (default, commit-addressed) or "incremental" (ref-addressed, branch-only). Routes the + # unit to the incremental delta-index path instead of the snapshot pre-clone/skip/retention flow. + index_strategy: str = "snapshot" @property def label(self) -> str: diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index 6f9115a..fd5eedc 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -262,10 +262,10 @@ def gather_intended_incremental_index_by_ref( Feeds the incremental stale-location sweep (Class D-I): content for a branch sitting in an index NOT in this set is stale from a crashed migration and should be reclaimed. Filters to - update_mode=="incremental" docs only, so snapshot markers (which always have git.commit) are + index_strategy=="incremental" docs only, so snapshot markers (which always have git.commit) are not double-counted. Returns {} if the refs index doesn't exist.""" out: dict[tuple[str, str, str, str], set[str]] = {} - body = {"query": {"term": {"update_mode": "incremental"}}} + body = {"query": {"term": {"index_strategy": "incremental"}}} src_fields = ["git.host", "git.org", "git.repo", "git.ref", "index_level", "index_suffix"] try: for hit in scan(es, index=REFS_ALIAS, query=body, _source=src_fields, preserve_order=False): @@ -414,13 +414,13 @@ def _enumerate_content_field( def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> list[str]: """Join-uniqueness gate (INV-011 backstop): verifies every content key maps to a correct - refs join doc. Split by content shape (no `update_mode` on content docs since d77726a): + refs join doc. Split by content shape (no `index_strategy` on content docs): - Snapshot (git.commit IS NOT NULL): each commit must resolve to ≥1 complete refs doc (presence check -- multi-ref-per-commit is legal; the snapshot FORK arm no longer joins so the uniqueness requirement there is already removed). - Incremental (git.ref IS NOT NULL): each ref must resolve to EXACTLY ONE refs doc with - `update_mode == "incremental"` -- the anti-fan-out invariant for the surviving join. + `index_strategy == "incremental"` -- the anti-fan-out invariant for the surviving join. Returns the sorted list of offending keys (commits/refs that fail their respective check); an empty list means the invariant holds.""" @@ -457,7 +457,7 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"terms": {"git.ref": sorted(refs)}}, - {"term": {"update_mode": "incremental"}}, + {"term": {"index_strategy": "incremental"}}, ]}}, aggs={"refs": {"terms": {"field": "git.ref", "size": len(refs)}}}, ) diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index 1e8ff21..5976e18 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -62,7 +62,7 @@ def test_content_tools_use_universal_ref_join_query(): for tid in _CONTENT_TOOL_IDS: query = tools[tid]["configuration"]["query"] params = tools[tid]["configuration"]["params"] - assert "update_mode" not in query, f"{tid} query has an update_mode conditional" + assert "index_strategy" not in query, f"{tid} query has an index_strategy conditional" # git.ref_key must not be used as a field or join key (comments may reference it by name) assert "git.ref_key" not in query, f"{tid} still uses git.ref_key as a field" assert "ON git.ref_key" not in query, f"{tid} still joins on git.ref_key" diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 99f699a..2f810f9 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -59,7 +59,7 @@ def test_missing_index_is_ignored(self): class TestStaleSnapshotMarkersForRef: def test_returns_complete_non_incremental_markers(self): - """Returns complete markers that are NOT update_mode=incremental (i.e. snapshot markers).""" + """Returns complete markers that are index_strategy=snapshot (i.e. snapshot markers).""" es = MagicMock() es.search.return_value = {"hits": {"hits": [ {"_id": "abc123", "_source": {"git": {"commit": "deadbeef"}}}, @@ -84,6 +84,7 @@ def test_query_scopes_to_host_org_repo_ref(self): assert {"term": {"git.repo": "widgets"}} in filt assert {"term": {"git.ref": "main"}} in filt assert {"term": {"status": "complete"}} in filt + assert {"term": {"index_strategy": "snapshot"}} in filt class TestMarkSnapshotMarkersStale: diff --git a/tests/test_config.py b/tests/test_config.py index 6f21ec9..9571df4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -53,7 +53,7 @@ def _git(host="github", org="acme", repo="widgets", ref_type="branch"): def _source(host="github", org="acme", repo="widgets", ref_type="branch", - match="main", since=None, retain=None, omit_match=False, update=None, index=None): + match="main", since=None, retain=None, omit_match=False, strategy=None, index=None): src = {"git": _git(host, org, repo, ref_type)} if not omit_match: src["match"] = match @@ -61,10 +61,12 @@ def _source(host="github", org="acme", repo="widgets", ref_type="branch", src["since"] = since if retain is not None: src["retain"] = retain - if update is not None: - src["update"] = update - if index is not None: - src["index"] = index + # strategy is a convenience shim: merges {"strategy": ...} into the index: block. + if strategy is not None or index is not None: + merged = dict(index or {}) + if strategy is not None: + merged["strategy"] = strategy + src["index"] = merged return src @@ -193,42 +195,47 @@ def test_versioned_patterns_agreeing_on_levels_is_fine(self): assert cfg.repos[0].selectors[0].levels == ("major", "minor", "patch") -class TestParseUpdateMode: +class TestParseIndexStrategy: def test_default_is_snapshot(self): cfg = _cfg([_source()]) - assert cfg.repos[0].selectors[0].update == "snapshot" + assert cfg.repos[0].selectors[0].index_strategy == "snapshot" def test_incremental_accepted_on_branch(self): - cfg = _cfg([_source(ref_type="branch", update="incremental")]) - assert cfg.repos[0].selectors[0].update == "incremental" + cfg = _cfg([_source(ref_type="branch", strategy="incremental")]) + assert cfg.repos[0].selectors[0].index_strategy == "incremental" def test_incremental_rejected_on_tag(self): with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): - _cfg([_source(ref_type="tag", match="v1.0.0", update="incremental")]) + _cfg([_source(ref_type="tag", match="v1.0.0", strategy="incremental")]) def test_incremental_rejected_on_commit(self): with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): - _cfg([_source(ref_type="commit", match="cfefb3b", update="incremental")]) + _cfg([_source(ref_type="commit", match="cfefb3b", strategy="incremental")]) - def test_invalid_mode_raises(self): + def test_invalid_strategy_raises(self): with pytest.raises(ValueError, match="must be one of"): - _cfg([_source(update="bogus")]) + _cfg([_source(strategy="bogus")]) def test_incremental_with_since_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'since'"): - _cfg([_source(ref_type="branch", update="incremental", since={"age": "1y"})]) + _cfg([_source(ref_type="branch", strategy="incremental", since={"age": "1y"})]) def test_incremental_with_retain_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'retain'"): - _cfg([_source(ref_type="branch", update="incremental", retain={"count": 5})]) + _cfg([_source(ref_type="branch", strategy="incremental", retain={"count": 5})]) def test_incremental_with_commit_level_index_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'index.level: commit'"): - _cfg([_source(ref_type="branch", update="incremental", index={"level": "commit"})]) + _cfg([_source(ref_type="branch", strategy="incremental", index={"level": "commit"})]) + + def test_top_level_update_key_raises(self): + with pytest.raises(ValueError, match="unknown keys"): + _cfg([{"git": {"host": "github", "org": "acme", "repo": "widgets", "ref_type": "branch"}, + "match": "main", "update": "incremental"}]) def test_incremental_with_repo_level_index_is_fine(self): - cfg = _cfg([_source(ref_type="branch", update="incremental", index={"level": "repo"})]) - assert cfg.repos[0].selectors[0].update == "incremental" + cfg = _cfg([_source(ref_type="branch", strategy="incremental", index={"level": "repo"})]) + assert cfg.repos[0].selectors[0].index_strategy == "incremental" class TestParseCommitSource: diff --git a/tests/test_documents.py b/tests/test_documents.py index 22cf9eb..0c2edf0 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -29,14 +29,14 @@ def _set_worker_ctx(host: str, org: str, repo: str, commit_sha: str, repo_dir, s # should leave behind for the rest of the pytest session. documents._WORKER_CTX.update( host=host, org=org, repo=repo, commit_sha=commit_sha, repo_dir=pathlib.Path(repo_dir), - symlink_paths=symlink_paths, mode="snapshot", + symlink_paths=symlink_paths, strategy="snapshot", ) def _set_worker_ctx_incremental(host: str, org: str, repo: str, ref: str, repo_dir, symlink_paths=frozenset()) -> None: documents._WORKER_CTX.update( host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), - symlink_paths=symlink_paths, mode="incremental", + symlink_paths=symlink_paths, strategy="incremental", ) diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py index 8cbabe9..12fd110 100644 --- a/tests/test_incremental_index.py +++ b/tests/test_incremental_index.py @@ -52,7 +52,7 @@ def test_first_index_does_full_rebuild(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - update="incremental") + index_strategy="incremental") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_called_once() @@ -75,7 +75,7 @@ def test_second_run_indexes_only_changed_paths(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - update="incremental") + index_strategy="incremental") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_not_called() @@ -95,7 +95,7 @@ def test_missing_diff_base_triggers_full_rebuild(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - update="incremental") + index_strategy="incremental") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_called_once() @@ -110,7 +110,7 @@ def test_no_change_skips_entirely(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - update="incremental") + index_strategy="incremental") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["index_incremental_paths"].assert_not_called() @@ -130,7 +130,7 @@ def test_failed_run_does_not_advance_commit(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - update="incremental") + index_strategy="incremental") try: index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) @@ -166,7 +166,7 @@ def test_suffix_change_forces_full_rebuild_and_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - update="incremental", index_level="repo", index_suffix="deploy") + index_strategy="incremental", index_level="repo", index_suffix="deploy") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Full rebuild path: delete_incremental_branch called at new routing, full tree indexed. @@ -193,7 +193,7 @@ def test_level_change_forces_full_rebuild_and_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - update="incremental", index_level="org", index_suffix=None) + index_strategy="incremental", index_level="org", index_suffix=None) index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) assert mocks["delete_incremental_branch"].call_count == 2 @@ -213,7 +213,7 @@ def test_no_changes_with_routing_change_still_migrates(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - update="incremental", index_level="repo", index_suffix="v2") + index_strategy="incremental", index_level="repo", index_suffix="v2") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Must NOT skip even though old_sha == new_sha. @@ -232,7 +232,7 @@ def test_same_routing_no_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - update="incremental", index_level="repo", index_suffix=None) + index_strategy="incremental", index_level="repo", index_suffix=None) index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Delta run: no full rebuild (delete_incremental_branch not called), no extra delete. diff --git a/tests/test_markers.py b/tests/test_markers.py index e1de463..6540780 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -414,7 +414,7 @@ def test_marker_carries_commit_no_ref_key(self): doc = es.index.call_args.kwargs["document"] assert doc["git"]["commit"] == OLD assert "ref_key" not in doc["git"] - assert doc["update_mode"] == "snapshot" + assert doc["index_strategy"] == "snapshot" def test_marker_id_is_hashed_not_the_commit(self): # _id is build_ref_id (BLAKE2b hash) -- one per (ref, commit), NOT the bare commit SHA. @@ -468,7 +468,7 @@ def test_incremental_marker_preserves_completed_commit_and_exposes_target(self): assert doc["status"] == "indexing" assert doc["git"]["commit"] == OLD # completed pointer unchanged (INV-006) assert doc["git"]["target_commit"] == NEW - assert doc["update_mode"] == "incremental" + assert doc["index_strategy"] == "incremental" assert es.index.call_args.kwargs["index"] == REFS_INDEX def test_incremental_marker_first_index_has_no_completed_commit(self): diff --git a/tests/test_uniqueness_gate.py b/tests/test_uniqueness_gate.py index 73ea52e..d87650a 100644 --- a/tests/test_uniqueness_gate.py +++ b/tests/test_uniqueness_gate.py @@ -1,7 +1,7 @@ """Tests for the post-index join-uniqueness gate: sourcerer.queries.check_join_uniqueness (INV-011) and its command.py wiring in _run_uniqueness_gate. Every ES call is mocked. -The gate is split by content shape (no update_mode on content docs since d77726a): +The gate is split by content shape (no index_strategy on content docs): - Snapshot (git.commit IS NOT NULL): each commit must have ≥1 complete refs doc. - Incremental (git.ref IS NOT NULL): each ref must have EXACTLY ONE incremental join doc. """ From 021eb591b1472991250a5e2b1c157562f529aa78 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Thu, 20 Aug 2026 09:19:09 -0400 Subject: [PATCH 18/29] Include git.sort in index sorting for files and lines. Enforce lowercase on index_level, index_suffix, and index_strategy in refs index. --- .../elastic/index_templates/sourcerer-v3-files.json | 2 ++ .../elastic/index_templates/sourcerer-v3-lines.json | 2 ++ .../elastic/index_templates/sourcerer-v3-refs.json | 9 ++++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json index eac08e3..80c9e6c 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json @@ -24,6 +24,7 @@ "git.org", "git.repo", "git.commit", + "git.ref", "file.path" ], "order": [ @@ -31,6 +32,7 @@ "asc", "asc", "asc", + "asc", "asc" ] } diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json index 04bafee..3a2623b 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json @@ -24,6 +24,7 @@ "git.org", "git.repo", "git.commit", + "git.ref", "file.path", "line.number" ], @@ -33,6 +34,7 @@ "asc", "asc", "asc", + "asc", "asc" ] } diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index bf85f7d..8191842 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -88,13 +88,16 @@ "type": "date" }, "index_level": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase" }, "index_suffix": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase" }, "index_strategy": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase" } } } From b2cf29a8ac81e7899cda3a5b1bebc6fcfc0f0ac6 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Thu, 20 Aug 2026 09:52:27 -0400 Subject: [PATCH 19/29] Update most kibana visualizations to filter by refs whose status is 'complete' --- .../kibana_saved_objects/export.ndjson | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/sourcerer/elastic/kibana_saved_objects/export.ndjson b/src/sourcerer/elastic/kibana_saved_objects/export.ndjson index b1cab79..85e05ee 100644 --- a/src/sourcerer/elastic/kibana_saved_objects/export.ndjson +++ b/src/sourcerer/elastic/kibana_saved_objects/export.ndjson @@ -1,13 +1,13 @@ -{"attributes":{"allowHidden":false,"fieldAttrs":"{}","fieldFormatMap":"{}","fields":"[]","name":"sourcerer-refs","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"git.commit_date","title":"sourcerer-refs"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"b576da63-38d3-418c-a541-06f87ebef58f","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM4NiwxMF0="} -{"attributes":{"color":"#FCD883","description":"","name":"Sourcerer"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","managed":false,"references":[],"type":"tag","typeMigrationVersion":"8.0.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzQ2MjU4LDEwXQ=="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Lines","operationType":"sum","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":2}}},"sourceField":"lines_count"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# lines indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"10307b0e-7f80-4095-bcaa-14adc9c41fac","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934664],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5MiwxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"72d88170-f38a-4d63-9d23-b48fedafe769":{"columnOrder":["gh0st0000-0000-0000-0000-000000000001","f2b9711d-0da6-4c79-8b66-f3176097b892","c07c2dd5-7d83-4924-ac44-9807d28cf11d","ff805016-2503-4100-ad05-e0cd8a29ead3","cb6b8976-ac47-4fc3-b887-e5988128269c","e6e8ddbe-0004-4982-8532-9f341343f74b","28c16acf-857e-41ae-9304-822350018ba6","e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b","5e6e2c2d-765b-4572-809c-e2f544371594","21887acc-0eac-4819-a90b-a6226cdcb3d7","a5fed915-466a-4b56-a0bd-6510f9b77a33"],"columns":{"21887acc-0eac-4819-a90b-a6226cdcb3d7":{"customLabel":true,"dataType":"date","filter":{"language":"kuery","query":"\"indexed_at\": *"},"isBucketed":false,"label":"Indexed at","operationType":"last_value","params":{"sortField":"git.commit_date"},"sourceField":"indexed_at"},"28c16acf-857e-41ae-9304-822350018ba6":{"customLabel":true,"dataType":"date","filter":{"language":"kuery","query":"\"git.commit_date\": *"},"isBucketed":false,"label":"Committed at","operationType":"last_value","params":{"showArrayValues":false,"sortField":"git.commit_date"},"sourceField":"git.commit_date"},"5e6e2c2d-765b-4572-809c-e2f544371594":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Lines","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"lines_count"},"a5fed915-466a-4b56-a0bd-6510f9b77a33":{"customLabel":true,"dataType":"string","filter":{"language":"kuery","query":"\"status\": *"},"isBucketed":false,"label":"Status","operationType":"last_value","params":{"sortField":"git.commit_date"},"sourceField":"status"},"c07c2dd5-7d83-4924-ac44-9807d28cf11d":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.repo"},"cb6b8976-ac47-4fc3-b887-e5988128269c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Type","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.ref_type"},"e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Files","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"files_count"},"e6e8ddbe-0004-4982-8532-9f341343f74b":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Commit","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.commit"},"f2b9711d-0da6-4c79-8b66-f3176097b892":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.org"},"ff805016-2503-4100-ad05-e0cd8a29ead3":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Ref","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.ref"},"gh0st0000-0000-0000-0000-000000000001":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Host","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.host"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"columns":[{"columnId":"gh0st0000-0000-0000-0000-000000000001","isMetric":false,"isTransposed":false},{"columnId":"f2b9711d-0da6-4c79-8b66-f3176097b892","isMetric":false,"isTransposed":false},{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","isMetric":true,"isTransposed":false},{"columnId":"a5fed915-466a-4b56-a0bd-6510f9b77a33","isMetric":true,"isTransposed":false},{"columnId":"e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b","isMetric":true,"isTransposed":false},{"columnId":"c07c2dd5-7d83-4924-ac44-9807d28cf11d","isMetric":false,"isTransposed":false},{"columnId":"ff805016-2503-4100-ad05-e0cd8a29ead3","isMetric":false,"isTransposed":false},{"columnId":"cb6b8976-ac47-4fc3-b887-e5988128269c","isMetric":false,"isTransposed":false},{"columnId":"21887acc-0eac-4819-a90b-a6226cdcb3d7","isMetric":true,"isTransposed":false},{"columnId":"28c16acf-857e-41ae-9304-822350018ba6","isMetric":true,"isTransposed":false,"width":113},{"columnId":"e6e8ddbe-0004-4982-8532-9f341343f74b","isMetric":false,"isTransposed":false}],"layerId":"72d88170-f38a-4d63-9d23-b48fedafe769","layerType":"data","showRowNumbers":true,"sorting":{"columnId":"28c16acf-857e-41ae-9304-822350018ba6","direction":"desc"}}},"title":"Indexed repos","version":2,"visualizationType":"lnsDatatable"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-11T21:19:34.310Z","created_by":"u_3477301940_cloud","id":"85ea5355-8301-455d-aaca-2063ba93ceb0","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-72d88170-f38a-4d63-9d23-b48fedafe769","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-11T21:19:34.310Z","updated_by":"u_3477301940_cloud","version":"WzEyMjYsMTFd"} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Orgs","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.org"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# orgs indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"3e86ded1-35d1-48cc-bae7-51338893a4cd","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934652],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM4OCwxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Repos","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.repo"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# repos indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"22ff37d8-794e-4067-9e39-b3ebb47b5db4","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934655],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM4OSwxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Refs","operationType":"count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.ref"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# refs indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"286f4d43-d0c8-4cb6-935b-b2c2a3eeefe2","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934658],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5MCwxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Files","operationType":"sum","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":2}}},"sourceField":"files_count"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# files indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"b674ebfd-a225-440c-bf40-a6c827e2bf1c","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934661],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5MSwxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b":{"columnOrder":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1","db869f77-feda-4a5a-8dc6-233b8436591c"],"columns":{"1861256b-df4b-4807-9157-12ec9b7de6ce":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"db869f77-feda-4a5a-8dc6-233b8436591c","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.org"},"4d6e7a79-1fc4-414f-b16a-86ee89fed2a1":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"db869f77-feda-4a5a-8dc6-233b8436591c","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.repo"},"db869f77-feda-4a5a-8dc6-233b8436591c":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Refs per repo","operationType":"count","params":{"emptyAsNull":true},"sourceField":"git.ref"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layers":[{"categoryDisplay":"default","colorMapping":{"assignments":[],"colorMode":{"type":"categorical"},"paletteId":"default","specialAssignments":[{"color":{"type":"loop"},"rules":[{"type":"other"}],"touched":false}]},"layerId":"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","layerType":"data","legendDisplay":"default","metrics":["db869f77-feda-4a5a-8dc6-233b8436591c"],"nestedLegend":false,"numberDisplay":"percent","primaryGroups":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1"]}],"shape":"treemap"}},"title":"# refs by org/repo","version":2,"visualizationType":"lnsPie"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"853efb3c-c141-497a-9f99-dff8c6eaf90a","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934667],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5MywxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b":{"columnOrder":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1","6d93874d-ceab-4271-9347-b7e9613c4bbc"],"columns":{"1861256b-df4b-4807-9157-12ec9b7de6ce":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"6d93874d-ceab-4271-9347-b7e9613c4bbc","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.org"},"4d6e7a79-1fc4-414f-b16a-86ee89fed2a1":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"6d93874d-ceab-4271-9347-b7e9613c4bbc","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.repo"},"6d93874d-ceab-4271-9347-b7e9613c4bbc":{"dataType":"number","isBucketed":false,"label":"Sum of lines_count","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"lines_count"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layers":[{"categoryDisplay":"default","colorMapping":{"assignments":[],"colorMode":{"type":"categorical"},"paletteId":"default","specialAssignments":[{"color":{"type":"loop"},"rules":[{"type":"other"}],"touched":false}]},"layerId":"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","layerType":"data","legendDisplay":"default","metrics":["6d93874d-ceab-4271-9347-b7e9613c4bbc"],"nestedLegend":false,"numberDisplay":"percent","primaryGroups":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1"]}],"shape":"treemap"}},"title":"# lines by org/repo","version":2,"visualizationType":"lnsPie"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"bb2c2db8-eda6-4127-ac97-89816eb00193","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934670],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5NCwxMF0="} -{"accessControl":{"accessMode":"default","owner":"u_RnkX2N-gU2tcwqOiST1LCMoTxDNJeOfiQwrhxwEzUoo_0"},"attributes":{"description":"","esqlApproximation":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"}}"},"optionsJSON":"{\"autoApplyFilters\":true,\"hidePanelBorders\":false,\"hidePanelTitles\":false,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"useMargins\":true}","panelsJSON":"[{\"type\":\"markdown\",\"embeddableConfig\":{\"hide_border\":true,\"content\":\"# Sourcerer indexing status\",\"settings\":{\"open_links_in_new_tab\":true}},\"panelIndex\":\"54f7337f-cbe3-4c82-82d6-aec205795654\",\"gridData\":{\"y\":0,\"x\":0,\"w\":18,\"h\":3,\"i\":\"54f7337f-cbe3-4c82-82d6-aec205795654\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"hide_title\":true,\"hide_border\":false},\"panelIndex\":\"015963bb-cbce-4f91-8bff-8a1aaddc6bbb\",\"gridData\":{\"y\":0,\"x\":18,\"w\":30,\"h\":26,\"i\":\"015963bb-cbce-4f91-8bff-8a1aaddc6bbb\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 2)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"67b82c98-6aff-42ab-b405-95412a3e9c58\",\"gridData\":{\"y\":3,\"x\":0,\"w\":3,\"h\":3,\"i\":\"67b82c98-6aff-42ab-b405-95412a3e9c58\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 1)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357\",\"gridData\":{\"y\":3,\"x\":3,\"w\":3,\"h\":3,\"i\":\"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 3)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d\",\"gridData\":{\"y\":3,\"x\":6,\"w\":3,\"h\":3,\"i\":\"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Files indexed\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"554c5394-bc05-44ca-84bb-95493167f04c\",\"gridData\":{\"y\":3,\"x\":9,\"w\":4,\"h\":3,\"i\":\"554c5394-bc05-44ca-84bb-95493167f04c\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"28f96a1f-5ba5-477f-bfb3-82ad6626994d\",\"gridData\":{\"y\":3,\"x\":13,\"w\":5,\"h\":3,\"i\":\"28f96a1f-5ba5-477f-bfb3-82ad6626994d\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"# refs by org/repo\",\"hide_border\":false},\"panelIndex\":\"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc\",\"gridData\":{\"y\":16,\"x\":0,\"w\":18,\"h\":10,\"i\":\"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"# lines of code by org/repo\",\"hide_border\":false},\"panelIndex\":\"d698f7cd-ee88-40a6-a826-a72826606507\",\"gridData\":{\"y\":6,\"x\":0,\"w\":18,\"h\":10,\"i\":\"d698f7cd-ee88-40a6-a826-a72826606507\"}}]","refreshInterval":{"pause":true,"value":1000},"timeFrom":"now-100y/d","timeRestore":true,"timeTo":"now","title":"[Sourcerer] Overview"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-11T15:14:23.909Z","id":"1bda5652-9556-4fc2-8ec1-d4e0c1e43eb7","managed":false,"references":[{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"},{"id":"85ea5355-8301-455d-aaca-2063ba93ceb0","name":"015963bb-cbce-4f91-8bff-8a1aaddc6bbb:savedObjectRef","type":"lens"},{"id":"3e86ded1-35d1-48cc-bae7-51338893a4cd","name":"67b82c98-6aff-42ab-b405-95412a3e9c58:savedObjectRef","type":"lens"},{"id":"22ff37d8-794e-4067-9e39-b3ebb47b5db4","name":"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357:savedObjectRef","type":"lens"},{"id":"286f4d43-d0c8-4cb6-935b-b2c2a3eeefe2","name":"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d:savedObjectRef","type":"lens"},{"id":"b674ebfd-a225-440c-bf40-a6c827e2bf1c","name":"554c5394-bc05-44ca-84bb-95493167f04c:savedObjectRef","type":"lens"},{"id":"10307b0e-7f80-4095-bcaa-14adc9c41fac","name":"28f96a1f-5ba5-477f-bfb3-82ad6626994d:savedObjectRef","type":"lens"},{"id":"853efb3c-c141-497a-9f99-dff8c6eaf90a","name":"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc:savedObjectRef","type":"lens"},{"id":"bb2c2db8-eda6-4127-ac97-89816eb00193","name":"d698f7cd-ee88-40a6-a826-a72826606507:savedObjectRef","type":"lens"}],"type":"dashboard","typeMigrationVersion":"10.3.0","updated_at":"2026-08-11T21:19:39.937Z","updated_by":"u_3477301940_cloud","version":"WzEyMjcsMTFd"} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Hosts","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.host"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# hosts indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"gh0st0000-0000-0000-0000-000000000004","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934683],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5NiwxMF0="} +{"attributes":{"color":"#FCD883","description":"","name":"Sourcerer"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:46:50.062Z","id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","managed":false,"references":[],"type":"tag","typeMigrationVersion":"8.0.0","updated_at":"2026-08-20T13:46:50.062Z","version":"WzY0NDgsNF0="} +{"attributes":{"allowHidden":false,"fieldAttrs":"{}","fieldFormatMap":"{}","fields":"[]","name":"sourcerer-refs","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"git.commit_date","title":"sourcerer-refs"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:46:50.062Z","id":"b576da63-38d3-418c-a541-06f87ebef58f","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2026-08-20T13:46:50.062Z","version":"WzYwNCw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"72d88170-f38a-4d63-9d23-b48fedafe769":{"columnOrder":["gh0st0000-0000-0000-0000-000000000001","f2b9711d-0da6-4c79-8b66-f3176097b892","c07c2dd5-7d83-4924-ac44-9807d28cf11d","ff805016-2503-4100-ad05-e0cd8a29ead3","cb6b8976-ac47-4fc3-b887-e5988128269c","e6e8ddbe-0004-4982-8532-9f341343f74b","28c16acf-857e-41ae-9304-822350018ba6","e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b","5e6e2c2d-765b-4572-809c-e2f544371594","21887acc-0eac-4819-a90b-a6226cdcb3d7","a5fed915-466a-4b56-a0bd-6510f9b77a33"],"columns":{"21887acc-0eac-4819-a90b-a6226cdcb3d7":{"customLabel":true,"dataType":"date","filter":{"language":"kuery","query":"\"indexed_at\": *"},"isBucketed":false,"label":"Indexed at","operationType":"last_value","params":{"sortField":"git.commit_date"},"sourceField":"indexed_at"},"28c16acf-857e-41ae-9304-822350018ba6":{"customLabel":true,"dataType":"date","filter":{"language":"kuery","query":"\"git.commit_date\": *"},"isBucketed":false,"label":"Committed at","operationType":"last_value","params":{"showArrayValues":false,"sortField":"git.commit_date"},"sourceField":"git.commit_date"},"5e6e2c2d-765b-4572-809c-e2f544371594":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Lines","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"lines_count"},"a5fed915-466a-4b56-a0bd-6510f9b77a33":{"customLabel":true,"dataType":"string","filter":{"language":"kuery","query":"\"status\": *"},"isBucketed":false,"label":"Status","operationType":"last_value","params":{"sortField":"git.commit_date"},"sourceField":"status"},"c07c2dd5-7d83-4924-ac44-9807d28cf11d":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.repo"},"cb6b8976-ac47-4fc3-b887-e5988128269c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Type","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.ref_type"},"e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Files","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"files_count"},"e6e8ddbe-0004-4982-8532-9f341343f74b":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Commit","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.commit"},"f2b9711d-0da6-4c79-8b66-f3176097b892":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.org"},"ff805016-2503-4100-ad05-e0cd8a29ead3":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Ref","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.ref"},"gh0st0000-0000-0000-0000-000000000001":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Host","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.host"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"columns":[{"columnId":"gh0st0000-0000-0000-0000-000000000001","isMetric":false,"isTransposed":false},{"columnId":"f2b9711d-0da6-4c79-8b66-f3176097b892","isMetric":false,"isTransposed":false},{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","isMetric":true,"isTransposed":false},{"columnId":"a5fed915-466a-4b56-a0bd-6510f9b77a33","isMetric":true,"isTransposed":false},{"columnId":"e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b","isMetric":true,"isTransposed":false},{"columnId":"c07c2dd5-7d83-4924-ac44-9807d28cf11d","isMetric":false,"isTransposed":false},{"columnId":"ff805016-2503-4100-ad05-e0cd8a29ead3","isMetric":false,"isTransposed":false},{"columnId":"cb6b8976-ac47-4fc3-b887-e5988128269c","isMetric":false,"isTransposed":false},{"columnId":"21887acc-0eac-4819-a90b-a6226cdcb3d7","isMetric":true,"isTransposed":false},{"columnId":"28c16acf-857e-41ae-9304-822350018ba6","isMetric":true,"isTransposed":false,"width":113},{"columnId":"e6e8ddbe-0004-4982-8532-9f341343f74b","isMetric":false,"isTransposed":false}],"layerId":"72d88170-f38a-4d63-9d23-b48fedafe769","layerType":"data","showRowNumbers":true,"sorting":{"columnId":"28c16acf-857e-41ae-9304-822350018ba6","direction":"desc"}}},"title":"Indexed repos","version":2,"visualizationType":"lnsDatatable"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:46:50.062Z","id":"85ea5355-8301-455d-aaca-2063ba93ceb0","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-72d88170-f38a-4d63-9d23-b48fedafe769","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:46:50.062Z","version":"WzYwNiw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Orgs","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.org"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"density":"compact","layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# orgs indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:49:10.261Z","created_by":"u_3477301940_cloud","id":"3e86ded1-35d1-48cc-bae7-51338893a4cd","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:49:10.261Z","updated_by":"u_3477301940_cloud","version":"WzYyMyw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Repos","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.repo"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"density":"compact","layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# repos indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:49:21.120Z","created_by":"u_3477301940_cloud","id":"22ff37d8-794e-4067-9e39-b3ebb47b5db4","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:49:21.120Z","updated_by":"u_3477301940_cloud","version":"WzYyNiw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Refs","operationType":"count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.ref"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"density":"compact","layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# refs indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:49:30.878Z","created_by":"u_3477301940_cloud","id":"286f4d43-d0c8-4cb6-935b-b2c2a3eeefe2","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:49:30.878Z","updated_by":"u_3477301940_cloud","version":"WzYyOCw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Files","operationType":"sum","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":2}}},"sourceField":"files_count"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"density":"compact","layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# files indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:49:51.503Z","created_by":"u_3477301940_cloud","id":"b674ebfd-a225-440c-bf40-a6c827e2bf1c","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:49:51.503Z","updated_by":"u_3477301940_cloud","version":"WzYzNSw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Lines","operationType":"sum","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":2}}},"sourceField":"lines_count"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"density":"compact","layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# lines indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:49:46.957Z","created_by":"u_3477301940_cloud","id":"10307b0e-7f80-4095-bcaa-14adc9c41fac","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:49:46.957Z","updated_by":"u_3477301940_cloud","version":"WzYzMyw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b":{"columnOrder":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1","db869f77-feda-4a5a-8dc6-233b8436591c"],"columns":{"1861256b-df4b-4807-9157-12ec9b7de6ce":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"db869f77-feda-4a5a-8dc6-233b8436591c","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.org"},"4d6e7a79-1fc4-414f-b16a-86ee89fed2a1":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"db869f77-feda-4a5a-8dc6-233b8436591c","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.repo"},"db869f77-feda-4a5a-8dc6-233b8436591c":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Refs per repo","operationType":"count","params":{"emptyAsNull":true},"sourceField":"git.ref"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"layers":[{"categoryDisplay":"default","colorMapping":{"assignments":[],"colorMode":{"type":"categorical"},"paletteId":"default","specialAssignments":[{"color":{"type":"loop"},"rules":[{"type":"other"}],"touched":false}]},"layerId":"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","layerType":"data","legendDisplay":"default","metrics":["db869f77-feda-4a5a-8dc6-233b8436591c"],"nestedLegend":false,"numberDisplay":"percent","primaryGroups":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1"]}],"shape":"treemap"}},"title":"# refs by org/repo","version":2,"visualizationType":"lnsPie"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:48:46.252Z","created_by":"u_3477301940_cloud","id":"853efb3c-c141-497a-9f99-dff8c6eaf90a","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:48:46.252Z","updated_by":"u_3477301940_cloud","version":"WzYxNyw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b":{"columnOrder":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1","6d93874d-ceab-4271-9347-b7e9613c4bbc"],"columns":{"1861256b-df4b-4807-9157-12ec9b7de6ce":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"6d93874d-ceab-4271-9347-b7e9613c4bbc","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.org"},"4d6e7a79-1fc4-414f-b16a-86ee89fed2a1":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"6d93874d-ceab-4271-9347-b7e9613c4bbc","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.repo"},"6d93874d-ceab-4271-9347-b7e9613c4bbc":{"dataType":"number","isBucketed":false,"label":"Sum of lines_count","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"lines_count"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"layers":[{"categoryDisplay":"default","colorMapping":{"assignments":[],"colorMode":{"type":"categorical"},"paletteId":"default","specialAssignments":[{"color":{"type":"loop"},"rules":[{"type":"other"}],"touched":false}]},"layerId":"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","layerType":"data","legendDisplay":"default","metrics":["6d93874d-ceab-4271-9347-b7e9613c4bbc"],"nestedLegend":false,"numberDisplay":"percent","primaryGroups":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1"]}],"shape":"treemap"}},"title":"# lines by org/repo","version":2,"visualizationType":"lnsPie"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:48:58.003Z","created_by":"u_3477301940_cloud","id":"bb2c2db8-eda6-4127-ac97-89816eb00193","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:48:58.003Z","updated_by":"u_3477301940_cloud","version":"WzYyMCw1XQ=="} +{"attributes":{"description":"","esqlApproximation":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"}}"},"optionsJSON":"{\"autoApplyFilters\":true,\"hidePanelBorders\":false,\"hidePanelTitles\":false,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"useMargins\":true}","panelsJSON":"[{\"type\":\"markdown\",\"embeddableConfig\":{\"hide_border\":true,\"content\":\"# Sourcerer indexing status\",\"settings\":{\"open_links_in_new_tab\":true}},\"panelIndex\":\"54f7337f-cbe3-4c82-82d6-aec205795654\",\"gridData\":{\"y\":0,\"x\":0,\"w\":18,\"h\":3,\"i\":\"54f7337f-cbe3-4c82-82d6-aec205795654\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"hide_title\":true,\"hide_border\":false},\"panelIndex\":\"015963bb-cbce-4f91-8bff-8a1aaddc6bbb\",\"gridData\":{\"y\":0,\"x\":18,\"w\":30,\"h\":26,\"i\":\"015963bb-cbce-4f91-8bff-8a1aaddc6bbb\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 2)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"67b82c98-6aff-42ab-b405-95412a3e9c58\",\"gridData\":{\"y\":3,\"x\":0,\"w\":3,\"h\":3,\"i\":\"67b82c98-6aff-42ab-b405-95412a3e9c58\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 1)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357\",\"gridData\":{\"y\":3,\"x\":3,\"w\":3,\"h\":3,\"i\":\"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 3)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d\",\"gridData\":{\"y\":3,\"x\":6,\"w\":3,\"h\":3,\"i\":\"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Files indexed\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"554c5394-bc05-44ca-84bb-95493167f04c\",\"gridData\":{\"y\":3,\"x\":9,\"w\":4,\"h\":3,\"i\":\"554c5394-bc05-44ca-84bb-95493167f04c\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"28f96a1f-5ba5-477f-bfb3-82ad6626994d\",\"gridData\":{\"y\":3,\"x\":13,\"w\":5,\"h\":3,\"i\":\"28f96a1f-5ba5-477f-bfb3-82ad6626994d\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"# refs by org/repo\",\"hide_border\":false},\"panelIndex\":\"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc\",\"gridData\":{\"y\":16,\"x\":0,\"w\":18,\"h\":10,\"i\":\"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"# lines of code by org/repo\",\"hide_border\":false},\"panelIndex\":\"d698f7cd-ee88-40a6-a826-a72826606507\",\"gridData\":{\"y\":6,\"x\":0,\"w\":18,\"h\":10,\"i\":\"d698f7cd-ee88-40a6-a826-a72826606507\"}}]","refreshInterval":{"pause":true,"value":1000},"timeFrom":"now-100y/d","timeRestore":true,"timeTo":"now","title":"[Sourcerer] Overview"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:46:50.062Z","id":"1bda5652-9556-4fc2-8ec1-d4e0c1e43eb7","managed":false,"references":[{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"},{"id":"85ea5355-8301-455d-aaca-2063ba93ceb0","name":"015963bb-cbce-4f91-8bff-8a1aaddc6bbb:savedObjectRef","type":"lens"},{"id":"3e86ded1-35d1-48cc-bae7-51338893a4cd","name":"67b82c98-6aff-42ab-b405-95412a3e9c58:savedObjectRef","type":"lens"},{"id":"22ff37d8-794e-4067-9e39-b3ebb47b5db4","name":"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357:savedObjectRef","type":"lens"},{"id":"286f4d43-d0c8-4cb6-935b-b2c2a3eeefe2","name":"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d:savedObjectRef","type":"lens"},{"id":"b674ebfd-a225-440c-bf40-a6c827e2bf1c","name":"554c5394-bc05-44ca-84bb-95493167f04c:savedObjectRef","type":"lens"},{"id":"10307b0e-7f80-4095-bcaa-14adc9c41fac","name":"28f96a1f-5ba5-477f-bfb3-82ad6626994d:savedObjectRef","type":"lens"},{"id":"853efb3c-c141-497a-9f99-dff8c6eaf90a","name":"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc:savedObjectRef","type":"lens"},{"id":"bb2c2db8-eda6-4127-ac97-89816eb00193","name":"d698f7cd-ee88-40a6-a826-a72826606507:savedObjectRef","type":"lens"}],"type":"dashboard","typeMigrationVersion":"10.3.0","updated_at":"2026-08-20T13:49:53.424Z","updated_by":"u_3477301940_cloud","version":"WzYzNyw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Hosts","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.host"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# hosts indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:46:50.062Z","id":"gh0st0000-0000-0000-0000-000000000004","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:46:50.062Z","version":"WzYxNCw1XQ=="} {"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":12,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file From e7d052c0ba1ce59007aa5ea5ef575e9ede323c9e Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Thu, 20 Aug 2026 10:06:48 -0400 Subject: [PATCH 20/29] Rename git.target_commit to git.commit_target and make it official field in the template for sourcerer-v3-refs. --- src/sourcerer/commands/index/command.py | 6 +++--- src/sourcerer/commands/index/markers.py | 20 +++++++++---------- .../index_templates/sourcerer-v3-refs.json | 4 ++++ tests/test_markers.py | 12 +++++------ 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index 3b63129..26e2deb 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -303,7 +303,7 @@ def index_incremental_branch_in_dir( reporter.set_stage(unit, "indexing") write_incremental_indexing(es, host, org, repo, branch, completed_commit=old_sha, - target_commit=new_sha, prior=prior, + commit_target=new_sha, prior=prior, index_level=level, index_suffix=suffix) try: full_rebuild = old_sha is None or force or routing_changed @@ -353,12 +353,12 @@ def index_incremental_branch_in_dir( index_level=old_level, index_suffix=old_suffix) except KeyboardInterrupt: write_incremental_failed(es, host, org, repo, branch, completed_commit=old_sha, - target_commit=new_sha, error="interrupted", prior=prior, + commit_target=new_sha, error="interrupted", prior=prior, index_level=level, index_suffix=suffix) raise except Exception as e: write_incremental_failed(es, host, org, repo, branch, completed_commit=old_sha, - target_commit=new_sha, error=str(e), prior=prior, + commit_target=new_sha, error=str(e), prior=prior, index_level=level, index_suffix=suffix) raise reporter.finish(unit, "indexed", indexed_files, indexed_lines) diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index 093c947..e9b72d9 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -573,7 +573,7 @@ def _build_incremental_join_doc( *, status: str, commit: str | None, - target_commit: str | None = None, + commit_target: str | None = None, commit_date_iso: str | None = None, files_count: int = 0, lines_count: int = 0, @@ -592,7 +592,7 @@ def _build_incremental_join_doc( "ref": ref, "ref_type": "branch", "commit": commit, - "target_commit": target_commit, + "commit_target": commit_target, "commit_date": commit_date_iso, }, "index_strategy": "incremental", @@ -615,15 +615,15 @@ def write_incremental_indexing( repo: str, ref: str, completed_commit: str | None, - target_commit: str, + commit_target: str, prior: dict | None = None, refresh: bool = False, index_level: str = "repo", index_suffix: str | None = None, ) -> None: """Publish `status: indexing`: the completed pointer (`git.commit`) stays at the LAST - completed SHA (or None on a first index) while `git.target_commit` advertises the candidate - SHA the run is advancing to. A failed run never overwrites `git.commit` with `target_commit` + completed SHA (or None on a first index) while `git.commit_target` advertises the candidate + SHA the run is advancing to. A failed run never overwrites `git.commit` with `commit_target` (INV-006) -- only `write_incremental_ready` does that, after delete+index+refresh succeed.""" prior = prior or {} pg = prior.get("git", {}) @@ -631,7 +631,7 @@ def write_incremental_indexing( host, org, repo, ref, status="indexing", commit=completed_commit, - target_commit=target_commit, + commit_target=commit_target, commit_date_iso=pg.get("commit_date"), files_count=prior.get("files_count", 0), lines_count=prior.get("lines_count", 0), @@ -659,14 +659,14 @@ def write_incremental_ready( index_level: str = "repo", index_suffix: str | None = None, ) -> None: - """Publish `status: complete` at the NEW completed commit, clearing `target_commit` and any + """Publish `status: complete` at the NEW completed commit, clearing `commit_target` and any prior failure fields. This is the pointer-advancing publication boundary (INV-006): callers must delete+index+refresh the content indices FIRST, then call this.""" doc = _build_incremental_join_doc( host, org, repo, ref, status="complete", commit=commit, - target_commit=None, + commit_target=None, commit_date_iso=commit_date_iso, files_count=files_count, lines_count=lines_count, @@ -687,7 +687,7 @@ def write_incremental_failed( repo: str, ref: str, completed_commit: str | None, - target_commit: str | None, + commit_target: str | None, error: str, prior: dict | None = None, refresh: bool = False, @@ -703,7 +703,7 @@ def write_incremental_failed( host, org, repo, ref, status="indexing", commit=completed_commit, - target_commit=target_commit, + commit_target=commit_target, commit_date_iso=pg.get("commit_date"), files_count=prior.get("files_count", 0), lines_count=prior.get("lines_count", 0), diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index 8191842..ffad527 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -68,6 +68,10 @@ }, "commit_date": { "type": "date" + }, + "commit_target": { + "type": "keyword", + "normalizer": "lowercase" } } }, diff --git a/tests/test_markers.py b/tests/test_markers.py index 6540780..6cb750f 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -463,18 +463,18 @@ class TestWriteIncrementalIndexing: def test_incremental_marker_preserves_completed_commit_and_exposes_target(self): es = MagicMock() write_incremental_indexing(es, "github", "acme", "widgets", "main", - completed_commit=OLD, target_commit=NEW) + completed_commit=OLD, commit_target=NEW) doc = _indexed_doc(es) assert doc["status"] == "indexing" assert doc["git"]["commit"] == OLD # completed pointer unchanged (INV-006) - assert doc["git"]["target_commit"] == NEW + assert doc["git"]["commit_target"] == NEW assert doc["index_strategy"] == "incremental" assert es.index.call_args.kwargs["index"] == REFS_INDEX def test_incremental_marker_first_index_has_no_completed_commit(self): es = MagicMock() write_incremental_indexing(es, "github", "acme", "widgets", "main", - completed_commit=None, target_commit=NEW) + completed_commit=None, commit_target=NEW) assert _indexed_doc(es)["git"]["commit"] is None def test_incremental_marker_carries_prior_counts(self): @@ -510,7 +510,7 @@ def test_incremental_marker_advances_commit_and_clears_target_and_error(self): doc = _indexed_doc(es) assert doc["status"] == "complete" assert doc["git"]["commit"] == NEW # advances only after a successful run (INV-006) - assert doc["git"]["target_commit"] is None + assert doc["git"]["commit_target"] is None assert doc["error"] is None and doc["failed_at"] is None assert doc["files_count"] == 5 and doc["lines_count"] == 99 assert es.index.call_args.kwargs["refresh"] is True # publication boundary @@ -529,11 +529,11 @@ class TestWriteIncrementalFailed: def test_incremental_marker_keeps_status_indexing_and_retains_old_pointer(self): es = MagicMock() write_incremental_failed(es, "github", "acme", "widgets", "main", completed_commit=OLD, - target_commit=NEW, error="boom") + commit_target=NEW, error="boom") doc = _indexed_doc(es) assert doc["status"] == "indexing" # not advanced -- a failed run leaves the prior state assert doc["git"]["commit"] == OLD - assert doc["git"]["target_commit"] == NEW + assert doc["git"]["commit_target"] == NEW assert doc["error"] == "boom" assert doc["failed_at"] is not None From 7c6cf1d201eeb60f9c5508798b2d84d16daefe19 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Fri, 21 Aug 2026 12:42:17 -0400 Subject: [PATCH 21/29] Rename index_mode and sources[i].index.mode to simply mode and sources[i].mode --- AGENTS.md | 19 +++-- README.md | 13 ++- sourcerer.example.yml | 15 ++-- specs/sourcerer-yml.md | 30 ++++--- src/sourcerer/commands/index/command.py | 6 +- src/sourcerer/commands/index/documents.py | 8 +- src/sourcerer/commands/index/markers.py | 12 +-- src/sourcerer/commands/index/selection.py | 37 +++++---- src/sourcerer/config.py | 79 +++++++++---------- .../index_templates/sourcerer-v3-refs.json | 2 +- src/sourcerer/progress.py | 8 +- src/sourcerer/queries.py | 10 +-- tests/test_agent_builder_tools.py | 3 +- tests/test_backfill.py | 4 +- tests/test_config.py | 38 +++++---- tests/test_documents.py | 4 +- tests/test_incremental_index.py | 18 ++--- tests/test_markers.py | 4 +- tests/test_uniqueness_gate.py | 2 +- 19 files changed, 155 insertions(+), 157 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b4d473c..bf01d22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,9 +57,9 @@ config's `sources:` is a YAML list, one entry per (host, org, repo, ref_type). S | `match` | yes | For `branch`/`tag`: pattern string or list of patterns matched against ref names (version DSL + glob), a ref matches if any pattern hits. For `commit`: a commit SHA/prefix string or list of them (see below). | | `since` | no | Index-side inclusion floor: the earliest commit to start indexing from. See below. Not valid for `git.ref_type: commit`. | | `retain` | no | Retention policy (see below). Omit to keep forever. For `git.ref_type: commit`, only `age` is valid. | -| `index.strategy` | no | `snapshot` (default) or `incremental` (branch-only). See below. | +| `mode` | no | `snapshot` (default) or `incremental` (branch-only). See below. | -#### `index.strategy` (snapshot vs. incremental) +#### `mode` (snapshot vs. incremental) `snapshot` (default): content is commit-addressed. A HEAD advance on a branch indexes a whole new snapshot under the new commit. @@ -80,8 +80,7 @@ succeed, so a crash mid-update leaves the prior commit and content in place. repo: serverless-gitops ref_type: branch match: main - index: - strategy: incremental + mode: incremental ``` #### `git.ref_type: commit` (pinning an explicit commit) @@ -377,15 +376,15 @@ creates one index/shard per commit — see `specs/sourcerer-yml.md` for the cave Content docs come in two disjoint shapes depending on how they were indexed: -- **Snapshot** (`index.strategy: snapshot`): content docs carry `git.commit` and no `git.ref`. The ref-name +- **Snapshot** (`mode: snapshot`): content docs carry `git.commit` and no `git.ref`. The ref-name marker in `sourcerer-v3-refs` (keyed by `build_ref_id`, one per snapshot source) carries the commit and status. `git.commit` on the content row is already the answer; no join is needed to resolve it. -- **Incremental** (`index.strategy: incremental`): content docs carry `git.ref` and no `git.commit`. A +- **Incremental** (`mode: incremental`): content docs carry `git.ref` and no `git.commit`. A dedicated refs join doc at `_id = build_ref_key(host,org,repo,ref)` (one per branch) holds the live HEAD commit and is advanced two-phase (INV-006). The join resolves `git.commit` from this doc. Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) uses -the same shape that handles both index strategies without fan-out: +the same shape that handles both modes without fan-out: ```esql FROM sourcerer-lines @@ -417,7 +416,7 @@ FROM sourcerer-lines // arm needs no join -- it just asserts status to match the incremental arm's column. // Incremental rows carry only git.ref; the join resolves the ref's current status from // its join doc. Safety of the incremental join (one doc per (host,org,repo,ref)) is -// enforced by the "one index strategy owns a ref name" invariant at index time. +// enforced by the "one mode owns a ref name" invariant at index time. | FORK ( WHERE git.commit IS NOT NULL | EVAL status = "complete" ) @@ -466,7 +465,7 @@ Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental |---|---| | `indexing` | A run is mid-flight. `indexing_started_at` is set; `indexed_at` is absent/null. Present on snapshot ref-name markers (written by `write_indexing_marker` just before ingest) and incremental join docs (written by `write_incremental_indexing`). A stale `indexing` doc whose `indexing_started_at` is older than the retry window (default 6 h) marks a crashed run and is treated as due for re-indexing. | | `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot markers) and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | -| `stale` | A snapshot marker superseded by an index strategy switch to incremental. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). Stale markers are invisible to all content tools (all gate on `status == "complete"`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | +| `stale` | A snapshot marker superseded by a mode switch to incremental. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). Stale markers are invisible to all content tools (all gate on `status == "complete"`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | #### Uniqueness gate (INV-011 backstop) @@ -476,7 +475,7 @@ Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental - **Snapshot** (git.commit IS NOT NULL in content): each distinct commit must have ≥1 complete refs doc (presence check — multi-ref-per-commit is legal). - **Incremental** (git.ref IS NOT NULL in content): each distinct ref must have **exactly one** - incremental join doc with `index_strategy == "incremental"` (anti-fan-out guard for the surviving join). + incremental join doc with `mode == "incremental"` (anti-fan-out guard for the surviving join). The gate is non-fatal (logs a warning, does not block): with the flip-status switchover in place, violations should only occur if a stale-flip was skipped or crashed mid-way; the next prune run diff --git a/README.md b/README.md index aba18c2..55d9358 100644 --- a/README.md +++ b/README.md @@ -85,12 +85,12 @@ Make sure you have [uv](https://docs.astral.sh/uv/) and [git](https://git-scm.co The [`sourcerer.yml` specification](specs/sourcerer-yml.md) has the full reference of fields supported by the configuration file. -### Snapshot vs. incremental indexing (`index.strategy`) +### Snapshot vs. incremental indexing (`mode`) -Each source can set `index.strategy: snapshot` (the default) or `index.strategy: incremental` -(branch-only). Every Agent Builder content tool takes the same `git_commit_ish` param either way -(a commit SHA or a branch/tag name, `*`/`?` wildcards supported) and resolves a commit the same -way regardless of strategy. +Each source can set `mode: snapshot` (the default) or `mode: incremental` (branch-only). Every +Agent Builder content tool takes the same `git_commit_ish` param either way (a commit SHA or a +branch/tag name, `*`/`?` wildcards supported) and resolves a commit the same way regardless of +mode. - **`snapshot`** (default): content is commit-addressed. Every ref (branch, tag, or pinned commit) that resolves to the same commit collapses to one snapshot. A moving branch's HEAD advance indexes @@ -107,8 +107,7 @@ way regardless of strategy. sources: - git: { host: "github", org: "elastic", repo: "serverless-gitops", ref_type: "branch" } match: "main" - index: - strategy: incremental + mode: incremental ``` Upgrading from a pre-`ref_key` install is automatic and invisible: every `index` run backfills diff --git a/sourcerer.example.yml b/sourcerer.example.yml index 88a7580..08aa8e6 100644 --- a/sourcerer.example.yml +++ b/sourcerer.example.yml @@ -184,20 +184,19 @@ sources: retain: count: 5 -# Incremental index strategy -- branch-only. Instead of a new commit-addressed snapshot on -# every HEAD advance, content is keyed by git.ref and stays in place: a HEAD advance -# re-indexes only the files `git diff` reports changed (a delta update), rather than the whole -# tree. Good for a fast-moving branch that deploys off main, where staying current matters -# more than retaining per-commit history. `since` and `retain` are not meaningful here (there -# is no per-commit history to filter/retain) and are rejected if given. +# Incremental mode -- branch-only. Instead of a new commit-addressed snapshot on every HEAD +# advance, content is keyed by git.ref and stays in place: a HEAD advance re-indexes only the +# files `git diff` reports changed (a delta update), rather than the whole tree. Good for a +# fast-moving branch that deploys off main, where staying current matters more than retaining +# per-commit history. `since` and `retain` are not meaningful here (there is no per-commit +# history to filter/retain) and are rejected if given. - git: host: github org: elastic repo: serverless-gitops ref_type: branch match: main - index: - strategy: incremental # default: snapshot + mode: incremental # default: snapshot # Feature/fix branches as of a week ago; keep the newest commit, prune > 1 month. - git: diff --git a/specs/sourcerer-yml.md b/specs/sourcerer-yml.md index c2a6570..b035b0c 100644 --- a/specs/sourcerer-yml.md +++ b/specs/sourcerer-yml.md @@ -31,6 +31,7 @@ performs its `setup`, `index`, and `prune` commands. |`sources[i].git.repo` |String |Yes || |`sources[i].git.ref_type` |String |Yes || |`sources[i].match` |String, Array[String]|No || +|`sources[i].mode` |String |No || |`sources[i].since` |Object |No || |`sources[i].since.age` |String |No || |`sources[i].since.date` |String |No || @@ -50,7 +51,6 @@ performs its `setup`, `index`, and `prune` commands. |`sources[i].index` |Object |No || |`sources[i].index.level` |String |No || |`sources[i].index.suffix` |String |No || -|`sources[i].index.strategy` |String |No || Notes: - Fields can be expressed either in nested format or flat dotted format. @@ -413,6 +413,22 @@ for indexing if they don't also qualify for pruning. - Type: String or Array[String] - Default: `null` (omitted) +### `sources[i].mode` + +Defines whether to index the content of each matching ref as an immutable commit +snapshot (`"snapshot"`) or maintain a single ref-addressed view that is updated +incrementally as the HEAD moves (`"incremental"`). Controls whether `since` and +`retain` apply (both are rejected when `mode` is `"incremental"`). + +- Required: No +- Type: String +- Default: `"snapshot"` +- Validation: + - Must be one of: `"snapshot"`, `"incremental"` + - `"incremental"` is only valid when `git.ref_type` is `"branch"` + - `"incremental"` cannot be combined with `since` or `retain` + - `"incremental"` cannot be combined with `index.level: commit` + ### `sources[i].since` Sets the earliest point in commit history to index. Exactly one child field can @@ -708,18 +724,6 @@ For instance: - Cannot contain uppercase characters or whitespace characters - An empty string (`""`) is treated as omitted (`null`) -### `sources[i].index.strategy` - -Defines whether to index the content of each matching ref as an immutable commit -snapshot (`"snapshot"`) or maintain a single ref-addressed view that is updated -incrementally as the HEAD moves (`"incremental"`). - -- Required: No -- Type: String -- Default: `"snapshot"` -- Validation: - - Must be one of: `"snapshot"`, `"incremental"` - ## Example Here are the full example contents of sourcerer.yml that will replace repos.yml diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index 26e2deb..0400625 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -278,7 +278,7 @@ def index_incremental_branch_in_dir( if reporter is None: reporter = ProgressReporter() if unit is None: - unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", index_strategy="incremental") + unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", mode="incremental") reporter.set_stage(unit, "checkout") checkout_branch(repo_dir, branch) @@ -626,8 +626,8 @@ def process_group(item: tuple[tuple[str, str, str], list[Unit]]) -> None: # reuse -- each is a standalone two-phase delta update against its own prior state # (see index_incremental_branch_in_dir). Processed here, before the snapshot-only # `group` continues below with incremental units filtered out. - incremental_units = [u for u in group if u.index_strategy == "incremental"] - group = [u for u in group if u.index_strategy != "incremental"] + incremental_units = [u for u in group if u.mode == "incremental"] + group = [u for u in group if u.mode != "incremental"] for unit in incremental_units: reporter.start(unit) if incremental_units: diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index 0a7107e..cd2f71b 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -268,7 +268,7 @@ def _init_worker( _WORKER_CTX.update( host=host, org=org, repo=repo, commit_sha=commit_sha, repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, - strategy="snapshot", + mode="snapshot", ) @@ -277,12 +277,12 @@ def _init_worker_incremental( index_level: str = "repo", index_suffix: str | None = None, ) -> None: """Same as `_init_worker`, but for the incremental (ref-addressed) path: `ref` replaces - `commit_sha` and `strategy` routes `_build_one_file_actions` to the incremental doc builders.""" + `commit_sha` and `mode` routes `_build_one_file_actions` to the incremental doc builders.""" signal.signal(signal.SIGINT, signal.SIG_IGN) _WORKER_CTX.update( host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, - strategy="incremental", + mode="incremental", ) @@ -303,7 +303,7 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: text. Runs in a worker process (see _init_worker for the shared context). Mirrors the old inline generator -- a binary file or one that can't be read yields only its file doc.""" ctx = _WORKER_CTX - incremental = ctx.get("strategy", "snapshot") == "incremental" + incremental = ctx.get("mode", "snapshot") == "incremental" host, org, repo = ctx["host"], ctx["org"], ctx["repo"] commit_sha = ctx.get("ref") if incremental else ctx["commit_sha"] level, suffix = ctx.get("index_level", "repo"), ctx.get("index_suffix") diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index e9b72d9..7777365 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -427,7 +427,7 @@ def write_indexing_marker( "commit": commit_sha, "commit_date": commit_date_iso, }, - "index_strategy": "snapshot", + "mode": "snapshot", "status": "indexing", "indexing_started_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "files_count": 0, @@ -475,7 +475,7 @@ def write_ref_marker( "commit": commit_sha, "commit_date": commit_date_iso, }, - "index_strategy": "snapshot", + "mode": "snapshot", "status": "complete", "files_count": files_count, "lines_count": lines_count, @@ -595,7 +595,7 @@ def _build_incremental_join_doc( "commit_target": commit_target, "commit_date": commit_date_iso, }, - "index_strategy": "incremental", + "mode": "incremental", "status": status, "files_count": files_count, "lines_count": lines_count, @@ -866,16 +866,16 @@ def apply_content_index_mapping(es: Elasticsearch, files_mapping: dict, lines_ma def stale_snapshot_markers_for_ref( es: Elasticsearch, host: str, org: str, repo: str, ref: str, ) -> list[dict]: - """Return any complete snapshot ref-name markers (index_strategy: "snapshot", status: "complete") + """Return any complete snapshot ref-name markers (mode: "snapshot", status: "complete") for (host, org, repo, ref). Used by the incremental index path to detect and mark stale snapshot - markers left behind by an index strategy switch from snapshot to incremental.""" + markers left behind by a mode switch from snapshot to incremental.""" query = {"bool": {"filter": [ {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"term": {"git.ref": ref}}, {"term": {"status": "complete"}}, - {"term": {"index_strategy": "snapshot"}}, + {"term": {"mode": "snapshot"}}, ]}} try: resp = es.search(index=REFS_ALIAS, size=100, query=query, source_includes=["git.commit"]) diff --git a/src/sourcerer/commands/index/selection.py b/src/sourcerer/commands/index/selection.py index ecb4b94..6d2d194 100644 --- a/src/sourcerer/commands/index/selection.py +++ b/src/sourcerer/commands/index/selection.py @@ -33,13 +33,12 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: # Phase 2's pre-clone skip check. fetched: dict[str, dict[str, str] | None] = {} seen: set[tuple[str, str]] = set() - # Maps (ref_type, name) -> index strategy for the winning selector, so we can detect when a - # second selector of a DIFFERENT strategy also claims the same ref. Mixed-strategy refs are - # unsafe: the incremental LOOKUP JOIN ON git.ref requires exactly one refs doc per - # (host,org,repo,ref), but a snapshot marker and an incremental join doc would both be present - # (fan-out). - seen_strategy: dict[tuple[str, str], str] = {} - strategy_conflicts: list[tuple[str, str, str, str]] = [] # (ref_type, name, strategy_a, strategy_b) + # Maps (ref_type, name) -> mode for the winning selector, so we can detect when a second + # selector of a DIFFERENT mode also claims the same ref. Mixed-mode refs are unsafe: the + # incremental LOOKUP JOIN ON git.ref requires exactly one refs doc per (host,org,repo,ref), + # but a snapshot marker and an incremental join doc would both be present (fan-out). + seen_mode: dict[tuple[str, str], str] = {} + mode_conflicts: list[tuple[str, str, str, str]] = [] # (ref_type, name, mode_a, mode_b) units: list[Unit] = [] for sel in cfg.selectors: rt = sel.ref_type @@ -51,11 +50,11 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: if (rt, prefix) in seen: continue seen.add((rt, prefix)) - seen_strategy[(rt, prefix)] = sel.index_strategy + seen_mode[(rt, prefix)] = sel.mode units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=prefix, kind=rt, index_level=sel.index_level, index_suffix=sel.index_suffix, - index_strategy=sel.index_strategy, + mode=sel.mode, )) continue if rt not in fetched: @@ -72,28 +71,28 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: continue # below the since version floor key = (rt, name) if key in seen: - # Already claimed by an earlier selector: check for a strategy conflict. - prior_strategy = seen_strategy[key] - if prior_strategy != sel.index_strategy: - strategy_conflicts.append((rt, name, prior_strategy, sel.index_strategy)) + # Already claimed by an earlier selector: check for a mode conflict. + prior_mode = seen_mode[key] + if prior_mode != sel.mode: + mode_conflicts.append((rt, name, prior_mode, sel.mode)) continue seen.add(key) - seen_strategy[key] = sel.index_strategy + seen_mode[key] = sel.mode units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=name, kind=rt, remote_sha=ref_map[name], index_level=sel.index_level, index_suffix=sel.index_suffix, - index_strategy=sel.index_strategy, + mode=sel.mode, )) - if strategy_conflicts: + if mode_conflicts: conflicts_str = ", ".join( - f"{rt}/{name} ({strategy_a} vs {strategy_b})" - for rt, name, strategy_a, strategy_b in strategy_conflicts + f"{rt}/{name} ({mode_a} vs {mode_b})" + for rt, name, mode_a, mode_b in mode_conflicts ) click.echo( f"Warning: {cfg.org}/{cfg.repo}: selectors claim the same ref(s) with different " - f"index strategies -- skipping all units for this repo to avoid fan-out: {conflicts_str}", + f"modes -- skipping all units for this repo to avoid fan-out: {conflicts_str}", err=True, ) return [] diff --git a/src/sourcerer/config.py b/src/sourcerer/config.py index a56bc09..19160df 100644 --- a/src/sourcerer/config.py +++ b/src/sourcerer/config.py @@ -282,13 +282,15 @@ class Selector: retain: Retain | None levels: tuple[str, ...] = () # numeric levels shared by the versioned match patterns schedule: Schedule | None = None # per-source schedule override (sources[i].schedule) - # sources[i].index routing + strategy (see specs/sourcerer-yml.md): which physical - # files/lines index this source's content docs land in, and whether indexing is - # commit-addressed snapshots or ref-addressed incremental deltas. Per-source, so two - # sources sharing a (host, org, repo) may route differently. + # sources[i].mode: the indexing mode for this source -- "snapshot" (default, commit-addressed) + # or "incremental" (ref-addressed, branch-only). Controls whether since/retain apply and routes + # the unit to the incremental delta-index path instead of the snapshot flow. + mode: str = "snapshot" # "snapshot" (default) or "incremental" (branch-only) + # sources[i].index routing (see specs/sourcerer-yml.md): which physical files/lines index this + # source's content docs land in. Per-source, so two sources sharing a (host, org, repo) may + # route differently. index_level: str = "repo" # "host" | "org" | "repo" | "commit" index_suffix: str | None = None # appended as ^{suffix}; None == no suffix - index_strategy: str = "snapshot" # "snapshot" (default) or "incremental" (branch-only) def matches(self, ref_type: str, ref: str) -> Version | None: if self.ref_type != ref_type: @@ -410,25 +412,23 @@ def _parse_commit_match(raw: dict, ctx: str) -> list[str]: _GIT_KEYS = {"host", "org", "repo", "ref_type"} _INDEX_LEVELS = ("host", "org", "repo", "commit") -_INDEX_STRATEGIES = ("snapshot", "incremental") +_MODES = ("snapshot", "incremental") # A suffix goes into a physical index name after a `^`, so it must be safe as an index-name # segment: the same characters forbidden in a host id, plus the `^` we use as the suffix delimiter. _FORBIDDEN_SUFFIX_CHARS = _FORBIDDEN_HOST_CHARS | {"^"} -def _parse_index(raw: dict, ctx: str) -> tuple[str, str | None, str]: - """Validate a source's `index:` block and return (level, suffix, strategy). +def _parse_index(raw: dict, ctx: str) -> tuple[str, str | None]: + """Validate a source's `index:` block and return (level, suffix). - `level` defaults to "repo"; `suffix` defaults to None; `strategy` defaults to "snapshot". - An empty-string suffix is treated as omitted (per the spec). The suffix charset mirrors the - host-id rules (lowercase, no whitespace, no index-name-forbidden chars) plus a ban on the `^` - delimiter itself. Strategy "incremental" is rejected with index.level "commit" here (within- - block check) because incremental content is ref-addressed and cannot form a commit-keyed name.""" + `level` defaults to "repo"; `suffix` defaults to None. An empty-string suffix is treated as + omitted (per the spec). The suffix charset mirrors the host-id rules (lowercase, no whitespace, + no index-name-forbidden chars) plus a ban on the `^` delimiter itself.""" if not isinstance(raw, dict): - raise ValueError(f"{ctx} index: must be a mapping with 'level', 'suffix', and/or 'strategy'") - unknown = set(raw) - {"level", "suffix", "strategy"} + raise ValueError(f"{ctx} index: must be a mapping with 'level' and/or 'suffix'") + unknown = set(raw) - {"level", "suffix"} if unknown: - raise ValueError(f"{ctx} index: unknown keys {sorted(unknown)} (use 'level', 'suffix', 'strategy')") + raise ValueError(f"{ctx} index: unknown keys {sorted(unknown)} (use 'level', 'suffix')") level = "repo" if raw.get("level") is not None: @@ -451,18 +451,7 @@ def _parse_index(raw: dict, ctx: str) -> tuple[str, str | None, str]: raise ValueError(f"{ctx} index.suffix: {s!r} must not contain whitespace") suffix = s - strategy = "snapshot" - if raw.get("strategy") is not None: - strategy = raw["strategy"] - if strategy not in _INDEX_STRATEGIES: - raise ValueError(f"{ctx} index.strategy: must be one of {list(_INDEX_STRATEGIES)} " - f"(got {strategy!r})") - if strategy == "incremental" and level == "commit": - # Incremental content is ref-addressed (no git.commit on content docs), so a - # commit-level index name -- which requires a commit sha -- can never be built for it. - raise ValueError(f"{ctx} index.strategy: 'incremental' cannot be combined with " - f"'index.level: commit'") - return level, suffix, strategy + return level, suffix def _parse_git_scope(raw: dict, ctx: str) -> tuple[str, str, str, str]: @@ -492,29 +481,40 @@ def _parse_git_scope(raw: dict, ctx: str) -> tuple[str, str, str, str]: def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: """Parse one `sources[i]` entry into (host, org, repo, Selector). The ref_type comes from the - `git` block; `match`/`since`/`retain` are top-level siblings.""" - unknown = set(raw) - {"git", "match", "since", "retain", "schedule", "index"} + `git` block; `match`/`since`/`retain`/`mode` are top-level siblings.""" + unknown = set(raw) - {"git", "match", "since", "retain", "schedule", "index", "mode"} if unknown: raise ValueError(f"{ctx}: unknown keys {sorted(unknown)}") host, org, repo, ref_type = _parse_git_scope(raw, ctx) - # Parse the index: block early so that index_strategy is available for the incremental - # constraints below (strategy and level are both validated inside _parse_index). - index_level, index_suffix, index_strategy = "repo", None, "snapshot" + # Parse mode early so it is available for the incremental constraints below. + mode = "snapshot" + if raw.get("mode") is not None: + mode = raw["mode"] + if mode not in _MODES: + raise ValueError(f"{ctx} mode: must be one of {list(_MODES)} (got {mode!r})") + + # Parse the index: block for routing (level + suffix only; mode is now top-level). + index_level, index_suffix = "repo", None if raw.get("index") is not None: - index_level, index_suffix, index_strategy = _parse_index(raw["index"], ctx) + index_level, index_suffix = _parse_index(raw["index"], ctx) - if index_strategy == "incremental": + if mode == "incremental": if ref_type != "branch": - raise ValueError(f"{ctx} index.strategy: 'incremental' is only valid for " + raise ValueError(f"{ctx} mode: 'incremental' is only valid for " f"git.ref_type: branch (got ref_type {ref_type!r})") # An incremental branch maintains a single mutable ref-addressed view with no per-commit # history for retention to trim and no inclusion floor to apply -- both since and retain # are meaningless here (see specs/incremental-indexing.md). if raw.get("since") is not None: - raise ValueError(f"{ctx}: 'index.strategy: incremental' cannot be combined with 'since'") + raise ValueError(f"{ctx}: 'mode: incremental' cannot be combined with 'since'") if raw.get("retain") is not None: - raise ValueError(f"{ctx}: 'index.strategy: incremental' cannot be combined with 'retain'") + raise ValueError(f"{ctx}: 'mode: incremental' cannot be combined with 'retain'") + if index_level == "commit": + # Incremental content is ref-addressed (no git.commit on content docs), so a + # commit-level index name -- which requires a commit sha -- can never be built for it. + raise ValueError(f"{ctx} mode: 'incremental' cannot be combined with " + f"'index.level: commit'") if ref_type == "commit": # A pinned commit has no enumerable name to pattern-match against (see selection.py), @@ -563,8 +563,7 @@ def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: selector = Selector(ref_type=ref_type, raw_patterns=patterns, compiled=compiled, since=since, retain=retain, levels=levels, schedule=schedule, - index_level=index_level, index_suffix=index_suffix, - index_strategy=index_strategy) + mode=mode, index_level=index_level, index_suffix=index_suffix) return host, org, repo, selector diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index ffad527..2134e98 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -99,7 +99,7 @@ "type": "keyword", "normalizer": "lowercase" }, - "index_strategy": { + "mode": { "type": "keyword", "normalizer": "lowercase" } diff --git a/src/sourcerer/progress.py b/src/sourcerer/progress.py index f08d45e..6b026f3 100644 --- a/src/sourcerer/progress.py +++ b/src/sourcerer/progress.py @@ -67,10 +67,10 @@ class Unit: # unit's content docs are written to; defaults reproduce the historical repo-level name. index_level: str = "repo" index_suffix: str | None = None - # sources[i].index.strategy carried from the selector that emitted this unit: "snapshot" - # (default, commit-addressed) or "incremental" (ref-addressed, branch-only). Routes the - # unit to the incremental delta-index path instead of the snapshot pre-clone/skip/retention flow. - index_strategy: str = "snapshot" + # sources[i].mode carried from the selector that emitted this unit: "snapshot" (default, + # commit-addressed) or "incremental" (ref-addressed, branch-only). Routes the unit to the + # incremental delta-index path instead of the snapshot pre-clone/skip/retention flow. + mode: str = "snapshot" @property def label(self) -> str: diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index fd5eedc..38ebb2f 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -262,10 +262,10 @@ def gather_intended_incremental_index_by_ref( Feeds the incremental stale-location sweep (Class D-I): content for a branch sitting in an index NOT in this set is stale from a crashed migration and should be reclaimed. Filters to - index_strategy=="incremental" docs only, so snapshot markers (which always have git.commit) are + mode=="incremental" docs only, so snapshot markers (which always have git.commit) are not double-counted. Returns {} if the refs index doesn't exist.""" out: dict[tuple[str, str, str, str], set[str]] = {} - body = {"query": {"term": {"index_strategy": "incremental"}}} + body = {"query": {"term": {"mode": "incremental"}}} src_fields = ["git.host", "git.org", "git.repo", "git.ref", "index_level", "index_suffix"] try: for hit in scan(es, index=REFS_ALIAS, query=body, _source=src_fields, preserve_order=False): @@ -414,13 +414,13 @@ def _enumerate_content_field( def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> list[str]: """Join-uniqueness gate (INV-011 backstop): verifies every content key maps to a correct - refs join doc. Split by content shape (no `index_strategy` on content docs): + refs join doc. Split by content shape (no `mode` on content docs): - Snapshot (git.commit IS NOT NULL): each commit must resolve to ≥1 complete refs doc (presence check -- multi-ref-per-commit is legal; the snapshot FORK arm no longer joins so the uniqueness requirement there is already removed). - Incremental (git.ref IS NOT NULL): each ref must resolve to EXACTLY ONE refs doc with - `index_strategy == "incremental"` -- the anti-fan-out invariant for the surviving join. + `mode == "incremental"` -- the anti-fan-out invariant for the surviving join. Returns the sorted list of offending keys (commits/refs that fail their respective check); an empty list means the invariant holds.""" @@ -457,7 +457,7 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"terms": {"git.ref": sorted(refs)}}, - {"term": {"index_strategy": "incremental"}}, + {"term": {"mode": "incremental"}}, ]}}, aggs={"refs": {"terms": {"field": "git.ref", "size": len(refs)}}}, ) diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index 5976e18..f59b39e 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -62,7 +62,8 @@ def test_content_tools_use_universal_ref_join_query(): for tid in _CONTENT_TOOL_IDS: query = tools[tid]["configuration"]["query"] params = tools[tid]["configuration"]["params"] - assert "index_strategy" not in query, f"{tid} query has an index_strategy conditional" + assert "index_strategy" not in query, f"{tid} query references the removed index_strategy field" + assert "mode" not in strip_esql_comments(query), f"{tid} query filters on the refs-only mode field" # git.ref_key must not be used as a field or join key (comments may reference it by name) assert "git.ref_key" not in query, f"{tid} still uses git.ref_key as a field" assert "ON git.ref_key" not in query, f"{tid} still joins on git.ref_key" diff --git a/tests/test_backfill.py b/tests/test_backfill.py index 2f810f9..d214ab6 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -59,7 +59,7 @@ def test_missing_index_is_ignored(self): class TestStaleSnapshotMarkersForRef: def test_returns_complete_non_incremental_markers(self): - """Returns complete markers that are index_strategy=snapshot (i.e. snapshot markers).""" + """Returns complete markers that are mode=snapshot (i.e. snapshot markers).""" es = MagicMock() es.search.return_value = {"hits": {"hits": [ {"_id": "abc123", "_source": {"git": {"commit": "deadbeef"}}}, @@ -84,7 +84,7 @@ def test_query_scopes_to_host_org_repo_ref(self): assert {"term": {"git.repo": "widgets"}} in filt assert {"term": {"git.ref": "main"}} in filt assert {"term": {"status": "complete"}} in filt - assert {"term": {"index_strategy": "snapshot"}} in filt + assert {"term": {"mode": "snapshot"}} in filt class TestMarkSnapshotMarkersStale: diff --git a/tests/test_config.py b/tests/test_config.py index 9571df4..11a4088 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -53,7 +53,7 @@ def _git(host="github", org="acme", repo="widgets", ref_type="branch"): def _source(host="github", org="acme", repo="widgets", ref_type="branch", - match="main", since=None, retain=None, omit_match=False, strategy=None, index=None): + match="main", since=None, retain=None, omit_match=False, mode=None, index=None): src = {"git": _git(host, org, repo, ref_type)} if not omit_match: src["match"] = match @@ -61,12 +61,10 @@ def _source(host="github", org="acme", repo="widgets", ref_type="branch", src["since"] = since if retain is not None: src["retain"] = retain - # strategy is a convenience shim: merges {"strategy": ...} into the index: block. - if strategy is not None or index is not None: - merged = dict(index or {}) - if strategy is not None: - merged["strategy"] = strategy - src["index"] = merged + if mode is not None: + src["mode"] = mode + if index is not None: + src["index"] = index return src @@ -195,38 +193,38 @@ def test_versioned_patterns_agreeing_on_levels_is_fine(self): assert cfg.repos[0].selectors[0].levels == ("major", "minor", "patch") -class TestParseIndexStrategy: +class TestParseMode: def test_default_is_snapshot(self): cfg = _cfg([_source()]) - assert cfg.repos[0].selectors[0].index_strategy == "snapshot" + assert cfg.repos[0].selectors[0].mode == "snapshot" def test_incremental_accepted_on_branch(self): - cfg = _cfg([_source(ref_type="branch", strategy="incremental")]) - assert cfg.repos[0].selectors[0].index_strategy == "incremental" + cfg = _cfg([_source(ref_type="branch", mode="incremental")]) + assert cfg.repos[0].selectors[0].mode == "incremental" def test_incremental_rejected_on_tag(self): with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): - _cfg([_source(ref_type="tag", match="v1.0.0", strategy="incremental")]) + _cfg([_source(ref_type="tag", match="v1.0.0", mode="incremental")]) def test_incremental_rejected_on_commit(self): with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): - _cfg([_source(ref_type="commit", match="cfefb3b", strategy="incremental")]) + _cfg([_source(ref_type="commit", match="cfefb3b", mode="incremental")]) - def test_invalid_strategy_raises(self): + def test_invalid_mode_raises(self): with pytest.raises(ValueError, match="must be one of"): - _cfg([_source(strategy="bogus")]) + _cfg([_source(mode="bogus")]) def test_incremental_with_since_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'since'"): - _cfg([_source(ref_type="branch", strategy="incremental", since={"age": "1y"})]) + _cfg([_source(ref_type="branch", mode="incremental", since={"age": "1y"})]) def test_incremental_with_retain_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'retain'"): - _cfg([_source(ref_type="branch", strategy="incremental", retain={"count": 5})]) + _cfg([_source(ref_type="branch", mode="incremental", retain={"count": 5})]) def test_incremental_with_commit_level_index_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'index.level: commit'"): - _cfg([_source(ref_type="branch", strategy="incremental", index={"level": "commit"})]) + _cfg([_source(ref_type="branch", mode="incremental", index={"level": "commit"})]) def test_top_level_update_key_raises(self): with pytest.raises(ValueError, match="unknown keys"): @@ -234,8 +232,8 @@ def test_top_level_update_key_raises(self): "match": "main", "update": "incremental"}]) def test_incremental_with_repo_level_index_is_fine(self): - cfg = _cfg([_source(ref_type="branch", strategy="incremental", index={"level": "repo"})]) - assert cfg.repos[0].selectors[0].index_strategy == "incremental" + cfg = _cfg([_source(ref_type="branch", mode="incremental", index={"level": "repo"})]) + assert cfg.repos[0].selectors[0].mode == "incremental" class TestParseCommitSource: diff --git a/tests/test_documents.py b/tests/test_documents.py index 0c2edf0..22cf9eb 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -29,14 +29,14 @@ def _set_worker_ctx(host: str, org: str, repo: str, commit_sha: str, repo_dir, s # should leave behind for the rest of the pytest session. documents._WORKER_CTX.update( host=host, org=org, repo=repo, commit_sha=commit_sha, repo_dir=pathlib.Path(repo_dir), - symlink_paths=symlink_paths, strategy="snapshot", + symlink_paths=symlink_paths, mode="snapshot", ) def _set_worker_ctx_incremental(host: str, org: str, repo: str, ref: str, repo_dir, symlink_paths=frozenset()) -> None: documents._WORKER_CTX.update( host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), - symlink_paths=symlink_paths, strategy="incremental", + symlink_paths=symlink_paths, mode="incremental", ) diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py index 12fd110..ce20092 100644 --- a/tests/test_incremental_index.py +++ b/tests/test_incremental_index.py @@ -52,7 +52,7 @@ def test_first_index_does_full_rebuild(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - index_strategy="incremental") + mode="incremental") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_called_once() @@ -75,7 +75,7 @@ def test_second_run_indexes_only_changed_paths(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - index_strategy="incremental") + mode="incremental") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_not_called() @@ -95,7 +95,7 @@ def test_missing_diff_base_triggers_full_rebuild(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - index_strategy="incremental") + mode="incremental") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_called_once() @@ -110,7 +110,7 @@ def test_no_change_skips_entirely(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - index_strategy="incremental") + mode="incremental") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["index_incremental_paths"].assert_not_called() @@ -130,7 +130,7 @@ def test_failed_run_does_not_advance_commit(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - index_strategy="incremental") + mode="incremental") try: index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) @@ -166,7 +166,7 @@ def test_suffix_change_forces_full_rebuild_and_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - index_strategy="incremental", index_level="repo", index_suffix="deploy") + mode="incremental", index_level="repo", index_suffix="deploy") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Full rebuild path: delete_incremental_branch called at new routing, full tree indexed. @@ -193,7 +193,7 @@ def test_level_change_forces_full_rebuild_and_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - index_strategy="incremental", index_level="org", index_suffix=None) + mode="incremental", index_level="org", index_suffix=None) index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) assert mocks["delete_incremental_branch"].call_count == 2 @@ -213,7 +213,7 @@ def test_no_changes_with_routing_change_still_migrates(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - index_strategy="incremental", index_level="repo", index_suffix="v2") + mode="incremental", index_level="repo", index_suffix="v2") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Must NOT skip even though old_sha == new_sha. @@ -232,7 +232,7 @@ def test_same_routing_no_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - index_strategy="incremental", index_level="repo", index_suffix=None) + mode="incremental", index_level="repo", index_suffix=None) index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Delta run: no full rebuild (delete_incremental_branch not called), no extra delete. diff --git a/tests/test_markers.py b/tests/test_markers.py index 6cb750f..19f49de 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -414,7 +414,7 @@ def test_marker_carries_commit_no_ref_key(self): doc = es.index.call_args.kwargs["document"] assert doc["git"]["commit"] == OLD assert "ref_key" not in doc["git"] - assert doc["index_strategy"] == "snapshot" + assert doc["mode"] == "snapshot" def test_marker_id_is_hashed_not_the_commit(self): # _id is build_ref_id (BLAKE2b hash) -- one per (ref, commit), NOT the bare commit SHA. @@ -468,7 +468,7 @@ def test_incremental_marker_preserves_completed_commit_and_exposes_target(self): assert doc["status"] == "indexing" assert doc["git"]["commit"] == OLD # completed pointer unchanged (INV-006) assert doc["git"]["commit_target"] == NEW - assert doc["index_strategy"] == "incremental" + assert doc["mode"] == "incremental" assert es.index.call_args.kwargs["index"] == REFS_INDEX def test_incremental_marker_first_index_has_no_completed_commit(self): diff --git a/tests/test_uniqueness_gate.py b/tests/test_uniqueness_gate.py index d87650a..dfc67ed 100644 --- a/tests/test_uniqueness_gate.py +++ b/tests/test_uniqueness_gate.py @@ -1,7 +1,7 @@ """Tests for the post-index join-uniqueness gate: sourcerer.queries.check_join_uniqueness (INV-011) and its command.py wiring in _run_uniqueness_gate. Every ES call is mocked. -The gate is split by content shape (no index_strategy on content docs): +The gate is split by content shape (no mode on content docs): - Snapshot (git.commit IS NOT NULL): each commit must have ≥1 complete refs doc. - Incremental (git.ref IS NOT NULL): each ref must have EXACTLY ONE incremental join doc. """ From e7b5a494e25c79f6b9d4ff60468939db08eb3f39 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Mon, 24 Aug 2026 07:06:04 -0700 Subject: [PATCH 22/29] Rename mode value of 'incremental' to 'head' --- AGENTS.md | 16 ++++++------ README.md | 10 +++---- sourcerer.example.yml | 4 +-- specs/sourcerer-yml.md | 12 ++++----- src/sourcerer/commands/index/command.py | 6 ++--- src/sourcerer/commands/index/documents.py | 4 +-- src/sourcerer/commands/index/markers.py | 2 +- src/sourcerer/config.py | 20 +++++++------- src/sourcerer/progress.py | 2 +- src/sourcerer/queries.py | 8 +++--- tests/test_config.py | 32 +++++++++++------------ tests/test_documents.py | 2 +- tests/test_incremental_index.py | 18 ++++++------- tests/test_markers.py | 2 +- 14 files changed, 69 insertions(+), 69 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index bf01d22..03b14fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,14 +57,14 @@ config's `sources:` is a YAML list, one entry per (host, org, repo, ref_type). S | `match` | yes | For `branch`/`tag`: pattern string or list of patterns matched against ref names (version DSL + glob), a ref matches if any pattern hits. For `commit`: a commit SHA/prefix string or list of them (see below). | | `since` | no | Index-side inclusion floor: the earliest commit to start indexing from. See below. Not valid for `git.ref_type: commit`. | | `retain` | no | Retention policy (see below). Omit to keep forever. For `git.ref_type: commit`, only `age` is valid. | -| `mode` | no | `snapshot` (default) or `incremental` (branch-only). See below. | +| `mode` | no | `snapshot` (default) or `head` (branch-only). See below. | -#### `mode` (snapshot vs. incremental) +#### `mode` (snapshot vs. head) `snapshot` (default): content is commit-addressed. A HEAD advance on a branch indexes a whole new snapshot under the new commit. -`incremental` (branch-only; rejects `since`/`retain` -- there is no per-commit history for either +`head` (branch-only; rejects `since`/`retain` -- there is no per-commit history for either to apply to): content is ref-addressed instead. A HEAD advance runs `git diff --name-status` between the previously-completed commit and the new tip and only deletes/ reindexes the paths git reports changed -- add/modify/delete/rename/copy -- instead of @@ -80,7 +80,7 @@ succeed, so a crash mid-update leaves the prior commit and content in place. repo: serverless-gitops ref_type: branch match: main - mode: incremental + mode: head ``` #### `git.ref_type: commit` (pinning an explicit commit) @@ -379,7 +379,7 @@ Content docs come in two disjoint shapes depending on how they were indexed: - **Snapshot** (`mode: snapshot`): content docs carry `git.commit` and no `git.ref`. The ref-name marker in `sourcerer-v3-refs` (keyed by `build_ref_id`, one per snapshot source) carries the commit and status. `git.commit` on the content row is already the answer; no join is needed to resolve it. -- **Incremental** (`mode: incremental`): content docs carry `git.ref` and no `git.commit`. A +- **Head** (`mode: head`): content docs carry `git.ref` and no `git.commit`. A dedicated refs join doc at `_id = build_ref_key(host,org,repo,ref)` (one per branch) holds the live HEAD commit and is advanced two-phase (INV-006). The join resolves `git.commit` from this doc. @@ -465,7 +465,7 @@ Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental |---|---| | `indexing` | A run is mid-flight. `indexing_started_at` is set; `indexed_at` is absent/null. Present on snapshot ref-name markers (written by `write_indexing_marker` just before ingest) and incremental join docs (written by `write_incremental_indexing`). A stale `indexing` doc whose `indexing_started_at` is older than the retry window (default 6 h) marks a crashed run and is treated as due for re-indexing. | | `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot markers) and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | -| `stale` | A snapshot marker superseded by a mode switch to incremental. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). Stale markers are invisible to all content tools (all gate on `status == "complete"`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | +| `stale` | A snapshot marker superseded by a mode switch to `head`. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). Stale markers are invisible to all content tools (all gate on `status == "complete"`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | #### Uniqueness gate (INV-011 backstop) @@ -474,8 +474,8 @@ Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental - **Snapshot** (git.commit IS NOT NULL in content): each distinct commit must have ≥1 complete refs doc (presence check — multi-ref-per-commit is legal). -- **Incremental** (git.ref IS NOT NULL in content): each distinct ref must have **exactly one** - incremental join doc with `mode == "incremental"` (anti-fan-out guard for the surviving join). +- **Head** (git.ref IS NOT NULL in content): each distinct ref must have **exactly one** + incremental join doc with `mode == "head"` (anti-fan-out guard for the surviving join). The gate is non-fatal (logs a warning, does not block): with the flip-status switchover in place, violations should only occur if a stale-flip was skipped or crashed mid-way; the next prune run diff --git a/README.md b/README.md index 55d9358..62ce091 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,9 @@ Make sure you have [uv](https://docs.astral.sh/uv/) and [git](https://git-scm.co The [`sourcerer.yml` specification](specs/sourcerer-yml.md) has the full reference of fields supported by the configuration file. -### Snapshot vs. incremental indexing (`mode`) +### Snapshot vs. head indexing (`mode`) -Each source can set `mode: snapshot` (the default) or `mode: incremental` (branch-only). Every +Each source can set `mode: snapshot` (the default) or `mode: head` (branch-only). Every Agent Builder content tool takes the same `git_commit_ish` param either way (a commit SHA or a branch/tag name, `*`/`?` wildcards supported) and resolves a commit the same way regardless of mode. @@ -95,19 +95,19 @@ mode. - **`snapshot`** (default): content is commit-addressed. Every ref (branch, tag, or pinned commit) that resolves to the same commit collapses to one snapshot. A moving branch's HEAD advance indexes a brand-new snapshot under the new commit. -- **`incremental`** (branch-only): content is ref-addressed instead. Content docs carry `git.ref` +- **`head`** (branch-only): content is ref-addressed instead. Content docs carry `git.ref` but no `git.commit` of their own — the branch's current commit lives only on its refs join doc, resolved at query time via a LOOKUP JOIN. A HEAD advance re-indexes only the files `git diff --name-status` reports changed (add/modify/delete/rename), not the whole tree, so staying current on a fast-moving branch (e.g. GitOps/IaC repos that deploy off `main`) is cheap. - `since` and `retain` don't apply to an incremental source (there is no per-commit history to + `since` and `retain` don't apply to a head-mode source (there is no per-commit history to filter or retain) and are rejected if given. ```yaml sources: - git: { host: "github", org: "elastic", repo: "serverless-gitops", ref_type: "branch" } match: "main" - mode: incremental + mode: head ``` Upgrading from a pre-`ref_key` install is automatic and invisible: every `index` run backfills diff --git a/sourcerer.example.yml b/sourcerer.example.yml index 08aa8e6..c36d0b9 100644 --- a/sourcerer.example.yml +++ b/sourcerer.example.yml @@ -184,7 +184,7 @@ sources: retain: count: 5 -# Incremental mode -- branch-only. Instead of a new commit-addressed snapshot on every HEAD +# Head mode -- branch-only. Instead of a new commit-addressed snapshot on every HEAD # advance, content is keyed by git.ref and stays in place: a HEAD advance re-indexes only the # files `git diff` reports changed (a delta update), rather than the whole tree. Good for a # fast-moving branch that deploys off main, where staying current matters more than retaining @@ -196,7 +196,7 @@ sources: repo: serverless-gitops ref_type: branch match: main - mode: incremental # default: snapshot + mode: head # default: snapshot # Feature/fix branches as of a week ago; keep the newest commit, prune > 1 month. - git: diff --git a/specs/sourcerer-yml.md b/specs/sourcerer-yml.md index b035b0c..31fa9bf 100644 --- a/specs/sourcerer-yml.md +++ b/specs/sourcerer-yml.md @@ -417,17 +417,17 @@ for indexing if they don't also qualify for pruning. Defines whether to index the content of each matching ref as an immutable commit snapshot (`"snapshot"`) or maintain a single ref-addressed view that is updated -incrementally as the HEAD moves (`"incremental"`). Controls whether `since` and -`retain` apply (both are rejected when `mode` is `"incremental"`). +incrementally as the HEAD moves (`"head"`). Controls whether `since` and +`retain` apply (both are rejected when `mode` is `"head"`). - Required: No - Type: String - Default: `"snapshot"` - Validation: - - Must be one of: `"snapshot"`, `"incremental"` - - `"incremental"` is only valid when `git.ref_type` is `"branch"` - - `"incremental"` cannot be combined with `since` or `retain` - - `"incremental"` cannot be combined with `index.level: commit` + - Must be one of: `"snapshot"`, `"head"` + - `"head"` is only valid when `git.ref_type` is `"branch"` + - `"head"` cannot be combined with `since` or `retain` + - `"head"` cannot be combined with `index.level: commit` ### `sources[i].since` diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index 0400625..2308613 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -278,7 +278,7 @@ def index_incremental_branch_in_dir( if reporter is None: reporter = ProgressReporter() if unit is None: - unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", mode="incremental") + unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", mode="head") reporter.set_stage(unit, "checkout") checkout_branch(repo_dir, branch) @@ -626,8 +626,8 @@ def process_group(item: tuple[tuple[str, str, str], list[Unit]]) -> None: # reuse -- each is a standalone two-phase delta update against its own prior state # (see index_incremental_branch_in_dir). Processed here, before the snapshot-only # `group` continues below with incremental units filtered out. - incremental_units = [u for u in group if u.mode == "incremental"] - group = [u for u in group if u.mode != "incremental"] + incremental_units = [u for u in group if u.mode == "head"] + group = [u for u in group if u.mode != "head"] for unit in incremental_units: reporter.start(unit) if incremental_units: diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index cd2f71b..e03d528 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -282,7 +282,7 @@ def _init_worker_incremental( _WORKER_CTX.update( host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, - mode="incremental", + mode="head", ) @@ -303,7 +303,7 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: text. Runs in a worker process (see _init_worker for the shared context). Mirrors the old inline generator -- a binary file or one that can't be read yields only its file doc.""" ctx = _WORKER_CTX - incremental = ctx.get("mode", "snapshot") == "incremental" + incremental = ctx.get("mode", "snapshot") == "head" host, org, repo = ctx["host"], ctx["org"], ctx["repo"] commit_sha = ctx.get("ref") if incremental else ctx["commit_sha"] level, suffix = ctx.get("index_level", "repo"), ctx.get("index_suffix") diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index 7777365..ba74f9d 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -595,7 +595,7 @@ def _build_incremental_join_doc( "commit_target": commit_target, "commit_date": commit_date_iso, }, - "mode": "incremental", + "mode": "head", "status": status, "files_count": files_count, "lines_count": lines_count, diff --git a/src/sourcerer/config.py b/src/sourcerer/config.py index 19160df..ed89e2d 100644 --- a/src/sourcerer/config.py +++ b/src/sourcerer/config.py @@ -283,9 +283,9 @@ class Selector: levels: tuple[str, ...] = () # numeric levels shared by the versioned match patterns schedule: Schedule | None = None # per-source schedule override (sources[i].schedule) # sources[i].mode: the indexing mode for this source -- "snapshot" (default, commit-addressed) - # or "incremental" (ref-addressed, branch-only). Controls whether since/retain apply and routes + # or "head" (ref-addressed, branch-only). Controls whether since/retain apply and routes # the unit to the incremental delta-index path instead of the snapshot flow. - mode: str = "snapshot" # "snapshot" (default) or "incremental" (branch-only) + mode: str = "snapshot" # "snapshot" (default) or "head" (branch-only) # sources[i].index routing (see specs/sourcerer-yml.md): which physical files/lines index this # source's content docs land in. Per-source, so two sources sharing a (host, org, repo) may # route differently. @@ -412,7 +412,7 @@ def _parse_commit_match(raw: dict, ctx: str) -> list[str]: _GIT_KEYS = {"host", "org", "repo", "ref_type"} _INDEX_LEVELS = ("host", "org", "repo", "commit") -_MODES = ("snapshot", "incremental") +_MODES = ("snapshot", "head") # A suffix goes into a physical index name after a `^`, so it must be safe as an index-name # segment: the same characters forbidden in a host id, plus the `^` we use as the suffix delimiter. _FORBIDDEN_SUFFIX_CHARS = _FORBIDDEN_HOST_CHARS | {"^"} @@ -499,21 +499,21 @@ def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: if raw.get("index") is not None: index_level, index_suffix = _parse_index(raw["index"], ctx) - if mode == "incremental": + if mode == "head": if ref_type != "branch": - raise ValueError(f"{ctx} mode: 'incremental' is only valid for " + raise ValueError(f"{ctx} mode: 'head' is only valid for " f"git.ref_type: branch (got ref_type {ref_type!r})") - # An incremental branch maintains a single mutable ref-addressed view with no per-commit + # A head-mode branch maintains a single mutable ref-addressed view with no per-commit # history for retention to trim and no inclusion floor to apply -- both since and retain # are meaningless here (see specs/incremental-indexing.md). if raw.get("since") is not None: - raise ValueError(f"{ctx}: 'mode: incremental' cannot be combined with 'since'") + raise ValueError(f"{ctx}: 'mode: head' cannot be combined with 'since'") if raw.get("retain") is not None: - raise ValueError(f"{ctx}: 'mode: incremental' cannot be combined with 'retain'") + raise ValueError(f"{ctx}: 'mode: head' cannot be combined with 'retain'") if index_level == "commit": - # Incremental content is ref-addressed (no git.commit on content docs), so a + # Head-mode content is ref-addressed (no git.commit on content docs), so a # commit-level index name -- which requires a commit sha -- can never be built for it. - raise ValueError(f"{ctx} mode: 'incremental' cannot be combined with " + raise ValueError(f"{ctx} mode: 'head' cannot be combined with " f"'index.level: commit'") if ref_type == "commit": diff --git a/src/sourcerer/progress.py b/src/sourcerer/progress.py index 6b026f3..888fb35 100644 --- a/src/sourcerer/progress.py +++ b/src/sourcerer/progress.py @@ -68,7 +68,7 @@ class Unit: index_level: str = "repo" index_suffix: str | None = None # sources[i].mode carried from the selector that emitted this unit: "snapshot" (default, - # commit-addressed) or "incremental" (ref-addressed, branch-only). Routes the unit to the + # commit-addressed) or "head" (ref-addressed, branch-only). Routes the unit to the # incremental delta-index path instead of the snapshot pre-clone/skip/retention flow. mode: str = "snapshot" diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index 38ebb2f..3ff40d7 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -262,10 +262,10 @@ def gather_intended_incremental_index_by_ref( Feeds the incremental stale-location sweep (Class D-I): content for a branch sitting in an index NOT in this set is stale from a crashed migration and should be reclaimed. Filters to - mode=="incremental" docs only, so snapshot markers (which always have git.commit) are + mode=="head" docs only, so snapshot markers (which always have git.commit) are not double-counted. Returns {} if the refs index doesn't exist.""" out: dict[tuple[str, str, str, str], set[str]] = {} - body = {"query": {"term": {"mode": "incremental"}}} + body = {"query": {"term": {"mode": "head"}}} src_fields = ["git.host", "git.org", "git.repo", "git.ref", "index_level", "index_suffix"] try: for hit in scan(es, index=REFS_ALIAS, query=body, _source=src_fields, preserve_order=False): @@ -420,7 +420,7 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> (presence check -- multi-ref-per-commit is legal; the snapshot FORK arm no longer joins so the uniqueness requirement there is already removed). - Incremental (git.ref IS NOT NULL): each ref must resolve to EXACTLY ONE refs doc with - `mode == "incremental"` -- the anti-fan-out invariant for the surviving join. + `mode == "head"` -- the anti-fan-out invariant for the surviving join. Returns the sorted list of offending keys (commits/refs that fail their respective check); an empty list means the invariant holds.""" @@ -457,7 +457,7 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"terms": {"git.ref": sorted(refs)}}, - {"term": {"mode": "incremental"}}, + {"term": {"mode": "head"}}, ]}}, aggs={"refs": {"terms": {"field": "git.ref", "size": len(refs)}}}, ) diff --git a/tests/test_config.py b/tests/test_config.py index 11a4088..cc7acab 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -198,42 +198,42 @@ def test_default_is_snapshot(self): cfg = _cfg([_source()]) assert cfg.repos[0].selectors[0].mode == "snapshot" - def test_incremental_accepted_on_branch(self): - cfg = _cfg([_source(ref_type="branch", mode="incremental")]) - assert cfg.repos[0].selectors[0].mode == "incremental" + def test_head_accepted_on_branch(self): + cfg = _cfg([_source(ref_type="branch", mode="head")]) + assert cfg.repos[0].selectors[0].mode == "head" - def test_incremental_rejected_on_tag(self): + def test_head_rejected_on_tag(self): with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): - _cfg([_source(ref_type="tag", match="v1.0.0", mode="incremental")]) + _cfg([_source(ref_type="tag", match="v1.0.0", mode="head")]) - def test_incremental_rejected_on_commit(self): + def test_head_rejected_on_commit(self): with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): - _cfg([_source(ref_type="commit", match="cfefb3b", mode="incremental")]) + _cfg([_source(ref_type="commit", match="cfefb3b", mode="head")]) def test_invalid_mode_raises(self): with pytest.raises(ValueError, match="must be one of"): _cfg([_source(mode="bogus")]) - def test_incremental_with_since_raises(self): + def test_head_with_since_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'since'"): - _cfg([_source(ref_type="branch", mode="incremental", since={"age": "1y"})]) + _cfg([_source(ref_type="branch", mode="head", since={"age": "1y"})]) - def test_incremental_with_retain_raises(self): + def test_head_with_retain_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'retain'"): - _cfg([_source(ref_type="branch", mode="incremental", retain={"count": 5})]) + _cfg([_source(ref_type="branch", mode="head", retain={"count": 5})]) - def test_incremental_with_commit_level_index_raises(self): + def test_head_with_commit_level_index_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'index.level: commit'"): - _cfg([_source(ref_type="branch", mode="incremental", index={"level": "commit"})]) + _cfg([_source(ref_type="branch", mode="head", index={"level": "commit"})]) def test_top_level_update_key_raises(self): with pytest.raises(ValueError, match="unknown keys"): _cfg([{"git": {"host": "github", "org": "acme", "repo": "widgets", "ref_type": "branch"}, "match": "main", "update": "incremental"}]) - def test_incremental_with_repo_level_index_is_fine(self): - cfg = _cfg([_source(ref_type="branch", mode="incremental", index={"level": "repo"})]) - assert cfg.repos[0].selectors[0].mode == "incremental" + def test_head_with_repo_level_index_is_fine(self): + cfg = _cfg([_source(ref_type="branch", mode="head", index={"level": "repo"})]) + assert cfg.repos[0].selectors[0].mode == "head" class TestParseCommitSource: diff --git a/tests/test_documents.py b/tests/test_documents.py index 22cf9eb..7362160 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -36,7 +36,7 @@ def _set_worker_ctx(host: str, org: str, repo: str, commit_sha: str, repo_dir, s def _set_worker_ctx_incremental(host: str, org: str, repo: str, ref: str, repo_dir, symlink_paths=frozenset()) -> None: documents._WORKER_CTX.update( host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), - symlink_paths=symlink_paths, mode="incremental", + symlink_paths=symlink_paths, mode="head", ) diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py index ce20092..949bfeb 100644 --- a/tests/test_incremental_index.py +++ b/tests/test_incremental_index.py @@ -52,7 +52,7 @@ def test_first_index_does_full_rebuild(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="incremental") + mode="head") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_called_once() @@ -75,7 +75,7 @@ def test_second_run_indexes_only_changed_paths(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="incremental") + mode="head") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_not_called() @@ -95,7 +95,7 @@ def test_missing_diff_base_triggers_full_rebuild(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="incremental") + mode="head") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_called_once() @@ -110,7 +110,7 @@ def test_no_change_skips_entirely(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="incremental") + mode="head") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["index_incremental_paths"].assert_not_called() @@ -130,7 +130,7 @@ def test_failed_run_does_not_advance_commit(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="incremental") + mode="head") try: index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) @@ -166,7 +166,7 @@ def test_suffix_change_forces_full_rebuild_and_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="incremental", index_level="repo", index_suffix="deploy") + mode="head", index_level="repo", index_suffix="deploy") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Full rebuild path: delete_incremental_branch called at new routing, full tree indexed. @@ -193,7 +193,7 @@ def test_level_change_forces_full_rebuild_and_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="incremental", index_level="org", index_suffix=None) + mode="head", index_level="org", index_suffix=None) index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) assert mocks["delete_incremental_branch"].call_count == 2 @@ -213,7 +213,7 @@ def test_no_changes_with_routing_change_still_migrates(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="incremental", index_level="repo", index_suffix="v2") + mode="head", index_level="repo", index_suffix="v2") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Must NOT skip even though old_sha == new_sha. @@ -232,7 +232,7 @@ def test_same_routing_no_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="incremental", index_level="repo", index_suffix=None) + mode="head", index_level="repo", index_suffix=None) index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Delta run: no full rebuild (delete_incremental_branch not called), no extra delete. diff --git a/tests/test_markers.py b/tests/test_markers.py index 19f49de..94d5dd4 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -468,7 +468,7 @@ def test_incremental_marker_preserves_completed_commit_and_exposes_target(self): assert doc["status"] == "indexing" assert doc["git"]["commit"] == OLD # completed pointer unchanged (INV-006) assert doc["git"]["commit_target"] == NEW - assert doc["mode"] == "incremental" + assert doc["mode"] == "head" assert es.index.call_args.kwargs["index"] == REFS_INDEX def test_incremental_marker_first_index_has_no_completed_commit(self): From f64062d1461144bfc8b2161b9450bf8c24fac9ad Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Mon, 24 Aug 2026 09:14:00 -0700 Subject: [PATCH 23/29] Rename mode value from 'head' to 'delta' (ref-topology-neutral; future-proofs for fast-moving tags) --- AGENTS.md | 16 ++++++------ README.md | 10 +++---- sourcerer.example.yml | 4 +-- specs/sourcerer-yml.md | 12 ++++----- src/sourcerer/commands/index/command.py | 6 ++--- src/sourcerer/commands/index/documents.py | 4 +-- src/sourcerer/commands/index/markers.py | 2 +- src/sourcerer/config.py | 20 +++++++------- src/sourcerer/progress.py | 2 +- src/sourcerer/queries.py | 8 +++--- tests/test_config.py | 32 +++++++++++------------ tests/test_documents.py | 2 +- tests/test_incremental_index.py | 18 ++++++------- tests/test_markers.py | 2 +- 14 files changed, 69 insertions(+), 69 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 03b14fd..7e723cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,14 +57,14 @@ config's `sources:` is a YAML list, one entry per (host, org, repo, ref_type). S | `match` | yes | For `branch`/`tag`: pattern string or list of patterns matched against ref names (version DSL + glob), a ref matches if any pattern hits. For `commit`: a commit SHA/prefix string or list of them (see below). | | `since` | no | Index-side inclusion floor: the earliest commit to start indexing from. See below. Not valid for `git.ref_type: commit`. | | `retain` | no | Retention policy (see below). Omit to keep forever. For `git.ref_type: commit`, only `age` is valid. | -| `mode` | no | `snapshot` (default) or `head` (branch-only). See below. | +| `mode` | no | `snapshot` (default) or `delta` (branch-only). See below. | -#### `mode` (snapshot vs. head) +#### `mode` (snapshot vs. delta) `snapshot` (default): content is commit-addressed. A HEAD advance on a branch indexes a whole new snapshot under the new commit. -`head` (branch-only; rejects `since`/`retain` -- there is no per-commit history for either +`delta` (branch-only; rejects `since`/`retain` -- there is no per-commit history for either to apply to): content is ref-addressed instead. A HEAD advance runs `git diff --name-status` between the previously-completed commit and the new tip and only deletes/ reindexes the paths git reports changed -- add/modify/delete/rename/copy -- instead of @@ -80,7 +80,7 @@ succeed, so a crash mid-update leaves the prior commit and content in place. repo: serverless-gitops ref_type: branch match: main - mode: head + mode: delta ``` #### `git.ref_type: commit` (pinning an explicit commit) @@ -379,7 +379,7 @@ Content docs come in two disjoint shapes depending on how they were indexed: - **Snapshot** (`mode: snapshot`): content docs carry `git.commit` and no `git.ref`. The ref-name marker in `sourcerer-v3-refs` (keyed by `build_ref_id`, one per snapshot source) carries the commit and status. `git.commit` on the content row is already the answer; no join is needed to resolve it. -- **Head** (`mode: head`): content docs carry `git.ref` and no `git.commit`. A +- **Delta** (`mode: delta`): content docs carry `git.ref` and no `git.commit`. A dedicated refs join doc at `_id = build_ref_key(host,org,repo,ref)` (one per branch) holds the live HEAD commit and is advanced two-phase (INV-006). The join resolves `git.commit` from this doc. @@ -465,7 +465,7 @@ Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental |---|---| | `indexing` | A run is mid-flight. `indexing_started_at` is set; `indexed_at` is absent/null. Present on snapshot ref-name markers (written by `write_indexing_marker` just before ingest) and incremental join docs (written by `write_incremental_indexing`). A stale `indexing` doc whose `indexing_started_at` is older than the retry window (default 6 h) marks a crashed run and is treated as due for re-indexing. | | `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot markers) and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | -| `stale` | A snapshot marker superseded by a mode switch to `head`. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). Stale markers are invisible to all content tools (all gate on `status == "complete"`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | +| `stale` | A snapshot marker superseded by a mode switch to `delta`. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). Stale markers are invisible to all content tools (all gate on `status == "complete"`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | #### Uniqueness gate (INV-011 backstop) @@ -474,8 +474,8 @@ Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental - **Snapshot** (git.commit IS NOT NULL in content): each distinct commit must have ≥1 complete refs doc (presence check — multi-ref-per-commit is legal). -- **Head** (git.ref IS NOT NULL in content): each distinct ref must have **exactly one** - incremental join doc with `mode == "head"` (anti-fan-out guard for the surviving join). +- **Delta** (git.ref IS NOT NULL in content): each distinct ref must have **exactly one** + incremental join doc with `mode == "delta"` (anti-fan-out guard for the surviving join). The gate is non-fatal (logs a warning, does not block): with the flip-status switchover in place, violations should only occur if a stale-flip was skipped or crashed mid-way; the next prune run diff --git a/README.md b/README.md index 62ce091..543911a 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,9 @@ Make sure you have [uv](https://docs.astral.sh/uv/) and [git](https://git-scm.co The [`sourcerer.yml` specification](specs/sourcerer-yml.md) has the full reference of fields supported by the configuration file. -### Snapshot vs. head indexing (`mode`) +### Snapshot vs. delta indexing (`mode`) -Each source can set `mode: snapshot` (the default) or `mode: head` (branch-only). Every +Each source can set `mode: snapshot` (the default) or `mode: delta` (branch-only). Every Agent Builder content tool takes the same `git_commit_ish` param either way (a commit SHA or a branch/tag name, `*`/`?` wildcards supported) and resolves a commit the same way regardless of mode. @@ -95,19 +95,19 @@ mode. - **`snapshot`** (default): content is commit-addressed. Every ref (branch, tag, or pinned commit) that resolves to the same commit collapses to one snapshot. A moving branch's HEAD advance indexes a brand-new snapshot under the new commit. -- **`head`** (branch-only): content is ref-addressed instead. Content docs carry `git.ref` +- **`delta`** (branch-only): content is ref-addressed instead. Content docs carry `git.ref` but no `git.commit` of their own — the branch's current commit lives only on its refs join doc, resolved at query time via a LOOKUP JOIN. A HEAD advance re-indexes only the files `git diff --name-status` reports changed (add/modify/delete/rename), not the whole tree, so staying current on a fast-moving branch (e.g. GitOps/IaC repos that deploy off `main`) is cheap. - `since` and `retain` don't apply to a head-mode source (there is no per-commit history to + `since` and `retain` don't apply to a delta-mode source (there is no per-commit history to filter or retain) and are rejected if given. ```yaml sources: - git: { host: "github", org: "elastic", repo: "serverless-gitops", ref_type: "branch" } match: "main" - mode: head + mode: delta ``` Upgrading from a pre-`ref_key` install is automatic and invisible: every `index` run backfills diff --git a/sourcerer.example.yml b/sourcerer.example.yml index c36d0b9..216d715 100644 --- a/sourcerer.example.yml +++ b/sourcerer.example.yml @@ -184,7 +184,7 @@ sources: retain: count: 5 -# Head mode -- branch-only. Instead of a new commit-addressed snapshot on every HEAD +# Delta mode -- branch-only. Instead of a new commit-addressed snapshot on every HEAD # advance, content is keyed by git.ref and stays in place: a HEAD advance re-indexes only the # files `git diff` reports changed (a delta update), rather than the whole tree. Good for a # fast-moving branch that deploys off main, where staying current matters more than retaining @@ -196,7 +196,7 @@ sources: repo: serverless-gitops ref_type: branch match: main - mode: head # default: snapshot + mode: delta # default: snapshot # Feature/fix branches as of a week ago; keep the newest commit, prune > 1 month. - git: diff --git a/specs/sourcerer-yml.md b/specs/sourcerer-yml.md index 31fa9bf..ecbd75e 100644 --- a/specs/sourcerer-yml.md +++ b/specs/sourcerer-yml.md @@ -417,17 +417,17 @@ for indexing if they don't also qualify for pruning. Defines whether to index the content of each matching ref as an immutable commit snapshot (`"snapshot"`) or maintain a single ref-addressed view that is updated -incrementally as the HEAD moves (`"head"`). Controls whether `since` and -`retain` apply (both are rejected when `mode` is `"head"`). +incrementally as the HEAD moves (`"delta"`). Controls whether `since` and +`retain` apply (both are rejected when `mode` is `"delta"`). - Required: No - Type: String - Default: `"snapshot"` - Validation: - - Must be one of: `"snapshot"`, `"head"` - - `"head"` is only valid when `git.ref_type` is `"branch"` - - `"head"` cannot be combined with `since` or `retain` - - `"head"` cannot be combined with `index.level: commit` + - Must be one of: `"snapshot"`, `"delta"` + - `"delta"` is only valid when `git.ref_type` is `"branch"` + - `"delta"` cannot be combined with `since` or `retain` + - `"delta"` cannot be combined with `index.level: commit` ### `sources[i].since` diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index 2308613..bf1d349 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -278,7 +278,7 @@ def index_incremental_branch_in_dir( if reporter is None: reporter = ProgressReporter() if unit is None: - unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", mode="head") + unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", mode="delta") reporter.set_stage(unit, "checkout") checkout_branch(repo_dir, branch) @@ -626,8 +626,8 @@ def process_group(item: tuple[tuple[str, str, str], list[Unit]]) -> None: # reuse -- each is a standalone two-phase delta update against its own prior state # (see index_incremental_branch_in_dir). Processed here, before the snapshot-only # `group` continues below with incremental units filtered out. - incremental_units = [u for u in group if u.mode == "head"] - group = [u for u in group if u.mode != "head"] + incremental_units = [u for u in group if u.mode == "delta"] + group = [u for u in group if u.mode != "delta"] for unit in incremental_units: reporter.start(unit) if incremental_units: diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index e03d528..fd0691d 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -282,7 +282,7 @@ def _init_worker_incremental( _WORKER_CTX.update( host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, - mode="head", + mode="delta", ) @@ -303,7 +303,7 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: text. Runs in a worker process (see _init_worker for the shared context). Mirrors the old inline generator -- a binary file or one that can't be read yields only its file doc.""" ctx = _WORKER_CTX - incremental = ctx.get("mode", "snapshot") == "head" + incremental = ctx.get("mode", "snapshot") == "delta" host, org, repo = ctx["host"], ctx["org"], ctx["repo"] commit_sha = ctx.get("ref") if incremental else ctx["commit_sha"] level, suffix = ctx.get("index_level", "repo"), ctx.get("index_suffix") diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index ba74f9d..da25957 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -595,7 +595,7 @@ def _build_incremental_join_doc( "commit_target": commit_target, "commit_date": commit_date_iso, }, - "mode": "head", + "mode": "delta", "status": status, "files_count": files_count, "lines_count": lines_count, diff --git a/src/sourcerer/config.py b/src/sourcerer/config.py index ed89e2d..b5f8e30 100644 --- a/src/sourcerer/config.py +++ b/src/sourcerer/config.py @@ -283,9 +283,9 @@ class Selector: levels: tuple[str, ...] = () # numeric levels shared by the versioned match patterns schedule: Schedule | None = None # per-source schedule override (sources[i].schedule) # sources[i].mode: the indexing mode for this source -- "snapshot" (default, commit-addressed) - # or "head" (ref-addressed, branch-only). Controls whether since/retain apply and routes + # or "delta" (ref-addressed, branch-only). Controls whether since/retain apply and routes # the unit to the incremental delta-index path instead of the snapshot flow. - mode: str = "snapshot" # "snapshot" (default) or "head" (branch-only) + mode: str = "snapshot" # "snapshot" (default) or "delta" (branch-only) # sources[i].index routing (see specs/sourcerer-yml.md): which physical files/lines index this # source's content docs land in. Per-source, so two sources sharing a (host, org, repo) may # route differently. @@ -412,7 +412,7 @@ def _parse_commit_match(raw: dict, ctx: str) -> list[str]: _GIT_KEYS = {"host", "org", "repo", "ref_type"} _INDEX_LEVELS = ("host", "org", "repo", "commit") -_MODES = ("snapshot", "head") +_MODES = ("snapshot", "delta") # A suffix goes into a physical index name after a `^`, so it must be safe as an index-name # segment: the same characters forbidden in a host id, plus the `^` we use as the suffix delimiter. _FORBIDDEN_SUFFIX_CHARS = _FORBIDDEN_HOST_CHARS | {"^"} @@ -499,21 +499,21 @@ def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: if raw.get("index") is not None: index_level, index_suffix = _parse_index(raw["index"], ctx) - if mode == "head": + if mode == "delta": if ref_type != "branch": - raise ValueError(f"{ctx} mode: 'head' is only valid for " + raise ValueError(f"{ctx} mode: 'delta' is only valid for " f"git.ref_type: branch (got ref_type {ref_type!r})") - # A head-mode branch maintains a single mutable ref-addressed view with no per-commit + # A delta-mode branch maintains a single mutable ref-addressed view with no per-commit # history for retention to trim and no inclusion floor to apply -- both since and retain # are meaningless here (see specs/incremental-indexing.md). if raw.get("since") is not None: - raise ValueError(f"{ctx}: 'mode: head' cannot be combined with 'since'") + raise ValueError(f"{ctx}: 'mode: delta' cannot be combined with 'since'") if raw.get("retain") is not None: - raise ValueError(f"{ctx}: 'mode: head' cannot be combined with 'retain'") + raise ValueError(f"{ctx}: 'mode: delta' cannot be combined with 'retain'") if index_level == "commit": - # Head-mode content is ref-addressed (no git.commit on content docs), so a + # Delta-mode content is ref-addressed (no git.commit on content docs), so a # commit-level index name -- which requires a commit sha -- can never be built for it. - raise ValueError(f"{ctx} mode: 'head' cannot be combined with " + raise ValueError(f"{ctx} mode: 'delta' cannot be combined with " f"'index.level: commit'") if ref_type == "commit": diff --git a/src/sourcerer/progress.py b/src/sourcerer/progress.py index 888fb35..bb07513 100644 --- a/src/sourcerer/progress.py +++ b/src/sourcerer/progress.py @@ -68,7 +68,7 @@ class Unit: index_level: str = "repo" index_suffix: str | None = None # sources[i].mode carried from the selector that emitted this unit: "snapshot" (default, - # commit-addressed) or "head" (ref-addressed, branch-only). Routes the unit to the + # commit-addressed) or "delta" (ref-addressed, branch-only). Routes the unit to the # incremental delta-index path instead of the snapshot pre-clone/skip/retention flow. mode: str = "snapshot" diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index 3ff40d7..d6a4141 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -262,10 +262,10 @@ def gather_intended_incremental_index_by_ref( Feeds the incremental stale-location sweep (Class D-I): content for a branch sitting in an index NOT in this set is stale from a crashed migration and should be reclaimed. Filters to - mode=="head" docs only, so snapshot markers (which always have git.commit) are + mode=="delta" docs only, so snapshot markers (which always have git.commit) are not double-counted. Returns {} if the refs index doesn't exist.""" out: dict[tuple[str, str, str, str], set[str]] = {} - body = {"query": {"term": {"mode": "head"}}} + body = {"query": {"term": {"mode": "delta"}}} src_fields = ["git.host", "git.org", "git.repo", "git.ref", "index_level", "index_suffix"] try: for hit in scan(es, index=REFS_ALIAS, query=body, _source=src_fields, preserve_order=False): @@ -420,7 +420,7 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> (presence check -- multi-ref-per-commit is legal; the snapshot FORK arm no longer joins so the uniqueness requirement there is already removed). - Incremental (git.ref IS NOT NULL): each ref must resolve to EXACTLY ONE refs doc with - `mode == "head"` -- the anti-fan-out invariant for the surviving join. + `mode == "delta"` -- the anti-fan-out invariant for the surviving join. Returns the sorted list of offending keys (commits/refs that fail their respective check); an empty list means the invariant holds.""" @@ -457,7 +457,7 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"terms": {"git.ref": sorted(refs)}}, - {"term": {"mode": "head"}}, + {"term": {"mode": "delta"}}, ]}}, aggs={"refs": {"terms": {"field": "git.ref", "size": len(refs)}}}, ) diff --git a/tests/test_config.py b/tests/test_config.py index cc7acab..54f8aea 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -198,42 +198,42 @@ def test_default_is_snapshot(self): cfg = _cfg([_source()]) assert cfg.repos[0].selectors[0].mode == "snapshot" - def test_head_accepted_on_branch(self): - cfg = _cfg([_source(ref_type="branch", mode="head")]) - assert cfg.repos[0].selectors[0].mode == "head" + def test_delta_accepted_on_branch(self): + cfg = _cfg([_source(ref_type="branch", mode="delta")]) + assert cfg.repos[0].selectors[0].mode == "delta" - def test_head_rejected_on_tag(self): + def test_delta_rejected_on_tag(self): with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): - _cfg([_source(ref_type="tag", match="v1.0.0", mode="head")]) + _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta")]) - def test_head_rejected_on_commit(self): + def test_delta_rejected_on_commit(self): with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): - _cfg([_source(ref_type="commit", match="cfefb3b", mode="head")]) + _cfg([_source(ref_type="commit", match="cfefb3b", mode="delta")]) def test_invalid_mode_raises(self): with pytest.raises(ValueError, match="must be one of"): _cfg([_source(mode="bogus")]) - def test_head_with_since_raises(self): + def test_delta_with_since_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'since'"): - _cfg([_source(ref_type="branch", mode="head", since={"age": "1y"})]) + _cfg([_source(ref_type="branch", mode="delta", since={"age": "1y"})]) - def test_head_with_retain_raises(self): + def test_delta_with_retain_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'retain'"): - _cfg([_source(ref_type="branch", mode="head", retain={"count": 5})]) + _cfg([_source(ref_type="branch", mode="delta", retain={"count": 5})]) - def test_head_with_commit_level_index_raises(self): + def test_delta_with_commit_level_index_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'index.level: commit'"): - _cfg([_source(ref_type="branch", mode="head", index={"level": "commit"})]) + _cfg([_source(ref_type="branch", mode="delta", index={"level": "commit"})]) def test_top_level_update_key_raises(self): with pytest.raises(ValueError, match="unknown keys"): _cfg([{"git": {"host": "github", "org": "acme", "repo": "widgets", "ref_type": "branch"}, "match": "main", "update": "incremental"}]) - def test_head_with_repo_level_index_is_fine(self): - cfg = _cfg([_source(ref_type="branch", mode="head", index={"level": "repo"})]) - assert cfg.repos[0].selectors[0].mode == "head" + def test_delta_with_repo_level_index_is_fine(self): + cfg = _cfg([_source(ref_type="branch", mode="delta", index={"level": "repo"})]) + assert cfg.repos[0].selectors[0].mode == "delta" class TestParseCommitSource: diff --git a/tests/test_documents.py b/tests/test_documents.py index 7362160..725a1ee 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -36,7 +36,7 @@ def _set_worker_ctx(host: str, org: str, repo: str, commit_sha: str, repo_dir, s def _set_worker_ctx_incremental(host: str, org: str, repo: str, ref: str, repo_dir, symlink_paths=frozenset()) -> None: documents._WORKER_CTX.update( host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), - symlink_paths=symlink_paths, mode="head", + symlink_paths=symlink_paths, mode="delta", ) diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py index 949bfeb..62bbb22 100644 --- a/tests/test_incremental_index.py +++ b/tests/test_incremental_index.py @@ -52,7 +52,7 @@ def test_first_index_does_full_rebuild(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="head") + mode="delta") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_called_once() @@ -75,7 +75,7 @@ def test_second_run_indexes_only_changed_paths(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="head") + mode="delta") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_not_called() @@ -95,7 +95,7 @@ def test_missing_diff_base_triggers_full_rebuild(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="head") + mode="delta") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_called_once() @@ -110,7 +110,7 @@ def test_no_change_skips_entirely(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="head") + mode="delta") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) mocks["index_incremental_paths"].assert_not_called() @@ -130,7 +130,7 @@ def test_failed_run_does_not_advance_commit(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="head") + mode="delta") try: index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) @@ -166,7 +166,7 @@ def test_suffix_change_forces_full_rebuild_and_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="head", index_level="repo", index_suffix="deploy") + mode="delta", index_level="repo", index_suffix="deploy") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Full rebuild path: delete_incremental_branch called at new routing, full tree indexed. @@ -193,7 +193,7 @@ def test_level_change_forces_full_rebuild_and_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="head", index_level="org", index_suffix=None) + mode="delta", index_level="org", index_suffix=None) index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) assert mocks["delete_incremental_branch"].call_count == 2 @@ -213,7 +213,7 @@ def test_no_changes_with_routing_change_still_migrates(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="head", index_level="repo", index_suffix="v2") + mode="delta", index_level="repo", index_suffix="v2") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Must NOT skip even though old_sha == new_sha. @@ -232,7 +232,7 @@ def test_same_routing_no_old_copy_delete(self): try: es = MagicMock() unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", - mode="head", index_level="repo", index_suffix=None) + mode="delta", index_level="repo", index_suffix=None) index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) # Delta run: no full rebuild (delete_incremental_branch not called), no extra delete. diff --git a/tests/test_markers.py b/tests/test_markers.py index 94d5dd4..831c0ee 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -468,7 +468,7 @@ def test_incremental_marker_preserves_completed_commit_and_exposes_target(self): assert doc["status"] == "indexing" assert doc["git"]["commit"] == OLD # completed pointer unchanged (INV-006) assert doc["git"]["commit_target"] == NEW - assert doc["mode"] == "head" + assert doc["mode"] == "delta" assert es.index.call_args.kwargs["index"] == REFS_INDEX def test_incremental_marker_first_index_has_no_completed_commit(self): From 578c101a2304d93910fc567348ac429dd6586e40 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Mon, 24 Aug 2026 09:55:31 -0700 Subject: [PATCH 24/29] Remove status == complete consistency guard from content tools --- AGENTS.md | 55 +++++++------------ .../elastic/agent_builder_tools/README.md | 28 ++-------- .../sourcerer.code.grep.yml | 29 ++-------- .../sourcerer.code.search.yml | 29 ++-------- .../sourcerer.files.cat.yml | 29 ++-------- .../sourcerer.files.head.yml | 26 ++------- .../sourcerer.files.ls.yml | 29 ++-------- .../sourcerer.files.read_lines.yml | 29 ++-------- .../sourcerer.files.tail.yml | 29 ++-------- .../sourcerer.files.tree.yml | 29 ++-------- .../sourcerer.files.wc.yml | 29 ++-------- .../sourcerer.repos.search.yml | 4 -- src/sourcerer/skills/ref-resolution/SKILL.md | 6 +- tests/test_agent_builder_tools.py | 26 ++++----- 14 files changed, 86 insertions(+), 291 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7e723cc..f358eb7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -395,7 +395,7 @@ FROM sourcerer-lines // both content-doc shapes in one pass. (git.commit IS NOT NULL AND git.commit IN ( FROM sourcerer-refs - | WHERE git.host LIKE ?git_host AND ... AND status == "complete" + | WHERE git.host LIKE ?git_host AND ... AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type @@ -404,40 +404,35 @@ FROM sourcerer-lines OR (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( FROM sourcerer-refs - | WHERE git.host LIKE ?git_host AND ... AND status == "complete" + | WHERE git.host LIKE ?git_host AND ... AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type | KEEP git.ref )) ) -// Branch by content-doc shape: snapshot rows already carry git.commit and status was -// pre-confirmed by the membership subquery above (status=="complete"), so the snapshot -// arm needs no join -- it just asserts status to match the incremental arm's column. -// Incremental rows carry only git.ref; the join resolves the ref's current status from -// its join doc. Safety of the incremental join (one doc per (host,org,repo,ref)) is -// enforced by the "one mode owns a ref name" invariant at index time. +// Branch by content-doc shape to resolve git.commit for incremental refs: +// Snapshot rows already carry git.commit (no join needed). +// Incremental rows carry only git.ref; the join resolves git.commit from the refs join doc. +// Safety of the incremental join (one doc per (host,org,repo,ref)) is enforced by +// the "one mode owns a ref name" invariant at index time. | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) -| WHERE status == "complete" ``` -**Snapshot arm**: no join needed. The commit already lives on the content row and was pre-confirmed -`complete` by the membership subquery; `EVAL status = "complete"` asserts the column so it matches -the incremental arm's shape. Critically, the snapshot arm never touches `sourcerer-refs`, so two -complete markers sharing the same commit (branch + same-named tag) do NOT fan out — they just produce -one row each in the pre-FORK membership filter, which deduplicates naturally. +**Snapshot rows** carry `git.commit` directly; they are matched by the first IN subquery and pass +through the FORK unchanged. Critically, the snapshot arm never touches `sourcerer-refs` at query +time, so two complete markers sharing the same commit (branch + same-named tag) do NOT fan out — +the commit set is resolved once and deduplicated naturally. -**Incremental arm**: joins `sourcerer-refs ON (git.host, git.org, git.repo, git.ref)`. This join is -safe (no fan-out) because there is always exactly one incremental join doc per `(host,org,repo,ref)`: -all three incremental writers use `_id = build_ref_key(...)` (overwrite-in-place), the runtime -strategy-conflict guard in `selection.py` prevents two selectors of different index strategies from -claiming the same ref name simultaneously, and the flip-status switchover marks any old snapshot -marker `"stale"` BEFORE the incremental join doc is published as `"complete"` — so the -two-complete-docs window never opens. +**Incremental rows** carry `git.ref` but no `git.commit`; they are matched by the second IN +subquery and then joined in the FORK incremental arm. The LOOKUP JOIN resolves `git.commit` from the +refs join doc so incremental rows carry a citable commit SHA in the output. The join is safe +(no fan-out) because there is always exactly one incremental join doc per `(host,org,repo,ref)`: +`_id = build_ref_key(...)` (overwrite-in-place) and the runtime mode-conflict guard in +`selection.py` prevent multiple concurrent join docs for the same ref. **Scoping params** (`git_commit`, `git_ref`, `git_ref_type`) are all optional (default `"*"`) and support `*`/`?` wildcards (filters use `LIKE`). For a normal content question, resolve a ref first @@ -448,14 +443,6 @@ content across all refs at once; because every content tool carries `git.commit` (and aggregations group `BY git.commit`), unpinned results stay attributable per commit rather than being blended — but a version-specific answer should still pin a ref. -**The post-FORK `| WHERE status == "complete"`** is an automatic consistency guard (no param): it -serves content only from a ref whose latest index is complete. For incremental content this excludes -the torn/partial-read window while a branch is mid-reindex — during a HEAD advance the branch's refs -join doc is `status: indexing` and its content is being mutated in place. For snapshot content this -is a no-op (status is already `"complete"` from the `EVAL` above). Trade-off: a *failed* incremental -run leaves the join doc at `status: indexing` with the prior commit's content still fully consistent; -the guard hides that content until the next successful run republishes `status: complete`. - #### `status` field values Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental join docs alike @@ -465,7 +452,7 @@ Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental |---|---| | `indexing` | A run is mid-flight. `indexing_started_at` is set; `indexed_at` is absent/null. Present on snapshot ref-name markers (written by `write_indexing_marker` just before ingest) and incremental join docs (written by `write_incremental_indexing`). A stale `indexing` doc whose `indexing_started_at` is older than the retry window (default 6 h) marks a crashed run and is treated as due for re-indexing. | | `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot markers) and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | -| `stale` | A snapshot marker superseded by a mode switch to `delta`. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). Stale markers are invisible to all content tools (all gate on `status == "complete"`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | +| `stale` | A snapshot marker superseded by a mode switch to `delta`. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | #### Uniqueness gate (INV-011 backstop) @@ -529,5 +516,5 @@ is a breaking change to the backing indices, with no config schema change: source. The old `sourcerer-v2-*` indices can be deleted once you have re-indexed. - **Agent Builder tools**: content tools (`sourcerer.code.*`, `sourcerer.files.*`) replace their `git_commit` param with `git_ref` (a commit SHA or a branch/tag name); `git_commit` survives - as an optional post-join consistency guard. Run `sourcerer setup` again to push the updated - tool definitions. \ No newline at end of file + as an optional filter alongside `git_ref` and `git_ref_type`. Run `sourcerer setup` again to push + the updated tool definitions. \ No newline at end of file diff --git a/src/sourcerer/elastic/agent_builder_tools/README.md b/src/sourcerer/elastic/agent_builder_tools/README.md index 8fd3518..d21c89e 100644 --- a/src/sourcerer/elastic/agent_builder_tools/README.md +++ b/src/sourcerer/elastic/agent_builder_tools/README.md @@ -23,8 +23,6 @@ Query snippet: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -34,7 +32,6 @@ Query snippet: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -47,36 +44,19 @@ Query snippet: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) // other filters -// Branch by content-doc shape: -// 1. Content for commit snapshots already carry git.commit, and status -// was confirmed by the membership subquery above (status=="complete"), -// so the snapshot arm needs no join; it just asserts status to match -// the incremental arm's column. -// 2. Content for incremental refs carry only git.ref. The join resolves -// the ref's current status from its join doc. This join assumes at -// most one doc per (host,org,repo,ref). That assumption requires a -// "one update mode owns a ref name" invariant (no repo may have both -// a snapshot and incremental source targeting the same ref name), -// which must be enforced separately (e.g. at config-validation time); -// nothing in this query itself enforces it. +// Branch by content-doc shape to resolve git.commit for incremental refs: +// Snapshot rows already carry git.commit (no join needed). +// Incremental rows carry only git.ref; the join resolves git.commit from the refs join doc. | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) -// Consistency guard: Only retrieve from a ref whose indexing is complete. -// Excludes torn/partial-read windows when an incrementally indexed ref is -// in the middle of an update. This is a no-op for the commit snapshot arm, -// whose status is always "complete" from the EVAL above. -| WHERE status == "complete" - // rest of query ``` diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml index a940aa5..940fc20 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml @@ -19,8 +19,6 @@ configuration: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -30,7 +28,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -43,37 +40,21 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) AND file.path LIKE ?file_path AND line.content RLIKE ?regex - // Branch by content-doc shape: - // 1. Content for commit snapshots already carry git.commit, and status - // was confirmed by the membership subquery above (status=="complete"), - // so the snapshot arm needs no join; it just asserts status to match - // the incremental arm's column. - // 2. Content for incremental refs carry only git.ref. The join resolves - // the ref's current status from its join doc. This join assumes at - // most one doc per (host,org,repo,ref). That assumption requires a - // "one update mode owns a ref name" invariant (no repo may have both - // a snapshot and incremental source targeting the same ref name), - // which must be enforced separately (e.g. at config-validation time); - // nothing in this query itself enforces it. + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: Only retrieve from a ref whose indexing is complete. - // Excludes torn/partial-read windows when an incrementally indexed ref is - // in the middle of an update. This is a no-op for the commit snapshot arm, - // whose status is always "complete" from the EVAL above. - | WHERE status == "complete" - // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml index 02a6548..1ae0075 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml @@ -19,8 +19,6 @@ configuration: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -30,7 +28,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -43,37 +40,21 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) AND file.path LIKE ?file_path AND MATCH(line.content.text, ?q) - // Branch by content-doc shape: - // 1. Content for commit snapshots already carry git.commit, and status - // was confirmed by the membership subquery above (status=="complete"), - // so the snapshot arm needs no join; it just asserts status to match - // the incremental arm's column. - // 2. Content for incremental refs carry only git.ref. The join resolves - // the ref's current status from its join doc. This join assumes at - // most one doc per (host,org,repo,ref). That assumption requires a - // "one update mode owns a ref name" invariant (no repo may have both - // a snapshot and incremental source targeting the same ref name), - // which must be enforced separately (e.g. at config-validation time); - // nothing in this query itself enforces it. + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: Only retrieve from a ref whose indexing is complete. - // Excludes torn/partial-read windows when an incrementally indexed ref is - // in the middle of an update. This is a no-op for the commit snapshot arm, - // whose status is always "complete" from the EVAL above. - | WHERE status == "complete" - // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml index 3e261d7..60debc3 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml @@ -19,8 +19,6 @@ configuration: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -30,7 +28,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -43,36 +40,20 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) AND file.path LIKE ?file_path - // Branch by content-doc shape: - // 1. Content for commit snapshots already carry git.commit, and status - // was confirmed by the membership subquery above (status=="complete"), - // so the snapshot arm needs no join; it just asserts status to match - // the incremental arm's column. - // 2. Content for incremental refs carry only git.ref. The join resolves - // the ref's current status from its join doc. This join assumes at - // most one doc per (host,org,repo,ref). That assumption requires a - // "one update mode owns a ref name" invariant (no repo may have both - // a snapshot and incremental source targeting the same ref name), - // which must be enforced separately (e.g. at config-validation time); - // nothing in this query itself enforces it. + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: Only retrieve from a ref whose indexing is complete. - // Excludes torn/partial-read windows when an incrementally indexed ref is - // in the middle of an update. This is a no-op for the commit snapshot arm, - // whose status is always "complete" from the EVAL above. - | WHERE status == "complete" - // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml index 9c4e6ab..1c292bf 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml @@ -19,8 +19,6 @@ configuration: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -30,7 +28,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -43,33 +40,20 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) AND file.path LIKE ?file_path - // Branch by content-doc shape: - // 1. Content for commit snapshots already carry git.commit, and status - // was confirmed by the membership subquery above (status=="complete"), - // so the snapshot arm needs no join; it just asserts status to match - // the incremental arm's column. - // 2. Content for incremental refs carry only git.ref. The join resolves - // the ref's current status from its join doc. Safety of the incremental - // join (one doc per (host,org,repo,ref)) is enforced by the "one update - // mode owns a ref name" invariant at index time. + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: Only retrieve from a ref whose indexing is complete. - // Excludes torn/partial-read windows when an incrementally indexed ref is - // in the middle of an update. This is a no-op for the commit snapshot arm, - // whose status is always "complete" from the EVAL above. - | WHERE status == "complete" - // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml index d761ac1..56e1ffa 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml @@ -19,8 +19,6 @@ configuration: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -30,7 +28,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -43,35 +40,19 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) - // Branch by content-doc shape: - // 1. Content for commit snapshots already carry git.commit, and status - // was confirmed by the membership subquery above (status=="complete"), - // so the snapshot arm needs no join; it just asserts status to match - // the incremental arm's column. - // 2. Content for incremental refs carry only git.ref. The join resolves - // the ref's current status from its join doc. This join assumes at - // most one doc per (host,org,repo,ref). That assumption requires a - // "one update mode owns a ref name" invariant (no repo may have both - // a snapshot and incremental source targeting the same ref name), - // which must be enforced separately (e.g. at config-validation time); - // nothing in this query itself enforces it. + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: Only retrieve from a ref whose indexing is complete. - // Excludes torn/partial-read windows when an incrementally indexed ref is - // in the middle of an update. This is a no-op for the commit snapshot arm, - // whose status is always "complete" from the EVAL above. - | WHERE status == "complete" - // Enforce glob depth for * and ** on file.path. // Without this, "src/*/Job.java" would match files at any depth, // not just three path segments, because a wildcard * in ES|QL diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml index a9fd956..a80cfe6 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml @@ -19,8 +19,6 @@ configuration: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -30,7 +28,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -43,7 +40,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) @@ -51,30 +47,15 @@ configuration: AND line.number >= ?line_number_start AND line.number <= ?line_number_end - // Branch by content-doc shape: - // 1. Content for commit snapshots already carry git.commit, and status - // was confirmed by the membership subquery above (status=="complete"), - // so the snapshot arm needs no join; it just asserts status to match - // the incremental arm's column. - // 2. Content for incremental refs carry only git.ref. The join resolves - // the ref's current status from its join doc. This join assumes at - // most one doc per (host,org,repo,ref). That assumption requires a - // "one update mode owns a ref name" invariant (no repo may have both - // a snapshot and incremental source targeting the same ref name), - // which must be enforced separately (e.g. at config-validation time); - // nothing in this query itself enforces it. + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: Only retrieve from a ref whose indexing is complete. - // Excludes torn/partial-read windows when an incrementally indexed ref is - // in the middle of an update. This is a no-op for the commit snapshot arm, - // whose status is always "complete" from the EVAL above. - | WHERE status == "complete" - // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml index 28df24c..188017c 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml @@ -19,8 +19,6 @@ configuration: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -30,7 +28,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -43,36 +40,20 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) AND file.path LIKE ?file_path - // Branch by content-doc shape: - // 1. Content for commit snapshots already carry git.commit, and status - // was confirmed by the membership subquery above (status=="complete"), - // so the snapshot arm needs no join; it just asserts status to match - // the incremental arm's column. - // 2. Content for incremental refs carry only git.ref. The join resolves - // the ref's current status from its join doc. This join assumes at - // most one doc per (host,org,repo,ref). That assumption requires a - // "one update mode owns a ref name" invariant (no repo may have both - // a snapshot and incremental source targeting the same ref name), - // which must be enforced separately (e.g. at config-validation time); - // nothing in this query itself enforces it. + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: Only retrieve from a ref whose indexing is complete. - // Excludes torn/partial-read windows when an incrementally indexed ref is - // in the middle of an update. This is a no-op for the commit snapshot arm, - // whose status is always "complete" from the EVAL above. - | WHERE status == "complete" - // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml index 5135d71..0c486d9 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml @@ -19,8 +19,6 @@ configuration: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -30,7 +28,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -43,35 +40,19 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) - // Branch by content-doc shape: - // 1. Content for commit snapshots already carry git.commit, and status - // was confirmed by the membership subquery above (status=="complete"), - // so the snapshot arm needs no join; it just asserts status to match - // the incremental arm's column. - // 2. Content for incremental refs carry only git.ref. The join resolves - // the ref's current status from its join doc. This join assumes at - // most one doc per (host,org,repo,ref). That assumption requires a - // "one update mode owns a ref name" invariant (no repo may have both - // a snapshot and incremental source targeting the same ref name), - // which must be enforced separately (e.g. at config-validation time); - // nothing in this query itself enforces it. + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: Only retrieve from a ref whose indexing is complete. - // Excludes torn/partial-read windows when an incrementally indexed ref is - // in the middle of an update. This is a no-op for the commit snapshot arm, - // whose status is always "complete" from the EVAL above. - | WHERE status == "complete" - // Split each file path into its segments | EVAL _segs = SPLIT(file.path, "/") | EVAL _file_segs = MV_COUNT(_segs) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml index bfeee6f..85bef11 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml @@ -19,8 +19,6 @@ configuration: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -30,7 +28,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -43,36 +40,20 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) AND file.path LIKE ?file_path - // Branch by content-doc shape: - // 1. Content for commit snapshots already carry git.commit, and status - // was confirmed by the membership subquery above (status=="complete"), - // so the snapshot arm needs no join; it just asserts status to match - // the incremental arm's column. - // 2. Content for incremental refs carry only git.ref. The join resolves - // the ref's current status from its join doc. This join assumes at - // most one doc per (host,org,repo,ref). That assumption requires a - // "one update mode owns a ref name" invariant (no repo may have both - // a snapshot and incremental source targeting the same ref name), - // which must be enforced separately (e.g. at config-validation time); - // nothing in this query itself enforces it. + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK - ( WHERE git.commit IS NOT NULL - | EVAL status = "complete" ) + ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) - // Consistency guard: Only retrieve from a ref whose indexing is complete. - // Excludes torn/partial-read windows when an incrementally indexed ref is - // in the middle of an update. This is a no-op for the commit snapshot arm, - // whose status is always "complete" from the EVAL above. - | WHERE status == "complete" - // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml index 68acbd7..b151040 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml @@ -19,8 +19,6 @@ configuration: // So resolution yields two membership sets off the same match: // 1. Matching commits, checked against snapshot-shaped rows // 2. Matching refs, checked against incremental-shaped rows - // Includes a consistency guard by only retrieving content from refs - // whose indexing is complete. (git.commit IS NOT NULL AND git.commit IN ( // Commit snapshots FROM sourcerer-refs @@ -30,7 +28,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.commit )) OR @@ -43,7 +40,6 @@ configuration: AND git.commit LIKE ?git_commit AND git.ref LIKE ?git_ref AND git.ref_type LIKE ?git_ref_type - AND status == "complete" | KEEP git.ref )) ) diff --git a/src/sourcerer/skills/ref-resolution/SKILL.md b/src/sourcerer/skills/ref-resolution/SKILL.md index 971076c..6a423f1 100644 --- a/src/sourcerer/skills/ref-resolution/SKILL.md +++ b/src/sourcerer/skills/ref-resolution/SKILL.md @@ -73,9 +73,9 @@ Once a ref is resolved above, pass the value straight through: itself as `git_commit_ish` (e.g. `main`) -- no commit needed, the query always resolves to whatever commit that branch is CURRENTLY at. -Internally, each tool resolves the citable commit via a FORK that branches on content-doc shape: -a `LOOKUP JOIN` on (`git.host`, `git.org`, `git.repo`, `git.commit`) for snapshot-shaped rows, or -on (`git.host`, `git.org`, `git.repo`, `git.ref`) for incremental-shaped rows. +Internally, each tool uses a FORK that branches on content-doc shape to resolve the citable commit: +snapshot-shaped rows already carry `git.commit` (no join needed), while incremental-shaped rows get +their `git.commit` resolved via a `LOOKUP JOIN` on (`git.host`, `git.org`, `git.repo`, `git.ref`). Read the resolved `git.commit` back from each result row (the content query's own join supplies it) for citations. Because incremental content overwrites in place, a branch query always returns diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index f59b39e..1f1420f 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -51,12 +51,13 @@ def test_git_host_filtered_before_git_org(): def test_content_tools_use_universal_ref_join_query(): # Every content tool uses a two-OR'd-IN subquery to scope rows to matching - # refs (git.commit OR git.ref), then a FORK to handle the two content shapes - # separately without fan-out: - # - Snapshot arm (git.commit IS NOT NULL): EVAL status = "complete" -- no join needed; the - # commit already lives on the content row, and status was pre-confirmed by the subquery. - # - Incremental arm (git.ref IS NOT NULL AND git.commit IS NULL): LOOKUP JOIN sourcerer-refs - # ON (host,org,repo,ref) to resolve status from the incremental join doc. + # refs (git.commit OR git.ref), then a FORK to resolve git.commit for both + # content shapes without fan-out: + # - Snapshot arm (git.commit IS NOT NULL): no join needed; git.commit already + # lives on the content row. + # - Incremental arm (git.ref IS NOT NULL AND git.commit IS NULL): LOOKUP JOIN + # sourcerer-refs ON (host,org,repo,ref) to resolve git.commit from the join doc. + # No status guard is applied anywhere in the query. # Ref scoping uses three separate params: git_commit, git_ref, git_ref_type. tools = _tools() for tid in _CONTENT_TOOL_IDS: @@ -72,10 +73,9 @@ def test_content_tools_use_universal_ref_join_query(): assert "git.commit LIKE ?git_commit" in query, f"{tid} missing git.commit LIKE ?git_commit" assert "git.ref LIKE ?git_ref" in query, f"{tid} missing git.ref LIKE ?git_ref" assert "git.ref_type LIKE ?git_ref_type" in query, f"{tid} missing git.ref_type LIKE ?git_ref_type" - # Snapshot arm: no join; asserts status = "complete" inline. + # Snapshot arm: no join; git.commit already on the content row. assert "git.commit IS NOT NULL" in query, f"{tid} missing snapshot FORK arm (git.commit IS NOT NULL)" - assert 'EVAL status = "complete"' in query, f"{tid} missing EVAL status = \"complete\" in snapshot arm" - # Incremental arm: join on the 4-tuple (no ref_key). + # Incremental arm: join on the 4-tuple (no ref_key) to resolve git.commit. assert "git.ref IS NOT NULL" in query, f"{tid} missing incremental FORK arm (git.ref IS NOT NULL)" assert "LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref" in query, \ f"{tid} missing the incremental join on (host,org,repo,ref)" @@ -90,10 +90,10 @@ def test_content_tools_use_universal_ref_join_query(): # No collapsed git_commit_ish param. assert "git_commit_ish" not in params, f"{tid} still exposes git_commit_ish as a param" assert "?git_commit_ish" not in query, f"{tid} still references ?git_commit_ish" - # The post-FORK status guard must appear after the join (defense-in-depth; free no-op for - # snapshot arm since status is already "complete" from the EVAL). - assert '| WHERE status == "complete"' in query, f"{tid} missing the post-FORK status guard" - assert query.index("LOOKUP JOIN sourcerer-refs") < query.index('WHERE status == "complete"') + # No status guard: the post-FORK status filter and status-related EVALs have been removed. + assert '| WHERE status == "complete"' not in query, f"{tid} still has the removed status guard" + assert 'EVAL status = "complete"' not in query, f"{tid} still has the removed EVAL status" + assert "status" not in strip_esql_comments(query), f"{tid} still references status in query body" def test_refs_list_does_not_surface_ref_key(): From 4ddd2285554adb19093e1ca9f63866211a872814 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Mon, 24 Aug 2026 10:34:34 -0700 Subject: [PATCH 25/29] Add tag support to delta-mode indexing (mode: delta, ref_type: tag) Delta mode now accepts git.ref_type: tag in addition to branch. This enables cheap indexing of fast-moving tags like elastic/kibana's deploy@{major} Serverless promotion tags: instead of minting a full snapshot per force-update, a SHA-to-SHA diff touches only changed paths. Changes: - config.py: lift branch-only restriction to accept branch or tag - utils.py: build_ref_key now takes ref_type, making branch and same-named tag produce distinct join-doc _ids - markers.py: thread ref_type through write_incremental_*, read, delete_incremental_paths/branch, and count functions; replace hardcoded "branch" literal in join doc body - documents.py: add ref_type to build_incremental_file_doc, iter_incremental_line_docs, _init_worker_incremental, and index_incremental_paths; carry it into content doc git.ref_type field and make_doc_id call - command.py: derive ref_type from unit.kind; dispatch checkout_ref for tags vs checkout_branch for branches; thread ref_type into all callers - queries.py: add git.ref_type to prune sweep tuple keys (5-tuple) and check_join_uniqueness composite agg; add _enumerate_incremental_content_ref_pairs helper - planner.py: update OrphanPlan and plan_orphans type annotations to 5-tuple (host, org, repo, ref_type, ref) - prune/execute.py: unpack ref_type from 5-tuple and add to delete-by-query filter so tag cleanup never touches same-named branch - 9 content tool YAMLs: extend LOOKUP JOIN to include git.ref_type, preventing join fan-out when a branch and tag share a name in delta mode - index templates (files + lines): add git.ref_type keyword mapping - Tests: update all markers/documents/config/incremental_index tests for new ref_type params; add tag-kind test variants and checkout dispatch assertions (254 tests pass) - Docs: update AGENTS.md, README.md, sourcerer.example.yml, specs/sourcerer-yml.md, and progress.py comment from "branch-only" to "branch or tag" Co-Authored-By: Claude Opus 4.8 --- AGENTS.md | 11 +- README.md | 13 +- sourcerer.example.yml | 8 +- specs/sourcerer-yml.md | 2 +- src/sourcerer/commands/index/command.py | 65 +++++---- src/sourcerer/commands/index/documents.py | 85 ++++++++---- src/sourcerer/commands/index/markers.py | 74 +++++++---- src/sourcerer/commands/prune/execute.py | 7 +- src/sourcerer/config.py | 10 +- .../sourcerer.code.grep.yml | 2 +- .../sourcerer.code.search.yml | 2 +- .../sourcerer.files.cat.yml | 2 +- .../sourcerer.files.head.yml | 2 +- .../sourcerer.files.ls.yml | 2 +- .../sourcerer.files.read_lines.yml | 2 +- .../sourcerer.files.tail.yml | 2 +- .../sourcerer.files.tree.yml | 2 +- .../sourcerer.files.wc.yml | 2 +- .../index_templates/sourcerer-v3-files.json | 3 + .../index_templates/sourcerer-v3-lines.json | 3 + src/sourcerer/planner.py | 46 ++++--- src/sourcerer/progress.py | 2 +- src/sourcerer/queries.py | 125 +++++++++++++----- src/sourcerer/utils.py | 15 ++- tests/test_config.py | 24 +++- tests/test_documents.py | 35 +++-- tests/test_incremental_index.py | 112 +++++++++++++++- tests/test_markers.py | 65 ++++++--- 28 files changed, 507 insertions(+), 216 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f358eb7..b0cf864 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,22 +57,27 @@ config's `sources:` is a YAML list, one entry per (host, org, repo, ref_type). S | `match` | yes | For `branch`/`tag`: pattern string or list of patterns matched against ref names (version DSL + glob), a ref matches if any pattern hits. For `commit`: a commit SHA/prefix string or list of them (see below). | | `since` | no | Index-side inclusion floor: the earliest commit to start indexing from. See below. Not valid for `git.ref_type: commit`. | | `retain` | no | Retention policy (see below). Omit to keep forever. For `git.ref_type: commit`, only `age` is valid. | -| `mode` | no | `snapshot` (default) or `delta` (branch-only). See below. | +| `mode` | no | `snapshot` (default) or `delta` (branch or tag). See below. | #### `mode` (snapshot vs. delta) `snapshot` (default): content is commit-addressed. A HEAD advance on a branch indexes a whole new snapshot under the new commit. -`delta` (branch-only; rejects `since`/`retain` -- there is no per-commit history for either +`delta` (branch or tag; rejects `since`/`retain` -- there is no per-commit history for either to apply to): content is ref-addressed instead. A HEAD advance runs `git diff --name-status` between the previously-completed commit and the new tip and only deletes/ reindexes the paths git reports changed -- add/modify/delete/rename/copy -- instead of reindexing the whole tree. A missing diff base (force-push, GC'd, or the first index) rebuilds -the whole branch namespace. The refs join doc publishes `status: indexing` before any content +the whole ref namespace. The refs join doc publishes `status: indexing` before any content change and `status: complete` (with the new commit) only after the deletes/indexes/refresh all succeed, so a crash mid-update leaves the prior commit and content in place. +Delta mode is especially useful for fast-moving tags that are force-updated many times a day +(e.g. `deploy@8`-style Serverless promotion tags): snapshot mode would mint a fresh full snapshot +per force-update; delta mode diffs only what changed, keeping indexing cost proportional to the +diff size regardless of tag-move frequency. + ```yaml - git: host: github diff --git a/README.md b/README.md index 543911a..bb631ed 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ The [`sourcerer.yml` specification](specs/sourcerer-yml.md) has the full referen ### Snapshot vs. delta indexing (`mode`) -Each source can set `mode: snapshot` (the default) or `mode: delta` (branch-only). Every +Each source can set `mode: snapshot` (the default) or `mode: delta` (branch or tag). Every Agent Builder content tool takes the same `git_commit_ish` param either way (a commit SHA or a branch/tag name, `*`/`?` wildcards supported) and resolves a commit the same way regardless of mode. @@ -95,13 +95,14 @@ mode. - **`snapshot`** (default): content is commit-addressed. Every ref (branch, tag, or pinned commit) that resolves to the same commit collapses to one snapshot. A moving branch's HEAD advance indexes a brand-new snapshot under the new commit. -- **`delta`** (branch-only): content is ref-addressed instead. Content docs carry `git.ref` - but no `git.commit` of their own — the branch's current commit lives only on its refs join doc, +- **`delta`** (branch or tag): content is ref-addressed instead. Content docs carry `git.ref` + but no `git.commit` of their own — the ref's current commit lives only on its refs join doc, resolved at query time via a LOOKUP JOIN. A HEAD advance re-indexes only the files `git diff --name-status` reports changed (add/modify/delete/rename), not the whole tree, so - staying current on a fast-moving branch (e.g. GitOps/IaC repos that deploy off `main`) is cheap. - `since` and `retain` don't apply to a delta-mode source (there is no per-commit history to - filter or retain) and are rejected if given. + staying current on a fast-moving branch or tag is cheap. Particularly useful for fast-moving + tags that are force-updated frequently (e.g. `deploy@8`-style Serverless promotion tags) where + snapshot mode would mint a full snapshot per move. `since` and `retain` don't apply to a + delta-mode source (there is no per-commit history to filter or retain) and are rejected if given. ```yaml sources: diff --git a/sourcerer.example.yml b/sourcerer.example.yml index 216d715..7632785 100644 --- a/sourcerer.example.yml +++ b/sourcerer.example.yml @@ -184,12 +184,12 @@ sources: retain: count: 5 -# Delta mode -- branch-only. Instead of a new commit-addressed snapshot on every HEAD +# Delta mode -- branch or tag. Instead of a new commit-addressed snapshot on every HEAD # advance, content is keyed by git.ref and stays in place: a HEAD advance re-indexes only the # files `git diff` reports changed (a delta update), rather than the whole tree. Good for a -# fast-moving branch that deploys off main, where staying current matters more than retaining -# per-commit history. `since` and `retain` are not meaningful here (there is no per-commit -# history to filter/retain) and are rejected if given. +# fast-moving branch or force-updated tag (e.g. `deploy@8`-style Serverless promotion tags) +# where staying current matters more than retaining per-commit history. `since` and `retain` +# are not meaningful here (there is no per-commit history to filter/retain) and are rejected if given. - git: host: github org: elastic diff --git a/specs/sourcerer-yml.md b/specs/sourcerer-yml.md index ecbd75e..bea68b2 100644 --- a/specs/sourcerer-yml.md +++ b/specs/sourcerer-yml.md @@ -425,7 +425,7 @@ incrementally as the HEAD moves (`"delta"`). Controls whether `since` and - Default: `"snapshot"` - Validation: - Must be one of: `"snapshot"`, `"delta"` - - `"delta"` is only valid when `git.ref_type` is `"branch"` + - `"delta"` is only valid when `git.ref_type` is `"branch"` or `"tag"` - `"delta"` cannot be combined with `since` or `retain` - `"delta"` cannot be combined with `index.level: commit` diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index bf1d349..e1e10e4 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -259,17 +259,18 @@ def index_incremental_branch_in_dir( reporter: ProgressReporter | None = None, unit: Unit | None = None, ) -> None: - """Advance one incremental (ref-addressed) branch source in an already-cloned `repo_dir`. + """Advance one incremental (ref-addressed) branch or tag source in an already-cloned + `repo_dir`. - Reads the branch's prior completed commit (its refs join doc, `_id = ref_key`), checks out - the fetched branch tip, and either: + Reads the ref's prior completed commit (its refs join doc, `_id = ref_key`), checks out + the fetched tip, and either: - does nothing (already at the completed commit and not `--force`), - does a full rebuild (first index, `--force`, or a missing diff base -- INV-007): delete - the whole branch namespace, then index every currently-tracked path, or + the whole ref namespace, then index every currently-tracked path, or - does a delta update: `git diff --name-status` (via `plan_changes`) between the prior and new commit, deleting only the paths git reports removed/changed and (re)indexing only the - paths git reports added/changed (INV-008 -- scoped by the exact (host,org,repo,ref) tuple, - never a whole namespace sweep). + paths git reports added/changed (INV-008 -- scoped by the exact + (host,org,repo,ref_type,ref) 5-tuple, never a whole namespace sweep). The refs join doc is published `indexing` before any mutation and `complete` only after the content deletes/indexes and a refresh all succeed (INV-006); a raised exception instead records `write_incremental_failed` and leaves the completed pointer untouched, then @@ -280,12 +281,17 @@ def index_incremental_branch_in_dir( if unit is None: unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", mode="delta") + ref_type = unit.kind # "branch" or "tag" + reporter.set_stage(unit, "checkout") - checkout_branch(repo_dir, branch) + if ref_type == "tag": + checkout_ref(repo_dir, branch) + else: + checkout_branch(repo_dir, branch) new_sha = resolve_commit(repo_dir) commit_date_iso = commit_date(repo_dir) - prior = read_incremental_ref(es, host, org, repo, branch) + prior = read_incremental_ref(es, host, org, repo, ref_type, branch) old_sha = None if force else (prior.get("git", {}).get("commit") if prior else None) level = unit.index_level @@ -302,7 +308,7 @@ def index_incremental_branch_in_dir( return reporter.set_stage(unit, "indexing") - write_incremental_indexing(es, host, org, repo, branch, completed_commit=old_sha, + write_incremental_indexing(es, host, org, repo, ref_type, branch, completed_commit=old_sha, commit_target=new_sha, prior=prior, index_level=level, index_suffix=suffix) try: @@ -312,52 +318,53 @@ def index_incremental_branch_in_dir( full_rebuild = plan.base_missing if full_rebuild: - delete_incremental_branch(es, host, org, repo, branch, index_level=level, index_suffix=suffix) + delete_incremental_branch(es, host, org, repo, branch, + ref_type=ref_type, index_level=level, index_suffix=suffix) reporter.set_total_files(unit, count_tracked_files(repo_dir)) indexed_files, indexed_lines = index_incremental_paths( es, host, org, repo, repo_dir, branch, None, on_progress=lambda f, l: reporter.update_counts(unit, f, l), - index_level=level, index_suffix=suffix, + index_level=level, index_suffix=suffix, ref_type=ref_type, ) else: - delete_incremental_paths(es, host, org, repo, branch, plan.delete_paths, + delete_incremental_paths(es, host, org, repo, ref_type, branch, plan.delete_paths, index_level=level, index_suffix=suffix) reporter.set_total_files(unit, len(plan.index_paths)) indexed_files, indexed_lines = index_incremental_paths( es, host, org, repo, repo_dir, branch, plan.index_paths, on_progress=lambda f, l: reporter.update_counts(unit, f, l), - index_level=level, index_suffix=suffix, + index_level=level, index_suffix=suffix, ref_type=ref_type, ) refresh_incremental_content(es, host, org, repo, index_level=level, index_suffix=suffix) files_count, lines_count = count_incremental_branch_docs( - es, host, org, repo, branch, index_level=level, index_suffix=suffix, + es, host, org, repo, branch, ref_type=ref_type, index_level=level, index_suffix=suffix, ) # Mode-switch: flip any complete snapshot markers for this (host,org,repo,ref) to # "stale" BEFORE publishing the incremental join doc as "complete". This ensures the # two-complete-docs fan-out window (one snapshot + one incremental marker both matching - # LOOKUP JOIN ON git.ref) never opens. Stale content is reclaimed by prune. + # LOOKUP JOIN ON git.ref, git.ref_type) never opens. Stale content is reclaimed by prune. mark_snapshot_markers_stale(es, host, org, repo, branch) - write_incremental_ready(es, host, org, repo, branch, new_sha, commit_date_iso, + write_incremental_ready(es, host, org, repo, ref_type, branch, new_sha, commit_date_iso, files_count, lines_count, index_level=level, index_suffix=suffix) # Migration cleanup (write-new -> flip join doc -> delete-old): now that the join doc is - # complete and points at the new routing, delete this branch's docs from the old physical - # index. Scoped to the exact (host,org,repo,ref) 4-term filter so a sibling source that - # still lives in the old index is never touched. A crash between the ready write above - # and this delete leaves stale-location incremental docs in the old index; prune's - # incremental stale-location sweep (Class D-I) reclaims them. + # complete and points at the new routing, delete this ref's docs from the old physical + # index. Scoped to the exact (host,org,repo,ref_type,ref) 5-term filter so a sibling + # source that still lives in the old index is never touched. A crash between the ready + # write above and this delete leaves stale-location incremental docs in the old index; + # prune's incremental stale-location sweep (Class D-I) reclaims them. if routing_changed: old_level, old_suffix = old_routing delete_incremental_branch(es, host, org, repo, branch, - index_level=old_level, index_suffix=old_suffix) + ref_type=ref_type, index_level=old_level, index_suffix=old_suffix) except KeyboardInterrupt: - write_incremental_failed(es, host, org, repo, branch, completed_commit=old_sha, + write_incremental_failed(es, host, org, repo, ref_type, branch, completed_commit=old_sha, commit_target=new_sha, error="interrupted", prior=prior, index_level=level, index_suffix=suffix) raise except Exception as e: - write_incremental_failed(es, host, org, repo, branch, completed_commit=old_sha, + write_incremental_failed(es, host, org, repo, ref_type, branch, completed_commit=old_sha, commit_target=new_sha, error=str(e), prior=prior, index_level=level, index_suffix=suffix) raise @@ -621,11 +628,11 @@ def process_group(item: tuple[tuple[str, str, str], list[Unit]]) -> None: (host, org, repo), group = item clone_url = hosts[host].clone_url(org, repo) - # Incremental branch units are split from the snapshot pre-clone/skip/retention flow - # entirely: no cohort retention, no `since` history walk, no commit-addressed content - # reuse -- each is a standalone two-phase delta update against its own prior state - # (see index_incremental_branch_in_dir). Processed here, before the snapshot-only - # `group` continues below with incremental units filtered out. + # Delta-mode units (branches and tags) are split from the snapshot pre-clone/skip/ + # retention flow entirely: no cohort retention, no `since` history walk, no + # commit-addressed content reuse -- each is a standalone two-phase delta update + # against its own prior state (see index_incremental_branch_in_dir). Processed here, + # before the snapshot-only `group` continues below with delta units filtered out. incremental_units = [u for u in group if u.mode == "delta"] group = [u for u in group if u.mode != "delta"] for unit in incremental_units: diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index fd0691d..aa63099 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -150,6 +150,7 @@ def build_incremental_file_doc( host: str, org: str, repo: str, + ref_type: str, ref: str, rel_path: str, abs_path: pathlib.Path, @@ -159,9 +160,11 @@ def build_incremental_file_doc( target_path: str | None = None, target_size: int | None = None, ) -> tuple[str, dict]: - """Ref-addressed (incremental) file doc: carries `git.ref` but no `git.commit`; `_id` is - stable across commits (derived from the branch name, not the commit), so a modified file's - doc overwrites in place on the next HEAD advance rather than minting a new id.""" + """Ref-addressed (incremental) file doc: carries `git.ref` and `git.ref_type` but no + `git.commit`; `_id` is stable across commits (derived from ref_type + ref name, not the + commit SHA), so a modified file's doc overwrites in place on the next HEAD advance rather + than minting a new id. Including ref_type in the id keeps a same-named branch and tag in + delta mode in non-overlapping id spaces.""" p = pathlib.PurePosixPath(rel_path) directory = "" if str(p.parent) == "." else str(p.parent) extension = p.suffix.lstrip(".") or None @@ -198,10 +201,11 @@ def build_incremental_file_doc( "org": org, "repo": repo, "ref": ref, + "ref_type": ref_type, }, "file": file_fields, } - _id = make_doc_id(host, org, repo, "branch", ref, rel_path) + _id = make_doc_id(host, org, repo, ref_type, ref, rel_path) return _id, doc @@ -209,6 +213,7 @@ def iter_incremental_line_docs( host: str, org: str, repo: str, + ref_type: str, ref: str, rel_path: str, content: str, @@ -218,7 +223,8 @@ def iter_incremental_line_docs( target_size: int | None = None, attributes: list[str] | None = None, ) -> Iterator[tuple[str, dict]]: - """Ref-addressed (incremental) line docs -- same shape as `build_incremental_file_doc`.""" + """Ref-addressed (incremental) line docs -- same shape as `build_incremental_file_doc`: + carries `git.ref` and `git.ref_type` but no `git.commit`.""" p = pathlib.PurePosixPath(rel_path) directory = "" if str(p.parent) == "." else str(p.parent) extension = p.suffix.lstrip(".") or None @@ -242,11 +248,12 @@ def iter_incremental_line_docs( "org": org, "repo": repo, "ref": ref, + "ref_type": ref_type, }, "file": file_fields, } for line_num, line_content in enumerate(content.splitlines(), start=1): - _id = make_doc_id(host, org, repo, "branch", ref, rel_path, str(line_num)) + _id = make_doc_id(host, org, repo, ref_type, ref, rel_path, str(line_num)) yield _id, {**base, "line": {"number": line_num, "content": line_content}} @@ -273,14 +280,17 @@ def _init_worker( def _init_worker_incremental( - host: str, org: str, repo: str, ref: str, repo_dir: str, symlink_paths: frozenset[str] = frozenset(), + host: str, org: str, repo: str, ref_type: str, ref: str, repo_dir: str, + symlink_paths: frozenset[str] = frozenset(), index_level: str = "repo", index_suffix: str | None = None, ) -> None: - """Same as `_init_worker`, but for the incremental (ref-addressed) path: `ref` replaces - `commit_sha` and `mode` routes `_build_one_file_actions` to the incremental doc builders.""" + """Same as `_init_worker`, but for the incremental (ref-addressed) path: `ref_type`+`ref` + replace `commit_sha` and `mode` routes `_build_one_file_actions` to the incremental doc + builders.""" signal.signal(signal.SIGINT, signal.SIG_IGN) _WORKER_CTX.update( - host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), + host=host, org=org, repo=repo, ref_type=ref_type, ref=ref, + repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, mode="delta", ) @@ -305,7 +315,9 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: ctx = _WORKER_CTX incremental = ctx.get("mode", "snapshot") == "delta" host, org, repo = ctx["host"], ctx["org"], ctx["repo"] - commit_sha = ctx.get("ref") if incremental else ctx["commit_sha"] + ref_type = ctx.get("ref_type", "branch") + ref = ctx.get("ref") + commit_sha = ref if incremental else ctx["commit_sha"] level, suffix = ctx.get("index_level", "repo"), ctx.get("index_suffix") abs_path = ctx["repo_dir"] / rel_path # Incremental content is ref-addressed (no commit-level index name); the physical index name @@ -351,26 +363,42 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: else: git_target_path = None git_target_size = None - doc_builder = build_incremental_file_doc if incremental else build_file_doc - file_id, file_doc = doc_builder( - host, org, repo, commit_sha, rel_path, abs_path, binary=binary, - is_symlink=True if is_git_symlink else None, - target_path=git_target_path, - target_size=git_target_size, - ) + if incremental: + file_id, file_doc = build_incremental_file_doc( + host, org, repo, ref_type, ref, rel_path, abs_path, binary=binary, + is_symlink=True if is_git_symlink else None, + target_path=git_target_path, + target_size=git_target_size, + ) + else: + file_id, file_doc = build_file_doc( + host, org, repo, commit_sha, rel_path, abs_path, binary=binary, + is_symlink=True if is_git_symlink else None, + target_path=git_target_path, + target_size=git_target_size, + ) actions = [{"_index": f_index, "_id": file_id, "_source": file_doc}] if raw is None or binary: return actions content = raw.decode("utf-8", errors="surrogateescape") ff = file_doc["file"] - line_iter = iter_incremental_line_docs if incremental else iter_line_docs - for line_id, line_doc in line_iter( - host, org, repo, commit_sha, rel_path, content, - size=ff["size"], - target_path=ff.get("target_path"), - target_size=ff.get("target_size"), - attributes=ff.get("attributes"), - ): + if incremental: + line_docs = iter_incremental_line_docs( + host, org, repo, ref_type, ref, rel_path, content, + size=ff["size"], + target_path=ff.get("target_path"), + target_size=ff.get("target_size"), + attributes=ff.get("attributes"), + ) + else: + line_docs = iter_line_docs( + host, org, repo, commit_sha, rel_path, content, + size=ff["size"], + target_path=ff.get("target_path"), + target_size=ff.get("target_size"), + attributes=ff.get("attributes"), + ) + for line_id, line_doc in line_docs: actions.append({"_index": l_index, "_id": line_id, "_source": line_doc}) return actions @@ -478,8 +506,9 @@ def index_incremental_paths( on_progress: Callable[[int, int], None] | None = None, index_level: str = "repo", index_suffix: str | None = None, + ref_type: str = "branch", ) -> tuple[int, int]: - """Index a set of paths for an incremental (ref-addressed) branch source. + """Index a set of paths for an incremental (ref-addressed) branch or tag source. `rel_paths=None` walks the whole checked-out tree (first index / full rebuild, e.g. when a diff base is unavailable). A given `rel_paths` list indexes only those paths -- the delta @@ -498,7 +527,7 @@ def index_incremental_paths( with ProcessPoolExecutor( max_workers=max(1, t.index_workers), initializer=_init_worker_incremental, - initargs=(host, org, repo, ref, str(repo_dir), symlink_paths, index_level, index_suffix), + initargs=(host, org, repo, ref_type, ref, str(repo_dir), symlink_paths, index_level, index_suffix), ) as executor: def _batched(items: Iterator[str], n: int) -> Iterator[list[str]]: it = iter(items) diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index da25957..13ca0d2 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -540,23 +540,27 @@ def pre_clone_skip( # --- incremental refs join docs, keyed by `_id = build_ref_key(...)` ---------------------- -# One document per incremental branch (INV-004): the branch's single join doc lives at -# `_id = {host}~{org}~{repo}~{ref}` (constructed by build_ref_key, a plain tilde-joined -# string -- not a stored field) and its `git.commit` is the branch's live HEAD, advanced -# only by a two-phase indexing -> complete publication (INV-006). This is a DISTINCT id space -# from `build_ref_id`'s hashed, append-only ref-name markers above; a join doc's `_id` is a -# plain, unhashed build_ref_key() string, which a `build_ref_id` hash can never collide with. -# build_ref_key is still used as the `_id` constructor even though git.ref_key is no longer -# a stored field -- the id itself remains the stable overwrite key for each branch. +# One document per incremental ref (INV-004): a delta-mode ref's single join doc lives at +# `_id = {host}~{org}~{repo}~{ref_type}~{ref}` (constructed by build_ref_key, a plain +# tilde-joined string -- not a stored field) and its `git.commit` is the ref's live target +# commit, advanced only by a two-phase indexing -> complete publication (INV-006). ref_type +# ("branch" or "tag") is part of the key so a same-named branch and tag each get a distinct +# join doc. This is a DISTINCT id space from `build_ref_id`'s hashed, append-only ref-name +# markers above; a join doc's `_id` is a plain, unhashed build_ref_key() string, which a +# `build_ref_id` hash can never collide with. build_ref_key is still used as the `_id` +# constructor even though git.ref_key is no longer a stored field -- the id itself remains the +# stable overwrite key for each delta-mode ref. ERROR_MAX_LEN = 2000 # bound stored failure text so a giant git/ES error can't bloat the doc -def read_incremental_ref(es: Elasticsearch, host: str, org: str, repo: str, ref: str) -> dict | None: - """The branch's incremental join doc `_source`, or None if never indexed. A real-time GET +def read_incremental_ref( + es: Elasticsearch, host: str, org: str, repo: str, ref_type: str, ref: str, +) -> dict | None: + """The ref's incremental join doc `_source`, or None if never indexed. A real-time GET (by `_id = ref_key`), so it reflects the last write even without a refresh.""" try: - return es.get(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref))["_source"] + return es.get(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref))["_source"] except NotFoundError: return None @@ -569,6 +573,7 @@ def _build_incremental_join_doc( host: str, org: str, repo: str, + ref_type: str, ref: str, *, status: str, @@ -590,7 +595,7 @@ def _build_incremental_join_doc( "org": org, "repo": repo, "ref": ref, - "ref_type": "branch", + "ref_type": ref_type, "commit": commit, "commit_target": commit_target, "commit_date": commit_date_iso, @@ -613,6 +618,7 @@ def write_incremental_indexing( host: str, org: str, repo: str, + ref_type: str, ref: str, completed_commit: str | None, commit_target: str, @@ -628,7 +634,7 @@ def write_incremental_indexing( prior = prior or {} pg = prior.get("git", {}) doc = _build_incremental_join_doc( - host, org, repo, ref, + host, org, repo, ref_type, ref, status="indexing", commit=completed_commit, commit_target=commit_target, @@ -642,7 +648,7 @@ def write_incremental_indexing( index_level=index_level, index_suffix=index_suffix, ) - es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref), document=doc, refresh=refresh) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref), document=doc, refresh=refresh) def write_incremental_ready( @@ -650,6 +656,7 @@ def write_incremental_ready( host: str, org: str, repo: str, + ref_type: str, ref: str, commit: str, commit_date_iso: str | None, @@ -663,7 +670,7 @@ def write_incremental_ready( prior failure fields. This is the pointer-advancing publication boundary (INV-006): callers must delete+index+refresh the content indices FIRST, then call this.""" doc = _build_incremental_join_doc( - host, org, repo, ref, + host, org, repo, ref_type, ref, status="complete", commit=commit, commit_target=None, @@ -677,7 +684,7 @@ def write_incremental_ready( index_level=index_level, index_suffix=index_suffix, ) - es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref), document=doc, refresh=refresh) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref), document=doc, refresh=refresh) def write_incremental_failed( @@ -685,6 +692,7 @@ def write_incremental_failed( host: str, org: str, repo: str, + ref_type: str, ref: str, completed_commit: str | None, commit_target: str | None, @@ -700,7 +708,7 @@ def write_incremental_failed( prior = prior or {} pg = prior.get("git", {}) doc = _build_incremental_join_doc( - host, org, repo, ref, + host, org, repo, ref_type, ref, status="indexing", commit=completed_commit, commit_target=commit_target, @@ -714,7 +722,7 @@ def write_incremental_failed( index_level=index_level, index_suffix=index_suffix, ) - es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref), document=doc, refresh=refresh) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref), document=doc, refresh=refresh) def _delete_by_query_sync(es: Elasticsearch, index: str, query: dict, refresh: bool) -> None: @@ -741,16 +749,17 @@ def delete_incremental_paths( host: str, org: str, repo: str, + ref_type: str, ref: str, paths, index_level: str = "repo", index_suffix: str | None = None, refresh: bool = False, ) -> None: - """Synchronously delete the file and line docs for `paths` on this exact branch. Scoped by - the exact (git.host, git.org, git.repo, git.ref) 4-term filter (INV-008: one branch's docs - can never bleed into another's) plus a `file.path` terms filter, never a wildcard. A no-op - for an empty path set.""" + """Synchronously delete the file and line docs for `paths` on this exact ref. Scoped by + the exact (git.host, git.org, git.repo, git.ref_type, git.ref) 5-term filter (INV-008: + one ref's docs can never bleed into another's) plus a `file.path` terms filter, never a + wildcard. A no-op for an empty path set.""" paths = list(paths) if not paths: return @@ -760,6 +769,7 @@ def delete_incremental_paths( {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, + {"term": {"git.ref_type": ref_type}}, {"term": {"git.ref": ref}}, {"terms": {"file.path": paths}}, ] @@ -778,17 +788,20 @@ def delete_incremental_branch( org: str, repo: str, ref: str, + ref_type: str = "branch", index_level: str = "repo", index_suffix: str | None = None, refresh: bool = False, ) -> None: - """Delete EVERY incremental content doc for this branch (full namespace), scoped by the - exact (git.host, git.org, git.repo, git.ref) 4-term filter (INV-008). Used for the initial - index and the missing-diff-base rebuild (INV-007).""" + """Delete EVERY incremental content doc for this ref (full namespace), scoped by the exact + (git.host, git.org, git.repo, git.ref_type, git.ref) 5-term filter (INV-008). Used for + the initial index and the missing-diff-base rebuild (INV-007). `ref_type` defaults to + "branch" for back-compat with existing callers that pass `ref` positionally.""" query = {"bool": {"filter": [ {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, + {"term": {"git.ref_type": ref_type}}, {"term": {"git.ref": ref}}, ]}} for index in ( @@ -800,16 +813,19 @@ def delete_incremental_branch( def count_incremental_branch_docs( es: Elasticsearch, host: str, org: str, repo: str, ref: str, + ref_type: str = "branch", index_level: str = "repo", index_suffix: str | None = None, ) -> tuple[int, int]: - """Authoritative (files, lines) totals for a branch's current incremental view, counted by - exact (git.host, git.org, git.repo, git.ref). Call AFTER refreshing the content indices so - counts reflect the just-applied deletes and indexes -- these become the ready marker's - files_count/lines_count. Returns 0 for an index that does not exist yet.""" + """Authoritative (files, lines) totals for a ref's current incremental view, counted by + exact (git.host, git.org, git.repo, git.ref_type, git.ref). Call AFTER refreshing the + content indices so counts reflect the just-applied deletes and indexes -- these become the + ready marker's files_count/lines_count. Returns 0 for an index that does not exist yet. + `ref_type` defaults to "branch" for back-compat with existing callers.""" query = {"bool": {"filter": [ {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, + {"term": {"git.ref_type": ref_type}}, {"term": {"git.ref": ref}}, ]}} diff --git a/src/sourcerer/commands/prune/execute.py b/src/sourcerer/commands/prune/execute.py index 7c9ef89..d493e60 100644 --- a/src/sourcerer/commands/prune/execute.py +++ b/src/sourcerer/commands/prune/execute.py @@ -299,11 +299,11 @@ def execute_orphan_deletions(es: Elasticsearch, plan: OrphanPlan) -> tuple[int, pass # Class D-I: stale-location incremental content (ref-addressed, no git.commit). Mirrors Class D - # but keyed on (host, org, repo, ref) tuples -- the commit-keyed filter above cannot match - # incremental docs whose git.commit is absent. + # but keyed on (host, org, repo, ref_type, ref) tuples -- the commit-keyed filter above cannot + # match incremental docs whose git.commit is absent. for index_name, ref_tuples in plan.orphan_stale_incremental.items(): stale_dropped += len(ref_tuples) - for (host, org, repo, ref) in ref_tuples: + for (host, org, repo, ref_type, ref) in ref_tuples: try: es.delete_by_query( index=index_name, @@ -311,6 +311,7 @@ def execute_orphan_deletions(es: Elasticsearch, plan: OrphanPlan) -> tuple[int, {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, + {"term": {"git.ref_type": ref_type}}, {"term": {"git.ref": ref}}, ]}}, conflicts="proceed", diff --git a/src/sourcerer/config.py b/src/sourcerer/config.py index b5f8e30..c8a9355 100644 --- a/src/sourcerer/config.py +++ b/src/sourcerer/config.py @@ -283,9 +283,9 @@ class Selector: levels: tuple[str, ...] = () # numeric levels shared by the versioned match patterns schedule: Schedule | None = None # per-source schedule override (sources[i].schedule) # sources[i].mode: the indexing mode for this source -- "snapshot" (default, commit-addressed) - # or "delta" (ref-addressed, branch-only). Controls whether since/retain apply and routes + # or "delta" (ref-addressed, branch or tag). Controls whether since/retain apply and routes # the unit to the incremental delta-index path instead of the snapshot flow. - mode: str = "snapshot" # "snapshot" (default) or "delta" (branch-only) + mode: str = "snapshot" # "snapshot" (default) or "delta" (branch or tag) # sources[i].index routing (see specs/sourcerer-yml.md): which physical files/lines index this # source's content docs land in. Per-source, so two sources sharing a (host, org, repo) may # route differently. @@ -500,10 +500,10 @@ def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: index_level, index_suffix = _parse_index(raw["index"], ctx) if mode == "delta": - if ref_type != "branch": + if ref_type not in ("branch", "tag"): raise ValueError(f"{ctx} mode: 'delta' is only valid for " - f"git.ref_type: branch (got ref_type {ref_type!r})") - # A delta-mode branch maintains a single mutable ref-addressed view with no per-commit + f"git.ref_type: branch or tag (got ref_type {ref_type!r})") + # A delta-mode ref maintains a single mutable ref-addressed view with no per-commit # history for retention to trim and no inclusion floor to apply -- both since and retain # are meaningless here (see specs/incremental-indexing.md). if raw.get("since") is not None: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml index 940fc20..e61763e 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml @@ -53,7 +53,7 @@ configuration: | FORK ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml index 1ae0075..cd10d80 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml @@ -53,7 +53,7 @@ configuration: | FORK ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml index 60debc3..3cc23db 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml @@ -52,7 +52,7 @@ configuration: | FORK ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml index 1c292bf..c7697c5 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml @@ -52,7 +52,7 @@ configuration: | FORK ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml index 56e1ffa..de4e57a 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml @@ -51,7 +51,7 @@ configuration: | FORK ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) // Enforce glob depth for * and ** on file.path. // Without this, "src/*/Job.java" would match files at any depth, diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml index a80cfe6..0feaa55 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml @@ -54,7 +54,7 @@ configuration: | FORK ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml index 188017c..edff818 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml @@ -52,7 +52,7 @@ configuration: | FORK ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml index 0c486d9..d6ca1ed 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml @@ -51,7 +51,7 @@ configuration: | FORK ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) // Split each file path into its segments | EVAL _segs = SPLIT(file.path, "/") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml index 85bef11..a4f13b7 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml @@ -52,7 +52,7 @@ configuration: | FORK ( WHERE git.commit IS NOT NULL ) ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json index 80c9e6c..dce8b07 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json @@ -60,6 +60,9 @@ }, "ref": { "type": "keyword" + }, + "ref_type": { + "type": "keyword" } } }, diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json index 3a2623b..b289f65 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json @@ -102,6 +102,9 @@ }, "ref": { "type": "keyword" + }, + "ref_type": { + "type": "keyword" } } }, diff --git a/src/sourcerer/planner.py b/src/sourcerer/planner.py index d0662ce..bb0fe95 100644 --- a/src/sourcerer/planner.py +++ b/src/sourcerer/planner.py @@ -361,26 +361,28 @@ def orphan_stale_content( def orphan_stale_incremental_content( - incremental_content_by_index: dict[str, set[tuple[str, str, str, str]]], - intended_incremental_index_by_ref: dict[tuple[str, str, str, str], set[str]], + incremental_content_by_index: dict[str, set[tuple[str, str, str, str, str]]], + intended_incremental_index_by_ref: dict[tuple[str, str, str, str, str], set[str]], skip_indices: set[str], -) -> dict[str, set[tuple[str, str, str, str]]]: - """Class D-I orphans: incremental content docs sitting in a physical index that the branch's +) -> dict[str, set[tuple[str, str, str, str, str]]]: + """Class D-I orphans: incremental content docs sitting in a physical index that the ref's join doc no longer intends. This is the incremental migration backstop -- an index.level/suffix - change re-homes a branch's content to a new index and flips its join doc there; if a crash + change re-homes a ref's content to a new index and flips its join doc there; if a crash happens before the old copy is deleted, the old-location docs survive with no join doc referencing that location. - `incremental_content_by_index` maps a physical index name -> the set of (host, org, repo, ref) - tuples with incremental content docs in it. `intended_incremental_index_by_ref` maps a ref - tuple -> the set of index names its join doc intends (reconstructed from index_level/index_suffix - with commit=None). `skip_indices` excludes indices already going away via a Class-A whole-index + `incremental_content_by_index` maps a physical index name -> the set of (host, org, repo, + ref_type, ref) tuples with incremental content docs in it. + `intended_incremental_index_by_ref` maps a (host, org, repo, ref_type, ref) tuple -> the set + of index names its join doc intends (reconstructed from index_level/index_suffix with + commit=None). `skip_indices` excludes indices already going away via a Class-A whole-index DELETE. - Returns {index_name -> set of (host, org, repo, ref) tuples to delete-by-query from that index}. - A ref with NO join doc at all is not flagged here -- that is a different category; this class is - specifically 'has a join doc, but content lives somewhere the join doc doesn't intend'.""" - out: dict[str, set[tuple[str, str, str, str]]] = {} + Returns {index_name -> set of (host, org, repo, ref_type, ref) tuples to delete-by-query + from that index}. A ref with NO join doc at all is not flagged here -- that is a different + category; this class is specifically 'has a join doc, but content lives somewhere the join + doc doesn't intend'.""" + out: dict[str, set[tuple[str, str, str, str, str]]] = {} for index_name, ref_tuples in incremental_content_by_index.items(): if index_name in skip_indices: continue @@ -409,11 +411,11 @@ class OrphanPlan: # empty so callers/tests without empty-index data don't need to supply it. empty_index_names: list[str] = field(default_factory=list) # Class D-I -> delete_by_query per index for incremental (ref-addressed, commit-less) content - # sitting in a physical index its branch's join doc no longer intends. The incremental mirror + # sitting in a physical index its ref's join doc no longer intends. The incremental mirror # of Class D, since the commit-keyed Class-D sweep cannot see incremental docs. Value is - # {index_name -> set of (host, org, repo, ref) tuples to reclaim from that index}. Defaults - # empty so callers/tests without incremental location data don't need to supply it. - orphan_stale_incremental: dict[str, set[tuple[str, str, str, str]]] = field(default_factory=dict) + # {index_name -> set of (host, org, repo, ref_type, ref) tuples to reclaim from that index}. + # Defaults empty so callers/tests without incremental location data don't need to supply it. + orphan_stale_incremental: dict[str, set[tuple[str, str, str, str, str]]] = field(default_factory=dict) def plan_orphans( @@ -423,8 +425,8 @@ def plan_orphans( content_by_index_commit: dict[str, set[tuple[str, str, str, str]]] | None = None, intended_index_by_commit: dict[tuple[str, str, str, str], set[str]] | None = None, empty_index_names: list[str] | None = None, - incremental_content_by_index: dict[str, set[tuple[str, str, str, str]]] | None = None, - intended_incremental_index_by_ref: dict[tuple[str, str, str, str], set[str]] | None = None, + incremental_content_by_index: dict[str, set[tuple[str, str, str, str, str]]] | None = None, + intended_incremental_index_by_ref: dict[tuple[str, str, str, str, str], set[str]] | None = None, ) -> OrphanPlan: """Combine the orphan classes into one plan from cheap snapshots: the physical index names, the distinct (host, org, repo, commit) tuples in refs, and the distinct (host, org, repo, commit) @@ -471,9 +473,9 @@ def plan_orphans( ) # Class D-I: stale-location incremental content (ref-addressed, no git.commit). Mirrors Class D - # but keyed on (host, org, repo, ref) tuples from the branch's join doc. Also skip Class-A - # indices since they'll be deleted whole. - orphan_stale_incremental: dict[str, set[tuple[str, str, str, str]]] = {} + # but keyed on (host, org, repo, ref_type, ref) tuples from the ref's join doc. Also skip + # Class-A indices since they'll be deleted whole. + orphan_stale_incremental: dict[str, set[tuple[str, str, str, str, str]]] = {} if incremental_content_by_index is not None and intended_incremental_index_by_ref is not None: orphan_stale_incremental = orphan_stale_incremental_content( incremental_content_by_index, intended_incremental_index_by_ref, diff --git a/src/sourcerer/progress.py b/src/sourcerer/progress.py index bb07513..52e32b6 100644 --- a/src/sourcerer/progress.py +++ b/src/sourcerer/progress.py @@ -68,7 +68,7 @@ class Unit: index_level: str = "repo" index_suffix: str | None = None # sources[i].mode carried from the selector that emitted this unit: "snapshot" (default, - # commit-addressed) or "delta" (ref-addressed, branch-only). Routes the unit to the + # commit-addressed) or "delta" (ref-addressed, branch or tag). Routes the unit to the # incremental delta-index path instead of the snapshot pre-clone/skip/retention flow. mode: str = "snapshot" diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index d6a4141..44fb495 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -254,29 +254,31 @@ def gather_intended_index_by_commit( def gather_intended_incremental_index_by_ref( es: Elasticsearch, -) -> dict[tuple[str, str, str, str], set[str]]: - """For every (host, org, repo, ref) with an incremental join doc, the set of physical content - index names that join doc intends -- reconstructed from its index_level/index_suffix via - files_index/lines_index with commit=None (incremental content is ref-addressed, not - commit-addressed). +) -> dict[tuple[str, str, str, str, str], set[str]]: + """For every (host, org, repo, ref_type, ref) with an incremental join doc, the set of + physical content index names that join doc intends -- reconstructed from its + index_level/index_suffix via files_index/lines_index with commit=None (incremental content + is ref-addressed, not commit-addressed). - Feeds the incremental stale-location sweep (Class D-I): content for a branch sitting in an + Feeds the incremental stale-location sweep (Class D-I): content for a ref sitting in an index NOT in this set is stale from a crashed migration and should be reclaimed. Filters to - mode=="delta" docs only, so snapshot markers (which always have git.commit) are - not double-counted. Returns {} if the refs index doesn't exist.""" - out: dict[tuple[str, str, str, str], set[str]] = {} + mode=="delta" docs only, so snapshot markers (which always have git.commit) are not + double-counted. Returns {} if the refs index doesn't exist.""" + out: dict[tuple[str, str, str, str, str], set[str]] = {} body = {"query": {"term": {"mode": "delta"}}} - src_fields = ["git.host", "git.org", "git.repo", "git.ref", "index_level", "index_suffix"] + src_fields = ["git.host", "git.org", "git.repo", "git.ref_type", "git.ref", + "index_level", "index_suffix"] try: for hit in scan(es, index=REFS_ALIAS, query=body, _source=src_fields, preserve_order=False): src = hit["_source"] g = src.get("git", {}) - host, org, repo, ref = g.get("host"), g.get("org"), g.get("repo"), g.get("ref") - if not (host and org and repo and ref): + host, org, repo = g.get("host"), g.get("org"), g.get("repo") + ref_type, ref = g.get("ref_type"), g.get("ref") + if not (host and org and repo and ref_type and ref): continue level = src.get("index_level") or "repo" suffix = src.get("index_suffix") or None - key = (host, org, repo, ref) + key = (host, org, repo, ref_type, ref) intended = out.setdefault(key, set()) intended.add(files_index(host, org, repo, None, level, suffix)) intended.add(lines_index(host, org, repo, None, level, suffix)) @@ -287,15 +289,15 @@ def gather_intended_incremental_index_by_ref( def gather_incremental_content_by_index( es: Elasticsearch, index_names: list[str], -) -> dict[str, set[tuple[str, str, str, str]]]: - """Per physical index, the distinct (host, org, repo, ref) tuples with incremental content - docs in it (docs that have git.ref and a null/absent git.commit). +) -> dict[str, set[tuple[str, str, str, str, str]]]: + """Per physical index, the distinct (host, org, repo, ref_type, ref) tuples with incremental + content docs in it (docs that have git.ref and a null/absent git.commit). Feeds the incremental stale-location sweep (Class D-I) in planner.orphan_stale_incremental_content: to decide a doc is stale we must know WHICH physical index holds it AND which ref it belongs to, - so this enumerates each backing index by name via a composite aggregation over git.ref. - Empty/missing indices contribute nothing.""" - out: dict[str, set[tuple[str, str, str, str]]] = {} + so this enumerates each backing index by name via a composite aggregation over git.ref_type + + git.ref. Empty/missing indices contribute nothing.""" + out: dict[str, set[tuple[str, str, str, str, str]]] = {} for name in index_names: tuples = _composite_incremental_ref_tuples(es, name) if tuples: @@ -305,10 +307,10 @@ def gather_incremental_content_by_index( def _composite_incremental_ref_tuples( es: Elasticsearch, index: str, -) -> set[tuple[str, str, str, str]]: - """Distinct (host, org, repo, ref) tuples from incremental content docs (git.ref present, - git.commit absent) in `index`. Returns empty set if the index doesn't exist.""" - out: set[tuple[str, str, str, str]] = set() +) -> set[tuple[str, str, str, str, str]]: + """Distinct (host, org, repo, ref_type, ref) tuples from incremental content docs (git.ref + present, git.commit absent) in `index`. Returns empty set if the index doesn't exist.""" + out: set[tuple[str, str, str, str, str]] = set() after: dict | None = None while True: composite: dict = { @@ -317,6 +319,7 @@ def _composite_incremental_ref_tuples( {"host": {"terms": {"field": "git.host"}}}, {"org": {"terms": {"field": "git.org"}}}, {"repo": {"terms": {"field": "git.repo"}}}, + {"ref_type": {"terms": {"field": "git.ref_type"}}}, {"ref": {"terms": {"field": "git.ref"}}}, ], } @@ -335,7 +338,8 @@ def _composite_incremental_ref_tuples( if not buckets: return out for b in buckets: - out.add((b["key"]["host"], b["key"]["org"], b["key"]["repo"], b["key"]["ref"])) + out.add((b["key"]["host"], b["key"]["org"], b["key"]["repo"], + b["key"]["ref_type"], b["key"]["ref"])) after = agg.get("after_key") if after is None: return out @@ -412,6 +416,53 @@ def _enumerate_content_field( return out +def _enumerate_incremental_content_ref_pairs( + es: Elasticsearch, host: str, org: str, repo: str, +) -> set[tuple[str, str]]: + """Every distinct (git.ref_type, git.ref) pair in this repo's incremental content docs + (docs that have git.ref and no git.commit), via paginated composite aggregation.""" + out: set[tuple[str, str]] = set() + for index in (FILES_ALIAS, LINES_ALIAS): + after: dict | None = None + while True: + composite: dict = { + "size": _COMPOSITE_PAGE_SIZE, + "sources": [ + {"ref_type": {"terms": {"field": "git.ref_type"}}}, + {"ref": {"terms": {"field": "git.ref"}}}, + ], + } + if after is not None: + composite["after"] = after + query = {"bool": { + "filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"exists": {"field": "git.ref"}}, + ], + "must_not": [{"exists": {"field": "git.commit"}}], + }} + try: + resp = es.search( + index=index, size=0, + query=query, + aggs={"pairs": {"composite": composite}}, + ) + except NotFoundError: + break + agg = resp["aggregations"]["pairs"] + buckets = agg["buckets"] + if not buckets: + break + for b in buckets: + out.add((b["key"]["ref_type"], b["key"]["ref"])) + after = agg.get("after_key") + if after is None: + break + return out + + def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> list[str]: """Join-uniqueness gate (INV-011 backstop): verifies every content key maps to a correct refs join doc. Split by content shape (no `mode` on content docs): @@ -446,9 +497,11 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> found_commits = set() offending.extend(sorted(commits - found_commits)) - # --- incremental: each ref must have EXACTLY ONE incremental join doc --- - refs = _enumerate_content_field(es, host, org, repo, "git.ref") - if refs: + # --- incremental: each (ref_type, ref) pair must have EXACTLY ONE incremental join doc --- + # Content docs carry both git.ref and git.ref_type; a same-named branch and tag are distinct + # (ref_type, ref) pairs and are each allowed exactly one join doc. + ref_pairs = _enumerate_incremental_content_ref_pairs(es, host, org, repo) + if ref_pairs: try: resp = es.search( index=REFS_ALIAS, size=0, @@ -456,15 +509,23 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, - {"terms": {"git.ref": sorted(refs)}}, + {"terms": {"git.ref": sorted({r for _, r in ref_pairs})}}, {"term": {"mode": "delta"}}, ]}}, - aggs={"refs": {"terms": {"field": "git.ref", "size": len(refs)}}}, + aggs={"ref_pairs": {"composite": {"size": 1000, "sources": [ + {"ref_type": {"terms": {"field": "git.ref_type"}}}, + {"ref": {"terms": {"field": "git.ref"}}}, + ]}}}, ) - ref_counts = {b["key"]: b["doc_count"] for b in resp["aggregations"]["refs"]["buckets"]} + pair_counts = { + (b["key"]["ref_type"], b["key"]["ref"]): b["doc_count"] + for b in resp["aggregations"]["ref_pairs"]["buckets"] + } except NotFoundError: - ref_counts = {} - offending.extend(sorted(ref for ref in refs if ref_counts.get(ref, 0) != 1)) + pair_counts = {} + offending.extend( + sorted(f"{rt}/{ref}" for rt, ref in ref_pairs if pair_counts.get((rt, ref), 0) != 1) + ) return sorted(offending) diff --git a/src/sourcerer/utils.py b/src/sourcerer/utils.py index 1b06c85..db023e7 100644 --- a/src/sourcerer/utils.py +++ b/src/sourcerer/utils.py @@ -17,19 +17,24 @@ ID_DIGEST_SIZE = 16 -def build_ref_key(host: str, org: str, repo: str, ref: str) -> str: - """Deterministic `_id` string for incremental join docs: `{host}~{org}~{repo}~{ref}` - (host/org/repo lowercased, ref case-preserved). +def build_ref_key(host: str, org: str, repo: str, ref_type: str, ref: str) -> str: + """Deterministic `_id` string for incremental join docs: + `{host}~{org}~{repo}~{ref_type}~{ref}` (host/org/repo lowercased, ref case-preserved). + + `ref_type` is "branch" or "tag"; folding it in keeps a same-named branch and tag in delta + mode as two distinct join docs with non-overlapping id spaces (mirrors `build_ref_id` in + markers.py, which already includes ref_type for snapshot marker ids). Used exclusively as the Elasticsearch `_id` for the incremental refs join doc. Not a stored field -- `git.ref_key` was removed from all index mappings and content builders. The string - is opaque to queries; the join uses `(git.host, git.org, git.repo, git.ref)` natively. + is opaque to queries; the join uses `(git.host, git.org, git.repo, git.ref, git.ref_type)` + natively. `~` is safe as a delimiter because it is illegal in git ref names (see `git check-ref-format`) and matches the index-name segment delimiter used for host/org/repo elsewhere (see `indices.py`), so it cannot collide with any joined value. """ - return "~".join((host.lower(), org.lower(), repo.lower(), ref)) + return "~".join((host.lower(), org.lower(), repo.lower(), ref_type, ref)) def make_doc_id(*parts: str) -> str: diff --git a/tests/test_config.py b/tests/test_config.py index 54f8aea..f44f4ac 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -202,12 +202,12 @@ def test_delta_accepted_on_branch(self): cfg = _cfg([_source(ref_type="branch", mode="delta")]) assert cfg.repos[0].selectors[0].mode == "delta" - def test_delta_rejected_on_tag(self): - with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): - _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta")]) + def test_delta_accepted_on_tag(self): + cfg = _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta")]) + assert cfg.repos[0].selectors[0].mode == "delta" def test_delta_rejected_on_commit(self): - with pytest.raises(ValueError, match="only valid for git.ref_type: branch"): + with pytest.raises(ValueError, match="only valid for git.ref_type: branch or tag"): _cfg([_source(ref_type="commit", match="cfefb3b", mode="delta")]) def test_invalid_mode_raises(self): @@ -218,14 +218,26 @@ def test_delta_with_since_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'since'"): _cfg([_source(ref_type="branch", mode="delta", since={"age": "1y"})]) + def test_delta_tag_with_since_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'since'"): + _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta", since={"age": "1y"})]) + def test_delta_with_retain_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'retain'"): _cfg([_source(ref_type="branch", mode="delta", retain={"count": 5})]) + def test_delta_tag_with_retain_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'retain'"): + _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta", retain={"count": 5})]) + def test_delta_with_commit_level_index_raises(self): with pytest.raises(ValueError, match="cannot be combined with 'index.level: commit'"): _cfg([_source(ref_type="branch", mode="delta", index={"level": "commit"})]) + def test_delta_tag_with_commit_level_index_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'index.level: commit'"): + _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta", index={"level": "commit"})]) + def test_top_level_update_key_raises(self): with pytest.raises(ValueError, match="unknown keys"): _cfg([{"git": {"host": "github", "org": "acme", "repo": "widgets", "ref_type": "branch"}, @@ -235,6 +247,10 @@ def test_delta_with_repo_level_index_is_fine(self): cfg = _cfg([_source(ref_type="branch", mode="delta", index={"level": "repo"})]) assert cfg.repos[0].selectors[0].mode == "delta" + def test_delta_tag_with_repo_level_index_is_fine(self): + cfg = _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta", index={"level": "repo"})]) + assert cfg.repos[0].selectors[0].mode == "delta" + class TestParseCommitSource: def test_full_sha_accepted(self): diff --git a/tests/test_documents.py b/tests/test_documents.py index 725a1ee..4d99791 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -33,9 +33,11 @@ def _set_worker_ctx(host: str, org: str, repo: str, commit_sha: str, repo_dir, s ) -def _set_worker_ctx_incremental(host: str, org: str, repo: str, ref: str, repo_dir, symlink_paths=frozenset()) -> None: +def _set_worker_ctx_incremental(host: str, org: str, repo: str, ref: str, repo_dir, + symlink_paths=frozenset(), ref_type: str = "branch") -> None: documents._WORKER_CTX.update( - host=host, org=org, repo=repo, ref=ref, repo_dir=pathlib.Path(repo_dir), + host=host, org=org, repo=repo, ref_type=ref_type, ref=ref, + repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, mode="delta", ) @@ -181,41 +183,51 @@ def test_no_optional_fields_when_omitted(self): class TestIncrementalDocs: def test_ref_field_set_no_ref_key(self, tmp_path): - # Incremental docs carry git.ref (the branch name) but no git.ref_key (field removed). + # Incremental docs carry git.ref (the ref name) and git.ref_type but no git.ref_key. p = tmp_path / "a.txt" p.write_text("hello") - _id, doc = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + _id, doc = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) assert doc["git"]["ref"] == "main" + assert doc["git"]["ref_type"] == "branch" assert "ref_key" not in doc["git"] def test_no_commit_field(self, tmp_path): p = tmp_path / "a.txt" p.write_text("hello") - _id, doc = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + _id, doc = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) assert "commit" not in doc["git"] def test_id_stable_across_commits(self, tmp_path): # The whole point of ref-addressing: the id does not depend on the commit, only the - # ref, so a modified file's doc overwrites in place rather than minting a new id. + # ref_type+ref, so a modified file's doc overwrites in place rather than minting a new id. p = tmp_path / "a.txt" p.write_text("hello") - id1, _ = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + id1, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) p.write_text("hello world -- content changed, same ref/path") - id2, _ = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + id2, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) assert id1 == id2 def test_id_differs_from_snapshot_id(self, tmp_path): p = tmp_path / "a.txt" p.write_text("hello") snap_id, _ = build_file_doc("github", "acme", "widgets", "deadbeef", "a.txt", p) - incr_id, _ = build_incremental_file_doc("github", "acme", "widgets", "main", "a.txt", p) + incr_id, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) assert snap_id != incr_id + def test_branch_and_tag_same_name_have_distinct_ids(self, tmp_path): + # A same-named branch and tag in delta mode must produce distinct content ids (INV-004). + p = tmp_path / "a.txt" + p.write_text("hello") + branch_id, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "deploy", "a.txt", p) + tag_id, _ = build_incremental_file_doc("github", "acme", "widgets", "tag", "deploy", "a.txt", p) + assert branch_id != tag_id + def test_line_docs_ref_and_no_commit_no_ref_key(self): - # Incremental line docs carry git.ref, no git.commit, no git.ref_key. - docs = list(iter_incremental_line_docs("github", "acme", "widgets", "main", "a.txt", "one\ntwo")) + # Incremental line docs carry git.ref and git.ref_type, no git.commit, no git.ref_key. + docs = list(iter_incremental_line_docs("github", "acme", "widgets", "branch", "main", "a.txt", "one\ntwo")) for _id, d in docs: assert d["git"]["ref"] == "main" + assert d["git"]["ref_type"] == "branch" assert "commit" not in d["git"] assert "ref_key" not in d["git"] @@ -224,6 +236,7 @@ def test_worker_ctx_routes_to_incremental_builders(self, tmp_path): _set_worker_ctx_incremental("github", "acme", "widgets", "main", tmp_path) actions = _build_one_file_actions("a.txt") assert actions[0]["_source"]["git"]["ref"] == "main" + assert actions[0]["_source"]["git"]["ref_type"] == "branch" assert "commit" not in actions[0]["_source"]["git"] assert "ref_key" not in actions[0]["_source"]["git"] diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py index 62bbb22..f8534db 100644 --- a/tests/test_incremental_index.py +++ b/tests/test_incremental_index.py @@ -21,6 +21,7 @@ def _patch_common(prior=None, plan=None): returning the patcher context managers as a dict of MagicMocks keyed by name.""" patchers = { "checkout_branch": patch("sourcerer.commands.index.command.checkout_branch"), + "checkout_ref": patch("sourcerer.commands.index.command.checkout_ref"), "resolve_commit": patch("sourcerer.commands.index.command.resolve_commit", return_value=NEW), "commit_date": patch("sourcerer.commands.index.command.commit_date", return_value="2026-01-01T00:00:00+00:00"), "read_incremental_ref": patch("sourcerer.commands.index.command.read_incremental_ref", return_value=prior), @@ -55,6 +56,9 @@ def test_first_index_does_full_rebuild(self): mode="delta") index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", reporter=ProgressReporter(), unit=unit) + # Branch: uses checkout_branch, NOT checkout_ref. + mocks["checkout_branch"].assert_called_once() + mocks["checkout_ref"].assert_not_called() mocks["delete_incremental_branch"].assert_called_once() mocks["index_incremental_paths"].assert_called_once() # rel_paths (4th positional after repo_dir/branch) is None -> full tree walk. @@ -62,7 +66,9 @@ def test_first_index_does_full_rebuild(self): assert call_args[0][6] is None mocks["delete_incremental_paths"].assert_not_called() mocks["write_incremental_ready"].assert_called_once() - assert mocks["write_incremental_ready"].call_args[0][5] == NEW + # write_incremental_ready(es, host, org, repo, ref_type, ref, commit, ...) + # ref_type at [4], ref at [5], commit (sha) at [6] + assert mocks["write_incremental_ready"].call_args[0][6] == NEW finally: _stop(patchers) @@ -80,7 +86,9 @@ def test_second_run_indexes_only_changed_paths(self): reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_not_called() mocks["delete_incremental_paths"].assert_called_once() - assert mocks["delete_incremental_paths"].call_args[0][5] == ["gone.txt"] + # delete_incremental_paths(es, host, org, repo, ref_type, ref, paths, ...) + # ref_type at [4], ref at [5], paths at [6] + assert mocks["delete_incremental_paths"].call_args[0][6] == ["gone.txt"] mocks["index_incremental_paths"].assert_called_once() call_args = mocks["index_incremental_paths"].call_args assert call_args[0][6] == ["new.txt"] @@ -239,3 +247,103 @@ def test_same_routing_no_old_copy_delete(self): mocks["delete_incremental_branch"].assert_not_called() finally: _stop(patchers) + + +class TestIncrementalIndexTagFirstRun: + """Mirror of TestIncrementalIndexFirstRun / TestIncrementalIndexDeltaRun for tag Units. + Confirms that: + - checkout_ref is used instead of checkout_branch for tags (git.ref_type: tag) + - The overall orchestration path (first run → full rebuild; second run → delta) is identical. + """ + + def test_tag_first_index_uses_checkout_ref(self): + """First run for a tag Unit: checkout_ref called, full tree walk.""" + patchers, mocks = _patch_common(prior=None) + try: + es = MagicMock() + unit = Unit(host="github", org="elastic", repo="kibana", + ref="deploy@8", kind="tag", mode="delta") + index_incremental_branch_in_dir( + es, "github", "elastic", "kibana", "/repo", "deploy@8", + reporter=ProgressReporter(), unit=unit, + ) + # Tag: uses checkout_ref, NOT checkout_branch. + mocks["checkout_ref"].assert_called_once() + mocks["checkout_branch"].assert_not_called() + mocks["delete_incremental_branch"].assert_called_once() + mocks["index_incremental_paths"].assert_called_once() + # Full tree walk: rel_paths arg is None. + call_args = mocks["index_incremental_paths"].call_args + assert call_args[0][6] is None + mocks["write_incremental_ready"].assert_called_once() + # write_incremental_ready(es, host, org, repo, ref_type, ref, commit, ...) + assert mocks["write_incremental_ready"].call_args[0][6] == NEW + finally: + _stop(patchers) + + def test_tag_delta_run_indexes_only_changed_paths(self): + """Second run for a tag Unit: delta diff applied, full rebuild skipped.""" + prior = {"git": {"commit": OLD}} + plan = ChangePlan(delete_paths=["gone.ts"], index_paths=["new.ts"]) + patchers, mocks = _patch_common(prior=prior, plan=plan) + try: + es = MagicMock() + unit = Unit(host="github", org="elastic", repo="kibana", + ref="deploy@8", kind="tag", mode="delta") + index_incremental_branch_in_dir( + es, "github", "elastic", "kibana", "/repo", "deploy@8", + reporter=ProgressReporter(), unit=unit, + ) + mocks["checkout_ref"].assert_called_once() + mocks["checkout_branch"].assert_not_called() + mocks["delete_incremental_branch"].assert_not_called() + mocks["delete_incremental_paths"].assert_called_once() + # delete_incremental_paths(es, host, org, repo, ref_type, ref, paths, ...) + assert mocks["delete_incremental_paths"].call_args[0][6] == ["gone.ts"] + mocks["index_incremental_paths"].assert_called_once() + assert mocks["index_incremental_paths"].call_args[0][6] == ["new.ts"] + mocks["write_incremental_ready"].assert_called_once() + finally: + _stop(patchers) + + def test_tag_missing_diff_base_triggers_full_rebuild(self): + """Force-moved tag whose old target is gone → base_missing → full rebuild (INV-007).""" + prior = {"git": {"commit": OLD}} + plan = ChangePlan(base_missing=True) + patchers, mocks = _patch_common(prior=prior, plan=plan) + try: + es = MagicMock() + unit = Unit(host="github", org="elastic", repo="kibana", + ref="deploy@8", kind="tag", mode="delta") + index_incremental_branch_in_dir( + es, "github", "elastic", "kibana", "/repo", "deploy@8", + reporter=ProgressReporter(), unit=unit, + ) + mocks["checkout_ref"].assert_called_once() + mocks["delete_incremental_branch"].assert_called_once() + assert mocks["index_incremental_paths"].call_args[0][6] is None + finally: + _stop(patchers) + + def test_tag_ref_type_reaches_write_ready_call(self): + """ref_type='tag' flows through to write_incremental_ready positional args.""" + patchers, mocks = _patch_common(prior=None) + try: + es = MagicMock() + unit = Unit(host="github", org="elastic", repo="kibana", + ref="deploy@8", kind="tag", mode="delta") + index_incremental_branch_in_dir( + es, "github", "elastic", "kibana", "/repo", "deploy@8", + reporter=ProgressReporter(), unit=unit, + ) + ready_args = mocks["write_incremental_ready"].call_args[0] + # write_incremental_ready(es, host, org, repo, ref_type, ref, sha, ...) + # positional index 4 is ref_type + assert ready_args[4] == "tag", ( + f"Expected ref_type='tag' at pos 4 of write_incremental_ready call, got {ready_args}" + ) + assert ready_args[5] == "deploy@8", ( + f"Expected ref='deploy@8' at pos 5, got {ready_args}" + ) + finally: + _stop(patchers) diff --git a/tests/test_markers.py b/tests/test_markers.py index 831c0ee..5045114 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -450,44 +450,50 @@ def test_marker_status_complete(self): class TestIncrementalRefKeyIdentity: def test_id_is_ref_key_not_a_hash(self): es = MagicMock() - write_incremental_indexing(es, "github", "acme", "widgets", "main", OLD, NEW) - assert es.index.call_args.kwargs["id"] == build_ref_key("github", "acme", "widgets", "main") + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", OLD, NEW) + assert es.index.call_args.kwargs["id"] == build_ref_key("github", "acme", "widgets", "branch", "main") def test_stable_across_calls_commit_independent(self): - a = build_ref_key("github", "acme", "widgets", "main") - b = build_ref_key("github", "acme", "widgets", "main") - assert a == b # one document per branch, no commit folded in + a = build_ref_key("github", "acme", "widgets", "branch", "main") + b = build_ref_key("github", "acme", "widgets", "branch", "main") + assert a == b # one document per ref, no commit folded in + + def test_branch_and_tag_same_name_have_distinct_keys(self): + branch_key = build_ref_key("github", "acme", "widgets", "branch", "deploy") + tag_key = build_ref_key("github", "acme", "widgets", "tag", "deploy") + assert branch_key != tag_key # ref_type distinguishes them class TestWriteIncrementalIndexing: def test_incremental_marker_preserves_completed_commit_and_exposes_target(self): es = MagicMock() - write_incremental_indexing(es, "github", "acme", "widgets", "main", + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", completed_commit=OLD, commit_target=NEW) doc = _indexed_doc(es) assert doc["status"] == "indexing" assert doc["git"]["commit"] == OLD # completed pointer unchanged (INV-006) assert doc["git"]["commit_target"] == NEW assert doc["mode"] == "delta" + assert doc["git"]["ref_type"] == "branch" assert es.index.call_args.kwargs["index"] == REFS_INDEX def test_incremental_marker_first_index_has_no_completed_commit(self): es = MagicMock() - write_incremental_indexing(es, "github", "acme", "widgets", "main", + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", completed_commit=None, commit_target=NEW) assert _indexed_doc(es)["git"]["commit"] is None def test_incremental_marker_carries_prior_counts(self): es = MagicMock() prior = {"files_count": 12, "lines_count": 340, "git": {"commit_date": "2026-01-01T00:00:00+00:00"}} - write_incremental_indexing(es, "github", "acme", "widgets", "main", OLD, NEW, prior=prior) + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", OLD, NEW, prior=prior) doc = _indexed_doc(es) assert doc["files_count"] == 12 and doc["lines_count"] == 340 assert doc["git"]["commit_date"] == "2026-01-01T00:00:00+00:00" def test_incremental_indexing_carries_routing(self): es = MagicMock() - write_incremental_indexing(es, "github", "acme", "widgets", "main", OLD, NEW, + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", OLD, NEW, index_level="commit", index_suffix="s1") doc = _indexed_doc(es) assert doc["index_level"] == "commit" @@ -495,16 +501,29 @@ def test_incremental_indexing_carries_routing(self): def test_incremental_indexing_default_routing(self): es = MagicMock() - write_incremental_indexing(es, "github", "acme", "widgets", "main", OLD, NEW) + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", OLD, NEW) doc = _indexed_doc(es) assert doc["index_level"] == "repo" assert doc["index_suffix"] is None + def test_tag_ref_type_propagates_to_doc(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "tag", "deploy@1", + completed_commit=None, commit_target=NEW) + doc = _indexed_doc(es) + assert doc["git"]["ref_type"] == "tag" + assert doc["git"]["ref"] == "deploy@1" + # Join doc id must differ from a same-named branch's + branch_id = build_ref_key("github", "acme", "widgets", "branch", "deploy@1") + tag_id = build_ref_key("github", "acme", "widgets", "tag", "deploy@1") + assert es.index.call_args.kwargs["id"] == tag_id + assert tag_id != branch_id + class TestWriteIncrementalReady: def test_incremental_marker_advances_commit_and_clears_target_and_error(self): es = MagicMock() - write_incremental_ready(es, "github", "acme", "widgets", "main", commit=NEW, + write_incremental_ready(es, "github", "acme", "widgets", "branch", "main", commit=NEW, commit_date_iso="2026-02-02T00:00:00+00:00", files_count=5, lines_count=99) doc = _indexed_doc(es) @@ -517,7 +536,7 @@ def test_incremental_marker_advances_commit_and_clears_target_and_error(self): def test_incremental_ready_carries_routing(self): es = MagicMock() - write_incremental_ready(es, "github", "acme", "widgets", "main", commit=NEW, + write_incremental_ready(es, "github", "acme", "widgets", "branch", "main", commit=NEW, commit_date_iso=None, files_count=1, lines_count=1, index_level="commit", index_suffix="s1") doc = _indexed_doc(es) @@ -528,8 +547,8 @@ def test_incremental_ready_carries_routing(self): class TestWriteIncrementalFailed: def test_incremental_marker_keeps_status_indexing_and_retains_old_pointer(self): es = MagicMock() - write_incremental_failed(es, "github", "acme", "widgets", "main", completed_commit=OLD, - commit_target=NEW, error="boom") + write_incremental_failed(es, "github", "acme", "widgets", "branch", "main", + completed_commit=OLD, commit_target=NEW, error="boom") doc = _indexed_doc(es) assert doc["status"] == "indexing" # not advanced -- a failed run leaves the prior state assert doc["git"]["commit"] == OLD @@ -539,12 +558,12 @@ def test_incremental_marker_keeps_status_indexing_and_retains_old_pointer(self): def test_incremental_marker_error_text_is_bounded(self): es = MagicMock() - write_incremental_failed(es, "github", "acme", "widgets", "main", OLD, NEW, error="x" * 5000) + write_incremental_failed(es, "github", "acme", "widgets", "branch", "main", OLD, NEW, error="x" * 5000) assert len(_indexed_doc(es)["error"]) == ERROR_MAX_LEN def test_incremental_failed_carries_routing(self): es = MagicMock() - write_incremental_failed(es, "github", "acme", "widgets", "main", OLD, NEW, error="boom", + write_incremental_failed(es, "github", "acme", "widgets", "branch", "main", OLD, NEW, error="boom", index_level="commit", index_suffix="s1") doc = _indexed_doc(es) assert doc["index_level"] == "commit" @@ -555,26 +574,26 @@ class TestReadIncrementalRef: def test_returns_source(self): es = MagicMock() es.get.return_value = {"_source": {"status": "ready", "git": {"commit": NEW}}} - assert read_incremental_ref(es, "github", "acme", "widgets", "main") == ( + assert read_incremental_ref(es, "github", "acme", "widgets", "branch", "main") == ( {"status": "ready", "git": {"commit": NEW}} ) - assert es.get.call_args.kwargs["id"] == build_ref_key("github", "acme", "widgets", "main") + assert es.get.call_args.kwargs["id"] == build_ref_key("github", "acme", "widgets", "branch", "main") def test_missing_returns_none(self): es = MagicMock() es.get.side_effect = _not_found() - assert read_incremental_ref(es, "github", "acme", "widgets", "main") is None + assert read_incremental_ref(es, "github", "acme", "widgets", "branch", "main") is None class TestDeleteIncrementalPaths: def test_empty_paths_is_noop(self): es = MagicMock() - delete_incremental_paths(es, "github", "acme", "widgets", "main", []) + delete_incremental_paths(es, "github", "acme", "widgets", "branch", "main", []) es.delete_by_query.assert_not_called() def test_scoped_to_exact_ref_key_and_paths(self): es = MagicMock() - delete_incremental_paths(es, "github", "acme", "widgets", "main", ["a.txt", "b.txt"]) + delete_incremental_paths(es, "github", "acme", "widgets", "branch", "main", ["a.txt", "b.txt"]) assert es.delete_by_query.call_count == 2 # files + lines indices for call in es.delete_by_query.call_args_list: query = call.kwargs["query"] @@ -582,13 +601,14 @@ def test_scoped_to_exact_ref_key_and_paths(self): assert {"term": {"git.host": "github"}} in filt assert {"term": {"git.org": "acme"}} in filt assert {"term": {"git.repo": "widgets"}} in filt + assert {"term": {"git.ref_type": "branch"}} in filt assert {"term": {"git.ref": "main"}} in filt assert {"terms": {"file.path": ["a.txt", "b.txt"]}} in filt def test_missing_index_is_ignored(self): es = MagicMock() es.delete_by_query.side_effect = _not_found() - delete_incremental_paths(es, "github", "acme", "widgets", "main", ["a.txt"]) # no raise + delete_incremental_paths(es, "github", "acme", "widgets", "branch", "main", ["a.txt"]) # no raise class TestDeleteIncrementalBranch: @@ -602,6 +622,7 @@ def test_scoped_to_exact_ref_key_only(self): assert {"term": {"git.host": "github"}} in filt assert {"term": {"git.org": "acme"}} in filt assert {"term": {"git.repo": "widgets"}} in filt + assert {"term": {"git.ref_type": "branch"}} in filt assert {"term": {"git.ref": "main"}} in filt assert not any("ref_key" in str(f) for f in filt) From e2c0a8256183de2fb7703b7d1d7f4283b90200e9 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Mon, 24 Aug 2026 10:37:24 -0700 Subject: [PATCH 26/29] Add git.ref_type to index sorting in files and lines indices --- src/sourcerer/elastic/index_templates/sourcerer-v3-files.json | 2 ++ src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json index dce8b07..4654e4d 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json @@ -25,6 +25,7 @@ "git.repo", "git.commit", "git.ref", + "git.ref_type", "file.path" ], "order": [ @@ -33,6 +34,7 @@ "asc", "asc", "asc", + "asc", "asc" ] } diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json index b289f65..30f0219 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json @@ -25,6 +25,7 @@ "git.repo", "git.commit", "git.ref", + "git.ref_type", "file.path", "line.number" ], @@ -35,6 +36,7 @@ "asc", "asc", "asc", + "asc", "asc" ] } From 438fd66c20886e3a995de9d63580aa302b3c1c10 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Mon, 24 Aug 2026 13:57:03 -0700 Subject: [PATCH 27/29] Make git.ref_pattern (sources[i].match) the stream identity, not a duplicate of git.ref. Drop git.ref from delta content docs and index sort. Replace error/failed_at with status:'failed' --- src/sourcerer/commands/index/command.py | 84 ++++++--- src/sourcerer/commands/index/documents.py | 54 +++--- src/sourcerer/commands/index/markers.py | 124 +++++++------ src/sourcerer/commands/index/selection.py | 38 +++- src/sourcerer/commands/prune/execute.py | 8 +- src/sourcerer/config.py | 17 ++ .../elastic/agent_builder_tools/README.md | 19 +- .../sourcerer.code.grep.yml | 10 +- .../sourcerer.code.search.yml | 10 +- .../sourcerer.files.cat.yml | 10 +- .../sourcerer.files.head.yml | 10 +- .../sourcerer.files.ls.yml | 10 +- .../sourcerer.files.read_lines.yml | 10 +- .../sourcerer.files.tail.yml | 10 +- .../sourcerer.files.tree.yml | 10 +- .../sourcerer.files.wc.yml | 10 +- .../sourcerer.refs.list.yml | 9 +- .../sourcerer.repos.search.yml | 6 +- .../index_templates/sourcerer-v3-files.json | 5 +- .../index_templates/sourcerer-v3-lines.json | 5 +- .../index_templates/sourcerer-v3-refs.json | 11 +- src/sourcerer/progress.py | 6 + src/sourcerer/queries.py | 49 +++--- tests/test_agent_builder_tools.py | 21 ++- tests/test_backfill.py | 2 +- tests/test_config.py | 35 ++++ tests/test_documents.py | 50 ++++-- tests/test_incremental_index.py | 135 +++++++++----- tests/test_markers.py | 88 ++++++++-- tests/test_selection.py | 164 ++++++++++++++++++ tests/test_uniqueness_gate.py | 154 +++++++++------- tests/test_utils.py | 24 ++- 32 files changed, 863 insertions(+), 335 deletions(-) create mode 100644 tests/test_selection.py diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index e1e10e4..92167f1 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -28,6 +28,7 @@ from ...indices import files_index, lines_index from ...queries import check_join_uniqueness from ...utils import ES_ERRORS, make_client +from ...version import compile_pattern, match_version from ..prune import command as prune_cmd from ..prune.execute import delete_commit_from_indices from .documents import index_incremental_paths, index_repo @@ -223,7 +224,8 @@ def index_ref_in_dir( # that this scope is currently being indexed by another run and skip it. # The terminal write_ref_marker (status:'complete') overwrites this doc in place. write_indexing_marker(es, host, org, repo, ref_type, ref_for_id, commit_sha, - commit_date_iso, index_level=level, index_suffix=suffix) + commit_date_iso, index_level=level, index_suffix=suffix, + ref_pattern=unit.ref_pattern or ref_for_id) files_count, lines_count = index_repo( es, host, org, repo, repo_dir, commit_sha, on_progress=lambda f, l: reporter.update_counts(unit, f, l), @@ -234,7 +236,8 @@ def index_ref_in_dir( # old copy is deleted, so a crash between here and the delete below leaves stale (not missing) # data that the prune stale-location sweep reclaims. write_ref_marker(es, host, org, repo, ref_type, ref_for_id, commit_sha, commit_date_iso, - files_count, lines_count, index_level=level, index_suffix=suffix) + files_count, lines_count, index_level=level, index_suffix=suffix, + ref_pattern=unit.ref_pattern or ref_for_id) if migrating: # Reconstruct the OLD index name from the prior marker's routing and drop this commit's # stale copy there. Commit-safety (another surviving ref sharing the commit) is respected @@ -283,15 +286,42 @@ def index_incremental_branch_in_dir( ref_type = unit.kind # "branch" or "tag" + # Stream identity: for a delta-tag stream `unit.ref_pattern` holds the literal match-pattern + # string (e.g. "deploy@{major}") and is the stable _id key for the refs join doc and the + # `git.ref_pattern` stored on all content docs. For branches and non-stream tags + # ref_pattern == branch (the concrete ref name). + ref_pattern = unit.ref_pattern or branch # stable identity (pattern for tag streams, branch otherwise) + + # For a delta-tag stream resolve the CONCRETE newest tag post-clone (commit dates are + # available here via `ref_dates`). `ref` then holds the concrete tag while `ref_pattern` + # (the pattern) stays the identity for all content/marker scoping. For branches ref == ref_pattern. reporter.set_stage(unit, "checkout") if ref_type == "tag": - checkout_ref(repo_dir, branch) + cp = compile_pattern(branch) + dates = ref_dates(repo_dir) + tag_matches = [ + (ts, name) + for (kind, name), ts in dates.items() + if kind == "tag" and match_version(cp, name) is not None + ] + if not tag_matches: + # Pattern matched tags in Phase 1 but none are present in the clone. This can happen + # if all matching tags were deleted between ls-remote and the clone fetch. Skip. + reporter.finish(unit, "no-changes", detail="no matching tags in clone") + return + ref = max(tag_matches)[1] # newest by creatordate timestamp — concrete resolved tag + checkout_ref(repo_dir, ref) else: - checkout_branch(repo_dir, branch) + ref = branch # branch name is both the concrete ref and the identity + checkout_branch(repo_dir, ref) + # Expose the concrete resolved ref in the unit so the reporter displays it. + unit.ref = ref + new_sha = resolve_commit(repo_dir) commit_date_iso = commit_date(repo_dir) - prior = read_incremental_ref(es, host, org, repo, ref_type, branch) + # Read the prior join doc keyed on `ref_pattern` (the stable identity, unchanged across promotions). + prior = read_incremental_ref(es, host, org, repo, ref_type, ref_pattern) old_sha = None if force else (prior.get("git", {}).get("commit") if prior else None) level = unit.index_level @@ -308,8 +338,11 @@ def index_incremental_branch_in_dir( return reporter.set_stage(unit, "indexing") - write_incremental_indexing(es, host, org, repo, ref_type, branch, completed_commit=old_sha, - commit_target=new_sha, prior=prior, + # `ref` = concrete tag/branch (payload stored as git.ref in the join doc and content docs). + # `ref_pattern` = stream identity (pattern for tag streams; == ref for branches). + # Content docs (files/lines) are keyed on `ref_pattern` for _id stability across promotions. + write_incremental_indexing(es, host, org, repo, ref_type, ref, completed_commit=old_sha, + commit_target=new_sha, ref_pattern=ref_pattern, prior=prior, index_level=level, index_suffix=suffix) try: full_rebuild = old_sha is None or force or routing_changed @@ -318,54 +351,55 @@ def index_incremental_branch_in_dir( full_rebuild = plan.base_missing if full_rebuild: - delete_incremental_branch(es, host, org, repo, branch, + # Content scoped on `ref_pattern` (the stream identity). + delete_incremental_branch(es, host, org, repo, ref_pattern, ref_type=ref_type, index_level=level, index_suffix=suffix) reporter.set_total_files(unit, count_tracked_files(repo_dir)) indexed_files, indexed_lines = index_incremental_paths( - es, host, org, repo, repo_dir, branch, None, + es, host, org, repo, repo_dir, ref_pattern, None, on_progress=lambda f, l: reporter.update_counts(unit, f, l), index_level=level, index_suffix=suffix, ref_type=ref_type, ) else: - delete_incremental_paths(es, host, org, repo, ref_type, branch, plan.delete_paths, + delete_incremental_paths(es, host, org, repo, ref_type, ref_pattern, plan.delete_paths, index_level=level, index_suffix=suffix) reporter.set_total_files(unit, len(plan.index_paths)) indexed_files, indexed_lines = index_incremental_paths( - es, host, org, repo, repo_dir, branch, plan.index_paths, + es, host, org, repo, repo_dir, ref_pattern, plan.index_paths, on_progress=lambda f, l: reporter.update_counts(unit, f, l), index_level=level, index_suffix=suffix, ref_type=ref_type, ) refresh_incremental_content(es, host, org, repo, index_level=level, index_suffix=suffix) files_count, lines_count = count_incremental_branch_docs( - es, host, org, repo, branch, ref_type=ref_type, index_level=level, index_suffix=suffix, + es, host, org, repo, ref_pattern, ref_type=ref_type, index_level=level, index_suffix=suffix, ) # Mode-switch: flip any complete snapshot markers for this (host,org,repo,ref) to # "stale" BEFORE publishing the incremental join doc as "complete". This ensures the # two-complete-docs fan-out window (one snapshot + one incremental marker both matching - # LOOKUP JOIN ON git.ref, git.ref_type) never opens. Stale content is reclaimed by prune. - mark_snapshot_markers_stale(es, host, org, repo, branch) - write_incremental_ready(es, host, org, repo, ref_type, branch, new_sha, commit_date_iso, - files_count, lines_count, + # LOOKUP JOIN ON git.ref_pattern) never opens. Stale content is reclaimed by prune. + mark_snapshot_markers_stale(es, host, org, repo, ref_pattern) + write_incremental_ready(es, host, org, repo, ref_type, ref, new_sha, commit_date_iso, + files_count, lines_count, ref_pattern=ref_pattern, index_level=level, index_suffix=suffix) # Migration cleanup (write-new -> flip join doc -> delete-old): now that the join doc is # complete and points at the new routing, delete this ref's docs from the old physical - # index. Scoped to the exact (host,org,repo,ref_type,ref) 5-term filter so a sibling - # source that still lives in the old index is never touched. A crash between the ready - # write above and this delete leaves stale-location incremental docs in the old index; - # prune's incremental stale-location sweep (Class D-I) reclaims them. + # index. Scoped to the exact (host,org,repo,ref_type,git.ref_pattern) 5-term filter so a + # sibling source that still lives in the old index is never touched. A crash between the + # ready write above and this delete leaves stale-location incremental docs in the old + # index; prune's incremental stale-location sweep (Class D-I) reclaims them. if routing_changed: old_level, old_suffix = old_routing - delete_incremental_branch(es, host, org, repo, branch, + delete_incremental_branch(es, host, org, repo, ref_pattern, ref_type=ref_type, index_level=old_level, index_suffix=old_suffix) except KeyboardInterrupt: - write_incremental_failed(es, host, org, repo, ref_type, branch, completed_commit=old_sha, - commit_target=new_sha, error="interrupted", prior=prior, + write_incremental_failed(es, host, org, repo, ref_type, ref, completed_commit=old_sha, + commit_target=new_sha, ref_pattern=ref_pattern, error="interrupted", prior=prior, index_level=level, index_suffix=suffix) raise except Exception as e: - write_incremental_failed(es, host, org, repo, ref_type, branch, completed_commit=old_sha, - commit_target=new_sha, error=str(e), prior=prior, + write_incremental_failed(es, host, org, repo, ref_type, ref, completed_commit=old_sha, + commit_target=new_sha, ref_pattern=ref_pattern, error=str(e), prior=prior, index_level=level, index_suffix=suffix) raise reporter.finish(unit, "indexed", indexed_files, indexed_lines) diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index aa63099..ddd9201 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -151,7 +151,7 @@ def build_incremental_file_doc( org: str, repo: str, ref_type: str, - ref: str, + ref_pattern: str, rel_path: str, abs_path: pathlib.Path, *, @@ -160,11 +160,12 @@ def build_incremental_file_doc( target_path: str | None = None, target_size: int | None = None, ) -> tuple[str, dict]: - """Ref-addressed (incremental) file doc: carries `git.ref` and `git.ref_type` but no - `git.commit`; `_id` is stable across commits (derived from ref_type + ref name, not the - commit SHA), so a modified file's doc overwrites in place on the next HEAD advance rather - than minting a new id. Including ref_type in the id keeps a same-named branch and tag in - delta mode in non-overlapping id spaces.""" + """Ref-addressed (incremental) file doc: carries `git.ref_pattern` (stream identity) but no + `git.ref` (concrete) or `git.commit`. `git.ref` is intentionally omitted: because content + `_id` is keyed on `ref_pattern`, a delta run only re-writes CHANGED paths, so after a tag + stream promotion unchanged paths would retain a stale `git.ref`. The concrete ref lives only + on the refs join doc (written wholesale each run). `_id` is keyed on `ref_pattern` so that + delta-tag stream promotions overwrite the same doc in place rather than minting a new id.""" p = pathlib.PurePosixPath(rel_path) directory = "" if str(p.parent) == "." else str(p.parent) extension = p.suffix.lstrip(".") or None @@ -200,12 +201,12 @@ def build_incremental_file_doc( "host": host, "org": org, "repo": repo, - "ref": ref, + "ref_pattern": ref_pattern, "ref_type": ref_type, }, "file": file_fields, } - _id = make_doc_id(host, org, repo, ref_type, ref, rel_path) + _id = make_doc_id(host, org, repo, ref_type, ref_pattern, rel_path) return _id, doc @@ -214,7 +215,7 @@ def iter_incremental_line_docs( org: str, repo: str, ref_type: str, - ref: str, + ref_pattern: str, rel_path: str, content: str, *, @@ -224,7 +225,8 @@ def iter_incremental_line_docs( attributes: list[str] | None = None, ) -> Iterator[tuple[str, dict]]: """Ref-addressed (incremental) line docs -- same shape as `build_incremental_file_doc`: - carries `git.ref` and `git.ref_type` but no `git.commit`.""" + carries `git.ref_pattern` (stream identity) but no `git.ref` (concrete) or `git.commit`. + See `build_incremental_file_doc` for why `git.ref` is intentionally omitted.""" p = pathlib.PurePosixPath(rel_path) directory = "" if str(p.parent) == "." else str(p.parent) extension = p.suffix.lstrip(".") or None @@ -247,13 +249,13 @@ def iter_incremental_line_docs( "host": host, "org": org, "repo": repo, - "ref": ref, + "ref_pattern": ref_pattern, "ref_type": ref_type, }, "file": file_fields, } for line_num, line_content in enumerate(content.splitlines(), start=1): - _id = make_doc_id(host, org, repo, ref_type, ref, rel_path, str(line_num)) + _id = make_doc_id(host, org, repo, ref_type, ref_pattern, rel_path, str(line_num)) yield _id, {**base, "line": {"number": line_num, "content": line_content}} @@ -280,16 +282,18 @@ def _init_worker( def _init_worker_incremental( - host: str, org: str, repo: str, ref_type: str, ref: str, repo_dir: str, + host: str, org: str, repo: str, ref_type: str, ref_pattern: str, repo_dir: str, symlink_paths: frozenset[str] = frozenset(), index_level: str = "repo", index_suffix: str | None = None, ) -> None: - """Same as `_init_worker`, but for the incremental (ref-addressed) path: `ref_type`+`ref` - replace `commit_sha` and `mode` routes `_build_one_file_actions` to the incremental doc - builders.""" + """Same as `_init_worker`, but for the incremental (ref-addressed) path: `ref_type`+ + `ref_pattern` replace `commit_sha` and `mode` routes `_build_one_file_actions` to the + incremental doc builders. `ref_pattern` is the stream identity (== the concrete ref name for + non-stream refs; the literal pattern string for delta-tag streams). The concrete `ref` is NOT + stored here -- content docs carry only `ref_pattern`; `git.ref` lives on the refs join doc.""" signal.signal(signal.SIGINT, signal.SIG_IGN) _WORKER_CTX.update( - host=host, org=org, repo=repo, ref_type=ref_type, ref=ref, + host=host, org=org, repo=repo, ref_type=ref_type, ref_pattern=ref_pattern, repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, mode="delta", @@ -316,8 +320,8 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: incremental = ctx.get("mode", "snapshot") == "delta" host, org, repo = ctx["host"], ctx["org"], ctx["repo"] ref_type = ctx.get("ref_type", "branch") - ref = ctx.get("ref") - commit_sha = ref if incremental else ctx["commit_sha"] + ref_pattern = ctx["ref_pattern"] if incremental else None + commit_sha = ctx.get("commit_sha") level, suffix = ctx.get("index_level", "repo"), ctx.get("index_suffix") abs_path = ctx["repo_dir"] / rel_path # Incremental content is ref-addressed (no commit-level index name); the physical index name @@ -365,7 +369,7 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: git_target_size = None if incremental: file_id, file_doc = build_incremental_file_doc( - host, org, repo, ref_type, ref, rel_path, abs_path, binary=binary, + host, org, repo, ref_type, ref_pattern, rel_path, abs_path, binary=binary, is_symlink=True if is_git_symlink else None, target_path=git_target_path, target_size=git_target_size, @@ -384,7 +388,7 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: ff = file_doc["file"] if incremental: line_docs = iter_incremental_line_docs( - host, org, repo, ref_type, ref, rel_path, content, + host, org, repo, ref_type, ref_pattern, rel_path, content, size=ff["size"], target_path=ff.get("target_path"), target_size=ff.get("target_size"), @@ -501,7 +505,7 @@ def index_incremental_paths( org: str, repo: str, repo_dir: pathlib.Path, - ref: str, + ref_pattern: str, rel_paths: list[str] | None = None, on_progress: Callable[[int, int], None] | None = None, index_level: str = "repo", @@ -516,6 +520,10 @@ def index_incremental_paths( an incremental HEAD advance only touch the files git reports changed. Deletions for removed paths are the caller's responsibility (see `markers.delete_incremental_paths`) since they need no doc generation. Mirrors `index_repo`'s worker-pool ingest loop. + + `ref_pattern` is the stream identity key (the literal pattern for delta-tag streams, or the + concrete ref name for all other refs). The concrete `ref` is intentionally absent here -- + content docs carry only `git.ref_pattern`; `git.ref` lives only on the refs join doc. """ files_count = 0 lines_count = 0 @@ -527,7 +535,7 @@ def index_incremental_paths( with ProcessPoolExecutor( max_workers=max(1, t.index_workers), initializer=_init_worker_incremental, - initargs=(host, org, repo, ref_type, ref, str(repo_dir), symlink_paths, index_level, index_suffix), + initargs=(host, org, repo, ref_type, ref_pattern, str(repo_dir), symlink_paths, index_level, index_suffix), ) as executor: def _batched(items: Iterator[str], n: int) -> Iterator[list[str]]: it = iter(items) diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index 13ca0d2..592e48b 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -401,6 +401,7 @@ def write_indexing_marker( commit_date_iso: str | None, index_level: str = "repo", index_suffix: str | None = None, + ref_pattern: str | None = None, ) -> None: """Write a status:'indexing' marker for a ref that is about to be ingested. @@ -423,6 +424,7 @@ def write_indexing_marker( "org": org, "repo": repo, "ref": ref, + "ref_pattern": ref_pattern if ref_pattern is not None else ref, "ref_type": ref_type, "commit": commit_sha, "commit_date": commit_date_iso, @@ -452,6 +454,7 @@ def write_ref_marker( index_level: str = "repo", index_suffix: str | None = None, refresh: bool = False, + ref_pattern: str | None = None, ) -> None: # (ref, ref_type) replaces the old git.branch/git.tag fields: those were write-only and # fully reconstructable as `git.ref filtered by git.ref_type`. git.tag was an array that @@ -471,6 +474,7 @@ def write_ref_marker( "org": org, "repo": repo, "ref": ref, + "ref_pattern": ref_pattern if ref_pattern is not None else ref, "ref_type": ref_type, "commit": commit_sha, "commit_date": commit_date_iso, @@ -551,16 +555,16 @@ def pre_clone_skip( # constructor even though git.ref_key is no longer a stored field -- the id itself remains the # stable overwrite key for each delta-mode ref. -ERROR_MAX_LEN = 2000 # bound stored failure text so a giant git/ES error can't bloat the doc - def read_incremental_ref( - es: Elasticsearch, host: str, org: str, repo: str, ref_type: str, ref: str, + es: Elasticsearch, host: str, org: str, repo: str, ref_type: str, ref_pattern: str, ) -> dict | None: """The ref's incremental join doc `_source`, or None if never indexed. A real-time GET - (by `_id = ref_key`), so it reflects the last write even without a refresh.""" + (by `_id = build_ref_key(..., ref_pattern)` where `ref_pattern` is the stream identity + stored as `git.ref_pattern`). For delta-tag streams this is the pattern string + (e.g. "deploy@{major}"); for all other refs it equals the concrete ref name.""" try: - return es.get(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref))["_source"] + return es.get(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref_pattern))["_source"] except NotFoundError: return None @@ -575,6 +579,7 @@ def _build_incremental_join_doc( repo: str, ref_type: str, ref: str, + ref_pattern: str, *, status: str, commit: str | None, @@ -584,17 +589,20 @@ def _build_incremental_join_doc( lines_count: int = 0, indexed_at: str | None = None, indexing_started_at: str | None = None, - failed_at: str | None = None, - error: str | None = None, index_level: str = "repo", index_suffix: str | None = None, ) -> dict: + """Build an incremental join doc. `ref` is the CONCRETE resolved ref (e.g. the newest + matching tag for a delta-tag stream, or the branch name for a delta branch). `ref_pattern` + is the STREAM IDENTITY: the literal match-pattern string for a delta-tag stream, or equal to + `ref` for all other refs. Stored as `git.ref_pattern` and doubles as the stable `_id` key.""" return { "git": { "host": host, "org": org, "repo": repo, "ref": ref, + "ref_pattern": ref_pattern, "ref_type": ref_type, "commit": commit, "commit_target": commit_target, @@ -606,8 +614,6 @@ def _build_incremental_join_doc( "lines_count": lines_count, "indexed_at": indexed_at, "indexing_started_at": indexing_started_at, - "failed_at": failed_at, - "error": error[:ERROR_MAX_LEN] if error else None, "index_level": index_level, "index_suffix": index_suffix, } @@ -622,6 +628,7 @@ def write_incremental_indexing( ref: str, completed_commit: str | None, commit_target: str, + ref_pattern: str | None = None, prior: dict | None = None, refresh: bool = False, index_level: str = "repo", @@ -630,11 +637,16 @@ def write_incremental_indexing( """Publish `status: indexing`: the completed pointer (`git.commit`) stays at the LAST completed SHA (or None on a first index) while `git.commit_target` advertises the candidate SHA the run is advancing to. A failed run never overwrites `git.commit` with `commit_target` - (INV-006) -- only `write_incremental_ready` does that, after delete+index+refresh succeed.""" + (INV-006) -- only `write_incremental_ready` does that, after delete+index+refresh succeed. + + `ref` is the CONCRETE resolved ref (git.ref payload); `ref_pattern` is the STREAM IDENTITY + stored as `git.ref_pattern` and used as the doc _id. For delta-tag streams these differ + (e.g. ref="deploy@1788000000", ref_pattern="deploy@{major}"). Defaults to `ref`.""" + ref_pattern = ref_pattern or ref prior = prior or {} pg = prior.get("git", {}) doc = _build_incremental_join_doc( - host, org, repo, ref_type, ref, + host, org, repo, ref_type, ref, ref_pattern, status="indexing", commit=completed_commit, commit_target=commit_target, @@ -643,12 +655,10 @@ def write_incremental_indexing( lines_count=prior.get("lines_count", 0), indexed_at=prior.get("indexed_at"), indexing_started_at=_now_iso(), - failed_at=prior.get("failed_at"), - error=prior.get("error"), index_level=index_level, index_suffix=index_suffix, ) - es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref), document=doc, refresh=refresh) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref_pattern), document=doc, refresh=refresh) def write_incremental_ready( @@ -662,15 +672,20 @@ def write_incremental_ready( commit_date_iso: str | None, files_count: int, lines_count: int, + ref_pattern: str | None = None, refresh: bool = True, index_level: str = "repo", index_suffix: str | None = None, ) -> None: """Publish `status: complete` at the NEW completed commit, clearing `commit_target` and any prior failure fields. This is the pointer-advancing publication boundary (INV-006): callers - must delete+index+refresh the content indices FIRST, then call this.""" + must delete+index+refresh the content indices FIRST, then call this. + + `ref` is the CONCRETE resolved ref (git.ref payload); `ref_pattern` is the STREAM IDENTITY + (stored as `git.ref_pattern`, used as the doc _id). Defaults to `ref`.""" + ref_pattern = ref_pattern or ref doc = _build_incremental_join_doc( - host, org, repo, ref_type, ref, + host, org, repo, ref_type, ref, ref_pattern, status="complete", commit=commit, commit_target=None, @@ -679,12 +694,10 @@ def write_incremental_ready( lines_count=lines_count, indexed_at=_now_iso(), indexing_started_at=None, - failed_at=None, - error=None, index_level=index_level, index_suffix=index_suffix, ) - es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref), document=doc, refresh=refresh) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref_pattern), document=doc, refresh=refresh) def write_incremental_failed( @@ -697,32 +710,34 @@ def write_incremental_failed( completed_commit: str | None, commit_target: str | None, error: str, + ref_pattern: str | None = None, prior: dict | None = None, refresh: bool = False, index_level: str = "repo", index_suffix: str | None = None, ) -> None: - """Record a failed update WITHOUT advancing the completed pointer: status stays `indexing`, - `git.commit` remains the last completed SHA, and a bounded `error`/`failed_at` are stored for - diagnosis (INV-006). The next run retries old -> current and clears these on success.""" + """Record a failed update WITHOUT advancing the completed pointer: status becomes `failed`, + `git.commit` remains the last completed SHA (INV-006). The next run retries old -> current. + + `ref` is the CONCRETE resolved ref (git.ref payload); `ref_pattern` is the STREAM IDENTITY + (stored as `git.ref_pattern`, used as the doc _id). Defaults to `ref`.""" + ref_pattern = ref_pattern or ref prior = prior or {} pg = prior.get("git", {}) doc = _build_incremental_join_doc( - host, org, repo, ref_type, ref, - status="indexing", + host, org, repo, ref_type, ref, ref_pattern, + status="failed", commit=completed_commit, commit_target=commit_target, commit_date_iso=pg.get("commit_date"), files_count=prior.get("files_count", 0), lines_count=prior.get("lines_count", 0), indexed_at=prior.get("indexed_at"), - indexing_started_at=prior.get("indexing_started_at") or _now_iso(), - failed_at=_now_iso(), - error=error, + indexing_started_at=None, index_level=index_level, index_suffix=index_suffix, ) - es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref), document=doc, refresh=refresh) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref_pattern), document=doc, refresh=refresh) def _delete_by_query_sync(es: Elasticsearch, index: str, query: dict, refresh: bool) -> None: @@ -750,16 +765,16 @@ def delete_incremental_paths( org: str, repo: str, ref_type: str, - ref: str, + ref_pattern: str, paths, index_level: str = "repo", index_suffix: str | None = None, refresh: bool = False, ) -> None: """Synchronously delete the file and line docs for `paths` on this exact ref. Scoped by - the exact (git.host, git.org, git.repo, git.ref_type, git.ref) 5-term filter (INV-008: - one ref's docs can never bleed into another's) plus a `file.path` terms filter, never a - wildcard. A no-op for an empty path set.""" + the exact (git.host, git.org, git.repo, git.ref_type, git.ref_pattern) 5-term filter + (INV-008: one ref's docs can never bleed into another's) plus a `file.path` terms filter. + A no-op for an empty path set.""" paths = list(paths) if not paths: return @@ -770,7 +785,7 @@ def delete_incremental_paths( {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"term": {"git.ref_type": ref_type}}, - {"term": {"git.ref": ref}}, + {"term": {"git.ref_pattern": ref_pattern}}, {"terms": {"file.path": paths}}, ] } @@ -787,22 +802,22 @@ def delete_incremental_branch( host: str, org: str, repo: str, - ref: str, + ref_pattern: str, ref_type: str = "branch", index_level: str = "repo", index_suffix: str | None = None, refresh: bool = False, ) -> None: """Delete EVERY incremental content doc for this ref (full namespace), scoped by the exact - (git.host, git.org, git.repo, git.ref_type, git.ref) 5-term filter (INV-008). Used for - the initial index and the missing-diff-base rebuild (INV-007). `ref_type` defaults to - "branch" for back-compat with existing callers that pass `ref` positionally.""" + (git.host, git.org, git.repo, git.ref_type, git.ref_pattern) 5-term filter (INV-008). Used + for the initial index and the missing-diff-base rebuild (INV-007). `ref_type` defaults to + "branch" for back-compat with existing callers that pass `ref_pattern` positionally.""" query = {"bool": {"filter": [ {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"term": {"git.ref_type": ref_type}}, - {"term": {"git.ref": ref}}, + {"term": {"git.ref_pattern": ref_pattern}}, ]}} for index in ( files_index(host, org, repo, None, index_level, index_suffix), @@ -812,21 +827,20 @@ def delete_incremental_branch( def count_incremental_branch_docs( - es: Elasticsearch, host: str, org: str, repo: str, ref: str, + es: Elasticsearch, host: str, org: str, repo: str, ref_pattern: str, ref_type: str = "branch", index_level: str = "repo", index_suffix: str | None = None, ) -> tuple[int, int]: """Authoritative (files, lines) totals for a ref's current incremental view, counted by - exact (git.host, git.org, git.repo, git.ref_type, git.ref). Call AFTER refreshing the - content indices so counts reflect the just-applied deletes and indexes -- these become the - ready marker's files_count/lines_count. Returns 0 for an index that does not exist yet. - `ref_type` defaults to "branch" for back-compat with existing callers.""" + exact (git.host, git.org, git.repo, git.ref_type, git.ref_pattern). Call AFTER refreshing + the content indices so counts reflect the just-applied deletes and indexes -- these become the + ready marker's files_count/lines_count. Returns 0 for an index that does not exist yet.""" query = {"bool": {"filter": [ {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"term": {"git.ref_type": ref_type}}, - {"term": {"git.ref": ref}}, + {"term": {"git.ref_pattern": ref_pattern}}, ]}} def _count(index: str) -> int: @@ -880,16 +894,16 @@ def apply_content_index_mapping(es: Elasticsearch, files_mapping: dict, lines_ma def stale_snapshot_markers_for_ref( - es: Elasticsearch, host: str, org: str, repo: str, ref: str, + es: Elasticsearch, host: str, org: str, repo: str, ref_pattern: str, ) -> list[dict]: """Return any complete snapshot ref-name markers (mode: "snapshot", status: "complete") - for (host, org, repo, ref). Used by the incremental index path to detect and mark stale snapshot - markers left behind by a mode switch from snapshot to incremental.""" + for (host, org, repo, ref_pattern). Used by the incremental index path to detect and mark stale + snapshot markers left behind by a mode switch from snapshot to incremental.""" query = {"bool": {"filter": [ {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, - {"term": {"git.ref": ref}}, + {"term": {"git.ref_pattern": ref_pattern}}, {"term": {"status": "complete"}}, {"term": {"mode": "snapshot"}}, ]}} @@ -901,17 +915,17 @@ def stale_snapshot_markers_for_ref( def mark_snapshot_markers_stale( - es: Elasticsearch, host: str, org: str, repo: str, ref: str, + es: Elasticsearch, host: str, org: str, repo: str, ref_pattern: str, ) -> int: - """Flip any complete snapshot markers for this (host, org, repo, ref) to status:"stale", - making them invisible to all content tools without deleting them immediately. Content - reclamation is deferred to the prune command's stale-marker step. Returns the count of - markers flipped. + """Flip any complete snapshot markers for this (host, org, repo, ref_pattern) to + status:"stale", making them invisible to all content tools without deleting them immediately. + Content reclamation is deferred to the prune command's stale-marker step. Returns the count + of markers flipped. ORDER: callers must call this BEFORE publishing the incremental join doc as "complete", so the two-complete-docs fan-out window (one snapshot + one incremental, both reachable by - the LOOKUP JOIN ON git.ref) never opens.""" - markers = stale_snapshot_markers_for_ref(es, host, org, repo, ref) + the LOOKUP JOIN ON git.ref_pattern) never opens.""" + markers = stale_snapshot_markers_for_ref(es, host, org, repo, ref_pattern) for hit in markers: try: es.update(index=REFS_INDEX, id=hit["_id"], doc={"status": "stale"}) diff --git a/src/sourcerer/commands/index/selection.py b/src/sourcerer/commands/index/selection.py index 6d2d194..c3dda93 100644 --- a/src/sourcerer/commands/index/selection.py +++ b/src/sourcerer/commands/index/selection.py @@ -16,6 +16,7 @@ from ...hosts import Host from ...planner import Marker, plan_repo from ...progress import Unit +from ...version import match_version from .git import _commit_date_of, list_remote_ref_names, list_remote_refs @@ -54,7 +55,7 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=prefix, kind=rt, index_level=sel.index_level, index_suffix=sel.index_suffix, - mode=sel.mode, + mode=sel.mode, ref_pattern=prefix, )) continue if rt not in fetched: @@ -63,10 +64,39 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: if ref_map is None: continue # ls-remote failed for this ref type, skip floor = sel.since_version_floor() # version-based `since: {ref}`, name-only + + # Delta-mode tag selectors are "moving streams": each raw pattern string is a single + # logical stream whose identity is the pattern itself (not any concrete tag name). One + # stream unit is emitted per raw pattern that matches at least one remote tag; the + # concrete newest-committed tag is resolved post-clone (ls-remote lacks dates). The + # per-name loop below handles all other cases (snapshot tags, all branches). + if sel.mode == "delta" and rt == "tag": + for pattern, cp in zip(sel.raw_patterns, sel.compiled): + key = (rt, pattern) + # Only emit a stream unit if this specific pattern matches at least one remote tag. + has_match = any(match_version(cp, name) is not None for name in ref_map) + if not has_match: + continue # pattern matches nothing remotely -- no stream to emit + if key in seen: + prior_mode = seen_mode[key] + if prior_mode != sel.mode: + mode_conflicts.append((rt, pattern, prior_mode, sel.mode)) + continue + seen.add(key) + seen_mode[key] = sel.mode + units.append(Unit( + host=cfg.host, org=cfg.org, repo=cfg.repo, ref=pattern, kind=rt, + remote_sha=None, # resolved post-clone via ref_dates + index_level=sel.index_level, index_suffix=sel.index_suffix, + mode=sel.mode, ref_pattern=pattern, + )) + continue + for name in sorted(ref_map): - v = sel.matches(rt, name) - if v is None: + matched = sel.match_pattern(rt, name) + if matched is None: continue + pattern, v = matched if floor is not None and v.components < floor: continue # below the since version floor key = (rt, name) @@ -82,7 +112,7 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: host=cfg.host, org=cfg.org, repo=cfg.repo, ref=name, kind=rt, remote_sha=ref_map[name], index_level=sel.index_level, index_suffix=sel.index_suffix, - mode=sel.mode, + mode=sel.mode, ref_pattern=pattern, )) if mode_conflicts: diff --git a/src/sourcerer/commands/prune/execute.py b/src/sourcerer/commands/prune/execute.py index d493e60..3dd1404 100644 --- a/src/sourcerer/commands/prune/execute.py +++ b/src/sourcerer/commands/prune/execute.py @@ -299,11 +299,11 @@ def execute_orphan_deletions(es: Elasticsearch, plan: OrphanPlan) -> tuple[int, pass # Class D-I: stale-location incremental content (ref-addressed, no git.commit). Mirrors Class D - # but keyed on (host, org, repo, ref_type, ref) tuples -- the commit-keyed filter above cannot - # match incremental docs whose git.commit is absent. + # but keyed on (host, org, repo, ref_type, ref_pattern) tuples -- the commit-keyed filter above + # cannot match incremental docs whose git.commit is absent. for index_name, ref_tuples in plan.orphan_stale_incremental.items(): stale_dropped += len(ref_tuples) - for (host, org, repo, ref_type, ref) in ref_tuples: + for (host, org, repo, ref_type, ref_pattern) in ref_tuples: try: es.delete_by_query( index=index_name, @@ -312,7 +312,7 @@ def execute_orphan_deletions(es: Elasticsearch, plan: OrphanPlan) -> tuple[int, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"term": {"git.ref_type": ref_type}}, - {"term": {"git.ref": ref}}, + {"term": {"git.ref_pattern": ref_pattern}}, ]}}, conflicts="proceed", refresh=False, diff --git a/src/sourcerer/config.py b/src/sourcerer/config.py index c8a9355..97c3dcd 100644 --- a/src/sourcerer/config.py +++ b/src/sourcerer/config.py @@ -309,6 +309,23 @@ def matches(self, ref_type: str, ref: str) -> Version | None: return v return None + def match_pattern(self, ref_type: str, ref: str) -> tuple[str, Version] | None: + """Like matches(), but also returns the raw match pattern that matched `ref`. + Used to set Unit.ref_pattern to the raw sources[i].match string (not the concrete ref).""" + if self.ref_type != ref_type: + return None + if self.ref_type == "commit": + ref_l = ref.lower() + for p in self.raw_patterns: + if ref_l.startswith(p): + return p, Version(ref=ref, components=(), prerelease="") + return None + for pattern, cp in zip(self.raw_patterns, self.compiled): + v = match_version(cp, ref) + if v is not None: + return pattern, v + return None + def since_version_floor(self) -> tuple[int, ...] | None: """If `since` is a ref anchor that denotes a version under this (versioned) selector, the inclusive version floor to index from -- name-only, so it's applied in Phase 1 and diff --git a/src/sourcerer/elastic/agent_builder_tools/README.md b/src/sourcerer/elastic/agent_builder_tools/README.md index d21c89e..b338854 100644 --- a/src/sourcerer/elastic/agent_builder_tools/README.md +++ b/src/sourcerer/elastic/agent_builder_tools/README.md @@ -35,27 +35,32 @@ Query snippet: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( - // Incremental refs + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs: content docs carry git.ref_pattern = stream identity (pattern for + // delta-tag streams, branch name otherwise). The refs-side join key is git.ref_pattern + // (same field, same value on both sides). The ?git_ref param can be the pattern, the + // concrete tag, or a wildcard — both git.ref_pattern and git.ref are checked. FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) // other filters // Branch by content-doc shape to resolve git.commit for incremental refs: // Snapshot rows already carry git.commit (no join needed). -// Incremental rows carry only git.ref; the join resolves git.commit from the refs join doc. +// Incremental rows carry git.ref_pattern (stream identity) and git.ref (concrete resolved ref); +// the LOOKUP JOIN resolves git.commit from the refs join doc using git.ref_pattern as the join +// key — the same field with the same value on both content and refs sides (no RENAME needed). | FORK ( WHERE git.commit IS NOT NULL ) - ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // rest of query ``` diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml index e61763e..b842966 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml @@ -31,16 +31,16 @@ configuration: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( // Incremental refs FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) AND file.path LIKE ?file_path @@ -52,8 +52,8 @@ configuration: // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK ( WHERE git.commit IS NOT NULL ) - ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml index cd10d80..60f4046 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml @@ -31,16 +31,16 @@ configuration: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( // Incremental refs FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) AND file.path LIKE ?file_path @@ -52,8 +52,8 @@ configuration: // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK ( WHERE git.commit IS NOT NULL ) - ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml index 3cc23db..6d7beed 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml @@ -31,16 +31,16 @@ configuration: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( // Incremental refs FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) AND file.path LIKE ?file_path @@ -51,8 +51,8 @@ configuration: // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK ( WHERE git.commit IS NOT NULL ) - ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml index c7697c5..6842035 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml @@ -31,16 +31,16 @@ configuration: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( // Incremental refs FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) AND file.path LIKE ?file_path @@ -51,8 +51,8 @@ configuration: // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK ( WHERE git.commit IS NOT NULL ) - ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml index de4e57a..b95d8f9 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml @@ -31,16 +31,16 @@ configuration: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( // Incremental refs FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) @@ -50,8 +50,8 @@ configuration: // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK ( WHERE git.commit IS NOT NULL ) - ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Enforce glob depth for * and ** on file.path. // Without this, "src/*/Job.java" would match files at any depth, diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml index 0feaa55..de88728 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml @@ -31,16 +31,16 @@ configuration: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( // Incremental refs FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) AND file.path LIKE ?file_path @@ -53,8 +53,8 @@ configuration: // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK ( WHERE git.commit IS NOT NULL ) - ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml index edff818..4b0de7c 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml @@ -31,16 +31,16 @@ configuration: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( // Incremental refs FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) AND file.path LIKE ?file_path @@ -51,8 +51,8 @@ configuration: // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK ( WHERE git.commit IS NOT NULL ) - ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml index d6ca1ed..5745ab3 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml @@ -31,16 +31,16 @@ configuration: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( // Incremental refs FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) @@ -50,8 +50,8 @@ configuration: // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK ( WHERE git.commit IS NOT NULL ) - ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Split each file path into its segments | EVAL _segs = SPLIT(file.path, "/") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml index a4f13b7..a1129dd 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml @@ -31,16 +31,16 @@ configuration: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( // Incremental refs FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) AND file.path LIKE ?file_path @@ -51,8 +51,8 @@ configuration: // git.commit from the refs join doc (one doc per (host,org,repo,ref)). | FORK ( WHERE git.commit IS NOT NULL ) - ( WHERE git.ref IS NOT NULL AND git.commit IS NULL - | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref, git.ref_type ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml index 750957a..7989353 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml @@ -10,13 +10,18 @@ configuration: AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + // `git.ref_pattern` is the stream identity (pattern for delta-tag streams, ref name otherwise). + // `git.ref` is the CONCRETE resolved ref (newest matching tag for delta-tag streams). + // Allow filtering by either so users can select by pattern, concrete tag, or wildcard. + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type AND status LIKE ?status // Format the response | SORT indexed_at DESC - | KEEP git.host, git.org, git.repo, git.ref, git.ref_type, git.commit, git.commit_date, status, files_count, lines_count, indexed_at, indexing_started_at + // Include `git.ref_pattern` (stream identity / join key) alongside git.ref (concrete resolved ref). + // For non-stream refs git.ref_pattern == git.ref. For delta-tag streams they differ. + | KEEP git.host, git.org, git.repo, git.ref, git.ref_pattern, git.ref_type, git.commit, git.commit_date, status, files_count, lines_count, indexed_at, indexing_started_at | LIMIT 1000000 params: git_host: diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml index b151040..7bc1f8f 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml @@ -31,16 +31,16 @@ configuration: | KEEP git.commit )) OR - (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( // Incremental refs FROM sourcerer-refs | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type - | KEEP git.ref + | KEEP git.ref_pattern )) ) AND file.path LIKE ?file_path diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json index 4654e4d..ba4e8a3 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json @@ -24,7 +24,7 @@ "git.org", "git.repo", "git.commit", - "git.ref", + "git.ref_pattern", "git.ref_type", "file.path" ], @@ -63,6 +63,9 @@ "ref": { "type": "keyword" }, + "ref_pattern": { + "type": "keyword" + }, "ref_type": { "type": "keyword" } diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json index 30f0219..56045ae 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json @@ -24,7 +24,7 @@ "git.org", "git.repo", "git.commit", - "git.ref", + "git.ref_pattern", "git.ref_type", "file.path", "line.number" @@ -105,6 +105,9 @@ "ref": { "type": "keyword" }, + "ref_pattern": { + "type": "keyword" + }, "ref_type": { "type": "keyword" } diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index 2134e98..c93a8aa 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -18,9 +18,13 @@ "git.host", "git.org", "git.repo", - "git.ref" + "git.ref", + "git.ref_pattern", + "git.ref_type" ], "order": [ + "asc", + "asc", "asc", "asc", "asc", @@ -58,6 +62,9 @@ "ref": { "type": "keyword" }, + "ref_pattern": { + "type": "keyword" + }, "ref_type": { "type": "keyword", "normalizer": "lowercase" @@ -106,4 +113,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/sourcerer/progress.py b/src/sourcerer/progress.py index 52e32b6..c91e935 100644 --- a/src/sourcerer/progress.py +++ b/src/sourcerer/progress.py @@ -71,6 +71,12 @@ class Unit: # commit-addressed) or "delta" (ref-addressed, branch or tag). Routes the unit to the # incremental delta-index path instead of the snapshot pre-clone/skip/retention flow. mode: str = "snapshot" + # Stream identity for delta-tag moving streams. For a delta-mode tag selector whose `match` + # pattern covers many concrete tags (e.g. "deploy@{major}"), `ref_pattern` holds the literal + # pattern string and is stored as `git.ref_pattern` on all content and refs docs. `ref` + # advances to the resolved concrete tag post-clone. For all other units (branches, snapshots, + # concrete tags) `ref_pattern` == `ref` so the split is transparent to non-stream code paths. + ref_pattern: str | None = None @property def label(self) -> str: diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index 44fb495..27992cd 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -266,14 +266,19 @@ def gather_intended_incremental_index_by_ref( double-counted. Returns {} if the refs index doesn't exist.""" out: dict[tuple[str, str, str, str, str], set[str]] = {} body = {"query": {"term": {"mode": "delta"}}} - src_fields = ["git.host", "git.org", "git.repo", "git.ref_type", "git.ref", + # `git.ref_pattern` is the stream identity that aligns with content docs' git.ref_pattern (the + # pattern for delta-tag streams, the branch name for delta branches). `git.ref` is the + # concrete resolved ref (e.g. newest tag) and is NOT the content key. Read `git.ref_pattern`. + src_fields = ["git.host", "git.org", "git.repo", "git.ref_type", "git.ref_pattern", "index_level", "index_suffix"] try: for hit in scan(es, index=REFS_ALIAS, query=body, _source=src_fields, preserve_order=False): src = hit["_source"] g = src.get("git", {}) host, org, repo = g.get("host"), g.get("org"), g.get("repo") - ref_type, ref = g.get("ref_type"), g.get("ref") + ref_type = g.get("ref_type") + # `git.ref_pattern` is the content-side identity (aligns with content docs' git.ref_pattern). + ref = g.get("ref_pattern") # nested under git if not (host and org and repo and ref_type and ref): continue level = src.get("index_level") or "repo" @@ -308,8 +313,9 @@ def gather_incremental_content_by_index( def _composite_incremental_ref_tuples( es: Elasticsearch, index: str, ) -> set[tuple[str, str, str, str, str]]: - """Distinct (host, org, repo, ref_type, ref) tuples from incremental content docs (git.ref - present, git.commit absent) in `index`. Returns empty set if the index doesn't exist.""" + """Distinct (host, org, repo, ref_type, ref_pattern) tuples from incremental content docs + (git.ref_pattern present, git.commit absent) in `index`. Returns empty set if the index + doesn't exist.""" out: set[tuple[str, str, str, str, str]] = set() after: dict | None = None while True: @@ -320,13 +326,13 @@ def _composite_incremental_ref_tuples( {"org": {"terms": {"field": "git.org"}}}, {"repo": {"terms": {"field": "git.repo"}}}, {"ref_type": {"terms": {"field": "git.ref_type"}}}, - {"ref": {"terms": {"field": "git.ref"}}}, + {"ref_pattern": {"terms": {"field": "git.ref_pattern"}}}, ], } if after is not None: composite["after"] = after - # Filter to docs that have git.ref but no git.commit (incremental content). - query = {"bool": {"filter": [{"exists": {"field": "git.ref"}}], + # Filter to docs that have git.ref_pattern but no git.commit (incremental content). + query = {"bool": {"filter": [{"exists": {"field": "git.ref_pattern"}}], "must_not": [{"exists": {"field": "git.commit"}}]}} try: resp = es.search(index=index, size=0, query=query, @@ -339,7 +345,7 @@ def _composite_incremental_ref_tuples( return out for b in buckets: out.add((b["key"]["host"], b["key"]["org"], b["key"]["repo"], - b["key"]["ref_type"], b["key"]["ref"])) + b["key"]["ref_type"], b["key"]["ref_pattern"])) after = agg.get("after_key") if after is None: return out @@ -419,8 +425,8 @@ def _enumerate_content_field( def _enumerate_incremental_content_ref_pairs( es: Elasticsearch, host: str, org: str, repo: str, ) -> set[tuple[str, str]]: - """Every distinct (git.ref_type, git.ref) pair in this repo's incremental content docs - (docs that have git.ref and no git.commit), via paginated composite aggregation.""" + """Every distinct (git.ref_type, git.ref_pattern) pair in this repo's incremental content docs + (docs that have git.ref_pattern and no git.commit), via paginated composite aggregation.""" out: set[tuple[str, str]] = set() for index in (FILES_ALIAS, LINES_ALIAS): after: dict | None = None @@ -429,7 +435,7 @@ def _enumerate_incremental_content_ref_pairs( "size": _COMPOSITE_PAGE_SIZE, "sources": [ {"ref_type": {"terms": {"field": "git.ref_type"}}}, - {"ref": {"terms": {"field": "git.ref"}}}, + {"ref_pattern": {"terms": {"field": "git.ref_pattern"}}}, ], } if after is not None: @@ -439,7 +445,7 @@ def _enumerate_incremental_content_ref_pairs( {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, - {"exists": {"field": "git.ref"}}, + {"exists": {"field": "git.ref_pattern"}}, ], "must_not": [{"exists": {"field": "git.commit"}}], }} @@ -456,7 +462,7 @@ def _enumerate_incremental_content_ref_pairs( if not buckets: break for b in buckets: - out.add((b["key"]["ref_type"], b["key"]["ref"])) + out.add((b["key"]["ref_type"], b["key"]["ref_pattern"])) after = agg.get("after_key") if after is None: break @@ -497,9 +503,12 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> found_commits = set() offending.extend(sorted(commits - found_commits)) - # --- incremental: each (ref_type, ref) pair must have EXACTLY ONE incremental join doc --- - # Content docs carry both git.ref and git.ref_type; a same-named branch and tag are distinct - # (ref_type, ref) pairs and are each allowed exactly one join doc. + # --- incremental: each (ref_type, ref_pattern) pair must have EXACTLY ONE incremental join doc --- + # Content docs carry git.ref_pattern (= the stream identity) and git.ref_type; a same-named + # branch and tag are distinct (ref_type, ref_pattern) pairs, each allowed exactly one join doc. + # On the refs side the join key is git.ref_pattern (= the stream identity), NOT git.ref + # (which holds the concrete resolved ref for delta-tag streams). Filter and aggregate by + # (git.ref_type, git.ref_pattern) so the pair-counts align with the content-side key. ref_pairs = _enumerate_incremental_content_ref_pairs(es, host, org, repo) if ref_pairs: try: @@ -509,22 +518,22 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> {"term": {"git.host": host}}, {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, - {"terms": {"git.ref": sorted({r for _, r in ref_pairs})}}, + {"terms": {"git.ref_pattern": sorted({r for _, r in ref_pairs})}}, {"term": {"mode": "delta"}}, ]}}, aggs={"ref_pairs": {"composite": {"size": 1000, "sources": [ {"ref_type": {"terms": {"field": "git.ref_type"}}}, - {"ref": {"terms": {"field": "git.ref"}}}, + {"ref_pattern": {"terms": {"field": "git.ref_pattern"}}}, ]}}}, ) pair_counts = { - (b["key"]["ref_type"], b["key"]["ref"]): b["doc_count"] + (b["key"]["ref_type"], b["key"]["ref_pattern"]): b["doc_count"] for b in resp["aggregations"]["ref_pairs"]["buckets"] } except NotFoundError: pair_counts = {} offending.extend( - sorted(f"{rt}/{ref}" for rt, ref in ref_pairs if pair_counts.get((rt, ref), 0) != 1) + sorted(f"{rt}/{ref_pattern}" for rt, ref_pattern in ref_pairs if pair_counts.get((rt, ref_pattern), 0) != 1) ) return sorted(offending) diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index 1f1420f..a38d2bd 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -51,12 +51,13 @@ def test_git_host_filtered_before_git_org(): def test_content_tools_use_universal_ref_join_query(): # Every content tool uses a two-OR'd-IN subquery to scope rows to matching - # refs (git.commit OR git.ref), then a FORK to resolve git.commit for both + # refs (git.commit OR git.ref_pattern), then a FORK to resolve git.commit for both # content shapes without fan-out: # - Snapshot arm (git.commit IS NOT NULL): no join needed; git.commit already # lives on the content row. - # - Incremental arm (git.ref IS NOT NULL AND git.commit IS NULL): LOOKUP JOIN - # sourcerer-refs ON (host,org,repo,ref) to resolve git.commit from the join doc. + # - Incremental arm (git.ref_pattern IS NOT NULL AND git.commit IS NULL): clean same-name + # LOOKUP JOIN sourcerer-refs ON (host,org,repo,git.ref_pattern,ref_type) to resolve + # git.commit from the join doc. No RENAME needed because both sides share git.ref_pattern. # No status guard is applied anywhere in the query. # Ref scoping uses three separate params: git_commit, git_ref, git_ref_type. tools = _tools() @@ -68,6 +69,11 @@ def test_content_tools_use_universal_ref_join_query(): # git.ref_key must not be used as a field or join key (comments may reference it by name) assert "git.ref_key" not in query, f"{tid} still uses git.ref_key as a field" assert "ON git.ref_key" not in query, f"{tid} still joins on git.ref_key" + # Old RENAME-based expression join must be gone + assert "RENAME git.ref AS _ref_id" not in query, \ + f"{tid} still has old RENAME git.ref AS _ref_id (replaced by git.ref_pattern join)" + assert "_ref_id == match" not in query, \ + f"{tid} still has old _ref_id == match expression join" # The membership subquery uses two OR'd IN paths (one for snapshot commits, one for # incremental refs), scoped by git_commit, git_ref, and git_ref_type params. assert "git.commit LIKE ?git_commit" in query, f"{tid} missing git.commit LIKE ?git_commit" @@ -75,10 +81,11 @@ def test_content_tools_use_universal_ref_join_query(): assert "git.ref_type LIKE ?git_ref_type" in query, f"{tid} missing git.ref_type LIKE ?git_ref_type" # Snapshot arm: no join; git.commit already on the content row. assert "git.commit IS NOT NULL" in query, f"{tid} missing snapshot FORK arm (git.commit IS NOT NULL)" - # Incremental arm: join on the 4-tuple (no ref_key) to resolve git.commit. - assert "git.ref IS NOT NULL" in query, f"{tid} missing incremental FORK arm (git.ref IS NOT NULL)" - assert "LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref" in query, \ - f"{tid} missing the incremental join on (host,org,repo,ref)" + # Incremental arm: clean same-name LOOKUP JOIN on git.ref_pattern (no RENAME). + assert "git.ref_pattern IS NOT NULL" in query, \ + f"{tid} missing incremental FORK arm (git.ref_pattern IS NOT NULL)" + assert "LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type" in query, \ + f"{tid} missing the new git.ref_pattern join on (host,org,repo,git.ref_pattern,git.ref_type)" # No ref_key param or join shape. assert "git_ref_key" not in params, f"{tid} still exposes git_ref_key as a param" assert "?git_ref_key" not in query, f"{tid} still references ?git_ref_key" diff --git a/tests/test_backfill.py b/tests/test_backfill.py index d214ab6..17cb1f4 100644 --- a/tests/test_backfill.py +++ b/tests/test_backfill.py @@ -82,7 +82,7 @@ def test_query_scopes_to_host_org_repo_ref(self): assert {"term": {"git.host": "github"}} in filt assert {"term": {"git.org": "acme"}} in filt assert {"term": {"git.repo": "widgets"}} in filt - assert {"term": {"git.ref": "main"}} in filt + assert {"term": {"git.ref_pattern": "main"}} in filt assert {"term": {"status": "complete"}} in filt assert {"term": {"mode": "snapshot"}} in filt diff --git a/tests/test_config.py b/tests/test_config.py index f44f4ac..50e3c2c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -347,6 +347,41 @@ def test_commit_prefix_matches_full_sha(self): assert sel.matches("commit", "deadbeef" * 5) is None +class TestSelectorMatchPattern: + def test_versioned_tag_returns_raw_pattern_and_version(self): + sel = _cfg([_source(ref_type="tag", match="v{major}.{minor}.{patch}")]).repos[0].selectors[0] + result = sel.match_pattern("tag", "v1.2.3") + assert result is not None + pattern, v = result + assert pattern == "v{major}.{minor}.{patch}" + assert v.components == (1, 2, 3) + + def test_ref_type_mismatch_returns_none(self): + sel = _cfg([_source(ref_type="branch", match="main")]).repos[0].selectors[0] + assert sel.match_pattern("tag", "main") is None + + def test_no_match_returns_none(self): + sel = _cfg([_source(ref_type="tag", match="v{major}.{minor}.{patch}")]).repos[0].selectors[0] + assert sel.match_pattern("tag", "unrelated-tag") is None + + def test_commit_prefix_returns_prefix_and_version(self): + sha = "cfefb3b2378ccbadefa7c8f4f9e21b3a1d2e5f60" + sel = _cfg([_source(ref_type="commit", match="cfefb3b")]).repos[0].selectors[0] + result = sel.match_pattern("commit", sha) + assert result is not None + prefix, v = result + assert prefix == "cfefb3b" + assert v.ref == sha + + def test_first_matching_pattern_wins(self): + # When multiple patterns match, the first raw pattern is returned. + sel = _cfg([_source(match=["main", "dev"])]).repos[0].selectors[0] + result = sel.match_pattern("branch", "main") + assert result is not None + pattern, _ = result + assert pattern == "main" + + class TestSinceVersionFloor: def test_full_ref_name(self): cfg = _cfg([_source(ref_type="tag", match="v{major}.{minor}.{patch}", since={"ref": "v8.17.0"})]) diff --git a/tests/test_documents.py b/tests/test_documents.py index 4d99791..74a898a 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -33,10 +33,12 @@ def _set_worker_ctx(host: str, org: str, repo: str, commit_sha: str, repo_dir, s ) -def _set_worker_ctx_incremental(host: str, org: str, repo: str, ref: str, repo_dir, +def _set_worker_ctx_incremental(host: str, org: str, repo: str, repo_dir, + ref_pattern: str = "main", symlink_paths=frozenset(), ref_type: str = "branch") -> None: documents._WORKER_CTX.update( - host=host, org=org, repo=repo, ref_type=ref_type, ref=ref, + host=host, org=org, repo=repo, ref_type=ref_type, + ref_pattern=ref_pattern, repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, mode="delta", ) @@ -182,13 +184,16 @@ def test_no_optional_fields_when_omitted(self): class TestIncrementalDocs: - def test_ref_field_set_no_ref_key(self, tmp_path): - # Incremental docs carry git.ref (the ref name) and git.ref_type but no git.ref_key. + def test_ref_pattern_set_no_ref_no_ref_key(self, tmp_path): + # Incremental docs carry git.ref_pattern (identity) and git.ref_type but NOT git.ref or + # git.ref_key. git.ref lives only on the refs join doc (written wholesale each run); + # content docs omit it to avoid staleness when unchanged paths survive a tag promotion. p = tmp_path / "a.txt" p.write_text("hello") _id, doc = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) - assert doc["git"]["ref"] == "main" + assert doc["git"]["ref_pattern"] == "main" assert doc["git"]["ref_type"] == "branch" + assert "ref" not in doc["git"] assert "ref_key" not in doc["git"] def test_no_commit_field(self, tmp_path): @@ -199,7 +204,7 @@ def test_no_commit_field(self, tmp_path): def test_id_stable_across_commits(self, tmp_path): # The whole point of ref-addressing: the id does not depend on the commit, only the - # ref_type+ref, so a modified file's doc overwrites in place rather than minting a new id. + # ref_type+ref_pattern, so a modified file's doc overwrites in place rather than minting a new id. p = tmp_path / "a.txt" p.write_text("hello") id1, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) @@ -207,6 +212,26 @@ def test_id_stable_across_commits(self, tmp_path): id2, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) assert id1 == id2 + def test_id_stable_across_tag_promotions(self, tmp_path): + # For delta-tag streams: promoting from deploy@1 to deploy@2 must not mint a new doc id. + # Content _id is keyed on ref_pattern (stream identity), not the concrete tag. + p = tmp_path / "a.txt" + p.write_text("hello") + id1, _ = build_incremental_file_doc("github", "acme", "widgets", "tag", "deploy@{major}", "a.txt", p) + id2, _ = build_incremental_file_doc("github", "acme", "widgets", "tag", "deploy@{major}", "a.txt", p) + assert id1 == id2, "doc id must be keyed on ref_pattern, not concrete ref" + + def test_delta_content_has_ref_pattern_not_ref(self, tmp_path): + # Content docs carry git.ref_pattern (stream identity) but NOT git.ref (concrete). + # After a tag-stream promotion, unchanged paths are NOT rewritten (delta only touches + # changed paths), so git.ref on content would be inconsistent/stale across a single ref. + # The concrete git.ref lives only on the refs join doc (one doc, rewritten each run). + p = tmp_path / "a.txt" + p.write_text("hello") + _id, doc = build_incremental_file_doc("github", "acme", "widgets", "tag", "deploy@{major}", "a.txt", p) + assert "ref" not in doc["git"], "git.ref must be absent from delta content docs" + assert doc["git"]["ref_pattern"] == "deploy@{major}" + def test_id_differs_from_snapshot_id(self, tmp_path): p = tmp_path / "a.txt" p.write_text("hello") @@ -222,21 +247,24 @@ def test_branch_and_tag_same_name_have_distinct_ids(self, tmp_path): tag_id, _ = build_incremental_file_doc("github", "acme", "widgets", "tag", "deploy", "a.txt", p) assert branch_id != tag_id - def test_line_docs_ref_and_no_commit_no_ref_key(self): - # Incremental line docs carry git.ref and git.ref_type, no git.commit, no git.ref_key. + def test_line_docs_ref_pattern_no_ref_no_commit_no_ref_key(self): + # Incremental line docs carry git.ref_pattern and git.ref_type; git.ref and git.commit + # are absent for the same reason as file docs (see test_delta_content_has_ref_pattern_not_ref). docs = list(iter_incremental_line_docs("github", "acme", "widgets", "branch", "main", "a.txt", "one\ntwo")) for _id, d in docs: - assert d["git"]["ref"] == "main" + assert d["git"]["ref_pattern"] == "main" assert d["git"]["ref_type"] == "branch" + assert "ref" not in d["git"] assert "commit" not in d["git"] assert "ref_key" not in d["git"] def test_worker_ctx_routes_to_incremental_builders(self, tmp_path): (tmp_path / "a.txt").write_text("one\ntwo\n") - _set_worker_ctx_incremental("github", "acme", "widgets", "main", tmp_path) + _set_worker_ctx_incremental("github", "acme", "widgets", tmp_path, ref_pattern="main") actions = _build_one_file_actions("a.txt") - assert actions[0]["_source"]["git"]["ref"] == "main" + assert actions[0]["_source"]["git"]["ref_pattern"] == "main" assert actions[0]["_source"]["git"]["ref_type"] == "branch" + assert "ref" not in actions[0]["_source"]["git"] assert "commit" not in actions[0]["_source"]["git"] assert "ref_key" not in actions[0]["_source"]["git"] diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py index f8534db..be233fe 100644 --- a/tests/test_incremental_index.py +++ b/tests/test_incremental_index.py @@ -16,12 +16,18 @@ NEW = "2222222222222222222222222222222222222222" -def _patch_common(prior=None, plan=None): +def _patch_common(prior=None, plan=None, ref_dates_return=None): """Patch every git/documents/markers side effect index_incremental_branch_in_dir calls, - returning the patcher context managers as a dict of MagicMocks keyed by name.""" + returning the patcher context managers as a dict of MagicMocks keyed by name. + + ref_dates_return: dict for ref_dates mock; default {} (branches never call ref_dates). + For tag stream tests pass e.g. {("tag", "deploy@1788"): 1788, ("tag", "deploy@1787"): 1787}. + """ patchers = { "checkout_branch": patch("sourcerer.commands.index.command.checkout_branch"), "checkout_ref": patch("sourcerer.commands.index.command.checkout_ref"), + "ref_dates": patch("sourcerer.commands.index.command.ref_dates", + return_value=ref_dates_return if ref_dates_return is not None else {}), "resolve_commit": patch("sourcerer.commands.index.command.resolve_commit", return_value=NEW), "commit_date": patch("sourcerer.commands.index.command.commit_date", return_value="2026-01-01T00:00:00+00:00"), "read_incremental_ref": patch("sourcerer.commands.index.command.read_incremental_ref", return_value=prior), @@ -61,7 +67,7 @@ def test_first_index_does_full_rebuild(self): mocks["checkout_ref"].assert_not_called() mocks["delete_incremental_branch"].assert_called_once() mocks["index_incremental_paths"].assert_called_once() - # rel_paths (4th positional after repo_dir/branch) is None -> full tree walk. + # rel_paths is None -> full tree walk. Signature: (es,host,org,repo,repo_dir,ref_pattern,rel_paths,...). call_args = mocks["index_incremental_paths"].call_args assert call_args[0][6] is None mocks["delete_incremental_paths"].assert_not_called() @@ -86,11 +92,12 @@ def test_second_run_indexes_only_changed_paths(self): reporter=ProgressReporter(), unit=unit) mocks["delete_incremental_branch"].assert_not_called() mocks["delete_incremental_paths"].assert_called_once() - # delete_incremental_paths(es, host, org, repo, ref_type, ref, paths, ...) - # ref_type at [4], ref at [5], paths at [6] + # delete_incremental_paths(es, host, org, repo, ref_type, ref_pattern, paths, ...) + # ref_type at [4], ref_pattern at [5], paths at [6] assert mocks["delete_incremental_paths"].call_args[0][6] == ["gone.txt"] mocks["index_incremental_paths"].assert_called_once() call_args = mocks["index_incremental_paths"].call_args + # index_incremental_paths(es, host, org, repo, repo_dir, ref_pattern, rel_paths, ...) assert call_args[0][6] == ["new.txt"] mocks["write_incremental_ready"].assert_called_once() finally: @@ -250,100 +257,140 @@ def test_same_routing_no_old_copy_delete(self): class TestIncrementalIndexTagFirstRun: - """Mirror of TestIncrementalIndexFirstRun / TestIncrementalIndexDeltaRun for tag Units. - Confirms that: - - checkout_ref is used instead of checkout_branch for tags (git.ref_type: tag) - - The overall orchestration path (first run → full rebuild; second run → delta) is identical. + """Delta-mode tag stream orchestration tests. + + A tag stream Unit carries `ref = ` (e.g. "deploy@{major}") as its stable + identity; ref_dates() resolves the newest concrete tag for checkout. All marker/content calls + receive the pattern; only checkout_ref receives the concrete tag name. """ - def test_tag_first_index_uses_checkout_ref(self): - """First run for a tag Unit: checkout_ref called, full tree walk.""" - patchers, mocks = _patch_common(prior=None) + # Two fake matching tags: 1788 is newer than 1787. + TAG_DATES = {("tag", "deploy@1788000000"): 1788000000, ("tag", "deploy@1787000000"): 1787000000} + NEWEST_TAG = "deploy@1788000000" + PATTERN = "deploy@{major}" + + def test_tag_first_index_uses_checkout_ref_with_newest_tag(self): + """First run: checkout_ref called with the NEWEST concrete tag; all stored refs use pattern.""" + patchers, mocks = _patch_common(prior=None, ref_dates_return=self.TAG_DATES) try: es = MagicMock() unit = Unit(host="github", org="elastic", repo="kibana", - ref="deploy@8", kind="tag", mode="delta") + ref=self.PATTERN, kind="tag", mode="delta") index_incremental_branch_in_dir( - es, "github", "elastic", "kibana", "/repo", "deploy@8", + es, "github", "elastic", "kibana", "/repo", self.PATTERN, reporter=ProgressReporter(), unit=unit, ) - # Tag: uses checkout_ref, NOT checkout_branch. - mocks["checkout_ref"].assert_called_once() + # Tag stream: checkout_ref called with the newest concrete tag, not the pattern. + mocks["checkout_ref"].assert_called_once_with("/repo", self.NEWEST_TAG) mocks["checkout_branch"].assert_not_called() mocks["delete_incremental_branch"].assert_called_once() mocks["index_incremental_paths"].assert_called_once() - # Full tree walk: rel_paths arg is None. - call_args = mocks["index_incremental_paths"].call_args - assert call_args[0][6] is None + # Full tree walk: rel_paths arg is None. Signature: (..., ref_pattern, rel_paths, ...) + assert mocks["index_incremental_paths"].call_args[0][6] is None mocks["write_incremental_ready"].assert_called_once() # write_incremental_ready(es, host, org, repo, ref_type, ref, commit, ...) assert mocks["write_incremental_ready"].call_args[0][6] == NEW finally: _stop(patchers) + def test_tag_stream_concrete_ref_and_match_pattern(self): + """refs join doc: git.ref = concrete tag (newest match), git.ref_pattern = pattern. + Content docs keyed on ref_pattern (pattern); only the refs join-doc git.ref advances.""" + patchers, mocks = _patch_common(prior=None, ref_dates_return=self.TAG_DATES) + try: + es = MagicMock() + unit = Unit(host="github", org="elastic", repo="kibana", + ref=self.PATTERN, kind="tag", mode="delta") + index_incremental_branch_in_dir( + es, "github", "elastic", "kibana", "/repo", self.PATTERN, + reporter=ProgressReporter(), unit=unit, + ) + ready_args = mocks["write_incremental_ready"].call_args[0] + ready_kwargs = mocks["write_incremental_ready"].call_args[1] + # write_incremental_ready(es, host, org, repo, ref_type, ref, sha, ..., ref_pattern=) + assert ready_args[4] == "tag" + # git.ref payload = the CONCRETE newest tag. + assert ready_args[5] == self.NEWEST_TAG, ( + f"git.ref must be the concrete tag '{self.NEWEST_TAG}', got {ready_args[5]!r}" + ) + # git.ref_pattern = the stable PATTERN (stream identity). + assert ready_kwargs.get("ref_pattern") == self.PATTERN, ( + f"ref_pattern kwarg must be the pattern '{self.PATTERN}', got {ready_kwargs.get('ref_pattern')!r}" + ) + # unit.ref is updated to the concrete tag for reporter display. + assert unit.ref == self.NEWEST_TAG, ( + f"unit.ref must advance to concrete tag '{self.NEWEST_TAG}', got {unit.ref!r}" + ) + # Content calls: arg[5] = ref_pattern = pattern (stream identity). The concrete ref is + # NOT passed to index_incremental_paths -- content docs carry only git.ref_pattern; + # git.ref (concrete) lives only on the refs join doc (write_incremental_ready above). + # index_incremental_paths(es, host, org, repo, repo_dir, ref_pattern, rel_paths, ...) + ref_pattern_arg = mocks["index_incremental_paths"].call_args[0][5] + assert ref_pattern_arg == self.PATTERN, ( + f"index_incremental_paths ref_pattern (arg 5) must be pattern '{self.PATTERN}', got {ref_pattern_arg!r}" + ) + finally: + _stop(patchers) + def test_tag_delta_run_indexes_only_changed_paths(self): - """Second run for a tag Unit: delta diff applied, full rebuild skipped.""" + """Second run: delta diff applied, full rebuild skipped; still uses pattern identity.""" prior = {"git": {"commit": OLD}} plan = ChangePlan(delete_paths=["gone.ts"], index_paths=["new.ts"]) - patchers, mocks = _patch_common(prior=prior, plan=plan) + patchers, mocks = _patch_common(prior=prior, plan=plan, ref_dates_return=self.TAG_DATES) try: es = MagicMock() unit = Unit(host="github", org="elastic", repo="kibana", - ref="deploy@8", kind="tag", mode="delta") + ref=self.PATTERN, kind="tag", mode="delta") index_incremental_branch_in_dir( - es, "github", "elastic", "kibana", "/repo", "deploy@8", + es, "github", "elastic", "kibana", "/repo", self.PATTERN, reporter=ProgressReporter(), unit=unit, ) - mocks["checkout_ref"].assert_called_once() + mocks["checkout_ref"].assert_called_once_with("/repo", self.NEWEST_TAG) mocks["checkout_branch"].assert_not_called() mocks["delete_incremental_branch"].assert_not_called() mocks["delete_incremental_paths"].assert_called_once() - # delete_incremental_paths(es, host, org, repo, ref_type, ref, paths, ...) + # delete_incremental_paths(es, host, org, repo, ref_type, ref_pattern, paths, ...) assert mocks["delete_incremental_paths"].call_args[0][6] == ["gone.ts"] mocks["index_incremental_paths"].assert_called_once() + # index_incremental_paths(es, host, org, repo, repo_dir, ref_pattern, rel_paths, ...) assert mocks["index_incremental_paths"].call_args[0][6] == ["new.ts"] mocks["write_incremental_ready"].assert_called_once() finally: _stop(patchers) def test_tag_missing_diff_base_triggers_full_rebuild(self): - """Force-moved tag whose old target is gone → base_missing → full rebuild (INV-007).""" + """Newest tag's diff base gone → base_missing → full rebuild (INV-007).""" prior = {"git": {"commit": OLD}} plan = ChangePlan(base_missing=True) - patchers, mocks = _patch_common(prior=prior, plan=plan) + patchers, mocks = _patch_common(prior=prior, plan=plan, ref_dates_return=self.TAG_DATES) try: es = MagicMock() unit = Unit(host="github", org="elastic", repo="kibana", - ref="deploy@8", kind="tag", mode="delta") + ref=self.PATTERN, kind="tag", mode="delta") index_incremental_branch_in_dir( - es, "github", "elastic", "kibana", "/repo", "deploy@8", + es, "github", "elastic", "kibana", "/repo", self.PATTERN, reporter=ProgressReporter(), unit=unit, ) - mocks["checkout_ref"].assert_called_once() + mocks["checkout_ref"].assert_called_once_with("/repo", self.NEWEST_TAG) mocks["delete_incremental_branch"].assert_called_once() assert mocks["index_incremental_paths"].call_args[0][6] is None finally: _stop(patchers) - def test_tag_ref_type_reaches_write_ready_call(self): - """ref_type='tag' flows through to write_incremental_ready positional args.""" - patchers, mocks = _patch_common(prior=None) + def test_tag_no_matching_tags_in_clone_skips(self): + """If ref_dates returns no tags matching the pattern, the unit is skipped gracefully.""" + patchers, mocks = _patch_common(prior=None, ref_dates_return={}) try: es = MagicMock() unit = Unit(host="github", org="elastic", repo="kibana", - ref="deploy@8", kind="tag", mode="delta") + ref=self.PATTERN, kind="tag", mode="delta") index_incremental_branch_in_dir( - es, "github", "elastic", "kibana", "/repo", "deploy@8", + es, "github", "elastic", "kibana", "/repo", self.PATTERN, reporter=ProgressReporter(), unit=unit, ) - ready_args = mocks["write_incremental_ready"].call_args[0] - # write_incremental_ready(es, host, org, repo, ref_type, ref, sha, ...) - # positional index 4 is ref_type - assert ready_args[4] == "tag", ( - f"Expected ref_type='tag' at pos 4 of write_incremental_ready call, got {ready_args}" - ) - assert ready_args[5] == "deploy@8", ( - f"Expected ref='deploy@8' at pos 5, got {ready_args}" - ) + mocks["checkout_ref"].assert_not_called() + mocks["index_incremental_paths"].assert_not_called() + mocks["write_incremental_ready"].assert_not_called() + assert unit.status == "no-changes" finally: _stop(patchers) diff --git a/tests/test_markers.py b/tests/test_markers.py index 5045114..cba86c1 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -11,7 +11,6 @@ # App packages from sourcerer.commands.index.markers import ( - ERROR_MAX_LEN, build_ref_id, commit_prefix_indexed, commits_with_content, @@ -28,6 +27,7 @@ write_incremental_failed, write_incremental_indexing, write_incremental_ready, + write_indexing_marker, write_ref_marker, ) from sourcerer.indices import FILES_ALIAS, REFS_ALIAS, REFS_INDEX @@ -446,6 +446,72 @@ def test_marker_status_complete(self): files_count=5, lines_count=100) assert es.index.call_args.kwargs["document"]["status"] == "complete" + def test_complete_marker_carries_ref_pattern(self): + # write_ref_marker (status:complete) must populate git.ref_pattern. When no ref_pattern + # arg is given (branch == pattern), the fallback is git.ref. + es = MagicMock() + write_ref_marker(es, "github", "acme", "widgets", "branch", "main", OLD, None, + files_count=1, lines_count=10) + doc = es.index.call_args.kwargs["document"] + assert doc["git"]["ref_pattern"] == "main" + + def test_complete_marker_explicit_ref_pattern_differs_from_ref(self): + # When a versioned snapshot tag has a pattern like "zentity-{major}.{minor}.{patch}", + # git.ref holds the concrete tag and git.ref_pattern holds the raw match pattern. + es = MagicMock() + write_ref_marker(es, "github", "zentity-io", "zentity", "tag", "zentity-1.7.0", OLD, None, + files_count=10, lines_count=200, + ref_pattern="zentity-{major}.{minor}.{patch}") + doc = es.index.call_args.kwargs["document"] + assert doc["git"]["ref"] == "zentity-1.7.0" + assert doc["git"]["ref_pattern"] == "zentity-{major}.{minor}.{patch}" + # _id still keys on build_ref_id(concrete ref + commit), not the pattern, + # so sibling tags sharing a ref_pattern get distinct docs. + from sourcerer.commands.index.markers import build_ref_id + expected_id = build_ref_id("github", "zentity-io", "zentity", "tag", "zentity-1.7.0", OLD) + assert es.index.call_args.kwargs["id"] == expected_id + + +class TestWriteIndexingMarker: + """write_indexing_marker is the in-progress (status:'indexing') snapshot writer. It must + populate git.ref_pattern from the very first write, so the field is never NULL in a snapshot + refs doc -- even while the ref is mid-index (before write_ref_marker's 'complete' overwrite).""" + + def test_indexing_marker_carries_ref_pattern(self): + # Fallback: when no ref_pattern arg is passed, git.ref_pattern == git.ref. + es = MagicMock() + write_indexing_marker(es, "github", "acme", "widgets", "branch", "main", OLD, None) + doc = es.index.call_args.kwargs["document"] + assert doc["git"]["ref_pattern"] == "main" + assert doc["git"]["ref"] == "main" + assert doc["git"]["ref_pattern"] == doc["git"]["ref"] + + def test_indexing_marker_explicit_ref_pattern_differs_from_ref(self): + # When a versioned snapshot tag has a pattern, the in-progress marker also carries it. + es = MagicMock() + write_indexing_marker(es, "github", "zentity-io", "zentity", "tag", "zentity-1.7.0", OLD, None, + ref_pattern="zentity-{major}.{minor}.{patch}") + doc = es.index.call_args.kwargs["document"] + assert doc["git"]["ref"] == "zentity-1.7.0" + assert doc["git"]["ref_pattern"] == "zentity-{major}.{minor}.{patch}" + + def test_indexing_marker_status_is_indexing(self): + es = MagicMock() + write_indexing_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None) + doc = es.index.call_args.kwargs["document"] + assert doc["status"] == "indexing" + assert doc["mode"] == "snapshot" + + def test_indexing_marker_id_matches_write_ref_marker_id(self): + # Both writers must use the same ref_id so write_ref_marker overwrites in place. + es = MagicMock() + write_indexing_marker(es, "github", "acme", "widgets", "tag", "v2.0.0", OLD, None) + indexing_id = es.index.call_args.kwargs["id"] + write_ref_marker(es, "github", "acme", "widgets", "tag", "v2.0.0", OLD, None, + files_count=1, lines_count=1) + complete_id = es.index.call_args.kwargs["id"] + assert indexing_id == complete_id + class TestIncrementalRefKeyIdentity: def test_id_is_ref_key_not_a_hash(self): @@ -530,7 +596,6 @@ def test_incremental_marker_advances_commit_and_clears_target_and_error(self): assert doc["status"] == "complete" assert doc["git"]["commit"] == NEW # advances only after a successful run (INV-006) assert doc["git"]["commit_target"] is None - assert doc["error"] is None and doc["failed_at"] is None assert doc["files_count"] == 5 and doc["lines_count"] == 99 assert es.index.call_args.kwargs["refresh"] is True # publication boundary @@ -545,21 +610,16 @@ def test_incremental_ready_carries_routing(self): class TestWriteIncrementalFailed: - def test_incremental_marker_keeps_status_indexing_and_retains_old_pointer(self): + def test_incremental_failed_sets_status_failed_and_retains_old_pointer(self): es = MagicMock() write_incremental_failed(es, "github", "acme", "widgets", "branch", "main", completed_commit=OLD, commit_target=NEW, error="boom") doc = _indexed_doc(es) - assert doc["status"] == "indexing" # not advanced -- a failed run leaves the prior state - assert doc["git"]["commit"] == OLD + assert doc["status"] == "failed" + assert doc["git"]["commit"] == OLD # completed pointer unchanged (INV-006) assert doc["git"]["commit_target"] == NEW - assert doc["error"] == "boom" - assert doc["failed_at"] is not None - - def test_incremental_marker_error_text_is_bounded(self): - es = MagicMock() - write_incremental_failed(es, "github", "acme", "widgets", "branch", "main", OLD, NEW, error="x" * 5000) - assert len(_indexed_doc(es)["error"]) == ERROR_MAX_LEN + assert "error" not in doc + assert "failed_at" not in doc def test_incremental_failed_carries_routing(self): es = MagicMock() @@ -602,7 +662,7 @@ def test_scoped_to_exact_ref_key_and_paths(self): assert {"term": {"git.org": "acme"}} in filt assert {"term": {"git.repo": "widgets"}} in filt assert {"term": {"git.ref_type": "branch"}} in filt - assert {"term": {"git.ref": "main"}} in filt + assert {"term": {"git.ref_pattern": "main"}} in filt assert {"terms": {"file.path": ["a.txt", "b.txt"]}} in filt def test_missing_index_is_ignored(self): @@ -623,7 +683,7 @@ def test_scoped_to_exact_ref_key_only(self): assert {"term": {"git.org": "acme"}} in filt assert {"term": {"git.repo": "widgets"}} in filt assert {"term": {"git.ref_type": "branch"}} in filt - assert {"term": {"git.ref": "main"}} in filt + assert {"term": {"git.ref_pattern": "main"}} in filt assert not any("ref_key" in str(f) for f in filt) def test_isolated_from_another_branch(self): diff --git a/tests/test_selection.py b/tests/test_selection.py new file mode 100644 index 0000000..fe97e12 --- /dev/null +++ b/tests/test_selection.py @@ -0,0 +1,164 @@ +"""Tests for _resolve_entry's delta-mode tag stream collapsing. + +A delta-mode tag selector whose match pattern covers multiple remote tags must emit exactly ONE +stream Unit per raw pattern (not one Unit per matching tag name). The Unit's ref is the literal +match pattern string; the concrete newest tag is resolved post-clone via ref_dates. +""" + +from unittest.mock import patch + +import yaml + +from sourcerer.commands.index.selection import _resolve_entry +from sourcerer.config import parse_config + + +def _resolve(source_yaml: str, remote_tags: dict[str, str], remote_branches: dict[str, str] | None = None): + """Run _resolve_entry with mocked ls-remote returning the given ref maps.""" + raw = yaml.safe_load(f""" +hosts: + - id: github + type: github + base_url: https://github.com +sources: +{source_yaml} +""") + cfg = parse_config(raw) + repo_cfg = cfg.repos[0] + host = cfg.hosts[repo_cfg.host] + + def _fake_list_remote_refs(url, kind): + if kind == "tags": + return remote_tags + return remote_branches or {} + + with patch("sourcerer.commands.index.selection.list_remote_refs", side_effect=_fake_list_remote_refs): + return _resolve_entry(repo_cfg, host) + + +class TestDeltaTagStreamSelection: + """Delta-mode tag selectors collapse to one stream Unit per pattern.""" + + MANY_TAGS = {f"deploy@{i}": f"sha{i}" for i in range(1, 10)} + + def test_delta_tag_glob_emits_one_unit_per_pattern(self): + """deploy@* matching 9 tags → 1 stream Unit, ref == pattern.""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@*" + mode: delta +""", remote_tags=self.MANY_TAGS) + assert len(units) == 1 + assert units[0].ref == "deploy@*" + assert units[0].kind == "tag" + assert units[0].mode == "delta" + assert units[0].remote_sha is None # resolved post-clone, not from ls-remote + + def test_delta_tag_stream_unit_carries_match_pattern(self): + """Stream unit has ref_pattern == pattern (stream identity, stable across tag promotions).""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@{major}" + mode: delta +""", remote_tags=self.MANY_TAGS) + assert len(units) == 1 + u = units[0] + assert u.ref == "deploy@{major}" + assert u.ref_pattern == "deploy@{major}", ( + f"Stream unit.ref_pattern must equal the pattern, got {u.ref_pattern!r}" + ) + + def test_delta_tag_version_pattern_emits_one_unit(self): + """deploy@{major} matching 9 deploy@N tags → 1 stream Unit.""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@{major}" + mode: delta +""", remote_tags=self.MANY_TAGS) + assert len(units) == 1 + assert units[0].ref == "deploy@{major}" + + def test_delta_tag_multi_pattern_emits_one_unit_per_pattern(self): + """Two raw patterns → two stream Units with distinct identities.""" + tags = {**{f"deploy@{i}": f"sha{i}" for i in range(1, 5)}, + **{f"release@{i}": f"rsha{i}" for i in range(1, 4)}} + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: ["deploy@*", "release@*"] + mode: delta +""", remote_tags=tags) + assert len(units) == 2 + refs = {u.ref for u in units} + assert refs == {"deploy@*", "release@*"} + + def test_delta_tag_pattern_with_no_matches_emits_nothing(self): + """If no remote tag matches the pattern, no stream Unit is emitted.""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "release@*" + mode: delta +""", remote_tags=self.MANY_TAGS) # only deploy@* tags, no release@* + assert units == [] + + def test_snapshot_tag_still_emits_one_unit_per_tag(self): + """snapshot mode is unchanged: one Unit per matching tag name.""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@*" +""", remote_tags={f"deploy@{i}": f"sha{i}" for i in range(1, 4)}) + assert len(units) == 3 + assert all(u.ref.startswith("deploy@") and not u.ref.endswith("*") for u in units) + assert all(u.mode == "snapshot" for u in units) + + def test_delta_branch_still_emits_one_unit_per_branch(self): + """delta mode on branches (not tags) is unchanged: one Unit per matching branch.""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: branch} + match: "main" + mode: delta +""", remote_tags={}, remote_branches={"main": "abc123"}) + assert len(units) == 1 + assert units[0].ref == "main" + assert units[0].remote_sha == "abc123" + + def test_snapshot_tag_units_use_raw_match_pattern(self): + """Snapshot tag units carry the raw match pattern in ref_pattern, not the concrete tag.""" + snapshot_units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@*" +""", remote_tags={f"deploy@{i}": f"sha{i}" for i in range(1, 4)}) + for u in snapshot_units: + assert u.ref_pattern == "deploy@*", ( + f"snapshot tag unit.ref_pattern must be the raw pattern, got {u.ref_pattern!r}" + ) + assert u.ref != u.ref_pattern, ( + f"snapshot tag unit.ref (concrete) should differ from ref_pattern (pattern), got ref={u.ref!r}" + ) + + def test_delta_branch_units_have_ref_pattern_equal_to_ref(self): + """Delta branch units: branch name is its own pattern, so ref_pattern == ref.""" + # Delta branch: ref_pattern == branch name (literal match, no version components). + branch_units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: branch} + match: "main" + mode: delta +""", remote_tags={}, remote_branches={"main": "abc123"}) + for u in branch_units: + assert u.ref_pattern == u.ref, f"branch unit.ref_pattern must == unit.ref, got ref_pattern={u.ref_pattern!r} ref={u.ref!r}" + + def test_two_delta_tag_sources_on_same_repo_produce_distinct_streams(self): + """Two delta-tag selectors on the same repo with different patterns are two streams.""" + tags = {**{f"deploy@{i}": f"sha{i}" for i in range(1, 4)}, + **{f"v1.{i}.0": f"vsha{i}" for i in range(1, 4)}} + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@*" + mode: delta + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "v{major}.{minor}.{patch}" + mode: delta +""", remote_tags=tags) + assert len(units) == 2 + refs = {u.ref for u in units} + assert "deploy@*" in refs + assert "v{major}.{minor}.{patch}" in refs diff --git a/tests/test_uniqueness_gate.py b/tests/test_uniqueness_gate.py index dfc67ed..d375c27 100644 --- a/tests/test_uniqueness_gate.py +++ b/tests/test_uniqueness_gate.py @@ -3,7 +3,8 @@ The gate is split by content shape (no mode on content docs): - Snapshot (git.commit IS NOT NULL): each commit must have ≥1 complete refs doc. - - Incremental (git.ref IS NOT NULL): each ref must have EXACTLY ONE incremental join doc. + - Incremental (git.ref_pattern IS NOT NULL): each (ref_type, ref_pattern) pair must have EXACTLY + ONE incremental join doc (keyed by git.ref_pattern; one join doc per stream identity). """ # Standard packages @@ -23,60 +24,90 @@ def _not_found() -> NotFoundError: return NotFoundError("index_not_found_exception", meta, None) -def _composite_resp(values: list[str]) -> dict: - """Build a composite agg response with the given values.""" - return {"aggregations": {"keys": {"buckets": [{"key": {"val": v}} for v in values]}}} +def _commit_enum_resp(commits: list[str]) -> dict: + """Response for _enumerate_content_field(git.commit) — composite agg 'keys'.""" + return {"aggregations": {"keys": {"buckets": [{"key": {"val": c}} for c in commits]}}} -def _terms_resp(counts: dict[str, int]) -> dict: - """Build a terms agg response mapping key -> doc_count.""" - return {"aggregations": { - "commits": {"buckets": [{"key": k, "doc_count": v} for k, v in counts.items()]}, - "refs": {"buckets": [{"key": k, "doc_count": v} for k, v in counts.items()]}, - }} +def _ref_enum_resp(ref_pairs: list[tuple[str, str]]) -> dict: + """Response for _enumerate_incremental_content_ref_pairs — composite agg 'pairs'. + Each pair is (ref_type, ref_pattern); content docs carry git.ref_pattern = the stream identity.""" + return {"aggregations": {"pairs": {"buckets": [ + {"key": {"ref_type": rt, "ref_pattern": r}} for rt, r in ref_pairs + ]}}} + + +def _snapshot_presence_resp(found_commits: list[str]) -> dict: + """Response for snapshot join-doc presence check — terms agg 'commits'.""" + return {"aggregations": {"commits": {"buckets": [ + {"key": c, "doc_count": 1} for c in found_commits + ]}}} + + +def _incremental_unique_resp(pair_counts: dict[tuple[str, str], int]) -> dict: + """Response for incremental uniqueness check — composite agg 'ref_pairs' on git.ref_pattern. + pair_counts is {(ref_type, ref_pattern): doc_count}.""" + return {"aggregations": {"ref_pairs": {"buckets": [ + {"key": {"ref_type": rt, "ref_pattern": r}, "doc_count": cnt} + for (rt, r), cnt in pair_counts.items() + ]}}} class TestCheckJoinUniqueness: """Tests for check_join_uniqueness: the combined snapshot + incremental gate.""" - def _make_es(self, snapshot_commits=(), snapshot_found=(), incremental_refs=(), incremental_counts=None): + def _make_es( + self, + snapshot_commits: list[str] = (), + snapshot_found: list[str] = (), + incremental_ref_pairs: list[tuple[str, str]] = (), + incremental_pair_counts: dict[tuple[str, str], int] | None = None, + ) -> MagicMock: """Build a mock ES with side_effects matching the exact call order of check_join_uniqueness: 1. _enumerate_content_field(git.commit): 1 search per index (FILES, LINES) + → composite agg 'keys', each bucket {'key': {'val': }} 2. if commits non-empty → snapshot presence check (1 search on sourcerer-refs) - 3. _enumerate_content_field(git.ref): 1 search per index (FILES, LINES) - 4. if refs non-empty → incremental uniqueness check (1 search on sourcerer-refs) - - The composite agg loop breaks on the first empty page (no after_key returned), so exactly - one search per index per field enumeration. + → terms agg 'commits' + 3. _enumerate_incremental_content_ref_pairs: 1 search per index (FILES, LINES) + → composite agg 'pairs', each bucket {'key': {'ref_type': ..., 'ref_pattern': ...}} + 4. if ref_pairs non-empty → incremental uniqueness check (1 search on sourcerer-refs) + → composite agg 'ref_pairs' (on git.ref_pattern), each bucket + {'key': {'ref_type': ..., 'ref_pattern': ...}, 'doc_count': N} + + The composite agg loop breaks on the first empty page, so exactly one search per index. """ es = MagicMock() - side_effects = [] + side_effects: list[dict] = [] # (1) enumerate git.commit: FILES then LINES - side_effects.append(_composite_resp(list(snapshot_commits))) # FILES git.commit - side_effects.append(_composite_resp(list(snapshot_commits))) # LINES git.commit + side_effects.append(_commit_enum_resp(list(snapshot_commits))) # FILES + side_effects.append(_commit_enum_resp(list(snapshot_commits))) # LINES # (2) snapshot join-doc presence check (only if commits found) if snapshot_commits: - found = {c: 1 for c in snapshot_found} - side_effects.append({"aggregations": {"commits": {"buckets": [ - {"key": k, "doc_count": v} for k, v in found.items() - ]}}}) - # (3) enumerate git.ref: FILES then LINES - side_effects.append(_composite_resp(list(incremental_refs))) # FILES git.ref - side_effects.append(_composite_resp(list(incremental_refs))) # LINES git.ref - # (4) incremental uniqueness check (only if refs found) - if incremental_refs: - counts = incremental_counts or {} - side_effects.append({"aggregations": {"refs": {"buckets": [ - {"key": k, "doc_count": v} for k, v in counts.items() - ]}}}) + side_effects.append(_snapshot_presence_resp(list(snapshot_found))) + # (3) enumerate incremental (ref_type, ref) pairs: FILES then LINES + side_effects.append(_ref_enum_resp(list(incremental_ref_pairs))) # FILES + side_effects.append(_ref_enum_resp(list(incremental_ref_pairs))) # LINES + # (4) incremental uniqueness check on refs-side `match` (only if pairs found) + if incremental_ref_pairs: + counts = incremental_pair_counts or {} + side_effects.append(_incremental_unique_resp(counts)) es.search.side_effect = side_effects return es def test_clean_repo_no_content(self): """No content at all → gate passes.""" es = MagicMock() - es.search.return_value = {"aggregations": {"keys": {"buckets": []}}} + # All composite enumeration calls return empty buckets. + # Steps 1+3: commit enum (keys agg, empty) and ref-pair enum (pairs agg, empty). + # We return a response that satisfies both agg names by making the mock return + # the correct format for each call in sequence. + es.search.side_effect = [ + _commit_enum_resp([]), # step 1a FILES git.commit + _commit_enum_resp([]), # step 1b LINES git.commit + _ref_enum_resp([]), # step 3a FILES ref pairs + _ref_enum_resp([]), # step 3b LINES ref pairs + ] assert check_join_uniqueness(es, "github", "acme", "widgets") == [] def test_clean_snapshot_all_present(self): @@ -99,28 +130,36 @@ def test_snapshot_missing_refs_doc_is_offending(self): def test_clean_incremental_exactly_one_join_doc(self): """Incremental ref with exactly one join doc → clean.""" es = self._make_es( - incremental_refs=["main"], - incremental_counts={"main": 1}, + incremental_ref_pairs=[("branch", "main")], + incremental_pair_counts={("branch", "main"): 1}, + ) + assert check_join_uniqueness(es, "github", "acme", "widgets") == [] + + def test_clean_incremental_tag_stream_one_join_doc(self): + """A delta-tag stream (pattern as identity) with one join doc → clean.""" + es = self._make_es( + incremental_ref_pairs=[("tag", "deploy@{major}")], + incremental_pair_counts={("tag", "deploy@{major}"): 1}, ) assert check_join_uniqueness(es, "github", "acme", "widgets") == [] def test_incremental_missing_join_doc_is_offending(self): """An incremental ref with no join doc is reported.""" es = self._make_es( - incremental_refs=["main"], - incremental_counts={}, # zero docs found + incremental_ref_pairs=[("branch", "main")], + incremental_pair_counts={}, # zero docs found ) result = check_join_uniqueness(es, "github", "acme", "widgets") - assert "main" in result + assert "branch/main" in result def test_incremental_duplicate_join_doc_is_offending(self): """An incremental ref with more than one join doc (e.g. stale snapshot marker) is reported.""" es = self._make_es( - incremental_refs=["main"], - incremental_counts={"main": 2}, # two docs — fan-out! + incremental_ref_pairs=[("branch", "main")], + incremental_pair_counts={("branch", "main"): 2}, # two docs — fan-out! ) result = check_join_uniqueness(es, "github", "acme", "widgets") - assert "main" in result + assert "branch/main" in result def test_missing_index_contributes_nothing(self): es = MagicMock() @@ -131,30 +170,25 @@ def test_missing_index_contributes_nothing(self): class TestRunUniquenessGate: def test_passes_silently_when_clean(self): es = MagicMock() - # No content: all composite aggs return empty - es.search.return_value = {"aggregations": {"keys": {"buckets": []}}} + # No content: all enumerations return empty pages. + es.search.side_effect = [ + _commit_enum_resp([]), # step 1a + _commit_enum_resp([]), # step 1b + _ref_enum_resp([]), # step 3a + _ref_enum_resp([]), # step 3b + ] assert _run_uniqueness_gate(es, "github", "acme", "widgets") is True def test_fails_and_reports_on_snapshot_violation(self, capsys): es = MagicMock() # Snapshot commit "aaa" exists in content but has no complete refs doc. - # sources structure: [{"val": {"terms": {"field": "git.commit"}}}] - def side_effect(*args, **kwargs): - aggs = kwargs.get("aggs", {}) - if "keys" in aggs and aggs["keys"].get("composite"): - sources = aggs["keys"]["composite"].get("sources", []) - # Each source is {"": {"terms": {"field": ""}}} - field = None - if sources: - src = sources[0] - for alias_val in src.values(): - field = alias_val.get("terms", {}).get("field") - if field == "git.commit": - return {"aggregations": {"keys": {"buckets": [{"key": {"val": "aaa"}}]}}} - return {"aggregations": {"keys": {"buckets": []}}} - # terms agg for join doc presence (snapshot or incremental) - return {"aggregations": {"commits": {"buckets": []}, "refs": {"buckets": []}}} - es.search.side_effect = side_effect + es.search.side_effect = [ + _commit_enum_resp(["aaa"]), # step 1a: FILES git.commit + _commit_enum_resp(["aaa"]), # step 1b: LINES git.commit + _snapshot_presence_resp([]), # step 2: no complete refs doc for "aaa" + _ref_enum_resp([]), # step 3a: no incremental refs + _ref_enum_resp([]), # step 3b + ] assert _run_uniqueness_gate(es, "github", "acme", "widgets") is False captured = capsys.readouterr() assert "aaa" in captured.err diff --git a/tests/test_utils.py b/tests/test_utils.py index 8521071..5e6f851 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -35,18 +35,30 @@ def test_non_utf8_parts_round_trip_via_surrogateescape(self): class TestBuildRefKey: def test_ref_key_incremental_shape(self): - assert build_ref_key("github", "elastic", "sourcerer", "main") == ( - "github~elastic~sourcerer~main" + assert build_ref_key("github", "elastic", "sourcerer", "branch", "main") == ( + "github~elastic~sourcerer~branch~main" ) def test_ref_key_lowercases_host_org_repo_preserves_ref_case(self): - assert build_ref_key("GitHub", "Elastic", "Sourcerer", "Feature/Mixed-Case") == ( - "github~elastic~sourcerer~Feature/Mixed-Case" + assert build_ref_key("GitHub", "Elastic", "Sourcerer", "branch", "Feature/Mixed-Case") == ( + "github~elastic~sourcerer~branch~Feature/Mixed-Case" + ) + + def test_ref_key_tag_stream_keys_on_pattern(self): + """A delta-tag stream's ref key uses the match pattern, not the concrete tag.""" + pattern = "deploy@{major}" + concrete = "deploy@1788000000" + assert build_ref_key("github", "elastic", "kibana", "tag", pattern) != ( + build_ref_key("github", "elastic", "kibana", "tag", concrete) + ) + # The stable identity (pattern) produces a consistent key. + assert build_ref_key("github", "elastic", "kibana", "tag", pattern) == ( + build_ref_key("github", "elastic", "kibana", "tag", pattern) ) def test_ref_key_deterministic(self): - assert build_ref_key("github", "acme", "widgets", "main") == build_ref_key( - "github", "acme", "widgets", "main" + assert build_ref_key("github", "acme", "widgets", "branch", "main") == build_ref_key( + "github", "acme", "widgets", "branch", "main" ) From 6c810b5b0e8cb5710b3464ea785396dfbda1a016 Mon Sep 17 00:00:00 2001 From: Dave Moore Date: Mon, 24 Aug 2026 17:17:47 -0700 Subject: [PATCH 28/29] Fix false failure report after indexing completes: don't falsely identify a commit as an orphan when its status is 'indexing' --- src/sourcerer/queries.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index 27992cd..ecb305b 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -483,7 +483,8 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> an empty list means the invariant holds.""" offending: list[str] = [] - # --- snapshot: each commit must have ≥1 complete refs doc --- + # --- snapshot: each commit must have ≥1 complete (or in-progress) refs doc --- + # Accept "indexing" status as non-offending: an active run holding this commit is not stale. commits = _enumerate_content_field(es, host, org, repo, "git.commit") if commits: try: @@ -494,7 +495,7 @@ def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> {"term": {"git.org": org}}, {"term": {"git.repo": repo}}, {"terms": {"git.commit": sorted(commits)}}, - {"term": {"status": "complete"}}, + {"terms": {"status": ["complete", "indexing"]}}, ]}}, aggs={"commits": {"terms": {"field": "git.commit", "size": len(commits)}}}, ) From 3e72d07047139382f313cdbee1ef5d14138a3c35 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 25 Aug 2026 16:05:43 -0600 Subject: [PATCH 29/29] Remove internal INV-00N invariant labels from code and tests The INV-00N references pointed at a private planning spec that outside readers of this repo cannot see or link to. Strip every label while keeping the plain-English rule each one annotated, so the comments stay self-explanatory on their own. --- AGENTS.md | 4 ++-- src/sourcerer/commands/index/command.py | 10 +++++----- src/sourcerer/commands/index/git.py | 2 +- src/sourcerer/commands/index/markers.py | 18 +++++++++--------- src/sourcerer/queries.py | 2 +- tests/test_documents.py | 2 +- tests/test_incremental_index.py | 4 ++-- tests/test_markers.py | 12 ++++++------ tests/test_uniqueness_gate.py | 2 +- 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b0cf864..c9e936c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -386,7 +386,7 @@ Content docs come in two disjoint shapes depending on how they were indexed: and status. `git.commit` on the content row is already the answer; no join is needed to resolve it. - **Delta** (`mode: delta`): content docs carry `git.ref` and no `git.commit`. A dedicated refs join doc at `_id = build_ref_key(host,org,repo,ref)` (one per branch) holds the - live HEAD commit and is advanced two-phase (INV-006). The join resolves `git.commit` from this doc. + live HEAD commit and is advanced two-phase. The join resolves `git.commit` from this doc. Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) uses the same shape that handles both modes without fan-out: @@ -459,7 +459,7 @@ Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental | `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot markers) and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | | `stale` | A snapshot marker superseded by a mode switch to `delta`. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | -#### Uniqueness gate (INV-011 backstop) +#### Uniqueness gate `_run_uniqueness_gate` (`commands/index/command.py`) runs after each index pass and calls `check_join_uniqueness` (`queries.py`) to verify: diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index 92167f1..74d77c7 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -61,7 +61,7 @@ def _run_uniqueness_gate(es: Elasticsearch, host: str, org: str, repo: str) -> bool: - """Post-index join-uniqueness gate (INV-011 backstop): every distinct content commit/ref in + """Post-index join-uniqueness gate: every distinct content commit/ref in this repo must resolve to a complete refs join doc. For snapshot content (git.commit IS NOT NULL) each commit must have at least one complete refs doc; for incremental (git.ref IS NOT NULL) each ref must have exactly one incremental join doc. Prints offenders to stderr and @@ -268,14 +268,14 @@ def index_incremental_branch_in_dir( Reads the ref's prior completed commit (its refs join doc, `_id = ref_key`), checks out the fetched tip, and either: - does nothing (already at the completed commit and not `--force`), - - does a full rebuild (first index, `--force`, or a missing diff base -- INV-007): delete + - does a full rebuild (first index, `--force`, or a missing diff base): delete the whole ref namespace, then index every currently-tracked path, or - does a delta update: `git diff --name-status` (via `plan_changes`) between the prior and new commit, deleting only the paths git reports removed/changed and (re)indexing only the - paths git reports added/changed (INV-008 -- scoped by the exact + paths git reports added/changed (scoped by the exact (host,org,repo,ref_type,ref) 5-tuple, never a whole namespace sweep). The refs join doc is published `indexing` before any mutation and `complete` only after the - content deletes/indexes and a refresh all succeed (INV-006); a raised exception instead + content deletes/indexes and a refresh all succeed; a raised exception instead records `write_incremental_failed` and leaves the completed pointer untouched, then re-raises so the caller's per-unit error handling reports it. """ @@ -953,7 +953,7 @@ def process_group(item: tuple[tuple[str, str, str], list[Unit]]) -> None: # errors are handled inside process_group and counted in `failures`. list(pool.map(process_group, groups.items())) - # Post-index uniqueness gate (INV-011), one distinct repo at a time, skipped on abort (the + # Post-index uniqueness gate, one distinct repo at a time, skipped on abort (the # plan is incomplete). Every offending repo's ref_key(s) are reported before exiting. if not _aborted.is_set(): distinct_repos = {(c.host, c.org, c.repo) for c in entries} diff --git a/src/sourcerer/commands/index/git.py b/src/sourcerer/commands/index/git.py index 5878a52..f824369 100644 --- a/src/sourcerer/commands/index/git.py +++ b/src/sourcerer/commands/index/git.py @@ -355,7 +355,7 @@ class ChangePlan: `index_paths` are the current tree paths to (re)index. A modified or type-changed file appears in BOTH (delete its stale docs, then re-index every current line). `base_missing` is True when the old commit object is unavailable locally -- the caller must then fall back - to full branch-namespace reconciliation instead of trusting an empty diff (INV-007).""" + to full branch-namespace reconciliation instead of trusting an empty diff.""" delete_paths: list[str] = field(default_factory=list) index_paths: list[str] = field(default_factory=list) diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index 592e48b..4881d10 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -544,10 +544,10 @@ def pre_clone_skip( # --- incremental refs join docs, keyed by `_id = build_ref_key(...)` ---------------------- -# One document per incremental ref (INV-004): a delta-mode ref's single join doc lives at +# One document per incremental ref: a delta-mode ref's single join doc lives at # `_id = {host}~{org}~{repo}~{ref_type}~{ref}` (constructed by build_ref_key, a plain # tilde-joined string -- not a stored field) and its `git.commit` is the ref's live target -# commit, advanced only by a two-phase indexing -> complete publication (INV-006). ref_type +# commit, advanced only by a two-phase indexing -> complete publication. ref_type # ("branch" or "tag") is part of the key so a same-named branch and tag each get a distinct # join doc. This is a DISTINCT id space from `build_ref_id`'s hashed, append-only ref-name # markers above; a join doc's `_id` is a plain, unhashed build_ref_key() string, which a @@ -637,7 +637,7 @@ def write_incremental_indexing( """Publish `status: indexing`: the completed pointer (`git.commit`) stays at the LAST completed SHA (or None on a first index) while `git.commit_target` advertises the candidate SHA the run is advancing to. A failed run never overwrites `git.commit` with `commit_target` - (INV-006) -- only `write_incremental_ready` does that, after delete+index+refresh succeed. + -- only `write_incremental_ready` does that, after delete+index+refresh succeed. `ref` is the CONCRETE resolved ref (git.ref payload); `ref_pattern` is the STREAM IDENTITY stored as `git.ref_pattern` and used as the doc _id. For delta-tag streams these differ @@ -678,7 +678,7 @@ def write_incremental_ready( index_suffix: str | None = None, ) -> None: """Publish `status: complete` at the NEW completed commit, clearing `commit_target` and any - prior failure fields. This is the pointer-advancing publication boundary (INV-006): callers + prior failure fields. This is the pointer-advancing publication boundary: callers must delete+index+refresh the content indices FIRST, then call this. `ref` is the CONCRETE resolved ref (git.ref payload); `ref_pattern` is the STREAM IDENTITY @@ -717,7 +717,7 @@ def write_incremental_failed( index_suffix: str | None = None, ) -> None: """Record a failed update WITHOUT advancing the completed pointer: status becomes `failed`, - `git.commit` remains the last completed SHA (INV-006). The next run retries old -> current. + `git.commit` remains the last completed SHA. The next run retries old -> current. `ref` is the CONCRETE resolved ref (git.ref payload); `ref_pattern` is the STREAM IDENTITY (stored as `git.ref_pattern`, used as the doc _id). Defaults to `ref`.""" @@ -773,7 +773,7 @@ def delete_incremental_paths( ) -> None: """Synchronously delete the file and line docs for `paths` on this exact ref. Scoped by the exact (git.host, git.org, git.repo, git.ref_type, git.ref_pattern) 5-term filter - (INV-008: one ref's docs can never bleed into another's) plus a `file.path` terms filter. + (one ref's docs can never bleed into another's) plus a `file.path` terms filter. A no-op for an empty path set.""" paths = list(paths) if not paths: @@ -809,8 +809,8 @@ def delete_incremental_branch( refresh: bool = False, ) -> None: """Delete EVERY incremental content doc for this ref (full namespace), scoped by the exact - (git.host, git.org, git.repo, git.ref_type, git.ref_pattern) 5-term filter (INV-008). Used - for the initial index and the missing-diff-base rebuild (INV-007). `ref_type` defaults to + (git.host, git.org, git.repo, git.ref_type, git.ref_pattern) 5-term filter. Used + for the initial index and the missing-diff-base rebuild. `ref_type` defaults to "branch" for back-compat with existing callers that pass `ref_pattern` positionally.""" query = {"bool": {"filter": [ {"term": {"git.host": host}}, @@ -860,7 +860,7 @@ def refresh_incremental_content( index_level: str = "repo", index_suffix: str | None = None, ) -> None: """Make the branch's just-written incremental content visible before the ready pointer is - published (INV-006: content refresh precedes the final refs write). Best-effort over + published (content refresh precedes the final refs write). Best-effort over missing indices.""" es.indices.refresh( index=[ diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index ecb305b..a9a6576 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -470,7 +470,7 @@ def _enumerate_incremental_content_ref_pairs( def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> list[str]: - """Join-uniqueness gate (INV-011 backstop): verifies every content key maps to a correct + """Join-uniqueness gate: verifies every content key maps to a correct refs join doc. Split by content shape (no `mode` on content docs): - Snapshot (git.commit IS NOT NULL): each commit must resolve to ≥1 complete refs doc diff --git a/tests/test_documents.py b/tests/test_documents.py index 74a898a..5a4bc56 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -240,7 +240,7 @@ def test_id_differs_from_snapshot_id(self, tmp_path): assert snap_id != incr_id def test_branch_and_tag_same_name_have_distinct_ids(self, tmp_path): - # A same-named branch and tag in delta mode must produce distinct content ids (INV-004). + # A same-named branch and tag in delta mode must produce distinct content ids. p = tmp_path / "a.txt" p.write_text("hello") branch_id, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "deploy", "a.txt", p) diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py index be233fe..24cc7df 100644 --- a/tests/test_incremental_index.py +++ b/tests/test_incremental_index.py @@ -154,7 +154,7 @@ def test_failed_run_does_not_advance_commit(self): pass mocks["write_incremental_ready"].assert_not_called() mocks["write_incremental_failed"].assert_called_once() - # The completed pointer stays at OLD -- a failed run must not advance it (INV-006). + # The completed pointer stays at OLD -- a failed run must not advance it. assert mocks["write_incremental_failed"].call_args.kwargs["completed_commit"] == OLD finally: _stop(patchers) @@ -359,7 +359,7 @@ def test_tag_delta_run_indexes_only_changed_paths(self): _stop(patchers) def test_tag_missing_diff_base_triggers_full_rebuild(self): - """Newest tag's diff base gone → base_missing → full rebuild (INV-007).""" + """Newest tag's diff base gone → base_missing → full rebuild.""" prior = {"git": {"commit": OLD}} plan = ChangePlan(base_missing=True) patchers, mocks = _patch_common(prior=prior, plan=plan, ref_dates_return=self.TAG_DATES) diff --git a/tests/test_markers.py b/tests/test_markers.py index cba86c1..fb27391 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -404,7 +404,7 @@ def _indexed_doc(es): class TestWriteRefMarker: - """write_ref_marker is the single refs doc per snapshot source (INV-004). git.ref_key has + """write_ref_marker is the single refs doc per snapshot source. git.ref_key has been removed; snapshot refs are identified by (host, org, repo, ref, commit) on the marker.""" def test_marker_carries_commit_no_ref_key(self): @@ -434,7 +434,7 @@ def test_default_write_does_not_refresh(self): def test_refresh_true_is_propagated(self): # write_ref_marker accepts refresh=True for callers that need the doc visible before - # the next gate (INV-011) runs. + # the next gate runs. es = MagicMock() write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, files_count=1, lines_count=1, refresh=True) @@ -537,7 +537,7 @@ def test_incremental_marker_preserves_completed_commit_and_exposes_target(self): completed_commit=OLD, commit_target=NEW) doc = _indexed_doc(es) assert doc["status"] == "indexing" - assert doc["git"]["commit"] == OLD # completed pointer unchanged (INV-006) + assert doc["git"]["commit"] == OLD # completed pointer unchanged assert doc["git"]["commit_target"] == NEW assert doc["mode"] == "delta" assert doc["git"]["ref_type"] == "branch" @@ -594,7 +594,7 @@ def test_incremental_marker_advances_commit_and_clears_target_and_error(self): files_count=5, lines_count=99) doc = _indexed_doc(es) assert doc["status"] == "complete" - assert doc["git"]["commit"] == NEW # advances only after a successful run (INV-006) + assert doc["git"]["commit"] == NEW # advances only after a successful run assert doc["git"]["commit_target"] is None assert doc["files_count"] == 5 and doc["lines_count"] == 99 assert es.index.call_args.kwargs["refresh"] is True # publication boundary @@ -616,7 +616,7 @@ def test_incremental_failed_sets_status_failed_and_retains_old_pointer(self): completed_commit=OLD, commit_target=NEW, error="boom") doc = _indexed_doc(es) assert doc["status"] == "failed" - assert doc["git"]["commit"] == OLD # completed pointer unchanged (INV-006) + assert doc["git"]["commit"] == OLD # completed pointer unchanged assert doc["git"]["commit_target"] == NEW assert "error" not in doc assert "failed_at" not in doc @@ -688,7 +688,7 @@ def test_scoped_to_exact_ref_key_only(self): def test_isolated_from_another_branch(self): # Two incremental branches indexed; deleting one's docs must never scope to the other's - # (host,org,repo,ref) quadruple (INV-008) -- asserted here at the query-construction level. + # (host,org,repo,ref) quadruple -- asserted here at the query-construction level. es_a = MagicMock() es_b = MagicMock() delete_incremental_branch(es_a, "github", "acme", "widgets", "main") diff --git a/tests/test_uniqueness_gate.py b/tests/test_uniqueness_gate.py index d375c27..a73574d 100644 --- a/tests/test_uniqueness_gate.py +++ b/tests/test_uniqueness_gate.py @@ -1,5 +1,5 @@ """Tests for the post-index join-uniqueness gate: sourcerer.queries.check_join_uniqueness -(INV-011) and its command.py wiring in _run_uniqueness_gate. Every ES call is mocked. +and its command.py wiring in _run_uniqueness_gate. Every ES call is mocked. The gate is split by content shape (no mode on content docs): - Snapshot (git.commit IS NOT NULL): each commit must have ≥1 complete refs doc.