diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index d954c0b..0a87578 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ "plugins": [ { "name": "sourcerer", - "version": "2.5.0", + "version": "3.0.0", "description": "Search indexed git repositories and generate responses with citations.", "source": "./", "strict": false, diff --git a/.gitignore b/.gitignore index 1c76771..9235135 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ vendor/ # Secrets & local config .env +sourcerer.yml # macOS -.DS_Store +.DS_Store \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index bb5c40f..c9e936c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ Commands: include them. - `sourcerer index / [-b ] [-t ] [-c ]` (single-repo path defaults to `git.host` = `github`) -- `sourcerer index --config [--prune] [--dry-run]` +- `sourcerer index --config [--prune] [--dry-run] [--no-backfill]` - `sourcerer prune [--config ] [--dry-run]` (config-driven retention prune is skipped without `--config`; the orphan sweep always runs) - `sourcerer mcp-proxy [-e ]` (run a stdio MCP proxy that forwards to the Kibana @@ -57,6 +57,36 @@ config's `sources:` is a YAML list, one entry per (host, org, repo, ref_type). S | `match` | yes | For `branch`/`tag`: pattern string or list of patterns matched against ref names (version DSL + glob), a ref matches if any pattern hits. For `commit`: a commit SHA/prefix string or list of them (see below). | | `since` | no | Index-side inclusion floor: the earliest commit to start indexing from. See below. Not valid for `git.ref_type: commit`. | | `retain` | no | Retention policy (see below). Omit to keep forever. For `git.ref_type: commit`, only `age` is valid. | +| `mode` | no | `snapshot` (default) or `delta` (branch or tag). See below. | + +#### `mode` (snapshot vs. delta) + +`snapshot` (default): content is commit-addressed. A HEAD advance on a branch +indexes a whole new snapshot under the new commit. + +`delta` (branch or tag; rejects `since`/`retain` -- there is no per-commit history for either +to apply to): content is ref-addressed instead. A HEAD advance runs `git diff +--name-status` between the previously-completed commit and the new tip and only deletes/ +reindexes the paths git reports changed -- add/modify/delete/rename/copy -- instead of +reindexing the whole tree. A missing diff base (force-push, GC'd, or the first index) rebuilds +the whole ref namespace. The refs join doc publishes `status: indexing` before any content +change and `status: complete` (with the new commit) only after the deletes/indexes/refresh all +succeed, so a crash mid-update leaves the prior commit and content in place. + +Delta mode is especially useful for fast-moving tags that are force-updated many times a day +(e.g. `deploy@8`-style Serverless promotion tags): snapshot mode would mint a fresh full snapshot +per force-update; delta mode diffs only what changed, keeping indexing cost proportional to the +diff size regardless of tag-move frequency. + +```yaml +- git: + host: github + org: elastic + repo: serverless-gitops + ref_type: branch + match: main + mode: delta +``` #### `git.ref_type: commit` (pinning an explicit commit) @@ -228,7 +258,7 @@ config files each driven by their own cron job. Run `sourcerer index --config` o cron** (e.g. every 5 minutes) and let the schedule config control the actual indexing cadence. **How the gate works**: on each invocation of `index --config`, before any ls-remote or clone -work, the command queries `sourcerer-v2-refs` to see when each source was last fully indexed +work, the command queries `sourcerer-v3-refs` to see when each source was last fully indexed (`status: complete`) and whether any ref in its scope is actively being indexed (`status: indexing`). Only sources whose schedule has fired since their last indexed run proceed to the expensive pipeline. Sources where another run is actively indexing are skipped. @@ -319,19 +349,19 @@ Content is addressed by **host + commit**, not by ref name. A file's bytes are f by `(git.host, git.org, git.repo, git.commit, file.path)`, so the same file reached via any ref collapses to a single doc (no per-ref duplication), while the same org/repo on two different git hosts stays distinct. `git.host` is a lowercase keyword, placed before `git.org` in every index -template's mappings and index sort. Backing indices are `sourcerer-v2-refs`, -`sourcerer-v2-files~{git.host}~{git.org}~{git.repo}`, and -`sourcerer-v2-lines~{git.host}~{git.org}~{git.repo}` (read via the unchanged `sourcerer-refs` / +template's mappings and index sort. Backing indices are `sourcerer-v3-refs`, +`sourcerer-v3-files~{git.host}~{git.org}~{git.repo}`, and +`sourcerer-v3-lines~{git.host}~{git.org}~{git.repo}` (read via the unchanged `sourcerer-refs` / `sourcerer-files` / `sourcerer-lines` aliases). **Index routing (`sources[i].index.level` / `index.suffix`).** A source can override the content index name: `level` (`host`/`org`/`repo` (default)/`commit`) chooses the granularity -(`sourcerer-v2-*~{host}` … `~{host}~{org}~{repo}~{commit}`), and `suffix` appends `^{suffix}` +(`sourcerer-v3-*~{host}` … `~{host}~{org}~{repo}~{commit}`), and `suffix` appends `^{suffix}` (e.g. `~{host}~{org}~{repo}^deploy`). Routing is **per-source**, so two sources of the same -`(host, org, repo)` may target different indices; the read aliases match `sourcerer-v2-files*` / -`sourcerer-v2-lines*`, so every leveled/suffixed index auto-joins them and agents are unaffected. +`(host, org, repo)` may target different indices; the read aliases match `sourcerer-v3-files*` / +`sourcerer-v3-lines*`, so every leveled/suffixed index auto-joins them and agents are unaffected. The name is built by `files_index`/`lines_index` in `src/sourcerer/indices.py`; each ref marker in -`sourcerer-v2-refs` records the source's `index_level`/`index_suffix` (semantic, not the resolved +`sourcerer-v3-refs` records the source's `index_level`/`index_suffix` (semantic, not the resolved name — so a future prefix bump stays correct). Changing a source's routing between runs triggers a **migration** (`sourcerer index` re-ingests at the new index, flips the marker, then deletes the old copy; `sourcerer prune` sweeps any crash-leftover as `orphan:stale-location`, and deletes any @@ -347,6 +377,102 @@ creates one index/shard per commit — see `specs/sourcerer-yml.md` for the cave resolve it to a commit via the refs index (the `sourcerer.refs.list` tool), then filter content by `git.host` + `git.commit`. +### Universal join query + +Content docs come in two disjoint shapes depending on how they were indexed: + +- **Snapshot** (`mode: snapshot`): content docs carry `git.commit` and no `git.ref`. The ref-name + marker in `sourcerer-v3-refs` (keyed by `build_ref_id`, one per snapshot source) carries the commit + and status. `git.commit` on the content row is already the answer; no join is needed to resolve it. +- **Delta** (`mode: delta`): content docs carry `git.ref` and no `git.commit`. A + dedicated refs join doc at `_id = build_ref_key(host,org,repo,ref)` (one per branch) holds the + live HEAD commit and is advanced two-phase. The join resolves `git.commit` from this doc. + +Every Agent Builder content tool (`sourcerer.code.*`, `sourcerer.files.*`) uses +the same shape that handles both modes without fan-out: + +```esql +FROM sourcerer-lines +| WHERE git.host LIKE ?git_host AND ... + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small sourcerer-refs + // index first (content docs carry no git.ref_type); the two membership sets handle + // both content-doc shapes in one pass. + (git.commit IS NOT NULL AND git.commit IN ( + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host AND ... + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref IS NOT NULL AND git.commit IS NULL AND git.ref IN ( + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host AND ... + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref + )) + ) +// Branch by content-doc shape to resolve git.commit for incremental refs: +// Snapshot rows already carry git.commit (no join needed). +// Incremental rows carry only git.ref; the join resolves git.commit from the refs join doc. +// Safety of the incremental join (one doc per (host,org,repo,ref)) is enforced by +// the "one mode owns a ref name" invariant at index time. +| FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref ) +``` + +**Snapshot rows** carry `git.commit` directly; they are matched by the first IN subquery and pass +through the FORK unchanged. Critically, the snapshot arm never touches `sourcerer-refs` at query +time, so two complete markers sharing the same commit (branch + same-named tag) do NOT fan out — +the commit set is resolved once and deduplicated naturally. + +**Incremental rows** carry `git.ref` but no `git.commit`; they are matched by the second IN +subquery and then joined in the FORK incremental arm. The LOOKUP JOIN resolves `git.commit` from the +refs join doc so incremental rows carry a citable commit SHA in the output. The join is safe +(no fan-out) because there is always exactly one incremental join doc per `(host,org,repo,ref)`: +`_id = build_ref_key(...)` (overwrite-in-place) and the runtime mode-conflict guard in +`selection.py` prevent multiple concurrent join docs for the same ref. + +**Scoping params** (`git_commit`, `git_ref`, `git_ref_type`) are all optional (default `"*"`) and +support `*`/`?` wildcards (filters use `LIKE`). For a normal content question, resolve a ref first +(see `src/sourcerer/skills/ref-resolution/SKILL.md`), then pass the result through the appropriate +param: a commit SHA goes to `git_commit`; a branch or tag name goes to `git_ref` (optionally narrow +further with `git_ref_type: branch` or `git_ref_type: tag`). Leaving all three at `"*"` matches +content across all refs at once; because every content tool carries `git.commit` through to output +(and aggregations group `BY git.commit`), unpinned results stay attributable per commit rather than +being blended — but a version-specific answer should still pin a ref. + +#### `status` field values + +Every `sourcerer-v3-refs` document — snapshot ref-name markers and incremental join docs alike +— carries a `status` field drawn from a three-value vocabulary: + +| Value | Meaning | +|---|---| +| `indexing` | A run is mid-flight. `indexing_started_at` is set; `indexed_at` is absent/null. Present on snapshot ref-name markers (written by `write_indexing_marker` just before ingest) and incremental join docs (written by `write_incremental_indexing`). A stale `indexing` doc whose `indexing_started_at` is older than the retry window (default 6 h) marks a crashed run and is treated as due for re-indexing. | +| `complete` | Fully indexed and ready to query. `indexed_at` is set; `indexing_started_at` is absent/null (the terminal write drops it). Written by `write_ref_marker` (snapshot markers) and `write_incremental_ready` (incremental join docs). The scheduler's "last indexed" aggregation and `sourcerer.refs.list`'s default `?status == "complete"` filter both use this value. | +| `stale` | A snapshot marker superseded by a mode switch to `delta`. Written by `mark_snapshot_markers_stale` (called BEFORE the incremental join doc is published as `complete`). The prune command reclaims their content and deletes the marker via `execute_stale_marker_deletions`. | + +#### Uniqueness gate + +`_run_uniqueness_gate` (`commands/index/command.py`) runs after each index pass and calls +`check_join_uniqueness` (`queries.py`) to verify: + +- **Snapshot** (git.commit IS NOT NULL in content): each distinct commit must have ≥1 complete refs + doc (presence check — multi-ref-per-commit is legal). +- **Delta** (git.ref IS NOT NULL in content): each distinct ref must have **exactly one** + incremental join doc with `mode == "delta"` (anti-fan-out guard for the surviving join). + +The gate is non-fatal (logs a warning, does not block): with the flip-status switchover in place, +violations should only occur if a stale-flip was skipped or crashed mid-way; the next prune run +reclaims the stale marker and resolves the violation automatically. + ## Releases `pyproject.toml` is the source of truth for the project version. Release version changes @@ -383,4 +509,17 @@ different git hosting providers. This is a breaking change: `sourcerer-v1-*` indices can be deleted once you have re-indexed. - **Citations**: `sourcerer setup --config sourcerer.yml` reads the config's `hosts:` section and generates one citation skill per host so the agent formats links correctly for each - provider. Run `setup` again whenever you add or customize a host. \ No newline at end of file + provider. Run `setup` again whenever you add or customize a host. + +### Upgrading from v2 to v3 + +v3.0.0 adds incremental (ref-addressed) branch indexing (`update: incremental`, see above). This +is a breaking change to the backing indices, with no config schema change: + +- **Indices**: backing indices are renamed `sourcerer-v2-*` to `sourcerer-v3-*`. There is no + automatic migration - run `sourcerer setup` to create the v3 templates, then re-index every + source. The old `sourcerer-v2-*` indices can be deleted once you have re-indexed. +- **Agent Builder tools**: content tools (`sourcerer.code.*`, `sourcerer.files.*`) replace their + `git_commit` param with `git_ref` (a commit SHA or a branch/tag name); `git_commit` survives + as an optional filter alongside `git_ref` and `git_ref_type`. Run `sourcerer setup` again to push + the updated tool definitions. \ No newline at end of file diff --git a/README.md b/README.md index 49812fc..bb631ed 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Make sure you have [uv](https://docs.astral.sh/uv/) and [git](https://git-scm.co 1. Install the `sourcerer` CLI: ```sh - uv tool install "git+https://github.com/elastic/sourcerer.git@v2.5.0" + uv tool install "git+https://github.com/elastic/sourcerer.git@v3.0.0" ``` 2. Add connection details. Create a `.env` in your working directory, then fill it in: ```sh @@ -85,6 +85,36 @@ Make sure you have [uv](https://docs.astral.sh/uv/) and [git](https://git-scm.co The [`sourcerer.yml` specification](specs/sourcerer-yml.md) has the full reference of fields supported by the configuration file. +### Snapshot vs. delta indexing (`mode`) + +Each source can set `mode: snapshot` (the default) or `mode: delta` (branch or tag). Every +Agent Builder content tool takes the same `git_commit_ish` param either way (a commit SHA or a +branch/tag name, `*`/`?` wildcards supported) and resolves a commit the same way regardless of +mode. + +- **`snapshot`** (default): content is commit-addressed. Every ref (branch, tag, or pinned commit) + that resolves to the same commit collapses to one snapshot. A moving branch's HEAD advance indexes + a brand-new snapshot under the new commit. +- **`delta`** (branch or tag): content is ref-addressed instead. Content docs carry `git.ref` + but no `git.commit` of their own — the ref's current commit lives only on its refs join doc, + resolved at query time via a LOOKUP JOIN. A HEAD advance re-indexes only the files + `git diff --name-status` reports changed (add/modify/delete/rename), not the whole tree, so + staying current on a fast-moving branch or tag is cheap. Particularly useful for fast-moving + tags that are force-updated frequently (e.g. `deploy@8`-style Serverless promotion tags) where + snapshot mode would mint a full snapshot per move. `since` and `retain` don't apply to a + delta-mode source (there is no per-commit history to filter or retain) and are rejected if given. + +```yaml +sources: +- git: { host: "github", org: "elastic", repo: "serverless-gitops", ref_type: "branch" } + match: "main" + mode: delta +``` + +Upgrading from a pre-`ref_key` install is automatic and invisible: every `index` run backfills +pre-existing snapshot content in place (idempotent -- a repeat run changes nothing) unless you +pass `--no-backfill`. + ### Cloning with SSH By default, `sourcerer` clones repos using HTTPS. You can override this in `sourcerer.yml` by setting the `urls.clone` of a Git host to an SSH-compatible URL template. @@ -316,10 +346,10 @@ claude plugin marketplace remove elastic-sourcerer ## Upgrades -To upgrade, reinstall from the desired release tag, replacing `v2.5.0` with the release you want: +To upgrade, reinstall from the desired release tag, replacing `v3.0.0` with the release you want: ```sh -uv tool install --reinstall "git+https://github.com/elastic/sourcerer.git@v2.5.0" +uv tool install --reinstall "git+https://github.com/elastic/sourcerer.git@v3.0.0" ``` Git tag installations remain pinned to that release. `uv tool upgrade sourcerer` does not automatically discover a newer GitHub tag. @@ -383,6 +413,7 @@ the repo root. uv reads `pyproject.toml`, provisions a matching Python, and sync dependencies into an isolated `./.venv` (gitignored) on first run: ```sh +uv sync --extra dev uv run sourcerer help uv run sourcerer setup uv run sourcerer index elastic/elasticsearch -b main @@ -406,7 +437,7 @@ uv run pytest tests/ #### Prepare a release ```sh -./scripts/release.sh prepare v2.5.0 +./scripts/release.sh prepare v3.0.0 ``` `prepare` bumps the version numbers in `pyproject.toml`, `uv.lock`, @@ -420,7 +451,7 @@ Then from an up-to-date `main` with no tracked changes, publish the tag to make an official release: ```sh -./scripts/release.sh publish v2.5.0 +./scripts/release.sh publish v3.0.0 ``` `publish` verifies that all version files are consistent, `main` matches diff --git a/pyproject.toml b/pyproject.toml index 10a9571..5da3049 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sourcerer" -version = "2.5.0" +version = "3.0.0" description = "Index and search source code in Elasticsearch." requires-python = ">=3.10" dependencies = [ diff --git a/sourcerer.example.yml b/sourcerer.example.yml index 242af3d..7632785 100644 --- a/sourcerer.example.yml +++ b/sourcerer.example.yml @@ -184,6 +184,20 @@ sources: retain: count: 5 +# Delta mode -- branch or tag. Instead of a new commit-addressed snapshot on every HEAD +# advance, content is keyed by git.ref and stays in place: a HEAD advance re-indexes only the +# files `git diff` reports changed (a delta update), rather than the whole tree. Good for a +# fast-moving branch or force-updated tag (e.g. `deploy@8`-style Serverless promotion tags) +# where staying current matters more than retaining per-commit history. `since` and `retain` +# are not meaningful here (there is no per-commit history to filter/retain) and are rejected if given. +- git: + host: github + org: elastic + repo: serverless-gitops + ref_type: branch + match: main + mode: delta # default: snapshot + # Feature/fix branches as of a week ago; keep the newest commit, prune > 1 month. - git: host: github @@ -225,7 +239,7 @@ sources: match: deploy@{major} index: level: repo # host | org | repo (default) | commit - suffix: deploy # -> sourcerer-v2-*~github~elastic~elasticsearch^deploy + suffix: deploy # -> sourcerer-v3-*~github~elastic~elasticsearch^deploy # A second host hosting a same-named org/repo is a distinct instance: - git: diff --git a/specs/sourcerer-yml.md b/specs/sourcerer-yml.md index 3e23979..bea68b2 100644 --- a/specs/sourcerer-yml.md +++ b/specs/sourcerer-yml.md @@ -31,6 +31,7 @@ performs its `setup`, `index`, and `prune` commands. |`sources[i].git.repo` |String |Yes || |`sources[i].git.ref_type` |String |Yes || |`sources[i].match` |String, Array[String]|No || +|`sources[i].mode` |String |No || |`sources[i].since` |Object |No || |`sources[i].since.age` |String |No || |`sources[i].since.date` |String |No || @@ -126,7 +127,7 @@ Elastic Agent Builder. URL template for repositories when running `git clone` during indexing. -The `git` fields from the `sourcerer-v2-refs` index can be referenced as +The `git` fields from the `sourcerer-v3-refs` index can be referenced as variables with curly braces (e.g. `{git.org}`, `{git.repo}`). Example: `https://github.com/{git.org}/{git.repo}.git` @@ -139,7 +140,7 @@ Example: `https://github.com/{git.org}/{git.repo}.git` URL template for citing links to a directory in a repo. Used by the citation skills in Agent Builder. -Fields from the `sourcerer-v2-files*` or `sourcerer-v2-lines*` indices can be +Fields from the `sourcerer-v3-files*` or `sourcerer-v3-lines*` indices can be referenced as variables with curly braces (e.g. `{git.org}`, `{file.directory}`). Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.directory}` @@ -152,7 +153,7 @@ Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.direct URL template for citing links to a file in a repo. Used by the citation skills in Elastic Agent Builder. -Fields from the `sourcerer-v2-files*` or `sourcerer-v2-lines*` indices can be +Fields from the `sourcerer-v3-files*` or `sourcerer-v3-lines*` indices can be referenced as variables with curly braces (e.g. `{git.org}`, `{file.path}`). Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.path}` @@ -165,7 +166,7 @@ Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.path}` URL template for citing links to a line of code in a repo. Used by the citation skills in Elastic Agent Builder. -Fields from the `sourcerer-v2-files*` or `sourcerer-v2-lines*` indices can be +Fields from the `sourcerer-v3-files*` or `sourcerer-v3-lines*` indices can be referenced as variables with curly braces (e.g. `{git.org}`, `{file.path}`, `{line.number}`). Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.path}#L{line.number}` @@ -178,7 +179,7 @@ Example: `https://github.com/{git.org}/{git.repo}/blob/{git.commit}/{file.path}# URL template for citing links to a range of lines of code in a repo. Used by the citation skills in Elastic Agent Builder. -Fields from the `sourcerer-v2-files*` or `sourcerer-v2-lines*` indices can be +Fields from the `sourcerer-v3-files*` or `sourcerer-v3-lines*` indices can be referenced as variables with curly braces (e.g. `{git.org}`, `{file.path}`) as well as fields returned by the `sourcerer.code.*` and `sourcerer.files.*` tools in Elastic Agent Builder (e.g. `{line.number_start}`, `{line.number_end}`). @@ -412,6 +413,22 @@ for indexing if they don't also qualify for pruning. - Type: String or Array[String] - Default: `null` (omitted) +### `sources[i].mode` + +Defines whether to index the content of each matching ref as an immutable commit +snapshot (`"snapshot"`) or maintain a single ref-addressed view that is updated +incrementally as the HEAD moves (`"delta"`). Controls whether `since` and +`retain` apply (both are rejected when `mode` is `"delta"`). + +- Required: No +- Type: String +- Default: `"snapshot"` +- Validation: + - Must be one of: `"snapshot"`, `"delta"` + - `"delta"` is only valid when `git.ref_type` is `"branch"` or `"tag"` + - `"delta"` cannot be combined with `since` or `retain` + - `"delta"` cannot be combined with `index.level: commit` + ### `sources[i].since` Sets the earliest point in commit history to index. Exactly one child field can @@ -599,7 +616,7 @@ indexing (if it doesn't also qualify for pruning). Accepts a 5-field cron expression or a duration string (see `schedules[i].schedule`). The "due" check compares against the **most recently completed** indexing run for -this source's scope `(host, org, repo, ref_type)` as recorded in `sourcerer-v2-refs`. +this source's scope `(host, org, repo, ref_type)` as recorded in `sourcerer-v3-refs`. A source is **not due** if another run has `status: indexing` for a ref in scope with `indexing_started_at` newer than 6 hours ago. This prevents redundant parallel work when the same config is invoked on a tight cron schedule. @@ -626,14 +643,14 @@ default repo-level index while its deploy tags go to a `^deploy`-suffixed index: ```yaml sources: - git: { host: github, org: elastic, repo: kibana, ref_type: tag } - match: "v{major}.{minor}.{patch}" # -> sourcerer-v2-*~github~elastic~kibana + match: "v{major}.{minor}.{patch}" # -> sourcerer-v3-*~github~elastic~kibana - git: { host: github, org: elastic, repo: kibana, ref_type: tag } match: "deploy@{major}" - index: { suffix: deploy } # -> sourcerer-v2-*~github~elastic~kibana^deploy + index: { suffix: deploy } # -> sourcerer-v3-*~github~elastic~kibana^deploy ``` Agents are unaffected by routing: they query the `sourcerer-files` / `sourcerer-lines` -read aliases, which span every `sourcerer-v2-files*` / `sourcerer-v2-lines*` index +read aliases, which span every `sourcerer-v3-files*` / `sourcerer-v3-lines*` index regardless of its level or suffix. **Changing routing on an already-indexed source (migration).** Because content @@ -666,10 +683,10 @@ Values of `level` and their effects on index names: |`level` |Index name | |----------|-------------------------------------------------------------| -|`"host"` |`sourcerer-v2-*~{git.host}` | -|`"org"` |`sourcerer-v2-*~{git.host}~{git.org}` | -|`"repo"` |`sourcerer-v2-*~{git.host}~{git.org}~{git.repo}` | -|`"commit"`|`sourcerer-v2-*~{git.host}~{git.org}~{git.repo}~{git.commit}`| +|`"host"` |`sourcerer-v3-*~{git.host}` | +|`"org"` |`sourcerer-v3-*~{git.host}~{git.org}` | +|`"repo"` |`sourcerer-v3-*~{git.host}~{git.org}~{git.repo}` | +|`"commit"`|`sourcerer-v3-*~{git.host}~{git.org}~{git.repo}~{git.commit}`| **Caveat — `"commit"`:** a commit-level index creates one physical index (and at least one shard) *per indexed commit*. On a repo/branch with many indexed commits @@ -692,12 +709,12 @@ a caret (`^`). Example index naming pattern where `sources[i].index.level` is `"repo"` and `sources[i].index.suffix` is `"deploy"`: -`sourcerer-v2-*~{git.host}~{git.org}~{git.repo}^deploy` +`sourcerer-v3-*~{git.host}~{git.org}~{git.repo}^deploy` For instance: -`sourcerer-v2-files~github~elastic~kibana^deploy` -`sourcerer-v2-lines~github~elastic~kibana^deploy` +`sourcerer-v3-files~github~elastic~kibana^deploy` +`sourcerer-v3-lines~github~elastic~kibana^deploy` - Required: No - Type: String diff --git a/src/sourcerer/commands/index/command.py b/src/sourcerer/commands/index/command.py index 808c3b7..74d77c7 100644 --- a/src/sourcerer/commands/index/command.py +++ b/src/sourcerer/commands/index/command.py @@ -26,10 +26,12 @@ from ...planner import Marker, plan_repo from ...progress import ProgressReporter, Unit, make_reporter from ...indices import files_index, lines_index +from ...queries import check_join_uniqueness from ...utils import ES_ERRORS, make_client +from ...version import compile_pattern, match_version from ..prune import command as prune_cmd from ..prune.execute import delete_commit_from_indices -from .documents import index_repo +from .documents import index_incremental_paths, index_repo from .git import ( checkout_branch, checkout_ref, @@ -37,6 +39,7 @@ count_tracked_files, default_branch, list_branch_commits, + plan_changes, prepared_repo, ref_dates, resolve_cache_root, @@ -44,9 +47,12 @@ _rev_info, ) from .markers import ( - build_ref_id, commits_with_content, content_present, fully_indexed_counts, - markers_status_by_id, _needs_index, pre_clone_skip, recorded_routing, - should_index, write_indexing_marker, write_ref_marker, + build_ref_id, commits_with_content, content_present, + count_incremental_branch_docs, delete_incremental_branch, delete_incremental_paths, + fully_indexed_counts, mark_snapshot_markers_stale, marker_routing, markers_status_by_id, + _needs_index, pre_clone_skip, read_incremental_ref, recorded_routing, + refresh_incremental_content, should_index, write_incremental_failed, + write_incremental_indexing, write_incremental_ready, write_indexing_marker, write_ref_marker, ) from .report import dry_run_config from .schedule import filter_config_by_schedule @@ -54,6 +60,24 @@ from .selection import _effective_since_floor, _load_config, _resolve_entry +def _run_uniqueness_gate(es: Elasticsearch, host: str, org: str, repo: str) -> bool: + """Post-index join-uniqueness gate: every distinct content commit/ref in + this repo must resolve to a complete refs join doc. For snapshot content (git.commit IS NOT + NULL) each commit must have at least one complete refs doc; for incremental (git.ref IS NOT + NULL) each ref must have exactly one incremental join doc. Prints offenders to stderr and + returns False; True (silent) when the invariant holds.""" + offending = check_join_uniqueness(es, host, org, repo) + if offending: + click.echo( + f"Warning: {host}/{org}/{repo}: {len(offending)} content key(s) with join-doc " + f"mismatch in sourcerer-v3-refs (run prune to clean stale markers): " + f"{', '.join(offending)}", + err=True, + ) + return False + return True + + def _branch_has_since(branch_name: str, cfg) -> bool: """True if any selector for this branch has a date/age/commit/ref `since` that would trigger a history walk. Version-based `since` (a tag name used as a floor for tag @@ -200,7 +224,8 @@ def index_ref_in_dir( # that this scope is currently being indexed by another run and skip it. # The terminal write_ref_marker (status:'complete') overwrites this doc in place. write_indexing_marker(es, host, org, repo, ref_type, ref_for_id, commit_sha, - commit_date_iso, index_level=level, index_suffix=suffix) + commit_date_iso, index_level=level, index_suffix=suffix, + ref_pattern=unit.ref_pattern or ref_for_id) files_count, lines_count = index_repo( es, host, org, repo, repo_dir, commit_sha, on_progress=lambda f, l: reporter.update_counts(unit, f, l), @@ -211,7 +236,8 @@ def index_ref_in_dir( # old copy is deleted, so a crash between here and the delete below leaves stale (not missing) # data that the prune stale-location sweep reclaims. write_ref_marker(es, host, org, repo, ref_type, ref_for_id, commit_sha, commit_date_iso, - files_count, lines_count, index_level=level, index_suffix=suffix) + files_count, lines_count, index_level=level, index_suffix=suffix, + ref_pattern=unit.ref_pattern or ref_for_id) if migrating: # Reconstruct the OLD index name from the prior marker's routing and drop this commit's # stale copy there. Commit-safety (another surviving ref sharing the commit) is respected @@ -225,6 +251,160 @@ def index_ref_in_dir( reporter.finish(unit, status, files_count, lines_count) +def index_incremental_branch_in_dir( + es: Elasticsearch, + host: str, + org: str, + repo: str, + repo_dir, + branch: str, + force: bool = False, + reporter: ProgressReporter | None = None, + unit: Unit | None = None, +) -> None: + """Advance one incremental (ref-addressed) branch or tag source in an already-cloned + `repo_dir`. + + Reads the ref's prior completed commit (its refs join doc, `_id = ref_key`), checks out + the fetched tip, and either: + - does nothing (already at the completed commit and not `--force`), + - does a full rebuild (first index, `--force`, or a missing diff base): delete + the whole ref namespace, then index every currently-tracked path, or + - does a delta update: `git diff --name-status` (via `plan_changes`) between the prior and + new commit, deleting only the paths git reports removed/changed and (re)indexing only the + paths git reports added/changed (scoped by the exact + (host,org,repo,ref_type,ref) 5-tuple, never a whole namespace sweep). + The refs join doc is published `indexing` before any mutation and `complete` only after the + content deletes/indexes and a refresh all succeed; a raised exception instead + records `write_incremental_failed` and leaves the completed pointer untouched, then + re-raises so the caller's per-unit error handling reports it. + """ + if reporter is None: + reporter = ProgressReporter() + if unit is None: + unit = Unit(host=host, org=org, repo=repo, ref=branch, kind="branch", mode="delta") + + ref_type = unit.kind # "branch" or "tag" + + # Stream identity: for a delta-tag stream `unit.ref_pattern` holds the literal match-pattern + # string (e.g. "deploy@{major}") and is the stable _id key for the refs join doc and the + # `git.ref_pattern` stored on all content docs. For branches and non-stream tags + # ref_pattern == branch (the concrete ref name). + ref_pattern = unit.ref_pattern or branch # stable identity (pattern for tag streams, branch otherwise) + + # For a delta-tag stream resolve the CONCRETE newest tag post-clone (commit dates are + # available here via `ref_dates`). `ref` then holds the concrete tag while `ref_pattern` + # (the pattern) stays the identity for all content/marker scoping. For branches ref == ref_pattern. + reporter.set_stage(unit, "checkout") + if ref_type == "tag": + cp = compile_pattern(branch) + dates = ref_dates(repo_dir) + tag_matches = [ + (ts, name) + for (kind, name), ts in dates.items() + if kind == "tag" and match_version(cp, name) is not None + ] + if not tag_matches: + # Pattern matched tags in Phase 1 but none are present in the clone. This can happen + # if all matching tags were deleted between ls-remote and the clone fetch. Skip. + reporter.finish(unit, "no-changes", detail="no matching tags in clone") + return + ref = max(tag_matches)[1] # newest by creatordate timestamp — concrete resolved tag + checkout_ref(repo_dir, ref) + else: + ref = branch # branch name is both the concrete ref and the identity + checkout_branch(repo_dir, ref) + # Expose the concrete resolved ref in the unit so the reporter displays it. + unit.ref = ref + + new_sha = resolve_commit(repo_dir) + commit_date_iso = commit_date(repo_dir) + + # Read the prior join doc keyed on `ref_pattern` (the stable identity, unchanged across promotions). + prior = read_incremental_ref(es, host, org, repo, ref_type, ref_pattern) + old_sha = None if force else (prior.get("git", {}).get("commit") if prior else None) + + level = unit.index_level + suffix = unit.index_suffix + + # Detect a routing (index.level / index.suffix) change from the prior completed run. When the + # routing changes we must migrate even if the commit hasn't advanced -- so routing_changed + # bypasses the no-changes early-return and forces a full rebuild into the new index (below). + old_routing = marker_routing(prior) if prior else None + routing_changed = old_routing is not None and old_routing != (level, suffix) + + if old_sha == new_sha and not force and not routing_changed: + reporter.finish(unit, "no-changes") + return + + reporter.set_stage(unit, "indexing") + # `ref` = concrete tag/branch (payload stored as git.ref in the join doc and content docs). + # `ref_pattern` = stream identity (pattern for tag streams; == ref for branches). + # Content docs (files/lines) are keyed on `ref_pattern` for _id stability across promotions. + write_incremental_indexing(es, host, org, repo, ref_type, ref, completed_commit=old_sha, + commit_target=new_sha, ref_pattern=ref_pattern, prior=prior, + index_level=level, index_suffix=suffix) + try: + full_rebuild = old_sha is None or force or routing_changed + if not full_rebuild: + plan = plan_changes(repo_dir, old_sha, new_sha) + full_rebuild = plan.base_missing + + if full_rebuild: + # Content scoped on `ref_pattern` (the stream identity). + delete_incremental_branch(es, host, org, repo, ref_pattern, + ref_type=ref_type, index_level=level, index_suffix=suffix) + reporter.set_total_files(unit, count_tracked_files(repo_dir)) + indexed_files, indexed_lines = index_incremental_paths( + es, host, org, repo, repo_dir, ref_pattern, None, + on_progress=lambda f, l: reporter.update_counts(unit, f, l), + index_level=level, index_suffix=suffix, ref_type=ref_type, + ) + else: + delete_incremental_paths(es, host, org, repo, ref_type, ref_pattern, plan.delete_paths, + index_level=level, index_suffix=suffix) + reporter.set_total_files(unit, len(plan.index_paths)) + indexed_files, indexed_lines = index_incremental_paths( + es, host, org, repo, repo_dir, ref_pattern, plan.index_paths, + on_progress=lambda f, l: reporter.update_counts(unit, f, l), + index_level=level, index_suffix=suffix, ref_type=ref_type, + ) + + refresh_incremental_content(es, host, org, repo, index_level=level, index_suffix=suffix) + files_count, lines_count = count_incremental_branch_docs( + es, host, org, repo, ref_pattern, ref_type=ref_type, index_level=level, index_suffix=suffix, + ) + # Mode-switch: flip any complete snapshot markers for this (host,org,repo,ref) to + # "stale" BEFORE publishing the incremental join doc as "complete". This ensures the + # two-complete-docs fan-out window (one snapshot + one incremental marker both matching + # LOOKUP JOIN ON git.ref_pattern) never opens. Stale content is reclaimed by prune. + mark_snapshot_markers_stale(es, host, org, repo, ref_pattern) + write_incremental_ready(es, host, org, repo, ref_type, ref, new_sha, commit_date_iso, + files_count, lines_count, ref_pattern=ref_pattern, + index_level=level, index_suffix=suffix) + # Migration cleanup (write-new -> flip join doc -> delete-old): now that the join doc is + # complete and points at the new routing, delete this ref's docs from the old physical + # index. Scoped to the exact (host,org,repo,ref_type,git.ref_pattern) 5-term filter so a + # sibling source that still lives in the old index is never touched. A crash between the + # ready write above and this delete leaves stale-location incremental docs in the old + # index; prune's incremental stale-location sweep (Class D-I) reclaims them. + if routing_changed: + old_level, old_suffix = old_routing + delete_incremental_branch(es, host, org, repo, ref_pattern, + ref_type=ref_type, index_level=old_level, index_suffix=old_suffix) + except KeyboardInterrupt: + write_incremental_failed(es, host, org, repo, ref_type, ref, completed_commit=old_sha, + commit_target=new_sha, ref_pattern=ref_pattern, error="interrupted", prior=prior, + index_level=level, index_suffix=suffix) + raise + except Exception as e: + write_incremental_failed(es, host, org, repo, ref_type, ref, completed_commit=old_sha, + commit_target=new_sha, ref_pattern=ref_pattern, error=str(e), prior=prior, + index_level=level, index_suffix=suffix) + raise + reporter.finish(unit, "indexed", indexed_files, indexed_lines) + + def index_one( es: Elasticsearch, host: str, @@ -346,6 +526,9 @@ def run( reporter.finish(unit, "error", detail=f"Elasticsearch request failed: {e}") sys.exit(1) + if not _run_uniqueness_gate(es, host, org, repo): + sys.exit(1) + def run_config( config_path: str, @@ -478,6 +661,49 @@ def process_group(item: tuple[tuple[str, str, str], list[Unit]]) -> None: return (host, org, repo), group = item clone_url = hosts[host].clone_url(org, repo) + + # Delta-mode units (branches and tags) are split from the snapshot pre-clone/skip/ + # retention flow entirely: no cohort retention, no `since` history walk, no + # commit-addressed content reuse -- each is a standalone two-phase delta update + # against its own prior state (see index_incremental_branch_in_dir). Processed here, + # before the snapshot-only `group` continues below with delta units filtered out. + incremental_units = [u for u in group if u.mode == "delta"] + group = [u for u in group if u.mode != "delta"] + for unit in incremental_units: + reporter.start(unit) + if incremental_units: + try: + with prepared_repo(host, org, repo, clone_url, cache_root, ephemeral) as repo_dir: + if repo_dir is None: + for unit in incremental_units: + reporter.finish( + unit, "locked", + detail="another sourcerer run holds this repo's cache lock", + ) + else: + for unit in incremental_units: + if _aborted.is_set(): + break + try: + index_incremental_branch_in_dir( + es, host, org, repo, repo_dir, unit.ref, force, reporter, unit, + ) + except KeyboardInterrupt: + break + except (FileNotFoundError, subprocess.CalledProcessError, + ValueError, *ES_ERRORS) as e: + with failures_lock: + failures += 1 + reporter.finish(unit, "error", detail=str(e)) + except (FileNotFoundError, subprocess.CalledProcessError, ValueError) as e: + for unit in incremental_units: + if unit.status is None: + with failures_lock: + failures += 1 + reporter.finish(unit, "error", detail=str(e)) + if not group: + return + # 2a. Cheap pre-clone skip for the whole group (no clone yet). # # Batched approach: for branch/tag units that carry a remote_sha from Phase 1's @@ -727,6 +953,14 @@ def process_group(item: tuple[tuple[str, str, str], list[Unit]]) -> None: # errors are handled inside process_group and counted in `failures`. list(pool.map(process_group, groups.items())) + # Post-index uniqueness gate, one distinct repo at a time, skipped on abort (the + # plan is incomplete). Every offending repo's ref_key(s) are reported before exiting. + if not _aborted.is_set(): + distinct_repos = {(c.host, c.org, c.repo) for c in entries} + for host, org, repo in sorted(distinct_repos): + if not _run_uniqueness_gate(es, host, org, repo): + failures += 1 + # Prune only after ALL indexing is complete, so a ref newly indexed this run is present in # the refs index before it's scored for retention (e.g. it can be the cohort-newest that # supersedes an older sibling). Skipped on abort: the plan is incomplete, so its retention diff --git a/src/sourcerer/commands/index/documents.py b/src/sourcerer/commands/index/documents.py index 9f0d9f2..ddd9201 100644 --- a/src/sourcerer/commands/index/documents.py +++ b/src/sourcerer/commands/index/documents.py @@ -146,6 +146,119 @@ def iter_line_docs( yield _id, {**base, "line": {"number": line_num, "content": line_content}} +def build_incremental_file_doc( + host: str, + org: str, + repo: str, + ref_type: str, + ref_pattern: str, + rel_path: str, + abs_path: pathlib.Path, + *, + binary: bool = False, + is_symlink: bool | None = None, + target_path: str | None = None, + target_size: int | None = None, +) -> tuple[str, dict]: + """Ref-addressed (incremental) file doc: carries `git.ref_pattern` (stream identity) but no + `git.ref` (concrete) or `git.commit`. `git.ref` is intentionally omitted: because content + `_id` is keyed on `ref_pattern`, a delta run only re-writes CHANGED paths, so after a tag + stream promotion unchanged paths would retain a stale `git.ref`. The concrete ref lives only + on the refs join doc (written wholesale each run). `_id` is keyed on `ref_pattern` so that + delta-tag stream promotions overwrite the same doc in place rather than minting a new id.""" + p = pathlib.PurePosixPath(rel_path) + directory = "" if str(p.parent) == "." else str(p.parent) + extension = p.suffix.lstrip(".") or None + _is_symlink = abs_path.is_symlink() if is_symlink is None else is_symlink + size = abs_path.lstat().st_size + file_fields: dict = { + "path": rel_path, + "directory": directory, + "name": p.name, + "extension": extension, + "size": size, + } + attrs = file_attributes(abs_path, binary=binary, is_symlink=_is_symlink) + if attrs: + file_fields["attributes"] = attrs + if _is_symlink: + if target_path is None: + try: + target_path = os.readlink(abs_path) + except OSError: + pass + if target_path is not None: + file_fields["target_path"] = target_path + if target_size is None: + try: + target_size = abs_path.stat().st_size + except OSError: + pass + if target_size is not None: + file_fields["target_size"] = target_size + doc = { + "git": { + "host": host, + "org": org, + "repo": repo, + "ref_pattern": ref_pattern, + "ref_type": ref_type, + }, + "file": file_fields, + } + _id = make_doc_id(host, org, repo, ref_type, ref_pattern, rel_path) + return _id, doc + + +def iter_incremental_line_docs( + host: str, + org: str, + repo: str, + ref_type: str, + ref_pattern: str, + rel_path: str, + content: str, + *, + size: int | None = None, + target_path: str | None = None, + target_size: int | None = None, + attributes: list[str] | None = None, +) -> Iterator[tuple[str, dict]]: + """Ref-addressed (incremental) line docs -- same shape as `build_incremental_file_doc`: + carries `git.ref_pattern` (stream identity) but no `git.ref` (concrete) or `git.commit`. + See `build_incremental_file_doc` for why `git.ref` is intentionally omitted.""" + p = pathlib.PurePosixPath(rel_path) + directory = "" if str(p.parent) == "." else str(p.parent) + extension = p.suffix.lstrip(".") or None + file_fields: dict = { + "path": rel_path, + "directory": directory, + "name": p.name, + "extension": extension, + } + if size is not None: + file_fields["size"] = size + if target_path is not None: + file_fields["target_path"] = target_path + if target_size is not None: + file_fields["target_size"] = target_size + if attributes is not None: + file_fields["attributes"] = attributes + base = { + "git": { + "host": host, + "org": org, + "repo": repo, + "ref_pattern": ref_pattern, + "ref_type": ref_type, + }, + "file": file_fields, + } + for line_num, line_content in enumerate(content.splitlines(), start=1): + _id = make_doc_id(host, org, repo, ref_type, ref_pattern, rel_path, str(line_num)) + yield _id, {**base, "line": {"number": line_num, "content": line_content}} + + # Per-(repo, commit, tag) context for the worker processes, set once per pool by _init_worker # so only the (small) file path crosses the process boundary on each task. _WORKER_CTX: dict = {} @@ -164,6 +277,26 @@ def _init_worker( _WORKER_CTX.update( host=host, org=org, repo=repo, commit_sha=commit_sha, repo_dir=pathlib.Path(repo_dir), symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, + mode="snapshot", + ) + + +def _init_worker_incremental( + host: str, org: str, repo: str, ref_type: str, ref_pattern: str, repo_dir: str, + symlink_paths: frozenset[str] = frozenset(), + index_level: str = "repo", index_suffix: str | None = None, +) -> None: + """Same as `_init_worker`, but for the incremental (ref-addressed) path: `ref_type`+ + `ref_pattern` replace `commit_sha` and `mode` routes `_build_one_file_actions` to the + incremental doc builders. `ref_pattern` is the stream identity (== the concrete ref name for + non-stream refs; the literal pattern string for delta-tag streams). The concrete `ref` is NOT + stored here -- content docs carry only `ref_pattern`; `git.ref` lives on the refs join doc.""" + signal.signal(signal.SIGINT, signal.SIG_IGN) + _WORKER_CTX.update( + host=host, org=org, repo=repo, ref_type=ref_type, ref_pattern=ref_pattern, + repo_dir=pathlib.Path(repo_dir), + symlink_paths=symlink_paths, index_level=index_level, index_suffix=index_suffix, + mode="delta", ) @@ -184,11 +317,17 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: text. Runs in a worker process (see _init_worker for the shared context). Mirrors the old inline generator -- a binary file or one that can't be read yields only its file doc.""" ctx = _WORKER_CTX - host, org, repo, commit_sha = ctx["host"], ctx["org"], ctx["repo"], ctx["commit_sha"] + incremental = ctx.get("mode", "snapshot") == "delta" + host, org, repo = ctx["host"], ctx["org"], ctx["repo"] + ref_type = ctx.get("ref_type", "branch") + ref_pattern = ctx["ref_pattern"] if incremental else None + commit_sha = ctx.get("commit_sha") level, suffix = ctx.get("index_level", "repo"), ctx.get("index_suffix") abs_path = ctx["repo_dir"] / rel_path - f_index = files_index(host, org, repo, commit_sha, level, suffix) - l_index = lines_index(host, org, repo, commit_sha, level, suffix) + # Incremental content is ref-addressed (no commit-level index name); the physical index name + # never includes the commit for this path since ref-keyed docs don't carry git.commit. + f_index = files_index(host, org, repo, None if incremental else commit_sha, level, suffix) + l_index = lines_index(host, org, repo, None if incremental else commit_sha, level, suffix) # Read the file's bytes once: detect binary from the first 8 KB, and (if text) decode the # same buffer for line splitting -- no second read of the file. Binary flag must be computed # before build_file_doc so it reaches file.attributes. @@ -228,24 +367,42 @@ def _build_one_file_actions(rel_path: str) -> list[dict]: else: git_target_path = None git_target_size = None - file_id, file_doc = build_file_doc( - host, org, repo, commit_sha, rel_path, abs_path, binary=binary, - is_symlink=True if is_git_symlink else None, - target_path=git_target_path, - target_size=git_target_size, - ) + if incremental: + file_id, file_doc = build_incremental_file_doc( + host, org, repo, ref_type, ref_pattern, rel_path, abs_path, binary=binary, + is_symlink=True if is_git_symlink else None, + target_path=git_target_path, + target_size=git_target_size, + ) + else: + file_id, file_doc = build_file_doc( + host, org, repo, commit_sha, rel_path, abs_path, binary=binary, + is_symlink=True if is_git_symlink else None, + target_path=git_target_path, + target_size=git_target_size, + ) actions = [{"_index": f_index, "_id": file_id, "_source": file_doc}] if raw is None or binary: return actions content = raw.decode("utf-8", errors="surrogateescape") ff = file_doc["file"] - for line_id, line_doc in iter_line_docs( - host, org, repo, commit_sha, rel_path, content, - size=ff["size"], - target_path=ff.get("target_path"), - target_size=ff.get("target_size"), - attributes=ff.get("attributes"), - ): + if incremental: + line_docs = iter_incremental_line_docs( + host, org, repo, ref_type, ref_pattern, rel_path, content, + size=ff["size"], + target_path=ff.get("target_path"), + target_size=ff.get("target_size"), + attributes=ff.get("attributes"), + ) + else: + line_docs = iter_line_docs( + host, org, repo, commit_sha, rel_path, content, + size=ff["size"], + target_path=ff.get("target_path"), + target_size=ff.get("target_size"), + attributes=ff.get("attributes"), + ) + for line_id, line_doc in line_docs: actions.append({"_index": l_index, "_id": line_id, "_source": line_doc}) return actions @@ -340,3 +497,86 @@ def generate_actions(): if on_progress is not None: on_progress(files_count, lines_count) return files_count, lines_count + + +def index_incremental_paths( + es: Elasticsearch, + host: str, + org: str, + repo: str, + repo_dir: pathlib.Path, + ref_pattern: str, + rel_paths: list[str] | None = None, + on_progress: Callable[[int, int], None] | None = None, + index_level: str = "repo", + index_suffix: str | None = None, + ref_type: str = "branch", +) -> tuple[int, int]: + """Index a set of paths for an incremental (ref-addressed) branch or tag source. + + `rel_paths=None` walks the whole checked-out tree (first index / full rebuild, e.g. when a + diff base is unavailable). A given `rel_paths` list indexes only those paths -- the delta + indexer's changed/added set (see `commands/index/git.py:plan_changes`), which is what makes + an incremental HEAD advance only touch the files git reports changed. Deletions for removed + paths are the caller's responsibility (see `markers.delete_incremental_paths`) since they + need no doc generation. Mirrors `index_repo`'s worker-pool ingest loop. + + `ref_pattern` is the stream identity key (the literal pattern for delta-tag streams, or the + concrete ref name for all other refs). The concrete `ref` is intentionally absent here -- + content docs carry only `git.ref_pattern`; `git.ref` lives only on the refs join doc. + """ + files_count = 0 + lines_count = 0 + f_index = files_index(host, org, repo, None, index_level, index_suffix) + + t = _tuning() + symlink_paths = get_symlink_paths(repo_dir) + paths = list(iter_tracked_files(repo_dir)) if rel_paths is None else list(rel_paths) + with ProcessPoolExecutor( + max_workers=max(1, t.index_workers), + initializer=_init_worker_incremental, + initargs=(host, org, repo, ref_type, ref_pattern, str(repo_dir), symlink_paths, index_level, index_suffix), + ) as executor: + def _batched(items: Iterator[str], n: int) -> Iterator[list[str]]: + it = iter(items) + while batch := list(islice(it, n)): + yield batch + + def generate_actions(): + batches = _batched(iter(paths), t.index_worker_chunksize) + max_inflight = max(1, t.index_workers) * 2 + inflight: deque = deque() + for batch in islice(batches, max_inflight): + inflight.append(executor.submit(build_file_actions, batch)) + while inflight: + file_actions = inflight.popleft().result() + for batch in islice(batches, 1): + inflight.append(executor.submit(build_file_actions, batch)) + yield from file_actions + + processed = 0 + try: + for _ok, info in es_parallel_bulk( + es, + generate_actions(), + thread_count=t.bulk_threads, + chunk_size=t.bulk_chunk_size, + max_chunk_bytes=t.bulk_max_bytes, + queue_size=t.bulk_queue_size, + ): + if _aborted.is_set(): + raise KeyboardInterrupt + meta = next(iter(info.values())) if info else {} + if meta.get("_index") == f_index: + files_count += 1 + else: + lines_count += 1 + processed += 1 + if on_progress is not None and processed % 1000 == 0: + on_progress(files_count, lines_count) + except KeyboardInterrupt: + executor.shutdown(wait=False, cancel_futures=True) + raise + if on_progress is not None: + on_progress(files_count, lines_count) + return files_count, lines_count diff --git a/src/sourcerer/commands/index/git.py b/src/sourcerer/commands/index/git.py index 73683fb..f824369 100644 --- a/src/sourcerer/commands/index/git.py +++ b/src/sourcerer/commands/index/git.py @@ -14,6 +14,7 @@ import tempfile import time from collections.abc import Iterator +from dataclasses import dataclass, field # App packages from ...queries import _parse_dt @@ -347,6 +348,112 @@ def count_tracked_files(repo_dir: pathlib.Path) -> int: return sum(1 for _ in iter_tracked_files(repo_dir)) +@dataclass +class ChangePlan: + """A pure plan for turning one incremental branch update (old commit -> new commit) into + Elasticsearch work. `delete_paths` are the prior file/line docs to remove synchronously; + `index_paths` are the current tree paths to (re)index. A modified or type-changed file + appears in BOTH (delete its stale docs, then re-index every current line). `base_missing` + is True when the old commit object is unavailable locally -- the caller must then fall back + to full branch-namespace reconciliation instead of trusting an empty diff.""" + + delete_paths: list[str] = field(default_factory=list) + index_paths: list[str] = field(default_factory=list) + base_missing: bool = False + + +def base_commit_available(repo_dir: pathlib.Path, old_sha: str) -> bool: + """True if `old_sha` resolves to a commit object present in the local clone. Uses + `git cat-file -e ^{commit}` so a tag or partial object still fails closed. A False + here means the diff base is gone (e.g. force-push, shallow clone) and the caller must + rebuild rather than treat the missing base as an empty diff.""" + try: + subprocess.run( + ["git", "-C", str(repo_dir), "cat-file", "-e", f"{old_sha}^{{commit}}"], + check=True, + capture_output=True, + ) + except (subprocess.CalledProcessError, OSError): + return False + return True + + +def _dedupe(paths: list[str]) -> list[str]: + """Order-preserving de-duplication (dict keeps first-seen order).""" + return list(dict.fromkeys(paths)) + + +def _parse_name_status_z(raw: bytes) -> tuple[list[str], list[str]]: + """Parse `git diff --name-status -z` output into (delete_paths, index_paths). + + The `-z` stream is a flat run of NUL-terminated tokens: a status token, then one path + (add/modify/delete/type-change) or two paths (rename/copy: old then new). Parsing by NUL + boundary -- never by whitespace -- preserves spaces, tabs, and unusual bytes in paths. + Mapping (rename detection is an optimization; delete+add of the same paths is equivalent): + A (add) -> index new + M (modify) -> delete + index (replace at file granularity) + T (type change) -> delete + index + D (delete) -> delete + R (rename) -> delete old + index new + C (copy) -> index new (source is unchanged, stays indexed) + An unrecognized status fails safe as delete + index of its path.""" + tokens = raw.split(b"\x00") + # Each real token is NUL-terminated, so a well-formed stream ends in an empty tail token; + # drop trailing empties so a complete record's last path isn't misread as "present but + # empty" and a truncated record is detected by running past the end. + while tokens and tokens[-1] == b"": + tokens.pop() + delete_paths: list[str] = [] + index_paths: list[str] = [] + i = 0 + n = len(tokens) + while i < n: + status = tokens[i] + if not status: + i += 1 + continue + code = status.decode("utf-8", errors="surrogateescape")[0] + if code in ("R", "C"): + if i + 2 >= n: + break # truncated record + old = tokens[i + 1].decode("utf-8", errors="surrogateescape") + new = tokens[i + 2].decode("utf-8", errors="surrogateescape") + i += 3 + if code == "R": + delete_paths.append(old) + index_paths.append(new) + else: + if i + 1 >= n: + break # truncated record + path = tokens[i + 1].decode("utf-8", errors="surrogateescape") + i += 2 + if code == "A": + index_paths.append(path) + elif code == "D": + delete_paths.append(path) + else: # M, T, or an unknown status -> replace the whole file + delete_paths.append(path) + index_paths.append(path) + return _dedupe(delete_paths), _dedupe(index_paths) + + +def plan_changes(repo_dir: pathlib.Path, old_sha: str, new_sha: str) -> ChangePlan: + """Build the ChangePlan for advancing an incremental branch from `old_sha` to `new_sha`. + Returns a `base_missing` plan (no paths) when the old commit is unavailable locally; the + caller then rebuilds the branch namespace. Otherwise runs a NUL-delimited name-status diff + with rename (-M) and copy (-C) detection and maps each record (see _parse_name_status_z).""" + if not base_commit_available(repo_dir, old_sha): + return ChangePlan(base_missing=True) + result = subprocess.run( + ["git", "-C", str(repo_dir), "diff", "--name-status", "-z", "-M", "-C", + old_sha, new_sha], + check=True, + capture_output=True, + ) + delete_paths, index_paths = _parse_name_status_z(result.stdout) + return ChangePlan(delete_paths=delete_paths, index_paths=index_paths) + + def _ls_remote(url: str, *patterns: str, flags: tuple[str, ...] = ()) -> str | None: """ Run `git ls-remote` against a remote without cloning. The URL must precede the ref diff --git a/src/sourcerer/commands/index/markers.py b/src/sourcerer/commands/index/markers.py index d3082af..4881d10 100644 --- a/src/sourcerer/commands/index/markers.py +++ b/src/sourcerer/commands/index/markers.py @@ -1,7 +1,7 @@ # sourcerer/commands/index/markers.py # Refs-index idempotency: content-addressing a ref's indexed state, the guards that decide # whether a ref needs (re)indexing, and writing the completion marker. Reads use the sourcerer -# aliases; writes use sourcerer-v2-refs and the physical per-repo content indices; +# aliases; writes use sourcerer-v3-refs and the physical per-repo content indices; # broader read-only queries across the whole cluster live in sourcerer/queries.py. # Standard packages @@ -11,8 +11,8 @@ from elasticsearch import Elasticsearch, NotFoundError # App packages -from ...indices import FILES_ALIAS, REFS_ALIAS, REFS_INDEX, files_index -from ...utils import make_doc_id +from ...indices import FILES_ALIAS, LINES_ALIAS, REFS_ALIAS, REFS_INDEX, files_index, lines_index +from ...utils import build_ref_key, make_doc_id from .git import resolve_remote @@ -401,6 +401,7 @@ def write_indexing_marker( commit_date_iso: str | None, index_level: str = "repo", index_suffix: str | None = None, + ref_pattern: str | None = None, ) -> None: """Write a status:'indexing' marker for a ref that is about to be ingested. @@ -423,10 +424,12 @@ def write_indexing_marker( "org": org, "repo": repo, "ref": ref, + "ref_pattern": ref_pattern if ref_pattern is not None else ref, "ref_type": ref_type, "commit": commit_sha, "commit_date": commit_date_iso, }, + "mode": "snapshot", "status": "indexing", "indexing_started_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "files_count": 0, @@ -450,6 +453,8 @@ def write_ref_marker( lines_count: int, index_level: str = "repo", index_suffix: str | None = None, + refresh: bool = False, + ref_pattern: str | None = None, ) -> None: # (ref, ref_type) replaces the old git.branch/git.tag fields: those were write-only and # fully reconstructable as `git.ref filtered by git.ref_type`. git.tag was an array that @@ -458,7 +463,7 @@ def write_ref_marker( # # index_level/index_suffix record this source's index.* routing (semantic, not the resolved # index name). The physical files/lines index is reconstructed on demand from git.host/org/ - # repo/commit + these two fields via indices.files_index/lines_index, so a v2->v3 prefix bump + # repo/commit + these two fields via indices.files_index/lines_index, so a v3->v4 prefix bump # stays correct and prune/migration can find (and clean up) exactly where content lives. # Legacy markers written before this feature omit both; readers fall back to the "repo"/None # defaults, which reconstruct to the historical repo-level name where that content actually is. @@ -469,10 +474,12 @@ def write_ref_marker( "org": org, "repo": repo, "ref": ref, + "ref_pattern": ref_pattern if ref_pattern is not None else ref, "ref_type": ref_type, "commit": commit_sha, "commit_date": commit_date_iso, }, + "mode": "snapshot", "status": "complete", "files_count": files_count, "lines_count": lines_count, @@ -480,7 +487,7 @@ def write_ref_marker( "index_level": index_level, "index_suffix": index_suffix, } - es.index(index=REFS_INDEX, id=ref_id, document=doc) + es.index(index=REFS_INDEX, id=ref_id, document=doc, refresh=refresh) def pre_clone_skip( @@ -536,6 +543,397 @@ def pre_clone_skip( return False, ref_for_id, remote_sha +# --- incremental refs join docs, keyed by `_id = build_ref_key(...)` ---------------------- +# One document per incremental ref: a delta-mode ref's single join doc lives at +# `_id = {host}~{org}~{repo}~{ref_type}~{ref}` (constructed by build_ref_key, a plain +# tilde-joined string -- not a stored field) and its `git.commit` is the ref's live target +# commit, advanced only by a two-phase indexing -> complete publication. ref_type +# ("branch" or "tag") is part of the key so a same-named branch and tag each get a distinct +# join doc. This is a DISTINCT id space from `build_ref_id`'s hashed, append-only ref-name +# markers above; a join doc's `_id` is a plain, unhashed build_ref_key() string, which a +# `build_ref_id` hash can never collide with. build_ref_key is still used as the `_id` +# constructor even though git.ref_key is no longer a stored field -- the id itself remains the +# stable overwrite key for each delta-mode ref. + + +def read_incremental_ref( + es: Elasticsearch, host: str, org: str, repo: str, ref_type: str, ref_pattern: str, +) -> dict | None: + """The ref's incremental join doc `_source`, or None if never indexed. A real-time GET + (by `_id = build_ref_key(..., ref_pattern)` where `ref_pattern` is the stream identity + stored as `git.ref_pattern`). For delta-tag streams this is the pattern string + (e.g. "deploy@{major}"); for all other refs it equals the concrete ref name.""" + try: + return es.get(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref_pattern))["_source"] + except NotFoundError: + return None + + +def _now_iso() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def _build_incremental_join_doc( + host: str, + org: str, + repo: str, + ref_type: str, + ref: str, + ref_pattern: str, + *, + status: str, + commit: str | None, + commit_target: str | None = None, + commit_date_iso: str | None = None, + files_count: int = 0, + lines_count: int = 0, + indexed_at: str | None = None, + indexing_started_at: str | None = None, + index_level: str = "repo", + index_suffix: str | None = None, +) -> dict: + """Build an incremental join doc. `ref` is the CONCRETE resolved ref (e.g. the newest + matching tag for a delta-tag stream, or the branch name for a delta branch). `ref_pattern` + is the STREAM IDENTITY: the literal match-pattern string for a delta-tag stream, or equal to + `ref` for all other refs. Stored as `git.ref_pattern` and doubles as the stable `_id` key.""" + return { + "git": { + "host": host, + "org": org, + "repo": repo, + "ref": ref, + "ref_pattern": ref_pattern, + "ref_type": ref_type, + "commit": commit, + "commit_target": commit_target, + "commit_date": commit_date_iso, + }, + "mode": "delta", + "status": status, + "files_count": files_count, + "lines_count": lines_count, + "indexed_at": indexed_at, + "indexing_started_at": indexing_started_at, + "index_level": index_level, + "index_suffix": index_suffix, + } + + +def write_incremental_indexing( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref_type: str, + ref: str, + completed_commit: str | None, + commit_target: str, + ref_pattern: str | None = None, + prior: dict | None = None, + refresh: bool = False, + index_level: str = "repo", + index_suffix: str | None = None, +) -> None: + """Publish `status: indexing`: the completed pointer (`git.commit`) stays at the LAST + completed SHA (or None on a first index) while `git.commit_target` advertises the candidate + SHA the run is advancing to. A failed run never overwrites `git.commit` with `commit_target` + -- only `write_incremental_ready` does that, after delete+index+refresh succeed. + + `ref` is the CONCRETE resolved ref (git.ref payload); `ref_pattern` is the STREAM IDENTITY + stored as `git.ref_pattern` and used as the doc _id. For delta-tag streams these differ + (e.g. ref="deploy@1788000000", ref_pattern="deploy@{major}"). Defaults to `ref`.""" + ref_pattern = ref_pattern or ref + prior = prior or {} + pg = prior.get("git", {}) + doc = _build_incremental_join_doc( + host, org, repo, ref_type, ref, ref_pattern, + status="indexing", + commit=completed_commit, + commit_target=commit_target, + commit_date_iso=pg.get("commit_date"), + files_count=prior.get("files_count", 0), + lines_count=prior.get("lines_count", 0), + indexed_at=prior.get("indexed_at"), + indexing_started_at=_now_iso(), + index_level=index_level, + index_suffix=index_suffix, + ) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref_pattern), document=doc, refresh=refresh) + + +def write_incremental_ready( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref_type: str, + ref: str, + commit: str, + commit_date_iso: str | None, + files_count: int, + lines_count: int, + ref_pattern: str | None = None, + refresh: bool = True, + index_level: str = "repo", + index_suffix: str | None = None, +) -> None: + """Publish `status: complete` at the NEW completed commit, clearing `commit_target` and any + prior failure fields. This is the pointer-advancing publication boundary: callers + must delete+index+refresh the content indices FIRST, then call this. + + `ref` is the CONCRETE resolved ref (git.ref payload); `ref_pattern` is the STREAM IDENTITY + (stored as `git.ref_pattern`, used as the doc _id). Defaults to `ref`.""" + ref_pattern = ref_pattern or ref + doc = _build_incremental_join_doc( + host, org, repo, ref_type, ref, ref_pattern, + status="complete", + commit=commit, + commit_target=None, + commit_date_iso=commit_date_iso, + files_count=files_count, + lines_count=lines_count, + indexed_at=_now_iso(), + indexing_started_at=None, + index_level=index_level, + index_suffix=index_suffix, + ) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref_pattern), document=doc, refresh=refresh) + + +def write_incremental_failed( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref_type: str, + ref: str, + completed_commit: str | None, + commit_target: str | None, + error: str, + ref_pattern: str | None = None, + prior: dict | None = None, + refresh: bool = False, + index_level: str = "repo", + index_suffix: str | None = None, +) -> None: + """Record a failed update WITHOUT advancing the completed pointer: status becomes `failed`, + `git.commit` remains the last completed SHA. The next run retries old -> current. + + `ref` is the CONCRETE resolved ref (git.ref payload); `ref_pattern` is the STREAM IDENTITY + (stored as `git.ref_pattern`, used as the doc _id). Defaults to `ref`.""" + ref_pattern = ref_pattern or ref + prior = prior or {} + pg = prior.get("git", {}) + doc = _build_incremental_join_doc( + host, org, repo, ref_type, ref, ref_pattern, + status="failed", + commit=completed_commit, + commit_target=commit_target, + commit_date_iso=pg.get("commit_date"), + files_count=prior.get("files_count", 0), + lines_count=prior.get("lines_count", 0), + indexed_at=prior.get("indexed_at"), + indexing_started_at=None, + index_level=index_level, + index_suffix=index_suffix, + ) + es.index(index=REFS_INDEX, id=build_ref_key(host, org, repo, ref_type, ref_pattern), document=doc, refresh=refresh) + + +def _delete_by_query_sync(es: Elasticsearch, index: str, query: dict, refresh: bool) -> None: + """Synchronous delete-by-query used by the incremental path. Unlike the async prune + deletion, this waits for completion so a subsequent re-index can't race a still-running + delete, and uses `conflicts="proceed"` so a concurrent version bump doesn't abort the batch. + A missing index (first index, before any content exists) is ignored.""" + try: + es.delete_by_query( + index=index, + query=query, + wait_for_completion=True, + conflicts="proceed", + refresh=refresh, + ignore_unavailable=True, + allow_no_indices=True, + ) + except NotFoundError: + pass + + +def delete_incremental_paths( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref_type: str, + ref_pattern: str, + paths, + index_level: str = "repo", + index_suffix: str | None = None, + refresh: bool = False, +) -> None: + """Synchronously delete the file and line docs for `paths` on this exact ref. Scoped by + the exact (git.host, git.org, git.repo, git.ref_type, git.ref_pattern) 5-term filter + (one ref's docs can never bleed into another's) plus a `file.path` terms filter. + A no-op for an empty path set.""" + paths = list(paths) + if not paths: + return + query = { + "bool": { + "filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.ref_type": ref_type}}, + {"term": {"git.ref_pattern": ref_pattern}}, + {"terms": {"file.path": paths}}, + ] + } + } + for index in ( + files_index(host, org, repo, None, index_level, index_suffix), + lines_index(host, org, repo, None, index_level, index_suffix), + ): + _delete_by_query_sync(es, index, query, refresh) + + +def delete_incremental_branch( + es: Elasticsearch, + host: str, + org: str, + repo: str, + ref_pattern: str, + ref_type: str = "branch", + index_level: str = "repo", + index_suffix: str | None = None, + refresh: bool = False, +) -> None: + """Delete EVERY incremental content doc for this ref (full namespace), scoped by the exact + (git.host, git.org, git.repo, git.ref_type, git.ref_pattern) 5-term filter. Used + for the initial index and the missing-diff-base rebuild. `ref_type` defaults to + "branch" for back-compat with existing callers that pass `ref_pattern` positionally.""" + query = {"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.ref_type": ref_type}}, + {"term": {"git.ref_pattern": ref_pattern}}, + ]}} + for index in ( + files_index(host, org, repo, None, index_level, index_suffix), + lines_index(host, org, repo, None, index_level, index_suffix), + ): + _delete_by_query_sync(es, index, query, refresh) + + +def count_incremental_branch_docs( + es: Elasticsearch, host: str, org: str, repo: str, ref_pattern: str, + ref_type: str = "branch", + index_level: str = "repo", index_suffix: str | None = None, +) -> tuple[int, int]: + """Authoritative (files, lines) totals for a ref's current incremental view, counted by + exact (git.host, git.org, git.repo, git.ref_type, git.ref_pattern). Call AFTER refreshing + the content indices so counts reflect the just-applied deletes and indexes -- these become the + ready marker's files_count/lines_count. Returns 0 for an index that does not exist yet.""" + query = {"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.ref_type": ref_type}}, + {"term": {"git.ref_pattern": ref_pattern}}, + ]}} + + def _count(index: str) -> int: + try: + return int(es.count(index=index, query=query)["count"]) + except NotFoundError: + return 0 + + return ( + _count(files_index(host, org, repo, None, index_level, index_suffix)), + _count(lines_index(host, org, repo, None, index_level, index_suffix)), + ) + + +def refresh_incremental_content( + es: Elasticsearch, host: str, org: str, repo: str, + index_level: str = "repo", index_suffix: str | None = None, +) -> None: + """Make the branch's just-written incremental content visible before the ready pointer is + published (content refresh precedes the final refs write). Best-effort over + missing indices.""" + es.indices.refresh( + index=[ + files_index(host, org, repo, None, index_level, index_suffix), + lines_index(host, org, repo, None, index_level, index_suffix), + ], + ignore_unavailable=True, + allow_no_indices=True, + ) + + +def apply_refs_index_mapping(es: Elasticsearch, mapping: dict) -> None: + """Apply an updated mapping to the physical REFS_INDEX. A `put_index_template` change alone + (see `setup`) only affects indices created AFTER the change -- an existing refs index may + need its mapping updated explicitly for new fields. A no-op if the index doesn't exist.""" + try: + es.indices.put_mapping(index=REFS_INDEX, properties=mapping.get("properties", {})) + except NotFoundError: + pass + + +def apply_content_index_mapping(es: Elasticsearch, files_mapping: dict, lines_mapping: dict) -> None: + """Apply the updated files/lines template mappings to every EXISTING physical content index + behind the read aliases. `put_mapping` against an alias updates every backing index it + resolves to. A no-op if neither alias has any backing index yet.""" + for alias, mapping in ((FILES_ALIAS, files_mapping), (LINES_ALIAS, lines_mapping)): + try: + es.indices.put_mapping(index=alias, properties=mapping.get("properties", {})) + except NotFoundError: + pass + + +def stale_snapshot_markers_for_ref( + es: Elasticsearch, host: str, org: str, repo: str, ref_pattern: str, +) -> list[dict]: + """Return any complete snapshot ref-name markers (mode: "snapshot", status: "complete") + for (host, org, repo, ref_pattern). Used by the incremental index path to detect and mark stale + snapshot markers left behind by a mode switch from snapshot to incremental.""" + query = {"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.ref_pattern": ref_pattern}}, + {"term": {"status": "complete"}}, + {"term": {"mode": "snapshot"}}, + ]}} + try: + resp = es.search(index=REFS_ALIAS, size=100, query=query, source_includes=["git.commit"]) + except NotFoundError: + return [] + return resp["hits"]["hits"] + + +def mark_snapshot_markers_stale( + es: Elasticsearch, host: str, org: str, repo: str, ref_pattern: str, +) -> int: + """Flip any complete snapshot markers for this (host, org, repo, ref_pattern) to + status:"stale", making them invisible to all content tools without deleting them immediately. + Content reclamation is deferred to the prune command's stale-marker step. Returns the count + of markers flipped. + + ORDER: callers must call this BEFORE publishing the incremental join doc as "complete", + so the two-complete-docs fan-out window (one snapshot + one incremental, both reachable by + the LOOKUP JOIN ON git.ref_pattern) never opens.""" + markers = stale_snapshot_markers_for_ref(es, host, org, repo, ref_pattern) + for hit in markers: + try: + es.update(index=REFS_INDEX, id=hit["_id"], doc={"status": "stale"}) + except NotFoundError: + pass # already gone; not an error + return len(markers) + + def resolve_head(es: Elasticsearch, host: str, org: str, repo: str, ref_type: str, ref: str) -> dict | None: """The current marker for a ref: the newest commit_date among its (possibly many) markers. For a branch with retained history this is its live tip; for a tag it's the diff --git a/src/sourcerer/commands/index/schedule.py b/src/sourcerer/commands/index/schedule.py index 972b651..8f36558 100644 --- a/src/sourcerer/commands/index/schedule.py +++ b/src/sourcerer/commands/index/schedule.py @@ -1,7 +1,7 @@ # sourcerer/commands/index/schedule.py # Schedule-gating logic for `sourcerer index --config`: determines which sources are due for # indexing based on their configured schedule and the state of refs already indexed in -# sourcerer-v2-refs (last completed-at and any active in-progress indexing runs). +# sourcerer-v3-refs (last completed-at and any active in-progress indexing runs). # # The gate runs BEFORE the expensive ls-remote / clone / ingest pipeline, so a source whose # schedule hasn't fired since its last indexed run is dropped before any network I/O. diff --git a/src/sourcerer/commands/index/selection.py b/src/sourcerer/commands/index/selection.py index 2a4f755..c3dda93 100644 --- a/src/sourcerer/commands/index/selection.py +++ b/src/sourcerer/commands/index/selection.py @@ -16,6 +16,7 @@ from ...hosts import Host from ...planner import Marker, plan_repo from ...progress import Unit +from ...version import match_version from .git import _commit_date_of, list_remote_ref_names, list_remote_refs @@ -33,6 +34,12 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: # Phase 2's pre-clone skip check. fetched: dict[str, dict[str, str] | None] = {} seen: set[tuple[str, str]] = set() + # Maps (ref_type, name) -> mode for the winning selector, so we can detect when a second + # selector of a DIFFERENT mode also claims the same ref. Mixed-mode refs are unsafe: the + # incremental LOOKUP JOIN ON git.ref requires exactly one refs doc per (host,org,repo,ref), + # but a snapshot marker and an incremental join doc would both be present (fan-out). + seen_mode: dict[tuple[str, str], str] = {} + mode_conflicts: list[tuple[str, str, str, str]] = [] # (ref_type, name, mode_a, mode_b) units: list[Unit] = [] for sel in cfg.selectors: rt = sel.ref_type @@ -44,9 +51,11 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: if (rt, prefix) in seen: continue seen.add((rt, prefix)) + seen_mode[(rt, prefix)] = sel.mode units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=prefix, kind=rt, index_level=sel.index_level, index_suffix=sel.index_suffix, + mode=sel.mode, ref_pattern=prefix, )) continue if rt not in fetched: @@ -55,21 +64,69 @@ def _resolve_entry(cfg: RepoConfig, host: Host) -> list[Unit]: if ref_map is None: continue # ls-remote failed for this ref type, skip floor = sel.since_version_floor() # version-based `since: {ref}`, name-only + + # Delta-mode tag selectors are "moving streams": each raw pattern string is a single + # logical stream whose identity is the pattern itself (not any concrete tag name). One + # stream unit is emitted per raw pattern that matches at least one remote tag; the + # concrete newest-committed tag is resolved post-clone (ls-remote lacks dates). The + # per-name loop below handles all other cases (snapshot tags, all branches). + if sel.mode == "delta" and rt == "tag": + for pattern, cp in zip(sel.raw_patterns, sel.compiled): + key = (rt, pattern) + # Only emit a stream unit if this specific pattern matches at least one remote tag. + has_match = any(match_version(cp, name) is not None for name in ref_map) + if not has_match: + continue # pattern matches nothing remotely -- no stream to emit + if key in seen: + prior_mode = seen_mode[key] + if prior_mode != sel.mode: + mode_conflicts.append((rt, pattern, prior_mode, sel.mode)) + continue + seen.add(key) + seen_mode[key] = sel.mode + units.append(Unit( + host=cfg.host, org=cfg.org, repo=cfg.repo, ref=pattern, kind=rt, + remote_sha=None, # resolved post-clone via ref_dates + index_level=sel.index_level, index_suffix=sel.index_suffix, + mode=sel.mode, ref_pattern=pattern, + )) + continue + for name in sorted(ref_map): - if (rt, name) in seen: - continue - v = sel.matches(rt, name) - if v is None: + matched = sel.match_pattern(rt, name) + if matched is None: continue + pattern, v = matched if floor is not None and v.components < floor: continue # below the since version floor - seen.add((rt, name)) + key = (rt, name) + if key in seen: + # Already claimed by an earlier selector: check for a mode conflict. + prior_mode = seen_mode[key] + if prior_mode != sel.mode: + mode_conflicts.append((rt, name, prior_mode, sel.mode)) + continue + seen.add(key) + seen_mode[key] = sel.mode units.append(Unit( host=cfg.host, org=cfg.org, repo=cfg.repo, ref=name, kind=rt, remote_sha=ref_map[name], index_level=sel.index_level, index_suffix=sel.index_suffix, + mode=sel.mode, ref_pattern=pattern, )) + if mode_conflicts: + conflicts_str = ", ".join( + f"{rt}/{name} ({mode_a} vs {mode_b})" + for rt, name, mode_a, mode_b in mode_conflicts + ) + click.echo( + f"Warning: {cfg.org}/{cfg.repo}: selectors claim the same ref(s) with different " + f"modes -- skipping all units for this repo to avoid fan-out: {conflicts_str}", + err=True, + ) + return [] + failed_kinds = sorted(k for k, v in fetched.items() if v is None) if failed_kinds: click.echo( diff --git a/src/sourcerer/commands/prune/command.py b/src/sourcerer/commands/prune/command.py index 33aea4b..4cf9dfa 100644 --- a/src/sourcerer/commands/prune/command.py +++ b/src/sourcerer/commands/prune/command.py @@ -22,7 +22,8 @@ from ...planner import Decision, Marker, plan_repo from ...queries import content_indices_for_commit, fetch_markers, resolve_content_commit from ...utils import ES_ERRORS, make_client -from .execute import delete_commit_content, execute_deletions, execute_orphan_deletions, plan_orphans_now +from .execute import (delete_commit_content, execute_deletions, execute_orphan_deletions, + execute_stale_marker_deletions, plan_orphans_now) from .report import _Row, _orphan_rows, _print, _ref_rows, _retention_rows @@ -102,6 +103,7 @@ def run(config_path=None, url=None, api_key=None, username=None, password=None, total_orphan_indices = total_orphan_content = total_orphan_markers = 0 total_orphan_stale = total_empty_indices = 0 + total_stale_markers = total_stale_commits = 0 if not dry_run: for cfg, decisions in repo_decisions: if any(d.action == "delete" for d in decisions): @@ -113,6 +115,16 @@ def run(config_path=None, url=None, api_key=None, username=None, password=None, failures[0] += 1 click.echo(f"{cfg.host}/{cfg.org}/{cfg.repo}: error deleting: {e}", err=True) + # Reclaim stale snapshot markers from mode-switches (snapshot → incremental). These are + # markers flipped to status="stale" by index_incremental_branch_in_dir before the new + # incremental join doc was published, so they are never visible to content tools but do + # hold snapshot content that may no longer be needed. + try: + total_stale_markers, total_stale_commits = execute_stale_marker_deletions(es) + except ES_ERRORS as e: + failures[0] += 1 + click.echo(f"error reclaiming stale markers: {e}", err=True) + if orphan_plan is not None: (total_orphan_indices, total_orphan_content, total_orphan_markers, total_orphan_stale, total_empty_indices) = _apply_orphan_plan(es, orphan_plan, failures) @@ -122,6 +134,7 @@ def run(config_path=None, url=None, api_key=None, username=None, password=None, elif not quiet: click.echo( f"Pruned {total_markers} marker(s) and {total_commits} commit(s) of content; " + f"reclaimed {total_stale_markers} stale marker(s) and {total_stale_commits} stale commit(s); " f"removed {total_orphan_indices} orphaned index(es), " f"{total_orphan_content} orphaned content commit(s), " f"{total_orphan_markers} orphaned marker commit(s), " diff --git a/src/sourcerer/commands/prune/execute.py b/src/sourcerer/commands/prune/execute.py index 47824c4..3dd1404 100644 --- a/src/sourcerer/commands/prune/execute.py +++ b/src/sourcerer/commands/prune/execute.py @@ -14,8 +14,10 @@ from ...indices import REFS_INDEX, files_index, lines_index from ...planner import OrphanPlan, content_delete_set, plan_orphans from ...queries import ( - empty_content_indices, enumerate_ref_tuples, gather_content_by_index, - gather_content_commit_tuples, gather_intended_index_by_commit, list_sourcerer_indices, + empty_content_indices, enumerate_ref_tuples, fetch_complete_commits_for_repo, + fetch_stale_markers, gather_content_by_index, gather_content_commit_tuples, + gather_incremental_content_by_index, gather_intended_incremental_index_by_ref, + gather_intended_index_by_commit, list_sourcerer_indices, ) @@ -146,6 +148,65 @@ def delete_index(es: Elasticsearch, name: str) -> bool: return False +def execute_stale_marker_deletions(es: Elasticsearch) -> tuple[int, int]: + """Reclaim snapshot content that was superseded by a mode switch to incremental indexing. + + The flip-status switchover in `index_incremental_branch_in_dir` marks old snapshot ref-name + markers as status="stale" BEFORE publishing the incremental join doc as "complete", so the + two-complete-docs fan-out window never opens. Stale markers carry git.commit (so their content + can be reclaimed) but are invisible to all content tools (which gate on status=="complete"). + + This function: + 1. Enumerates all status="stale" snapshot markers cluster-wide. + 2. Groups them by (host, org, repo) so the commit-safety guard can be applied per-repo. + 3. For each stale marker, drops its snapshot content ONLY if the commit is not referenced + by any surviving complete marker in the same repo (the same commit-safety guard used by + execute_deletions via content_delete_set). A commit shared with another snapshot ref is + left in place. + 4. Deletes the stale marker doc from sourcerer-v3-refs. + + Returns (stale_markers_deleted, stale_commits_content_dropped). + + Crash-safety: if a crash occurs between step 3 and step 4, the stale marker doc remains and + this function is idempotent -- it will reattempt the same cleanup on the next prune run. + Unreachable content left by a crashed step 3 is also reclaimed by the orphan sweep.""" + stale_hits = fetch_stale_markers(es) + if not stale_hits: + return (0, 0) + + # Group stale markers by repo for the per-repo commit-safety check. + from collections import defaultdict + by_repo: dict[tuple[str, str, str], list[dict]] = defaultdict(list) + for hit in stale_hits: + g = hit["_source"].get("git", {}) + host = g.get("host", "") + org = g.get("org", "") + repo = g.get("repo", "") + if host and org and repo: + by_repo[(host, org, repo)].append(hit) + + markers_deleted = 0 + commits_dropped = 0 + for (host, org, repo), hits in by_repo.items(): + # Fetch commits currently protected by any complete marker in this repo. + protected_commits = fetch_complete_commits_for_repo(es, host, org, repo) + for hit in hits: + commit = hit["_source"].get("git", {}).get("commit") + if commit and commit not in protected_commits: + # Safe to delete this commit's content. + delete_commit_content(es, host, org, repo, commit) + commits_dropped += 1 + # Delete the stale marker doc regardless (the content is either dropped or still + # protected by another marker, so the stale marker itself is never useful again). + try: + es.delete(index=REFS_INDEX, id=hit["_id"]) + except NotFoundError: + pass # already gone -- race with another prune run + markers_deleted += 1 + + return (markers_deleted, commits_dropped) + + def plan_orphans_now(es: Elasticsearch) -> OrphanPlan: """Take one read-only snapshot of the cluster (index names, ref tuples, content tuples) via the read helpers in sourcerer/queries.py, and compute the full orphan plan from it via @@ -159,13 +220,19 @@ def plan_orphans_now(es: Elasticsearch) -> OrphanPlan: # detection (the index.level/suffix migration backstop). content_by_index = gather_content_by_index(es, index_names) intended_by_commit = gather_intended_index_by_commit(es) + # Class D-I: incremental (ref-addressed, commit-less) stale-location detection -- the + # incremental mirror of Class D, since the commit-keyed sweep can't see incremental docs. + incremental_content_by_index = gather_incremental_content_by_index(es, index_names) + intended_incremental_by_ref = gather_intended_incremental_index_by_ref(es) # Class E: content indices already drained to zero docs (a fully-pruned repo, or a suffix # a->b migration that emptied ~repo^a while its identity still has markers at ~repo^b). empty = empty_content_indices(es, index_names) return plan_orphans(index_names, ref_tuples, content_tuples, content_by_index_commit=content_by_index, intended_index_by_commit=intended_by_commit, - empty_index_names=empty) + empty_index_names=empty, + incremental_content_by_index=incremental_content_by_index, + intended_incremental_index_by_ref=intended_incremental_by_ref) def execute_orphan_deletions(es: Elasticsearch, plan: OrphanPlan) -> tuple[int, int, int, int, int]: @@ -173,7 +240,7 @@ def execute_orphan_deletions(es: Elasticsearch, plan: OrphanPlan) -> tuple[int, indices and Class E empty indices -- both near-instant), then the per-repo content delete_by_query (Class B -- expensive, one call per content index per repo), then the per-index stale-location delete_by_query (Class D -- the index.level/suffix migration backstop, - one call per index holding stale docs), then a single delete_by_query against sourcerer-v2-refs + one call per index holding stale docs), then a single delete_by_query against sourcerer-v3-refs covering every orphaned marker tuple across every repo (Class C -- refs is tiny, so one combined query costs one merge cycle instead of one per repo). Repo keys are (host, org, repo). Returns (indices_deleted, content_commits_dropped, marker_commits_dropped, @@ -231,6 +298,30 @@ def execute_orphan_deletions(es: Elasticsearch, plan: OrphanPlan) -> tuple[int, except NotFoundError: pass + # Class D-I: stale-location incremental content (ref-addressed, no git.commit). Mirrors Class D + # but keyed on (host, org, repo, ref_type, ref_pattern) tuples -- the commit-keyed filter above + # cannot match incremental docs whose git.commit is absent. + for index_name, ref_tuples in plan.orphan_stale_incremental.items(): + stale_dropped += len(ref_tuples) + for (host, org, repo, ref_type, ref_pattern) in ref_tuples: + try: + es.delete_by_query( + index=index_name, + query={"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"git.ref_type": ref_type}}, + {"term": {"git.ref_pattern": ref_pattern}}, + ]}}, + conflicts="proceed", + refresh=False, + scroll_size=5000, + wait_for_completion=False, + ) + except NotFoundError: + pass + markers_dropped = sum(len(commits) for commits in plan.orphan_marker_commits.values()) if plan.orphan_marker_commits: should = [ diff --git a/src/sourcerer/commands/prune/report.py b/src/sourcerer/commands/prune/report.py index 69b349f..909d3b3 100644 --- a/src/sourcerer/commands/prune/report.py +++ b/src/sourcerer/commands/prune/report.py @@ -44,7 +44,7 @@ class _Row: without it. - orphan:content, orphan:marker -> "host/org/repo@commit" -- no ref exists for either case. - orphan:index -> the index's own name (e.g. - "sourcerer-v2-files~host~org~repo") -- not commit- or even repo-addressable, since a + "sourcerer-v3-files~host~org~repo") -- not commit- or even repo-addressable, since a host~org-level orphan index spans every repo under that host+org. - orphan:stale-location -> "@" -- location-specific, since the same commit may legitimately have content in another index; only this stale copy is deleted.""" diff --git a/src/sourcerer/config.py b/src/sourcerer/config.py index d4a853c..97c3dcd 100644 --- a/src/sourcerer/config.py +++ b/src/sourcerer/config.py @@ -282,9 +282,13 @@ class Selector: retain: Retain | None levels: tuple[str, ...] = () # numeric levels shared by the versioned match patterns schedule: Schedule | None = None # per-source schedule override (sources[i].schedule) + # sources[i].mode: the indexing mode for this source -- "snapshot" (default, commit-addressed) + # or "delta" (ref-addressed, branch or tag). Controls whether since/retain apply and routes + # the unit to the incremental delta-index path instead of the snapshot flow. + mode: str = "snapshot" # "snapshot" (default) or "delta" (branch or tag) # sources[i].index routing (see specs/sourcerer-yml.md): which physical files/lines index this # source's content docs land in. Per-source, so two sources sharing a (host, org, repo) may - # route to different indices (e.g. kibana release tags -> ~repo, deploy tags -> ~repo^deploy). + # route differently. index_level: str = "repo" # "host" | "org" | "repo" | "commit" index_suffix: str | None = None # appended as ^{suffix}; None == no suffix @@ -305,6 +309,23 @@ def matches(self, ref_type: str, ref: str) -> Version | None: return v return None + def match_pattern(self, ref_type: str, ref: str) -> tuple[str, Version] | None: + """Like matches(), but also returns the raw match pattern that matched `ref`. + Used to set Unit.ref_pattern to the raw sources[i].match string (not the concrete ref).""" + if self.ref_type != ref_type: + return None + if self.ref_type == "commit": + ref_l = ref.lower() + for p in self.raw_patterns: + if ref_l.startswith(p): + return p, Version(ref=ref, components=(), prerelease="") + return None + for pattern, cp in zip(self.raw_patterns, self.compiled): + v = match_version(cp, ref) + if v is not None: + return pattern, v + return None + def since_version_floor(self) -> tuple[int, ...] | None: """If `since` is a ref anchor that denotes a version under this (versioned) selector, the inclusive version floor to index from -- name-only, so it's applied in Phase 1 and @@ -408,16 +429,18 @@ def _parse_commit_match(raw: dict, ctx: str) -> list[str]: _GIT_KEYS = {"host", "org", "repo", "ref_type"} _INDEX_LEVELS = ("host", "org", "repo", "commit") +_MODES = ("snapshot", "delta") # A suffix goes into a physical index name after a `^`, so it must be safe as an index-name # segment: the same characters forbidden in a host id, plus the `^` we use as the suffix delimiter. _FORBIDDEN_SUFFIX_CHARS = _FORBIDDEN_HOST_CHARS | {"^"} def _parse_index(raw: dict, ctx: str) -> tuple[str, str | None]: - """Validate a source's `index:` block and return (level, suffix). `level` defaults to "repo"; - `suffix` defaults to None. An empty-string suffix is treated as omitted (per the spec). The - suffix charset mirrors the host-id rules (lowercase, no whitespace, no index-name-forbidden - chars) plus a ban on the `^` delimiter itself.""" + """Validate a source's `index:` block and return (level, suffix). + + `level` defaults to "repo"; `suffix` defaults to None. An empty-string suffix is treated as + omitted (per the spec). The suffix charset mirrors the host-id rules (lowercase, no whitespace, + no index-name-forbidden chars) plus a ban on the `^` delimiter itself.""" if not isinstance(raw, dict): raise ValueError(f"{ctx} index: must be a mapping with 'level' and/or 'suffix'") unknown = set(raw) - {"level", "suffix"} @@ -444,6 +467,7 @@ def _parse_index(raw: dict, ctx: str) -> tuple[str, str | None]: if any(c.isspace() for c in s): raise ValueError(f"{ctx} index.suffix: {s!r} must not contain whitespace") suffix = s + return level, suffix @@ -474,12 +498,41 @@ def _parse_git_scope(raw: dict, ctx: str) -> tuple[str, str, str, str]: def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: """Parse one `sources[i]` entry into (host, org, repo, Selector). The ref_type comes from the - `git` block; `match`/`since`/`retain` are top-level siblings.""" - unknown = set(raw) - {"git", "match", "since", "retain", "schedule", "index"} + `git` block; `match`/`since`/`retain`/`mode` are top-level siblings.""" + unknown = set(raw) - {"git", "match", "since", "retain", "schedule", "index", "mode"} if unknown: raise ValueError(f"{ctx}: unknown keys {sorted(unknown)}") host, org, repo, ref_type = _parse_git_scope(raw, ctx) + # Parse mode early so it is available for the incremental constraints below. + mode = "snapshot" + if raw.get("mode") is not None: + mode = raw["mode"] + if mode not in _MODES: + raise ValueError(f"{ctx} mode: must be one of {list(_MODES)} (got {mode!r})") + + # Parse the index: block for routing (level + suffix only; mode is now top-level). + index_level, index_suffix = "repo", None + if raw.get("index") is not None: + index_level, index_suffix = _parse_index(raw["index"], ctx) + + if mode == "delta": + if ref_type not in ("branch", "tag"): + raise ValueError(f"{ctx} mode: 'delta' is only valid for " + f"git.ref_type: branch or tag (got ref_type {ref_type!r})") + # A delta-mode ref maintains a single mutable ref-addressed view with no per-commit + # history for retention to trim and no inclusion floor to apply -- both since and retain + # are meaningless here (see specs/incremental-indexing.md). + if raw.get("since") is not None: + raise ValueError(f"{ctx}: 'mode: delta' cannot be combined with 'since'") + if raw.get("retain") is not None: + raise ValueError(f"{ctx}: 'mode: delta' cannot be combined with 'retain'") + if index_level == "commit": + # Delta-mode content is ref-addressed (no git.commit on content docs), so a + # commit-level index name -- which requires a commit sha -- can never be built for it. + raise ValueError(f"{ctx} mode: 'delta' cannot be combined with " + f"'index.level: commit'") + if ref_type == "commit": # A pinned commit has no enumerable name to pattern-match against (see selection.py), # so `match` holds literal SHA/prefix strings instead of version.py DSL patterns, and @@ -525,13 +578,9 @@ def _parse_source(raw: dict, ctx: str) -> tuple[str, str, str, Selector]: except ValueError as e: raise ValueError(f"{ctx} schedule: {e}") from e - index_level, index_suffix = "repo", None - if raw.get("index") is not None: - index_level, index_suffix = _parse_index(raw["index"], ctx) - selector = Selector(ref_type=ref_type, raw_patterns=patterns, compiled=compiled, since=since, retain=retain, levels=levels, schedule=schedule, - index_level=index_level, index_suffix=index_suffix) + mode=mode, index_level=index_level, index_suffix=index_suffix) return host, org, repo, selector diff --git a/src/sourcerer/elastic/agent_builder_tools/README.md b/src/sourcerer/elastic/agent_builder_tools/README.md index 2cec784..b338854 100644 --- a/src/sourcerer/elastic/agent_builder_tools/README.md +++ b/src/sourcerer/elastic/agent_builder_tools/README.md @@ -13,9 +13,55 @@ Query snippet: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs: content docs carry git.ref_pattern = stream identity (pattern for + // delta-tag streams, branch name otherwise). The refs-side join key is git.ref_pattern + // (same field, same value on both sides). The ?git_ref param can be the pattern, the + // concrete tag, or a wildcard — both git.ref_pattern and git.ref are checked. + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) // other filters +// Branch by content-doc shape to resolve git.commit for incremental refs: +// Snapshot rows already carry git.commit (no join needed). +// Incremental rows carry git.ref_pattern (stream identity) and git.ref (concrete resolved ref); +// the LOOKUP JOIN resolves git.commit from the refs join doc using git.ref_pattern as the join +// key — the same field with the same value on both content and refs sides (no RENAME needed). +| FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) + // rest of query ``` @@ -43,6 +89,16 @@ params: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" ``` ## Glob matching `file.path` @@ -57,7 +113,7 @@ Query snippet: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + // filter by git_commit, git_ref, and git_ref_type AND file.path LIKE ?file_path // other filters diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml index 7f67840..b842966 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.grep.yml @@ -9,10 +9,52 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) AND file.path LIKE ?file_path AND line.content RLIKE ?regex + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). + | FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -44,6 +86,16 @@ configuration: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to grep (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml index 7986bf5..60f4046 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.code.search.yml @@ -9,10 +9,52 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) AND file.path LIKE ?file_path AND MATCH(line.content.text, ?q) + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). + | FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -44,6 +86,16 @@ configuration: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to search (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml index 95d0d0b..6d7beed 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.cat.yml @@ -9,9 +9,51 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) AND file.path LIKE ?file_path + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). + | FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -62,6 +104,16 @@ configuration: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to cat (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml index 7879a3c..6842035 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.head.yml @@ -9,9 +9,51 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) AND file.path LIKE ?file_path + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). + | FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -61,6 +103,16 @@ configuration: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to head (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml index 0a68227..b95d8f9 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.ls.yml @@ -9,7 +9,49 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) + + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). + | FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Enforce glob depth for * and ** on file.path. // Without this, "src/*/Job.java" would match files at any depth, @@ -64,7 +106,11 @@ configuration: ) // Collapse to one row per unique entry - this turns raw paths into an ls listing. - | STATS files = COUNT(*), bytes = SUM(file.size) BY name, is_dir + // Group by the commit too: when git_commit/git_ref/git_ref_type match more than one ref (e.g. a + // wildcard or the default "*"), each ref must get its own listing rather than having + // its file counts/bytes summed together into a meaningless cross-ref total. + | STATS files = COUNT(*), bytes = SUM(file.size) + BY git.host, git.org, git.repo, git.commit, name, is_dir // Append a trailing "/" to directory entries so they read like ls output. | EVAL name = CASE(is_dir, CONCAT(name, "/"), name) @@ -72,8 +118,8 @@ configuration: // Format the response. Each row is a single entry name, the cheapest row // in this toolset, so limit defaults to a high ceiling that mainly exists // to opt out of ES|QL's implicit row cap rather than to trim real usage. - | SORT is_dir DESC, name - | KEEP name + | SORT git.host, git.org, git.repo, git.commit, is_dir DESC, name + | KEEP git.host, git.org, git.repo, git.commit, name | LIMIT ?limit params: git_host: @@ -96,6 +142,16 @@ configuration: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" file_path: type: string description: File path to ls (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml index e97eaac..de88728 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.read_lines.yml @@ -9,11 +9,53 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) AND file.path LIKE ?file_path AND line.number >= ?line_number_start AND line.number <= ?line_number_end + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). + | FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -64,6 +106,16 @@ configuration: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to read (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml index 4cf802a..4b0de7c 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tail.yml @@ -9,9 +9,51 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) AND file.path LIKE ?file_path + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). + | FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -61,6 +103,16 @@ configuration: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to tail (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml index f030602..5745ab3 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.tree.yml @@ -9,7 +9,49 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) + + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). + | FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) // Split each file path into its segments | EVAL _segs = SPLIT(file.path, "/") @@ -185,6 +227,16 @@ configuration: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" file_path: type: string description: Directory to tree, e.g. src or src/sourcerer. The tree is rooted there and recurses; depth is controlled by L, not by this pattern. May also be a glob to filter what is listed (e.g. src/*.xml lists only XML files under src, at any depth). A single * already matches across directories, so ** is never required. diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml index b729f41..a1129dd 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.files.wc.yml @@ -9,9 +9,51 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) AND file.path LIKE ?file_path + // Branch by content-doc shape to resolve git.commit for incremental refs: + // 1. Snapshot rows already carry git.commit on the content doc (no join needed). + // 2. Incremental rows carry only git.ref; the LOOKUP JOIN resolves the citable + // git.commit from the refs join doc (one doc per (host,org,repo,ref)). + | FORK + ( WHERE git.commit IS NOT NULL ) + ( WHERE git.ref_pattern IS NOT NULL AND git.commit IS NULL + | LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type ) + // Glob match (* and **) on file.path | EVAL _fp_is_recursive = ?file_path != REPLACE(?file_path, "[*][*]", "") | EVAL _fp_segs = LENGTH(?file_path) - LENGTH(REPLACE(?file_path, "/", "")) + 1 @@ -176,6 +218,16 @@ configuration: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to count (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml index 68fb3ed..7989353 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.refs.list.yml @@ -10,13 +10,18 @@ configuration: AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo AND git.commit LIKE ?git_commit - AND git.ref LIKE ?git_ref + // `git.ref_pattern` is the stream identity (pattern for delta-tag streams, ref name otherwise). + // `git.ref` is the CONCRETE resolved ref (newest matching tag for delta-tag streams). + // Allow filtering by either so users can select by pattern, concrete tag, or wildcard. + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) AND git.ref_type LIKE ?git_ref_type AND status LIKE ?status // Format the response | SORT indexed_at DESC - | KEEP git.host, git.org, git.repo, git.ref, git.ref_type, git.commit, git.commit_date, status, files_count, lines_count, indexed_at, indexing_started_at + // Include `git.ref_pattern` (stream identity / join key) alongside git.ref (concrete resolved ref). + // For non-stream refs git.ref_pattern == git.ref. For delta-tag streams they differ. + | KEEP git.host, git.org, git.repo, git.ref, git.ref_pattern, git.ref_type, git.commit, git.commit_date, status, files_count, lines_count, indexed_at, indexing_started_at | LIMIT 1000000 params: git_host: @@ -53,4 +58,4 @@ configuration: type: string description: Filter by ref status. "complete" = fully indexed (default); "indexing" = currently being indexed; "*" = all statuses. optional: true - defaultValue: "complete" + defaultValue: "complete" \ No newline at end of file diff --git a/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml b/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml index 356f14e..7bc1f8f 100644 --- a/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml +++ b/src/sourcerer/elastic/agent_builder_tools/sourcerer.repos.search.yml @@ -9,7 +9,40 @@ configuration: | WHERE git.host LIKE ?git_host AND git.org LIKE ?git_org AND git.repo LIKE ?git_repo - AND git.commit LIKE ?git_commit + AND ( + // Resolve git_commit, git_ref, and git_ref_type against the small + // sourcerer-refs index first, rather than evaluating the wildcard + // match per line against this (far larger) content index. + // Content docs come in two disjoint shapes: + // 1. Commit snapshots have git.commit and no git.ref + // 2. Incremental refs have git.ref and no git.commit + // So resolution yields two membership sets off the same match: + // 1. Matching commits, checked against snapshot-shaped rows + // 2. Matching refs, checked against incremental-shaped rows + (git.commit IS NOT NULL AND git.commit IN ( + // Commit snapshots + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND git.ref LIKE ?git_ref + AND git.ref_type LIKE ?git_ref_type + | KEEP git.commit + )) + OR + (git.ref_pattern IS NOT NULL AND git.commit IS NULL AND git.ref_pattern IN ( + // Incremental refs + FROM sourcerer-refs + | WHERE git.host LIKE ?git_host + AND git.org LIKE ?git_org + AND git.repo LIKE ?git_repo + AND git.commit LIKE ?git_commit + AND (git.ref_pattern LIKE ?git_ref OR git.ref LIKE ?git_ref) + AND git.ref_type LIKE ?git_ref_type + | KEEP git.ref_pattern + )) + ) AND file.path LIKE ?file_path AND MATCH(line.content.text, ?q) @@ -19,16 +52,26 @@ configuration: | EVAL _file_segs = MV_COUNT(SPLIT(file.path, "/")) | WHERE _fp_is_recursive OR _file_segs == _fp_segs - // Collapse matching lines to one row per repo + commit + // Collapse matching lines to one row per (repo, ref) with COALESCE(git.ref, git.commit). + // git.ref is checked first because it's the only field that distinguishes + // two incremental branches that happen to currently point at the same commit + // (this query never joins, so an incremental row's git.commit stays NULL + // throughout. There's nothing for the COALESCE to prefer it over anyway). + // For snapshot rows git.ref is NULL, so this falls through to git.commit, + // which is already a correct per-commit identity since snapshot content is + // naturally deduplicated by commit. Same two reasons as before: incremental + // rows have no commit of their own to group by, and two refs sharing a + // commit must not collapse into one bucket. + | EVAL _ref_ish = COALESCE(git.ref, git.commit) | STATS _commit_file_count_distinct = COUNT_DISTINCT(file.path), _commit_score_sum = SUM(_score) - BY git.host, git.org, git.repo, git.commit + BY git.host, git.org, git.repo, _ref_ish // Normalize by distinct files so breadth beats duplication | EVAL _commit_score_density = _commit_score_sum / SQRT(TO_DOUBLE(_commit_file_count_distinct)) - // Collapse commits to one row per repo + // Collapse refs to one row per repo | STATS _repo_ref_count = COUNT(*), _repo_score_sum = SUM(_commit_score_density) @@ -62,6 +105,16 @@ configuration: description: Filter by git commit(s) (supports * wildcards) optional: true defaultValue: "*" + git_ref: + type: string + description: Filter by ref name(s), e.g. "main" or "v1.*" (supports * and ? wildcards) + optional: true + defaultValue: "*" + git_ref_type: + type: string + description: Filter by ref type (can be "branch", "tag", "commit", or any with "*") + optional: true + defaultValue: "*" file_path: type: string description: File path(s) to search (supports * and ** glob syntax, e.g. src/test, src/*/resources, src/**, src/**/*.xml) diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v2-files.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json similarity index 85% rename from src/sourcerer/elastic/index_templates/sourcerer-v2-files.json rename to src/sourcerer/elastic/index_templates/sourcerer-v3-files.json index 9b0719b..ba4e8a3 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v2-files.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-files.json @@ -3,7 +3,7 @@ "description": "sourcerer-files" }, "index_patterns": [ - "sourcerer-v2-files*" + "sourcerer-v3-files*" ], "template": { "aliases": { @@ -24,6 +24,8 @@ "git.org", "git.repo", "git.commit", + "git.ref_pattern", + "git.ref_type", "file.path" ], "order": [ @@ -31,6 +33,8 @@ "asc", "asc", "asc", + "asc", + "asc", "asc" ] } @@ -55,6 +59,15 @@ "commit": { "type": "keyword", "normalizer": "lowercase" + }, + "ref": { + "type": "keyword" + }, + "ref_pattern": { + "type": "keyword" + }, + "ref_type": { + "type": "keyword" } } }, diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v2-lines.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json similarity index 91% rename from src/sourcerer/elastic/index_templates/sourcerer-v2-lines.json rename to src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json index 83410e0..56045ae 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v2-lines.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-lines.json @@ -3,7 +3,7 @@ "description": "sourcerer-lines" }, "index_patterns": [ - "sourcerer-v2-lines*" + "sourcerer-v3-lines*" ], "template": { "aliases": { @@ -24,6 +24,8 @@ "git.org", "git.repo", "git.commit", + "git.ref_pattern", + "git.ref_type", "file.path", "line.number" ], @@ -33,6 +35,8 @@ "asc", "asc", "asc", + "asc", + "asc", "asc" ] } @@ -97,6 +101,15 @@ "commit": { "type": "keyword", "normalizer": "lowercase" + }, + "ref": { + "type": "keyword" + }, + "ref_pattern": { + "type": "keyword" + }, + "ref_type": { + "type": "keyword" } } }, diff --git a/src/sourcerer/elastic/index_templates/sourcerer-v2-refs.json b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json similarity index 79% rename from src/sourcerer/elastic/index_templates/sourcerer-v2-refs.json rename to src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json index 61c23dd..c93a8aa 100644 --- a/src/sourcerer/elastic/index_templates/sourcerer-v2-refs.json +++ b/src/sourcerer/elastic/index_templates/sourcerer-v3-refs.json @@ -3,7 +3,7 @@ "description": "sourcerer-refs" }, "index_patterns": [ - "sourcerer-v2-refs*" + "sourcerer-v3-refs*" ], "template": { "aliases": { @@ -18,9 +18,13 @@ "git.host", "git.org", "git.repo", - "git.ref" + "git.ref", + "git.ref_pattern", + "git.ref_type" ], "order": [ + "asc", + "asc", "asc", "asc", "asc", @@ -56,8 +60,10 @@ "normalizer": "lowercase" }, "ref": { - "type": "keyword", - "normalizer": "lowercase" + "type": "keyword" + }, + "ref_pattern": { + "type": "keyword" }, "ref_type": { "type": "keyword", @@ -69,6 +75,10 @@ }, "commit_date": { "type": "date" + }, + "commit_target": { + "type": "keyword", + "normalizer": "lowercase" } } }, @@ -89,12 +99,18 @@ "type": "date" }, "index_level": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase" }, "index_suffix": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase" + }, + "mode": { + "type": "keyword", + "normalizer": "lowercase" } } } } -} \ No newline at end of file +} diff --git a/src/sourcerer/elastic/kibana_saved_objects/export.ndjson b/src/sourcerer/elastic/kibana_saved_objects/export.ndjson index b1cab79..85e05ee 100644 --- a/src/sourcerer/elastic/kibana_saved_objects/export.ndjson +++ b/src/sourcerer/elastic/kibana_saved_objects/export.ndjson @@ -1,13 +1,13 @@ -{"attributes":{"allowHidden":false,"fieldAttrs":"{}","fieldFormatMap":"{}","fields":"[]","name":"sourcerer-refs","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"git.commit_date","title":"sourcerer-refs"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"b576da63-38d3-418c-a541-06f87ebef58f","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM4NiwxMF0="} -{"attributes":{"color":"#FCD883","description":"","name":"Sourcerer"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","managed":false,"references":[],"type":"tag","typeMigrationVersion":"8.0.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzQ2MjU4LDEwXQ=="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Lines","operationType":"sum","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":2}}},"sourceField":"lines_count"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# lines indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"10307b0e-7f80-4095-bcaa-14adc9c41fac","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934664],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5MiwxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"72d88170-f38a-4d63-9d23-b48fedafe769":{"columnOrder":["gh0st0000-0000-0000-0000-000000000001","f2b9711d-0da6-4c79-8b66-f3176097b892","c07c2dd5-7d83-4924-ac44-9807d28cf11d","ff805016-2503-4100-ad05-e0cd8a29ead3","cb6b8976-ac47-4fc3-b887-e5988128269c","e6e8ddbe-0004-4982-8532-9f341343f74b","28c16acf-857e-41ae-9304-822350018ba6","e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b","5e6e2c2d-765b-4572-809c-e2f544371594","21887acc-0eac-4819-a90b-a6226cdcb3d7","a5fed915-466a-4b56-a0bd-6510f9b77a33"],"columns":{"21887acc-0eac-4819-a90b-a6226cdcb3d7":{"customLabel":true,"dataType":"date","filter":{"language":"kuery","query":"\"indexed_at\": *"},"isBucketed":false,"label":"Indexed at","operationType":"last_value","params":{"sortField":"git.commit_date"},"sourceField":"indexed_at"},"28c16acf-857e-41ae-9304-822350018ba6":{"customLabel":true,"dataType":"date","filter":{"language":"kuery","query":"\"git.commit_date\": *"},"isBucketed":false,"label":"Committed at","operationType":"last_value","params":{"showArrayValues":false,"sortField":"git.commit_date"},"sourceField":"git.commit_date"},"5e6e2c2d-765b-4572-809c-e2f544371594":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Lines","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"lines_count"},"a5fed915-466a-4b56-a0bd-6510f9b77a33":{"customLabel":true,"dataType":"string","filter":{"language":"kuery","query":"\"status\": *"},"isBucketed":false,"label":"Status","operationType":"last_value","params":{"sortField":"git.commit_date"},"sourceField":"status"},"c07c2dd5-7d83-4924-ac44-9807d28cf11d":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.repo"},"cb6b8976-ac47-4fc3-b887-e5988128269c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Type","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.ref_type"},"e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Files","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"files_count"},"e6e8ddbe-0004-4982-8532-9f341343f74b":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Commit","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.commit"},"f2b9711d-0da6-4c79-8b66-f3176097b892":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.org"},"ff805016-2503-4100-ad05-e0cd8a29ead3":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Ref","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.ref"},"gh0st0000-0000-0000-0000-000000000001":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Host","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.host"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"columns":[{"columnId":"gh0st0000-0000-0000-0000-000000000001","isMetric":false,"isTransposed":false},{"columnId":"f2b9711d-0da6-4c79-8b66-f3176097b892","isMetric":false,"isTransposed":false},{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","isMetric":true,"isTransposed":false},{"columnId":"a5fed915-466a-4b56-a0bd-6510f9b77a33","isMetric":true,"isTransposed":false},{"columnId":"e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b","isMetric":true,"isTransposed":false},{"columnId":"c07c2dd5-7d83-4924-ac44-9807d28cf11d","isMetric":false,"isTransposed":false},{"columnId":"ff805016-2503-4100-ad05-e0cd8a29ead3","isMetric":false,"isTransposed":false},{"columnId":"cb6b8976-ac47-4fc3-b887-e5988128269c","isMetric":false,"isTransposed":false},{"columnId":"21887acc-0eac-4819-a90b-a6226cdcb3d7","isMetric":true,"isTransposed":false},{"columnId":"28c16acf-857e-41ae-9304-822350018ba6","isMetric":true,"isTransposed":false,"width":113},{"columnId":"e6e8ddbe-0004-4982-8532-9f341343f74b","isMetric":false,"isTransposed":false}],"layerId":"72d88170-f38a-4d63-9d23-b48fedafe769","layerType":"data","showRowNumbers":true,"sorting":{"columnId":"28c16acf-857e-41ae-9304-822350018ba6","direction":"desc"}}},"title":"Indexed repos","version":2,"visualizationType":"lnsDatatable"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-11T21:19:34.310Z","created_by":"u_3477301940_cloud","id":"85ea5355-8301-455d-aaca-2063ba93ceb0","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-72d88170-f38a-4d63-9d23-b48fedafe769","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-11T21:19:34.310Z","updated_by":"u_3477301940_cloud","version":"WzEyMjYsMTFd"} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Orgs","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.org"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# orgs indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"3e86ded1-35d1-48cc-bae7-51338893a4cd","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934652],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM4OCwxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Repos","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.repo"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# repos indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"22ff37d8-794e-4067-9e39-b3ebb47b5db4","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934655],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM4OSwxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Refs","operationType":"count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.ref"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# refs indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"286f4d43-d0c8-4cb6-935b-b2c2a3eeefe2","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934658],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5MCwxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Files","operationType":"sum","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":2}}},"sourceField":"files_count"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# files indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"b674ebfd-a225-440c-bf40-a6c827e2bf1c","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934661],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5MSwxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b":{"columnOrder":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1","db869f77-feda-4a5a-8dc6-233b8436591c"],"columns":{"1861256b-df4b-4807-9157-12ec9b7de6ce":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"db869f77-feda-4a5a-8dc6-233b8436591c","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.org"},"4d6e7a79-1fc4-414f-b16a-86ee89fed2a1":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"db869f77-feda-4a5a-8dc6-233b8436591c","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.repo"},"db869f77-feda-4a5a-8dc6-233b8436591c":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Refs per repo","operationType":"count","params":{"emptyAsNull":true},"sourceField":"git.ref"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layers":[{"categoryDisplay":"default","colorMapping":{"assignments":[],"colorMode":{"type":"categorical"},"paletteId":"default","specialAssignments":[{"color":{"type":"loop"},"rules":[{"type":"other"}],"touched":false}]},"layerId":"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","layerType":"data","legendDisplay":"default","metrics":["db869f77-feda-4a5a-8dc6-233b8436591c"],"nestedLegend":false,"numberDisplay":"percent","primaryGroups":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1"]}],"shape":"treemap"}},"title":"# refs by org/repo","version":2,"visualizationType":"lnsPie"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"853efb3c-c141-497a-9f99-dff8c6eaf90a","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934667],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5MywxMF0="} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b":{"columnOrder":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1","6d93874d-ceab-4271-9347-b7e9613c4bbc"],"columns":{"1861256b-df4b-4807-9157-12ec9b7de6ce":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"6d93874d-ceab-4271-9347-b7e9613c4bbc","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.org"},"4d6e7a79-1fc4-414f-b16a-86ee89fed2a1":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"6d93874d-ceab-4271-9347-b7e9613c4bbc","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.repo"},"6d93874d-ceab-4271-9347-b7e9613c4bbc":{"dataType":"number","isBucketed":false,"label":"Sum of lines_count","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"lines_count"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layers":[{"categoryDisplay":"default","colorMapping":{"assignments":[],"colorMode":{"type":"categorical"},"paletteId":"default","specialAssignments":[{"color":{"type":"loop"},"rules":[{"type":"other"}],"touched":false}]},"layerId":"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","layerType":"data","legendDisplay":"default","metrics":["6d93874d-ceab-4271-9347-b7e9613c4bbc"],"nestedLegend":false,"numberDisplay":"percent","primaryGroups":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1"]}],"shape":"treemap"}},"title":"# lines by org/repo","version":2,"visualizationType":"lnsPie"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"bb2c2db8-eda6-4127-ac97-89816eb00193","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934670],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5NCwxMF0="} -{"accessControl":{"accessMode":"default","owner":"u_RnkX2N-gU2tcwqOiST1LCMoTxDNJeOfiQwrhxwEzUoo_0"},"attributes":{"description":"","esqlApproximation":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"}}"},"optionsJSON":"{\"autoApplyFilters\":true,\"hidePanelBorders\":false,\"hidePanelTitles\":false,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"useMargins\":true}","panelsJSON":"[{\"type\":\"markdown\",\"embeddableConfig\":{\"hide_border\":true,\"content\":\"# Sourcerer indexing status\",\"settings\":{\"open_links_in_new_tab\":true}},\"panelIndex\":\"54f7337f-cbe3-4c82-82d6-aec205795654\",\"gridData\":{\"y\":0,\"x\":0,\"w\":18,\"h\":3,\"i\":\"54f7337f-cbe3-4c82-82d6-aec205795654\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"hide_title\":true,\"hide_border\":false},\"panelIndex\":\"015963bb-cbce-4f91-8bff-8a1aaddc6bbb\",\"gridData\":{\"y\":0,\"x\":18,\"w\":30,\"h\":26,\"i\":\"015963bb-cbce-4f91-8bff-8a1aaddc6bbb\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 2)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"67b82c98-6aff-42ab-b405-95412a3e9c58\",\"gridData\":{\"y\":3,\"x\":0,\"w\":3,\"h\":3,\"i\":\"67b82c98-6aff-42ab-b405-95412a3e9c58\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 1)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357\",\"gridData\":{\"y\":3,\"x\":3,\"w\":3,\"h\":3,\"i\":\"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 3)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d\",\"gridData\":{\"y\":3,\"x\":6,\"w\":3,\"h\":3,\"i\":\"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Files indexed\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"554c5394-bc05-44ca-84bb-95493167f04c\",\"gridData\":{\"y\":3,\"x\":9,\"w\":4,\"h\":3,\"i\":\"554c5394-bc05-44ca-84bb-95493167f04c\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"28f96a1f-5ba5-477f-bfb3-82ad6626994d\",\"gridData\":{\"y\":3,\"x\":13,\"w\":5,\"h\":3,\"i\":\"28f96a1f-5ba5-477f-bfb3-82ad6626994d\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"# refs by org/repo\",\"hide_border\":false},\"panelIndex\":\"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc\",\"gridData\":{\"y\":16,\"x\":0,\"w\":18,\"h\":10,\"i\":\"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"# lines of code by org/repo\",\"hide_border\":false},\"panelIndex\":\"d698f7cd-ee88-40a6-a826-a72826606507\",\"gridData\":{\"y\":6,\"x\":0,\"w\":18,\"h\":10,\"i\":\"d698f7cd-ee88-40a6-a826-a72826606507\"}}]","refreshInterval":{"pause":true,"value":1000},"timeFrom":"now-100y/d","timeRestore":true,"timeTo":"now","title":"[Sourcerer] Overview"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-11T15:14:23.909Z","id":"1bda5652-9556-4fc2-8ec1-d4e0c1e43eb7","managed":false,"references":[{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"},{"id":"85ea5355-8301-455d-aaca-2063ba93ceb0","name":"015963bb-cbce-4f91-8bff-8a1aaddc6bbb:savedObjectRef","type":"lens"},{"id":"3e86ded1-35d1-48cc-bae7-51338893a4cd","name":"67b82c98-6aff-42ab-b405-95412a3e9c58:savedObjectRef","type":"lens"},{"id":"22ff37d8-794e-4067-9e39-b3ebb47b5db4","name":"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357:savedObjectRef","type":"lens"},{"id":"286f4d43-d0c8-4cb6-935b-b2c2a3eeefe2","name":"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d:savedObjectRef","type":"lens"},{"id":"b674ebfd-a225-440c-bf40-a6c827e2bf1c","name":"554c5394-bc05-44ca-84bb-95493167f04c:savedObjectRef","type":"lens"},{"id":"10307b0e-7f80-4095-bcaa-14adc9c41fac","name":"28f96a1f-5ba5-477f-bfb3-82ad6626994d:savedObjectRef","type":"lens"},{"id":"853efb3c-c141-497a-9f99-dff8c6eaf90a","name":"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc:savedObjectRef","type":"lens"},{"id":"bb2c2db8-eda6-4127-ac97-89816eb00193","name":"d698f7cd-ee88-40a6-a826-a72826606507:savedObjectRef","type":"lens"}],"type":"dashboard","typeMigrationVersion":"10.3.0","updated_at":"2026-08-11T21:19:39.937Z","updated_by":"u_3477301940_cloud","version":"WzEyMjcsMTFd"} -{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Hosts","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.host"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# hosts indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-09T15:44:27.347Z","id":"gh0st0000-0000-0000-0000-000000000004","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"sort":[1786290267347,8589934683],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-09T15:44:27.347Z","version":"WzM5NiwxMF0="} +{"attributes":{"color":"#FCD883","description":"","name":"Sourcerer"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:46:50.062Z","id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","managed":false,"references":[],"type":"tag","typeMigrationVersion":"8.0.0","updated_at":"2026-08-20T13:46:50.062Z","version":"WzY0NDgsNF0="} +{"attributes":{"allowHidden":false,"fieldAttrs":"{}","fieldFormatMap":"{}","fields":"[]","name":"sourcerer-refs","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"git.commit_date","title":"sourcerer-refs"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:46:50.062Z","id":"b576da63-38d3-418c-a541-06f87ebef58f","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2026-08-20T13:46:50.062Z","version":"WzYwNCw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"72d88170-f38a-4d63-9d23-b48fedafe769":{"columnOrder":["gh0st0000-0000-0000-0000-000000000001","f2b9711d-0da6-4c79-8b66-f3176097b892","c07c2dd5-7d83-4924-ac44-9807d28cf11d","ff805016-2503-4100-ad05-e0cd8a29ead3","cb6b8976-ac47-4fc3-b887-e5988128269c","e6e8ddbe-0004-4982-8532-9f341343f74b","28c16acf-857e-41ae-9304-822350018ba6","e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b","5e6e2c2d-765b-4572-809c-e2f544371594","21887acc-0eac-4819-a90b-a6226cdcb3d7","a5fed915-466a-4b56-a0bd-6510f9b77a33"],"columns":{"21887acc-0eac-4819-a90b-a6226cdcb3d7":{"customLabel":true,"dataType":"date","filter":{"language":"kuery","query":"\"indexed_at\": *"},"isBucketed":false,"label":"Indexed at","operationType":"last_value","params":{"sortField":"git.commit_date"},"sourceField":"indexed_at"},"28c16acf-857e-41ae-9304-822350018ba6":{"customLabel":true,"dataType":"date","filter":{"language":"kuery","query":"\"git.commit_date\": *"},"isBucketed":false,"label":"Committed at","operationType":"last_value","params":{"showArrayValues":false,"sortField":"git.commit_date"},"sourceField":"git.commit_date"},"5e6e2c2d-765b-4572-809c-e2f544371594":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Lines","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"lines_count"},"a5fed915-466a-4b56-a0bd-6510f9b77a33":{"customLabel":true,"dataType":"string","filter":{"language":"kuery","query":"\"status\": *"},"isBucketed":false,"label":"Status","operationType":"last_value","params":{"sortField":"git.commit_date"},"sourceField":"status"},"c07c2dd5-7d83-4924-ac44-9807d28cf11d":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.repo"},"cb6b8976-ac47-4fc3-b887-e5988128269c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Type","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.ref_type"},"e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Files","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"files_count"},"e6e8ddbe-0004-4982-8532-9f341343f74b":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Commit","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.commit"},"f2b9711d-0da6-4c79-8b66-f3176097b892":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.org"},"ff805016-2503-4100-ad05-e0cd8a29ead3":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Ref","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.ref"},"gh0st0000-0000-0000-0000-000000000001":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"Host","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":10000},"sourceField":"git.host"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"columns":[{"columnId":"gh0st0000-0000-0000-0000-000000000001","isMetric":false,"isTransposed":false},{"columnId":"f2b9711d-0da6-4c79-8b66-f3176097b892","isMetric":false,"isTransposed":false},{"columnId":"5e6e2c2d-765b-4572-809c-e2f544371594","isMetric":true,"isTransposed":false},{"columnId":"a5fed915-466a-4b56-a0bd-6510f9b77a33","isMetric":true,"isTransposed":false},{"columnId":"e5490b7d-42a1-45a4-b5bd-99ab8ce3b48b","isMetric":true,"isTransposed":false},{"columnId":"c07c2dd5-7d83-4924-ac44-9807d28cf11d","isMetric":false,"isTransposed":false},{"columnId":"ff805016-2503-4100-ad05-e0cd8a29ead3","isMetric":false,"isTransposed":false},{"columnId":"cb6b8976-ac47-4fc3-b887-e5988128269c","isMetric":false,"isTransposed":false},{"columnId":"21887acc-0eac-4819-a90b-a6226cdcb3d7","isMetric":true,"isTransposed":false},{"columnId":"28c16acf-857e-41ae-9304-822350018ba6","isMetric":true,"isTransposed":false,"width":113},{"columnId":"e6e8ddbe-0004-4982-8532-9f341343f74b","isMetric":false,"isTransposed":false}],"layerId":"72d88170-f38a-4d63-9d23-b48fedafe769","layerType":"data","showRowNumbers":true,"sorting":{"columnId":"28c16acf-857e-41ae-9304-822350018ba6","direction":"desc"}}},"title":"Indexed repos","version":2,"visualizationType":"lnsDatatable"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:46:50.062Z","id":"85ea5355-8301-455d-aaca-2063ba93ceb0","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-72d88170-f38a-4d63-9d23-b48fedafe769","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:46:50.062Z","version":"WzYwNiw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Orgs","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.org"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"density":"compact","layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# orgs indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:49:10.261Z","created_by":"u_3477301940_cloud","id":"3e86ded1-35d1-48cc-bae7-51338893a4cd","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:49:10.261Z","updated_by":"u_3477301940_cloud","version":"WzYyMyw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Repos","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.repo"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"density":"compact","layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# repos indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:49:21.120Z","created_by":"u_3477301940_cloud","id":"22ff37d8-794e-4067-9e39-b3ebb47b5db4","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:49:21.120Z","updated_by":"u_3477301940_cloud","version":"WzYyNiw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Refs","operationType":"count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.ref"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"density":"compact","layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# refs indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:49:30.878Z","created_by":"u_3477301940_cloud","id":"286f4d43-d0c8-4cb6-935b-b2c2a3eeefe2","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:49:30.878Z","updated_by":"u_3477301940_cloud","version":"WzYyOCw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Files","operationType":"sum","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":2}}},"sourceField":"files_count"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"density":"compact","layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# files indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:49:51.503Z","created_by":"u_3477301940_cloud","id":"b674ebfd-a225-440c-bf40-a6c827e2bf1c","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:49:51.503Z","updated_by":"u_3477301940_cloud","version":"WzYzNSw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Lines","operationType":"sum","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":2}}},"sourceField":"lines_count"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"density":"compact","layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# lines indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:49:46.957Z","created_by":"u_3477301940_cloud","id":"10307b0e-7f80-4095-bcaa-14adc9c41fac","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:49:46.957Z","updated_by":"u_3477301940_cloud","version":"WzYzMyw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b":{"columnOrder":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1","db869f77-feda-4a5a-8dc6-233b8436591c"],"columns":{"1861256b-df4b-4807-9157-12ec9b7de6ce":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"db869f77-feda-4a5a-8dc6-233b8436591c","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.org"},"4d6e7a79-1fc4-414f-b16a-86ee89fed2a1":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"db869f77-feda-4a5a-8dc6-233b8436591c","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.repo"},"db869f77-feda-4a5a-8dc6-233b8436591c":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Refs per repo","operationType":"count","params":{"emptyAsNull":true},"sourceField":"git.ref"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"layers":[{"categoryDisplay":"default","colorMapping":{"assignments":[],"colorMode":{"type":"categorical"},"paletteId":"default","specialAssignments":[{"color":{"type":"loop"},"rules":[{"type":"other"}],"touched":false}]},"layerId":"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","layerType":"data","legendDisplay":"default","metrics":["db869f77-feda-4a5a-8dc6-233b8436591c"],"nestedLegend":false,"numberDisplay":"percent","primaryGroups":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1"]}],"shape":"treemap"}},"title":"# refs by org/repo","version":2,"visualizationType":"lnsPie"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:48:46.252Z","created_by":"u_3477301940_cloud","id":"853efb3c-c141-497a-9f99-dff8c6eaf90a","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:48:46.252Z","updated_by":"u_3477301940_cloud","version":"WzYxNyw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b":{"columnOrder":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1","6d93874d-ceab-4271-9347-b7e9613c4bbc"],"columns":{"1861256b-df4b-4807-9157-12ec9b7de6ce":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.org","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"6d93874d-ceab-4271-9347-b7e9613c4bbc","type":"column"},"orderDirection":"desc","otherBucket":false,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.org"},"4d6e7a79-1fc4-414f-b16a-86ee89fed2a1":{"dataType":"string","isBucketed":true,"label":"Top 50 values of git.repo","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"6d93874d-ceab-4271-9347-b7e9613c4bbc","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":50},"sourceField":"git.repo"},"6d93874d-ceab-4271-9347-b7e9613c4bbc":{"dataType":"number","isBucketed":false,"label":"Sum of lines_count","operationType":"sum","params":{"emptyAsNull":true},"sourceField":"lines_count"}},"ignoreGlobalFilters":false,"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":"status : \"complete\" "},"visualization":{"layers":[{"categoryDisplay":"default","colorMapping":{"assignments":[],"colorMode":{"type":"categorical"},"paletteId":"default","specialAssignments":[{"color":{"type":"loop"},"rules":[{"type":"other"}],"touched":false}]},"layerId":"1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","layerType":"data","legendDisplay":"default","metrics":["6d93874d-ceab-4271-9347-b7e9613c4bbc"],"nestedLegend":false,"numberDisplay":"percent","primaryGroups":["1861256b-df4b-4807-9157-12ec9b7de6ce","4d6e7a79-1fc4-414f-b16a-86ee89fed2a1"]}],"shape":"treemap"}},"title":"# lines by org/repo","version":2,"visualizationType":"lnsPie"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:48:58.003Z","created_by":"u_3477301940_cloud","id":"bb2c2db8-eda6-4127-ac97-89816eb00193","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-1a03a10b-da8b-4d5a-b1d8-7b0be11ab59b","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:48:58.003Z","updated_by":"u_3477301940_cloud","version":"WzYyMCw1XQ=="} +{"attributes":{"description":"","esqlApproximation":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"}}"},"optionsJSON":"{\"autoApplyFilters\":true,\"hidePanelBorders\":false,\"hidePanelTitles\":false,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"useMargins\":true}","panelsJSON":"[{\"type\":\"markdown\",\"embeddableConfig\":{\"hide_border\":true,\"content\":\"# Sourcerer indexing status\",\"settings\":{\"open_links_in_new_tab\":true}},\"panelIndex\":\"54f7337f-cbe3-4c82-82d6-aec205795654\",\"gridData\":{\"y\":0,\"x\":0,\"w\":18,\"h\":3,\"i\":\"54f7337f-cbe3-4c82-82d6-aec205795654\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"hide_title\":true,\"hide_border\":false},\"panelIndex\":\"015963bb-cbce-4f91-8bff-8a1aaddc6bbb\",\"gridData\":{\"y\":0,\"x\":18,\"w\":30,\"h\":26,\"i\":\"015963bb-cbce-4f91-8bff-8a1aaddc6bbb\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 2)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"67b82c98-6aff-42ab-b405-95412a3e9c58\",\"gridData\":{\"y\":3,\"x\":0,\"w\":3,\"h\":3,\"i\":\"67b82c98-6aff-42ab-b405-95412a3e9c58\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 1)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357\",\"gridData\":{\"y\":3,\"x\":3,\"w\":3,\"h\":3,\"i\":\"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Lines indexed (copy 3)\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d\",\"gridData\":{\"y\":3,\"x\":6,\"w\":3,\"h\":3,\"i\":\"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"Files indexed\",\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"554c5394-bc05-44ca-84bb-95493167f04c\",\"gridData\":{\"y\":3,\"x\":9,\"w\":4,\"h\":3,\"i\":\"554c5394-bc05-44ca-84bb-95493167f04c\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"hide_title\":true,\"hide_border\":true},\"panelIndex\":\"28f96a1f-5ba5-477f-bfb3-82ad6626994d\",\"gridData\":{\"y\":3,\"x\":13,\"w\":5,\"h\":3,\"i\":\"28f96a1f-5ba5-477f-bfb3-82ad6626994d\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"# refs by org/repo\",\"hide_border\":false},\"panelIndex\":\"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc\",\"gridData\":{\"y\":16,\"x\":0,\"w\":18,\"h\":10,\"i\":\"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc\"}},{\"type\":\"vis\",\"embeddableConfig\":{\"title\":\"# lines of code by org/repo\",\"hide_border\":false},\"panelIndex\":\"d698f7cd-ee88-40a6-a826-a72826606507\",\"gridData\":{\"y\":6,\"x\":0,\"w\":18,\"h\":10,\"i\":\"d698f7cd-ee88-40a6-a826-a72826606507\"}}]","refreshInterval":{"pause":true,"value":1000},"timeFrom":"now-100y/d","timeRestore":true,"timeTo":"now","title":"[Sourcerer] Overview"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:46:50.062Z","id":"1bda5652-9556-4fc2-8ec1-d4e0c1e43eb7","managed":false,"references":[{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"},{"id":"85ea5355-8301-455d-aaca-2063ba93ceb0","name":"015963bb-cbce-4f91-8bff-8a1aaddc6bbb:savedObjectRef","type":"lens"},{"id":"3e86ded1-35d1-48cc-bae7-51338893a4cd","name":"67b82c98-6aff-42ab-b405-95412a3e9c58:savedObjectRef","type":"lens"},{"id":"22ff37d8-794e-4067-9e39-b3ebb47b5db4","name":"ffdc070e-bfd4-4393-a7e8-5d3d08ab5357:savedObjectRef","type":"lens"},{"id":"286f4d43-d0c8-4cb6-935b-b2c2a3eeefe2","name":"9dbb81e9-eb63-4c75-ab0e-5f892a4e253d:savedObjectRef","type":"lens"},{"id":"b674ebfd-a225-440c-bf40-a6c827e2bf1c","name":"554c5394-bc05-44ca-84bb-95493167f04c:savedObjectRef","type":"lens"},{"id":"10307b0e-7f80-4095-bcaa-14adc9c41fac","name":"28f96a1f-5ba5-477f-bfb3-82ad6626994d:savedObjectRef","type":"lens"},{"id":"853efb3c-c141-497a-9f99-dff8c6eaf90a","name":"5bf5915e-b7f4-42b5-ac70-7c7496f5f4bc:savedObjectRef","type":"lens"},{"id":"bb2c2db8-eda6-4127-ac97-89816eb00193","name":"d698f7cd-ee88-40a6-a826-a72826606507:savedObjectRef","type":"lens"}],"type":"dashboard","typeMigrationVersion":"10.3.0","updated_at":"2026-08-20T13:49:53.424Z","updated_by":"u_3477301940_cloud","version":"WzYzNyw1XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"a36ee7d3-7e48-4943-8c52-c29fe9727322":{"columnOrder":["7fdfb9a4-2933-468d-9fbe-bd6d302edae5"],"columns":{"7fdfb9a4-2933-468d-9fbe-bd6d302edae5":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"Hosts","operationType":"unique_count","params":{"emptyAsNull":true,"format":{"id":"number","params":{"compact":true,"decimals":0}}},"sourceField":"git.host"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"layerId":"a36ee7d3-7e48-4943-8c52-c29fe9727322","layerType":"data","metricAccessor":"7fdfb9a4-2933-468d-9fbe-bd6d302edae5"}},"title":"# hosts indexed","version":2,"visualizationType":"lnsMetric"},"coreMigrationVersion":"8.8.0","created_at":"2026-08-20T13:46:50.062Z","id":"gh0st0000-0000-0000-0000-000000000004","managed":false,"references":[{"id":"b576da63-38d3-418c-a541-06f87ebef58f","name":"indexpattern-datasource-layer-a36ee7d3-7e48-4943-8c52-c29fe9727322","type":"index-pattern"},{"id":"7e815a6e-246c-44b8-b4e7-00d17196d62f","name":"tag-ref-7e815a6e-246c-44b8-b4e7-00d17196d62f","type":"tag"}],"type":"lens","typeMigrationVersion":"10.1.0","updated_at":"2026-08-20T13:46:50.062Z","version":"WzYxNCw1XQ=="} {"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":12,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/src/sourcerer/indices.py b/src/sourcerer/indices.py index 9147f2b..a24b7d7 100644 --- a/src/sourcerer/indices.py +++ b/src/sourcerer/indices.py @@ -6,11 +6,12 @@ # command's logic. # # v2 (multi-host): content index names carry a leading git.host segment, so the same org/repo on -# two different hosts lands in distinct backing indices. See sourcerer/hosts.py. +# two different hosts lands in distinct backing indices. See sourcerer/hosts.py. Bumped to v3 for +# incremental (ref-addressed) branch indexing. -FILES_INDEX_PREFIX = "sourcerer-v2-files" -LINES_INDEX_PREFIX = "sourcerer-v2-lines" -REFS_INDEX = "sourcerer-v2-refs" +FILES_INDEX_PREFIX = "sourcerer-v3-files" +LINES_INDEX_PREFIX = "sourcerer-v3-lines" +REFS_INDEX = "sourcerer-v3-refs" # Read aliases span all versioned backing indices of their respective kinds. Writes, updates, # and deletes deliberately use the physical names above so a future index version can coexist @@ -56,7 +57,7 @@ def files_index( commit: str | None = None, level: str = "repo", suffix: str | None = None, ) -> str: """Files index name for a source's `index.level`/`index.suffix`. The 3-arg call reproduces the - historical repo-level name, e.g. sourcerer-v2-files~github~elastic~elasticsearch.""" + historical repo-level name, e.g. sourcerer-v3-files~github~elastic~elasticsearch.""" return _content_index(FILES_INDEX_PREFIX, host, org, repo, commit, level, suffix) @@ -65,5 +66,5 @@ def lines_index( commit: str | None = None, level: str = "repo", suffix: str | None = None, ) -> str: """Lines index name for a source's `index.level`/`index.suffix`. The 3-arg call reproduces the - historical repo-level name, e.g. sourcerer-v2-lines~github~elastic~elasticsearch.""" + historical repo-level name, e.g. sourcerer-v3-lines~github~elastic~elasticsearch.""" return _content_index(LINES_INDEX_PREFIX, host, org, repo, commit, level, suffix) diff --git a/src/sourcerer/planner.py b/src/sourcerer/planner.py index 8fe737a..bb0fe95 100644 --- a/src/sourcerer/planner.py +++ b/src/sourcerer/planner.py @@ -175,15 +175,15 @@ def content_delete_set(decisions: list[Decision]) -> set[str]: # C. orphan marker -- a commit marker in refs with no content docs at all in either # content index (content manually deleted, or a whole content # index/repo/org vanished without its markers being cleaned up). -# -> delete_by_query on sourcerer-v2-refs. +# -> delete_by_query on sourcerer-v3-refs. # # Today the CLI only ever produces host~org~repo-granularity indices (see files_index/lines_index # in sourcerer/indices.py); parse_index_name and orphan_indices also recognize the host-only, # host~org, and host~org~repo~commit levels so detection keeps working if a future granularity is # introduced, even though only one level is exercised in practice right now. -_FILES_PREFIX_DEFAULT = "sourcerer-v2-files" -_LINES_PREFIX_DEFAULT = "sourcerer-v2-lines" +_FILES_PREFIX_DEFAULT = "sourcerer-v3-files" +_LINES_PREFIX_DEFAULT = "sourcerer-v3-lines" @dataclass(frozen=True) @@ -205,7 +205,7 @@ def parse_index_name( """Inverse of files_index()/lines_index() (sourcerer/indices.py), extended to also recognize the host-only, host~org, and host~org~repo~commit granularities those builders don't produce today, plus an optional trailing `^{suffix}` (index.suffix). Returns None for anything that - doesn't fit the scheme (sourcerer-v2-refs, an unrelated index, or a malformed/empty segment) so + doesn't fit the scheme (sourcerer-v3-refs, an unrelated index, or a malformed/empty segment) so callers skip it rather than risk misclassifying it as an orphan. The `^suffix` is split off BEFORE the `~` segments so it can't corrupt the last segment @@ -312,7 +312,7 @@ def orphan_markers( This single per-commit check also covers a whole repo/org's content having vanished entirely (every one of that repo's ref commits shows up as "missing"), so it subsumes what would otherwise be separate org- and repo-level refs sweeps -- one combined delete_by_query - against sourcerer-v2-refs handles all of it. `skip_repos` excludes repos already going away + against sourcerer-v3-refs handles all of it. `skip_repos` excludes repos already going away via a Class-A index DELETE (their markers are dropped as part of that, not here). Repo keys are (host, org, repo).""" out: dict[tuple[str, str, str], set[str]] = {} @@ -360,6 +360,42 @@ def orphan_stale_content( return out +def orphan_stale_incremental_content( + incremental_content_by_index: dict[str, set[tuple[str, str, str, str, str]]], + intended_incremental_index_by_ref: dict[tuple[str, str, str, str, str], set[str]], + skip_indices: set[str], +) -> dict[str, set[tuple[str, str, str, str, str]]]: + """Class D-I orphans: incremental content docs sitting in a physical index that the ref's + join doc no longer intends. This is the incremental migration backstop -- an index.level/suffix + change re-homes a ref's content to a new index and flips its join doc there; if a crash + happens before the old copy is deleted, the old-location docs survive with no join doc + referencing that location. + + `incremental_content_by_index` maps a physical index name -> the set of (host, org, repo, + ref_type, ref) tuples with incremental content docs in it. + `intended_incremental_index_by_ref` maps a (host, org, repo, ref_type, ref) tuple -> the set + of index names its join doc intends (reconstructed from index_level/index_suffix with + commit=None). `skip_indices` excludes indices already going away via a Class-A whole-index + DELETE. + + Returns {index_name -> set of (host, org, repo, ref_type, ref) tuples to delete-by-query + from that index}. A ref with NO join doc at all is not flagged here -- that is a different + category; this class is specifically 'has a join doc, but content lives somewhere the join + doc doesn't intend'.""" + out: dict[str, set[tuple[str, str, str, str, str]]] = {} + for index_name, ref_tuples in incremental_content_by_index.items(): + if index_name in skip_indices: + continue + for ref_tuple in ref_tuples: + intended = intended_incremental_index_by_ref.get(ref_tuple) + # No join doc for this ref at all -> different sweep; leave it alone. + if not intended: + continue + if index_name not in intended: + out.setdefault(index_name, set()).add(ref_tuple) + return out + + @dataclass class OrphanPlan: orphan_index_names: list[str] # Class A -> DELETE {index} @@ -374,6 +410,12 @@ class OrphanPlan: # identity still has markers at ~repo^b). Disjoint from orphan_index_names (Class A). Defaults # empty so callers/tests without empty-index data don't need to supply it. empty_index_names: list[str] = field(default_factory=list) + # Class D-I -> delete_by_query per index for incremental (ref-addressed, commit-less) content + # sitting in a physical index its ref's join doc no longer intends. The incremental mirror + # of Class D, since the commit-keyed Class-D sweep cannot see incremental docs. Value is + # {index_name -> set of (host, org, repo, ref_type, ref) tuples to reclaim from that index}. + # Defaults empty so callers/tests without incremental location data don't need to supply it. + orphan_stale_incremental: dict[str, set[tuple[str, str, str, str, str]]] = field(default_factory=dict) def plan_orphans( @@ -383,6 +425,8 @@ def plan_orphans( content_by_index_commit: dict[str, set[tuple[str, str, str, str]]] | None = None, intended_index_by_commit: dict[tuple[str, str, str, str], set[str]] | None = None, empty_index_names: list[str] | None = None, + incremental_content_by_index: dict[str, set[tuple[str, str, str, str, str]]] | None = None, + intended_incremental_index_by_ref: dict[tuple[str, str, str, str, str], set[str]] | None = None, ) -> OrphanPlan: """Combine the orphan classes into one plan from cheap snapshots: the physical index names, the distinct (host, org, repo, commit) tuples in refs, and the distinct (host, org, repo, commit) @@ -392,7 +436,13 @@ def plan_orphans( `content_by_index_commit` (index name -> content commit tuples in it) and `intended_index_by_commit` (commit tuple -> index names its markers intend) enable Class-D stale-location detection (the index.level/suffix migration backstop). When omitted, Class D is - empty -- back-compat for callers that don't supply per-index location data.""" + empty -- back-compat for callers that don't supply per-index location data. + + `incremental_content_by_index` (index name -> incremental ref tuples in it) and + `intended_incremental_index_by_ref` (ref tuple -> index names its join doc intends) enable + Class-D-I stale-location detection for incremental (ref-addressed, commit-less) content -- the + incremental mirror of Class D since the commit-keyed sweep can't see incremental docs. When + omitted, Class D-I is empty -- back-compat for callers without incremental location data.""" ref_orgs = {(host, org) for host, org, _, _ in ref_commit_tuples} ref_repos = {(host, org, repo) for host, org, repo, _ in ref_commit_tuples} @@ -422,8 +472,19 @@ def plan_orphans( content_by_index_commit, intended_index_by_commit, skip_indices=orphaned_names, ) + # Class D-I: stale-location incremental content (ref-addressed, no git.commit). Mirrors Class D + # but keyed on (host, org, repo, ref_type, ref) tuples from the ref's join doc. Also skip + # Class-A indices since they'll be deleted whole. + orphan_stale_incremental: dict[str, set[tuple[str, str, str, str, str]]] = {} + if incremental_content_by_index is not None and intended_incremental_index_by_ref is not None: + orphan_stale_incremental = orphan_stale_incremental_content( + incremental_content_by_index, intended_incremental_index_by_ref, + skip_indices=orphaned_names, + ) + # Class E: empty content indices. Exclude any already slated for a Class-A DELETE (an index # that is both empty AND identity-orphaned only needs to be deleted once). empty = [n for n in (empty_index_names or []) if n not in orphaned_names] - return OrphanPlan(orphan_index_names, orphan_content, orphan_marker_commits, orphan_stale, empty) + return OrphanPlan(orphan_index_names, orphan_content, orphan_marker_commits, orphan_stale, + empty, orphan_stale_incremental) diff --git a/src/sourcerer/progress.py b/src/sourcerer/progress.py index 7ef8b97..c91e935 100644 --- a/src/sourcerer/progress.py +++ b/src/sourcerer/progress.py @@ -40,7 +40,7 @@ class Unit: `ref` may be None until a default branch is resolved. `kind` is one of branch|tag|commit|default. `status` is set once on completion to one of - indexed|skipped|tagged|recorded|error. + indexed|skipped|no-changes|tagged|recorded|error. """ host: str @@ -67,6 +67,16 @@ class Unit: # unit's content docs are written to; defaults reproduce the historical repo-level name. index_level: str = "repo" index_suffix: str | None = None + # sources[i].mode carried from the selector that emitted this unit: "snapshot" (default, + # commit-addressed) or "delta" (ref-addressed, branch or tag). Routes the unit to the + # incremental delta-index path instead of the snapshot pre-clone/skip/retention flow. + mode: str = "snapshot" + # Stream identity for delta-tag moving streams. For a delta-mode tag selector whose `match` + # pattern covers many concrete tags (e.g. "deploy@{major}"), `ref_pattern` holds the literal + # pattern string and is stored as `git.ref_pattern` on all content and refs docs. `ref` + # advances to the resolved concrete tag post-clone. For all other units (branches, snapshots, + # concrete tags) `ref_pattern` == `ref` so the split is transparent to non-stream code paths. + ref_pattern: str | None = None @property def label(self) -> str: @@ -184,6 +194,8 @@ def _completion_text(self, unit: Unit) -> str: return f"✓ {unit.label} - tagged existing content ({counts})" if unit.status == "recorded": return f"✓ {unit.label} - content already indexed, recorded ref ({counts})" + if unit.status == "no-changes": + return f"• {unit.label} - no changes, skipped" if unit.status == "skipped": return f"• {unit.label} - already indexed, skipped" if unit.status == "error": @@ -203,7 +215,7 @@ def _summary_text(self) -> str: files = sum(u.files for u in self.units) lines = sum(u.lines for u in self.units) order = [("indexed", "indexed"), ("tagged", "tagged"), ("recorded", "recorded"), - ("skipped", "skipped"), ("error", "failed")] + ("skipped", "skipped"), ("no-changes", "no changes"), ("error", "failed")] parts = [f"{by[k]} {label}" for k, label in order if by.get(k)] body = ", ".join(parts) or "nothing to do" return f"Done in {format_elapsed(time.monotonic() - self.start_time)} - {body}; {files:,} files, {lines:,} lines" diff --git a/src/sourcerer/queries.py b/src/sourcerer/queries.py index 4c3a580..a9a6576 100644 --- a/src/sourcerer/queries.py +++ b/src/sourcerer/queries.py @@ -65,6 +65,44 @@ def fetch_markers( return out +def fetch_stale_markers(es: Elasticsearch) -> list[dict]: + """Return all refs docs with status="stale" across all repos. Each element is the raw + Elasticsearch hit dict (keys: _id, _source). Called by the prune command to reclaim snapshot + content that was switched to incremental mode (the flip-status switchover writes "stale" + markers before publishing the incremental join doc as "complete", so these never re-appear in + any content-tool query, which all gate on status=="complete").""" + try: + hits = [] + for hit in scan(es, index=REFS_ALIAS, + query={"query": {"term": {"status": "stale"}}}, + preserve_order=False): + hits.append(hit) + return hits + except NotFoundError: + return [] + + +def fetch_complete_commits_for_repo(es: Elasticsearch, host: str, org: str, repo: str) -> set[str]: + """Return the set of git.commit values referenced by any complete (non-stale) marker in this + repo. Used by the stale-marker reclamation step to apply the commit-safety guard: a commit + still held by a surviving complete marker must not have its content deleted.""" + try: + resp = es.search( + index=REFS_ALIAS, size=0, + query={"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"term": {"status": "complete"}}, + {"exists": {"field": "git.commit"}}, + ]}}, + aggs={"commits": {"terms": {"field": "git.commit", "size": 10000}}}, + ) + return {b["key"] for b in resp["aggregations"]["commits"]["buckets"]} + except NotFoundError: + return set() + + # --- Orphan sweep: ES-facing read helpers -------------------------------------------------- # The detection logic itself (orphan_indices/orphan_content_commits/orphan_markers/ # plan_orphans) is pure and lives in planner.py; these are the thin, mockable, READ-ONLY @@ -214,6 +252,105 @@ def gather_intended_index_by_commit( return out +def gather_intended_incremental_index_by_ref( + es: Elasticsearch, +) -> dict[tuple[str, str, str, str, str], set[str]]: + """For every (host, org, repo, ref_type, ref) with an incremental join doc, the set of + physical content index names that join doc intends -- reconstructed from its + index_level/index_suffix via files_index/lines_index with commit=None (incremental content + is ref-addressed, not commit-addressed). + + Feeds the incremental stale-location sweep (Class D-I): content for a ref sitting in an + index NOT in this set is stale from a crashed migration and should be reclaimed. Filters to + mode=="delta" docs only, so snapshot markers (which always have git.commit) are not + double-counted. Returns {} if the refs index doesn't exist.""" + out: dict[tuple[str, str, str, str, str], set[str]] = {} + body = {"query": {"term": {"mode": "delta"}}} + # `git.ref_pattern` is the stream identity that aligns with content docs' git.ref_pattern (the + # pattern for delta-tag streams, the branch name for delta branches). `git.ref` is the + # concrete resolved ref (e.g. newest tag) and is NOT the content key. Read `git.ref_pattern`. + src_fields = ["git.host", "git.org", "git.repo", "git.ref_type", "git.ref_pattern", + "index_level", "index_suffix"] + try: + for hit in scan(es, index=REFS_ALIAS, query=body, _source=src_fields, preserve_order=False): + src = hit["_source"] + g = src.get("git", {}) + host, org, repo = g.get("host"), g.get("org"), g.get("repo") + ref_type = g.get("ref_type") + # `git.ref_pattern` is the content-side identity (aligns with content docs' git.ref_pattern). + ref = g.get("ref_pattern") # nested under git + if not (host and org and repo and ref_type and ref): + continue + level = src.get("index_level") or "repo" + suffix = src.get("index_suffix") or None + key = (host, org, repo, ref_type, ref) + intended = out.setdefault(key, set()) + intended.add(files_index(host, org, repo, None, level, suffix)) + intended.add(lines_index(host, org, repo, None, level, suffix)) + except NotFoundError: + return {} + return out + + +def gather_incremental_content_by_index( + es: Elasticsearch, index_names: list[str], +) -> dict[str, set[tuple[str, str, str, str, str]]]: + """Per physical index, the distinct (host, org, repo, ref_type, ref) tuples with incremental + content docs in it (docs that have git.ref and a null/absent git.commit). + + Feeds the incremental stale-location sweep (Class D-I) in planner.orphan_stale_incremental_content: + to decide a doc is stale we must know WHICH physical index holds it AND which ref it belongs to, + so this enumerates each backing index by name via a composite aggregation over git.ref_type + + git.ref. Empty/missing indices contribute nothing.""" + out: dict[str, set[tuple[str, str, str, str, str]]] = {} + for name in index_names: + tuples = _composite_incremental_ref_tuples(es, name) + if tuples: + out[name] = tuples + return out + + +def _composite_incremental_ref_tuples( + es: Elasticsearch, index: str, +) -> set[tuple[str, str, str, str, str]]: + """Distinct (host, org, repo, ref_type, ref_pattern) tuples from incremental content docs + (git.ref_pattern present, git.commit absent) in `index`. Returns empty set if the index + doesn't exist.""" + out: set[tuple[str, str, str, str, str]] = set() + after: dict | None = None + while True: + composite: dict = { + "size": _COMPOSITE_PAGE_SIZE, + "sources": [ + {"host": {"terms": {"field": "git.host"}}}, + {"org": {"terms": {"field": "git.org"}}}, + {"repo": {"terms": {"field": "git.repo"}}}, + {"ref_type": {"terms": {"field": "git.ref_type"}}}, + {"ref_pattern": {"terms": {"field": "git.ref_pattern"}}}, + ], + } + if after is not None: + composite["after"] = after + # Filter to docs that have git.ref_pattern but no git.commit (incremental content). + query = {"bool": {"filter": [{"exists": {"field": "git.ref_pattern"}}], + "must_not": [{"exists": {"field": "git.commit"}}]}} + try: + resp = es.search(index=index, size=0, query=query, + aggs={"tuples": {"composite": composite}}) + except NotFoundError: + return out + agg = resp["aggregations"]["tuples"] + buckets = agg["buckets"] + if not buckets: + return out + for b in buckets: + out.add((b["key"]["host"], b["key"]["org"], b["key"]["repo"], + b["key"]["ref_type"], b["key"]["ref_pattern"])) + after = agg.get("after_key") + if after is None: + return out + + _FULL_SHA_LEN = 40 _MIN_PREFIX_LEN = 7 @@ -244,6 +381,170 @@ def content_indices_for_commit( return sorted(names) +def _enumerate_content_field( + es: Elasticsearch, host: str, org: str, repo: str, field: str, +) -> set[str]: + """Every distinct value of `field` (git.commit or git.ref) in this repo's content, + restricted to docs where that field IS NOT NULL, via paginated composite aggregation.""" + filters = [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"exists": {"field": field}}, + ] + out: set[str] = set() + for index in (FILES_ALIAS, LINES_ALIAS): + after: dict | None = None + while True: + composite: dict = { + "size": _COMPOSITE_PAGE_SIZE, + "sources": [{"val": {"terms": {"field": field}}}], + } + if after is not None: + composite["after"] = after + try: + resp = es.search( + index=index, size=0, + query={"bool": {"filter": filters}}, + aggs={"keys": {"composite": composite}}, + ) + except NotFoundError: + break + agg = resp["aggregations"]["keys"] + buckets = agg["buckets"] + if not buckets: + break + for b in buckets: + out.add(b["key"]["val"]) + after = agg.get("after_key") + if after is None: + break + return out + + +def _enumerate_incremental_content_ref_pairs( + es: Elasticsearch, host: str, org: str, repo: str, +) -> set[tuple[str, str]]: + """Every distinct (git.ref_type, git.ref_pattern) pair in this repo's incremental content docs + (docs that have git.ref_pattern and no git.commit), via paginated composite aggregation.""" + out: set[tuple[str, str]] = set() + for index in (FILES_ALIAS, LINES_ALIAS): + after: dict | None = None + while True: + composite: dict = { + "size": _COMPOSITE_PAGE_SIZE, + "sources": [ + {"ref_type": {"terms": {"field": "git.ref_type"}}}, + {"ref_pattern": {"terms": {"field": "git.ref_pattern"}}}, + ], + } + if after is not None: + composite["after"] = after + query = {"bool": { + "filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"exists": {"field": "git.ref_pattern"}}, + ], + "must_not": [{"exists": {"field": "git.commit"}}], + }} + try: + resp = es.search( + index=index, size=0, + query=query, + aggs={"pairs": {"composite": composite}}, + ) + except NotFoundError: + break + agg = resp["aggregations"]["pairs"] + buckets = agg["buckets"] + if not buckets: + break + for b in buckets: + out.add((b["key"]["ref_type"], b["key"]["ref_pattern"])) + after = agg.get("after_key") + if after is None: + break + return out + + +def check_join_uniqueness(es: Elasticsearch, host: str, org: str, repo: str) -> list[str]: + """Join-uniqueness gate: verifies every content key maps to a correct + refs join doc. Split by content shape (no `mode` on content docs): + + - Snapshot (git.commit IS NOT NULL): each commit must resolve to ≥1 complete refs doc + (presence check -- multi-ref-per-commit is legal; the snapshot FORK arm no longer joins + so the uniqueness requirement there is already removed). + - Incremental (git.ref IS NOT NULL): each ref must resolve to EXACTLY ONE refs doc with + `mode == "delta"` -- the anti-fan-out invariant for the surviving join. + + Returns the sorted list of offending keys (commits/refs that fail their respective check); + an empty list means the invariant holds.""" + offending: list[str] = [] + + # --- snapshot: each commit must have ≥1 complete (or in-progress) refs doc --- + # Accept "indexing" status as non-offending: an active run holding this commit is not stale. + commits = _enumerate_content_field(es, host, org, repo, "git.commit") + if commits: + try: + resp = es.search( + index=REFS_ALIAS, size=0, + query={"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"terms": {"git.commit": sorted(commits)}}, + {"terms": {"status": ["complete", "indexing"]}}, + ]}}, + aggs={"commits": {"terms": {"field": "git.commit", "size": len(commits)}}}, + ) + found_commits = {b["key"] for b in resp["aggregations"]["commits"]["buckets"]} + except NotFoundError: + found_commits = set() + offending.extend(sorted(commits - found_commits)) + + # --- incremental: each (ref_type, ref_pattern) pair must have EXACTLY ONE incremental join doc --- + # Content docs carry git.ref_pattern (= the stream identity) and git.ref_type; a same-named + # branch and tag are distinct (ref_type, ref_pattern) pairs, each allowed exactly one join doc. + # On the refs side the join key is git.ref_pattern (= the stream identity), NOT git.ref + # (which holds the concrete resolved ref for delta-tag streams). Filter and aggregate by + # (git.ref_type, git.ref_pattern) so the pair-counts align with the content-side key. + ref_pairs = _enumerate_incremental_content_ref_pairs(es, host, org, repo) + if ref_pairs: + try: + resp = es.search( + index=REFS_ALIAS, size=0, + query={"bool": {"filter": [ + {"term": {"git.host": host}}, + {"term": {"git.org": org}}, + {"term": {"git.repo": repo}}, + {"terms": {"git.ref_pattern": sorted({r for _, r in ref_pairs})}}, + {"term": {"mode": "delta"}}, + ]}}, + aggs={"ref_pairs": {"composite": {"size": 1000, "sources": [ + {"ref_type": {"terms": {"field": "git.ref_type"}}}, + {"ref_pattern": {"terms": {"field": "git.ref_pattern"}}}, + ]}}}, + ) + pair_counts = { + (b["key"]["ref_type"], b["key"]["ref_pattern"]): b["doc_count"] + for b in resp["aggregations"]["ref_pairs"]["buckets"] + } + except NotFoundError: + pair_counts = {} + offending.extend( + sorted(f"{rt}/{ref_pattern}" for rt, ref_pattern in ref_pairs if pair_counts.get((rt, ref_pattern), 0) != 1) + ) + + return sorted(offending) + + +# Legacy alias: used by tests and any external callers referencing the old name. +# Prefer check_join_uniqueness for new code. +check_ref_key_uniqueness = check_join_uniqueness + + def resolve_content_commit( es: Elasticsearch, host: str, org: str, repo: str, prefix: str, ) -> set[str]: diff --git a/src/sourcerer/skills/ref-resolution/SKILL.md b/src/sourcerer/skills/ref-resolution/SKILL.md index d116faf..6a423f1 100644 --- a/src/sourcerer/skills/ref-resolution/SKILL.md +++ b/src/sourcerer/skills/ref-resolution/SKILL.md @@ -36,10 +36,10 @@ Disambiguate based on context: - **Comparison or history** (e.g. "how has X evolved across 8.x?"): resolve to *all matching commits*. Collect every stable tag in the range; query each one separately. Label findings clearly by version. ### Comparison across versions (e.g. "8.x vs 9.x", "before and after 8.17") -Resolve each ref independently using the steps above. Run content queries against each commit, then compare results. Label each finding with its version. +Resolve each ref independently using the steps above. Run content queries against each resolved ref (see "Pinning the ref" below), then compare results. Label each finding with its version. ### Explicit ref (branch name, exact tag, commit hash) -Use as given. If it is a branch, call `refs.list` with `git_ref_type: branch` to confirm it exists and retrieve its current commit. If it is a tag, confirm and get its commit. If it is a commit hash, use it directly - optionally confirm with `git_ref_type: commit` if it may be a pinned commit rather than one reached via a branch/tag. +Use as given. If it is a branch, call `refs.list` with `git_ref_type: branch` to confirm it exists and retrieve its row. If it is a tag, confirm it the same way. If it is a commit hash, you don't need a `refs.list` call to use it in a content query (a commit hash is already a valid `git_commit_ish` value) - optionally confirm with `git_ref_type: commit` if it may be a pinned commit rather than one reached via a branch/tag. ### Branch as of a specific date (e.g. "main as it was on 2024-03-01") When a branch was indexed with `since` (history walk), multiple snapshots of the branch exist — @@ -47,10 +47,38 @@ one per historical commit. Resolve "branch as of date D" like this: 1. Call `refs.list` with `git_ref_type: branch` and `git_ref: `. 2. From the results, filter to markers with `commit_date <= D` and `status: complete`. 3. Pick the marker with the **latest** `commit_date` among those (the branch state at the closest point on or before D). -4. Use that marker's `git.commit` for content queries. +4. Use that row for content queries as described below. If only one marker exists for the branch (tip-only indexing, no `since`), state that historical snapshots are unavailable for that branch. -## Pinning the commit -Once a ref resolves to a `git.commit`, use that commit in every subsequent `sourcerer.code.*` and `sourcerer.files.*` call for that repo and ref. Re-invoke this skill only when the question introduces a new or additional ref. +## Pinning the ref +Every content query (`sourcerer.code.*` and `sourcerer.files.*`) takes the same scoping param, +`git_commit_ish`: a commit SHA, or a branch/tag name (`*` and `?` wildcards are supported). The +tool matches it against whichever field the content actually carries (`git.commit` for a +`snapshot` source, `git.ref` for an `incremental` branch) and then joins to resolve the citable +commit -- there is no mode you need to reason about, and no separate commit param to supply. + +The param is optional (default `"*"`, matching every indexed ref), but for a version-specific +question you should still resolve and pin a ref: an unpinned `"*"` query returns matches from +*every* ref at once, which blends versions. Results stay attributable (each row carries its +`git.commit`), so `"*"` is fine for "does this symbol exist anywhere" style questions -- just not +for "how does X behave in 8.17". + +Once a ref is resolved above, pass the value straight through: +- Resolved to a commit (tags, one-off branch snapshots -- the common case): use that commit SHA + as `git_commit_ish`. +- Resolved to an incremental branch (a source configured with `update: incremental` in + `sourcerer.yml`; its `refs.list` row has `status: complete`, same as a snapshot ref): use the branch name + itself as `git_commit_ish` (e.g. `main`) -- no commit needed, the query always resolves to whatever + commit that branch is CURRENTLY at. + +Internally, each tool uses a FORK that branches on content-doc shape to resolve the citable commit: +snapshot-shaped rows already carry `git.commit` (no join needed), while incremental-shaped rows get +their `git.commit` resolved via a `LOOKUP JOIN` on (`git.host`, `git.org`, `git.repo`, `git.ref`). + +Read the resolved `git.commit` back from each result row (the content query's own join supplies +it) for citations. Because incremental content overwrites in place, a branch query always returns +its current HEAD -- if you need to confirm a branch hasn't advanced since you resolved it (e.g. a +long-running investigation), re-check `sourcerer.refs.list` for its current commit. Re-invoke this +skill only when the question introduces a new or additional ref. \ No newline at end of file diff --git a/src/sourcerer/utils.py b/src/sourcerer/utils.py index 6617722..db023e7 100644 --- a/src/sourcerer/utils.py +++ b/src/sourcerer/utils.py @@ -17,6 +17,26 @@ ID_DIGEST_SIZE = 16 +def build_ref_key(host: str, org: str, repo: str, ref_type: str, ref: str) -> str: + """Deterministic `_id` string for incremental join docs: + `{host}~{org}~{repo}~{ref_type}~{ref}` (host/org/repo lowercased, ref case-preserved). + + `ref_type` is "branch" or "tag"; folding it in keeps a same-named branch and tag in delta + mode as two distinct join docs with non-overlapping id spaces (mirrors `build_ref_id` in + markers.py, which already includes ref_type for snapshot marker ids). + + Used exclusively as the Elasticsearch `_id` for the incremental refs join doc. Not a stored + field -- `git.ref_key` was removed from all index mappings and content builders. The string + is opaque to queries; the join uses `(git.host, git.org, git.repo, git.ref, git.ref_type)` + natively. + + `~` is safe as a delimiter because it is illegal in git ref names (see + `git check-ref-format`) and matches the index-name segment delimiter used for + host/org/repo elsewhere (see `indices.py`), so it cannot collide with any joined value. + """ + return "~".join((host.lower(), org.lower(), repo.lower(), ref_type, ref)) + + def make_doc_id(*parts: str) -> str: """Deterministic, URL-safe document id: BLAKE2b hex of the NUL-joined fields. diff --git a/tests/test_agent_builder_tools.py b/tests/test_agent_builder_tools.py index cfed713..a38d2bd 100644 --- a/tests/test_agent_builder_tools.py +++ b/tests/test_agent_builder_tools.py @@ -4,6 +4,7 @@ # Standard packages import importlib.resources as resources +import re # Third-party packages import pytest @@ -40,6 +41,88 @@ def test_git_host_filtered_before_git_org(): f"{tid} filters git.org before git.host" +_CONTENT_TOOL_IDS = ( + "sourcerer.code.search", "sourcerer.code.grep", + "sourcerer.files.cat", "sourcerer.files.head", "sourcerer.files.tail", + "sourcerer.files.ls", "sourcerer.files.tree", + "sourcerer.files.read_lines", "sourcerer.files.wc", +) + + +def test_content_tools_use_universal_ref_join_query(): + # Every content tool uses a two-OR'd-IN subquery to scope rows to matching + # refs (git.commit OR git.ref_pattern), then a FORK to resolve git.commit for both + # content shapes without fan-out: + # - Snapshot arm (git.commit IS NOT NULL): no join needed; git.commit already + # lives on the content row. + # - Incremental arm (git.ref_pattern IS NOT NULL AND git.commit IS NULL): clean same-name + # LOOKUP JOIN sourcerer-refs ON (host,org,repo,git.ref_pattern,ref_type) to resolve + # git.commit from the join doc. No RENAME needed because both sides share git.ref_pattern. + # No status guard is applied anywhere in the query. + # Ref scoping uses three separate params: git_commit, git_ref, git_ref_type. + tools = _tools() + for tid in _CONTENT_TOOL_IDS: + query = tools[tid]["configuration"]["query"] + params = tools[tid]["configuration"]["params"] + assert "index_strategy" not in query, f"{tid} query references the removed index_strategy field" + assert "mode" not in strip_esql_comments(query), f"{tid} query filters on the refs-only mode field" + # git.ref_key must not be used as a field or join key (comments may reference it by name) + assert "git.ref_key" not in query, f"{tid} still uses git.ref_key as a field" + assert "ON git.ref_key" not in query, f"{tid} still joins on git.ref_key" + # Old RENAME-based expression join must be gone + assert "RENAME git.ref AS _ref_id" not in query, \ + f"{tid} still has old RENAME git.ref AS _ref_id (replaced by git.ref_pattern join)" + assert "_ref_id == match" not in query, \ + f"{tid} still has old _ref_id == match expression join" + # The membership subquery uses two OR'd IN paths (one for snapshot commits, one for + # incremental refs), scoped by git_commit, git_ref, and git_ref_type params. + assert "git.commit LIKE ?git_commit" in query, f"{tid} missing git.commit LIKE ?git_commit" + assert "git.ref LIKE ?git_ref" in query, f"{tid} missing git.ref LIKE ?git_ref" + assert "git.ref_type LIKE ?git_ref_type" in query, f"{tid} missing git.ref_type LIKE ?git_ref_type" + # Snapshot arm: no join; git.commit already on the content row. + assert "git.commit IS NOT NULL" in query, f"{tid} missing snapshot FORK arm (git.commit IS NOT NULL)" + # Incremental arm: clean same-name LOOKUP JOIN on git.ref_pattern (no RENAME). + assert "git.ref_pattern IS NOT NULL" in query, \ + f"{tid} missing incremental FORK arm (git.ref_pattern IS NOT NULL)" + assert "LOOKUP JOIN sourcerer-refs ON git.host, git.org, git.repo, git.ref_pattern, git.ref_type" in query, \ + f"{tid} missing the new git.ref_pattern join on (host,org,repo,git.ref_pattern,git.ref_type)" + # No ref_key param or join shape. + assert "git_ref_key" not in params, f"{tid} still exposes git_ref_key as a param" + assert "?git_ref_key" not in query, f"{tid} still references ?git_ref_key" + # git_commit, git_ref, git_ref_type are all optional with a "*" default. + for p in ("git_commit", "git_ref", "git_ref_type"): + assert p in params, f"{tid} missing param {p}" + assert params[p]["optional"] is True, f"{tid} param {p} is not optional" + assert params[p]["defaultValue"] == "*", f"{tid} param {p} defaultValue != '*'" + # No collapsed git_commit_ish param. + assert "git_commit_ish" not in params, f"{tid} still exposes git_commit_ish as a param" + assert "?git_commit_ish" not in query, f"{tid} still references ?git_commit_ish" + # No status guard: the post-FORK status filter and status-related EVALs have been removed. + assert '| WHERE status == "complete"' not in query, f"{tid} still has the removed status guard" + assert 'EVAL status = "complete"' not in query, f"{tid} still has the removed EVAL status" + assert "status" not in strip_esql_comments(query), f"{tid} still references status in query body" + + +def test_refs_list_does_not_surface_ref_key(): + # git.ref_key is purely an internal storage/join key -- never something an agent reads or + # constructs -- so refs.list must not surface it. + tool = _tools()["sourcerer.refs.list"] + query = tool["configuration"]["query"] + for line in query.splitlines(): + if line.strip().startswith("| KEEP"): + assert "ref_key" not in line + + +def test_refs_list_default_status_surfaces_all_refs(): + # Incremental join docs now use status:"complete" (same as snapshot), so the default + # ?status == "complete" correctly surfaces all indexed refs without a special-case. + tool = _tools()["sourcerer.refs.list"] + query = tool["configuration"]["query"] + assert 'status == "ready"' not in query # special-case removed; no longer needed + assert "status LIKE ?status" in query + assert tool["configuration"]["params"]["status"]["defaultValue"] == "complete" + + def test_output_keeps_git_host(): # Every tool that KEEPs git.org must also KEEP git.host (before it), so host reaches output. for tid, tool in _tools().items(): @@ -51,6 +134,35 @@ def test_output_keeps_git_host(): assert stripped.index("git.host") < stripped.index("git.org") +def test_content_tool_aggregation_is_ref_scoped(): + # git_commit/git_ref/git_ref_type all default to "*", so a content query can match more than + # one ref at once. That is only safe if aggregation never blends refs: every STATS in a content + # tool must carry git.commit in its BY grouping key, so per-ref counts/bytes/line-blobs stay + # separate rather than being summed or interleaved across commits. Guards the files.ls-style + # regression where a `BY name` grouping silently summed file counts across every matching ref. + tools = _tools() + for tid in _CONTENT_TOOL_IDS: + query = tools[tid]["configuration"]["query"] + # Walk each STATS block: from a "| STATS" line through its trailing "BY ..." clause(s). + lines = query.splitlines() + for i, line in enumerate(lines): + if not line.strip().startswith("| STATS"): + continue + # Collect this STATS block's text up to the next pipe command. + block = [line] + for nxt in lines[i + 1:]: + if nxt.strip().startswith("|"): + break + block.append(nxt) + block_text = "\n".join(block) + assert " BY " in block_text, f"{tid} has a STATS with no BY grouping" + by_clause = block_text.split(" BY ", 1)[1] + assert "git.commit" in by_clause, ( + f"{tid} STATS groups without git.commit -- would blend refs when git_commit/git_ref " + f"params match more than one ref" + ) + + # --------------------------------------------------------------------------- # strip_esql_comments tests # --------------------------------------------------------------------------- diff --git a/tests/test_backfill.py b/tests/test_backfill.py new file mode 100644 index 0000000..17cb1f4 --- /dev/null +++ b/tests/test_backfill.py @@ -0,0 +1,120 @@ +"""Tests for mapping helpers and the stale-marker switchover in markers.py. + +The ref_key backfill subsystem (backfill_snapshot_ref_keys, backfill_refs_join_docs, +backfill_repo, etc.) has been removed along with git.ref_key. This file tests the +surviving apply_*_index_mapping helpers and the new mark_snapshot_markers_stale helper +that handles mode-switch from snapshot to incremental. +""" + +# Standard packages +from unittest.mock import MagicMock, call + +# Third-party packages +from elastic_transport import ApiResponseMeta, HttpHeaders +from elasticsearch import NotFoundError + +# App packages +from sourcerer.commands.index.markers import ( + apply_content_index_mapping, + apply_refs_index_mapping, + mark_snapshot_markers_stale, + stale_snapshot_markers_for_ref, +) +from sourcerer.indices import FILES_ALIAS, LINES_ALIAS, REFS_INDEX + + +def _not_found() -> NotFoundError: + meta = ApiResponseMeta(status=404, http_version="1.1", headers=HttpHeaders({}), duration=0.0, node=None) + return NotFoundError("index_not_found_exception", meta, None) + + +class TestApplyContentIndexMapping: + def test_puts_mapping_on_both_aliases(self): + es = MagicMock() + apply_content_index_mapping( + es, + {"properties": {"git": {"properties": {"ref": {"type": "keyword"}}}}}, + {"properties": {"git": {"properties": {"ref": {"type": "keyword"}}}}}, + ) + indices = {c.kwargs["index"] for c in es.indices.put_mapping.call_args_list} + assert indices == {FILES_ALIAS, LINES_ALIAS} + + def test_missing_index_is_ignored(self): + es = MagicMock() + es.indices.put_mapping.side_effect = _not_found() + apply_content_index_mapping(es, {"properties": {}}, {"properties": {}}) # no raise + + +class TestApplyRefsIndexMapping: + def test_puts_mapping_on_refs_index(self): + es = MagicMock() + apply_refs_index_mapping(es, {"properties": {"status": {"type": "keyword"}}}) + assert es.indices.put_mapping.call_args.kwargs["index"] == REFS_INDEX + + def test_missing_index_is_ignored(self): + es = MagicMock() + es.indices.put_mapping.side_effect = _not_found() + apply_refs_index_mapping(es, {"properties": {}}) # no raise + + +class TestStaleSnapshotMarkersForRef: + def test_returns_complete_non_incremental_markers(self): + """Returns complete markers that are mode=snapshot (i.e. snapshot markers).""" + es = MagicMock() + es.search.return_value = {"hits": {"hits": [ + {"_id": "abc123", "_source": {"git": {"commit": "deadbeef"}}}, + ]}} + hits = stale_snapshot_markers_for_ref(es, "github", "acme", "widgets", "main") + assert len(hits) == 1 + assert hits[0]["_id"] == "abc123" + + def test_missing_index_returns_empty(self): + es = MagicMock() + es.search.side_effect = _not_found() + assert stale_snapshot_markers_for_ref(es, "github", "acme", "widgets", "main") == [] + + def test_query_scopes_to_host_org_repo_ref(self): + es = MagicMock() + es.search.return_value = {"hits": {"hits": []}} + stale_snapshot_markers_for_ref(es, "github", "acme", "widgets", "main") + query = es.search.call_args.kwargs["query"] + filt = query["bool"]["filter"] + assert {"term": {"git.host": "github"}} in filt + assert {"term": {"git.org": "acme"}} in filt + assert {"term": {"git.repo": "widgets"}} in filt + assert {"term": {"git.ref_pattern": "main"}} in filt + assert {"term": {"status": "complete"}} in filt + assert {"term": {"mode": "snapshot"}} in filt + + +class TestMarkSnapshotMarkersStale: + def test_flips_each_marker_to_stale(self): + """mark_snapshot_markers_stale flips every found marker to status:'stale'.""" + es = MagicMock() + es.search.return_value = {"hits": {"hits": [ + {"_id": "marker1", "_source": {"git": {"commit": "aaa"}}}, + {"_id": "marker2", "_source": {"git": {"commit": "aaa"}}}, + ]}} + count = mark_snapshot_markers_stale(es, "github", "acme", "widgets", "main") + assert count == 2 + assert es.update.call_count == 2 + ids_updated = {c.kwargs["id"] for c in es.update.call_args_list} + assert ids_updated == {"marker1", "marker2"} + for c in es.update.call_args_list: + assert c.kwargs["doc"] == {"status": "stale"} + + def test_no_markers_returns_zero(self): + es = MagicMock() + es.search.return_value = {"hits": {"hits": []}} + assert mark_snapshot_markers_stale(es, "github", "acme", "widgets", "main") == 0 + es.update.assert_not_called() + + def test_update_not_found_is_ignored(self): + """A marker that disappears between the search and the update is not an error.""" + es = MagicMock() + es.search.return_value = {"hits": {"hits": [ + {"_id": "gone", "_source": {"git": {"commit": "aaa"}}}, + ]}} + es.update.side_effect = _not_found() + count = mark_snapshot_markers_stale(es, "github", "acme", "widgets", "main") + assert count == 1 # 1 found, even if the update race-lost diff --git a/tests/test_config.py b/tests/test_config.py index 83736aa..50e3c2c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -53,7 +53,7 @@ def _git(host="github", org="acme", repo="widgets", ref_type="branch"): def _source(host="github", org="acme", repo="widgets", ref_type="branch", - match="main", since=None, retain=None, omit_match=False): + match="main", since=None, retain=None, omit_match=False, mode=None, index=None): src = {"git": _git(host, org, repo, ref_type)} if not omit_match: src["match"] = match @@ -61,6 +61,10 @@ def _source(host="github", org="acme", repo="widgets", ref_type="branch", src["since"] = since if retain is not None: src["retain"] = retain + if mode is not None: + src["mode"] = mode + if index is not None: + src["index"] = index return src @@ -189,6 +193,65 @@ def test_versioned_patterns_agreeing_on_levels_is_fine(self): assert cfg.repos[0].selectors[0].levels == ("major", "minor", "patch") +class TestParseMode: + def test_default_is_snapshot(self): + cfg = _cfg([_source()]) + assert cfg.repos[0].selectors[0].mode == "snapshot" + + def test_delta_accepted_on_branch(self): + cfg = _cfg([_source(ref_type="branch", mode="delta")]) + assert cfg.repos[0].selectors[0].mode == "delta" + + def test_delta_accepted_on_tag(self): + cfg = _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta")]) + assert cfg.repos[0].selectors[0].mode == "delta" + + def test_delta_rejected_on_commit(self): + with pytest.raises(ValueError, match="only valid for git.ref_type: branch or tag"): + _cfg([_source(ref_type="commit", match="cfefb3b", mode="delta")]) + + def test_invalid_mode_raises(self): + with pytest.raises(ValueError, match="must be one of"): + _cfg([_source(mode="bogus")]) + + def test_delta_with_since_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'since'"): + _cfg([_source(ref_type="branch", mode="delta", since={"age": "1y"})]) + + def test_delta_tag_with_since_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'since'"): + _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta", since={"age": "1y"})]) + + def test_delta_with_retain_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'retain'"): + _cfg([_source(ref_type="branch", mode="delta", retain={"count": 5})]) + + def test_delta_tag_with_retain_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'retain'"): + _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta", retain={"count": 5})]) + + def test_delta_with_commit_level_index_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'index.level: commit'"): + _cfg([_source(ref_type="branch", mode="delta", index={"level": "commit"})]) + + def test_delta_tag_with_commit_level_index_raises(self): + with pytest.raises(ValueError, match="cannot be combined with 'index.level: commit'"): + _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta", index={"level": "commit"})]) + + def test_top_level_update_key_raises(self): + with pytest.raises(ValueError, match="unknown keys"): + _cfg([{"git": {"host": "github", "org": "acme", "repo": "widgets", "ref_type": "branch"}, + "match": "main", "update": "incremental"}]) + + def test_delta_with_repo_level_index_is_fine(self): + cfg = _cfg([_source(ref_type="branch", mode="delta", index={"level": "repo"})]) + assert cfg.repos[0].selectors[0].mode == "delta" + + def test_delta_tag_with_repo_level_index_is_fine(self): + cfg = _cfg([_source(ref_type="tag", match="v1.0.0", mode="delta", index={"level": "repo"})]) + assert cfg.repos[0].selectors[0].mode == "delta" + + class TestParseCommitSource: def test_full_sha_accepted(self): cfg = _cfg([_source(ref_type="commit", match="a" * 40)]) @@ -284,6 +347,41 @@ def test_commit_prefix_matches_full_sha(self): assert sel.matches("commit", "deadbeef" * 5) is None +class TestSelectorMatchPattern: + def test_versioned_tag_returns_raw_pattern_and_version(self): + sel = _cfg([_source(ref_type="tag", match="v{major}.{minor}.{patch}")]).repos[0].selectors[0] + result = sel.match_pattern("tag", "v1.2.3") + assert result is not None + pattern, v = result + assert pattern == "v{major}.{minor}.{patch}" + assert v.components == (1, 2, 3) + + def test_ref_type_mismatch_returns_none(self): + sel = _cfg([_source(ref_type="branch", match="main")]).repos[0].selectors[0] + assert sel.match_pattern("tag", "main") is None + + def test_no_match_returns_none(self): + sel = _cfg([_source(ref_type="tag", match="v{major}.{minor}.{patch}")]).repos[0].selectors[0] + assert sel.match_pattern("tag", "unrelated-tag") is None + + def test_commit_prefix_returns_prefix_and_version(self): + sha = "cfefb3b2378ccbadefa7c8f4f9e21b3a1d2e5f60" + sel = _cfg([_source(ref_type="commit", match="cfefb3b")]).repos[0].selectors[0] + result = sel.match_pattern("commit", sha) + assert result is not None + prefix, v = result + assert prefix == "cfefb3b" + assert v.ref == sha + + def test_first_matching_pattern_wins(self): + # When multiple patterns match, the first raw pattern is returned. + sel = _cfg([_source(match=["main", "dev"])]).repos[0].selectors[0] + result = sel.match_pattern("branch", "main") + assert result is not None + pattern, _ = result + assert pattern == "main" + + class TestSinceVersionFloor: def test_full_ref_name(self): cfg = _cfg([_source(ref_type="tag", match="v{major}.{minor}.{patch}", since={"ref": "v8.17.0"})]) diff --git a/tests/test_documents.py b/tests/test_documents.py index 0f29d41..5a4bc56 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -14,7 +14,9 @@ _build_one_file_actions, build_file_actions, build_file_doc, + build_incremental_file_doc, file_attributes, + iter_incremental_line_docs, iter_line_docs, ) from sourcerer.indices import files_index, lines_index @@ -27,7 +29,18 @@ def _set_worker_ctx(host: str, org: str, repo: str, commit_sha: str, repo_dir, s # should leave behind for the rest of the pytest session. documents._WORKER_CTX.update( host=host, org=org, repo=repo, commit_sha=commit_sha, repo_dir=pathlib.Path(repo_dir), - symlink_paths=symlink_paths, + symlink_paths=symlink_paths, mode="snapshot", + ) + + +def _set_worker_ctx_incremental(host: str, org: str, repo: str, repo_dir, + ref_pattern: str = "main", + symlink_paths=frozenset(), ref_type: str = "branch") -> None: + documents._WORKER_CTX.update( + host=host, org=org, repo=repo, ref_type=ref_type, + ref_pattern=ref_pattern, + repo_dir=pathlib.Path(repo_dir), + symlink_paths=symlink_paths, mode="delta", ) @@ -70,6 +83,7 @@ def test_git_fields(self, tmp_path): p.write_text("hello") _id, doc = build_file_doc("github", "acme", "widgets", "deadbeef", "a.txt", p) assert doc["git"] == {"host": "github", "org": "acme", "repo": "widgets", "commit": "deadbeef"} + assert "ref_key" not in doc["git"] def test_host_changes_id(self, tmp_path): p = tmp_path / "a.txt" @@ -120,6 +134,13 @@ def test_broken_symlink_has_target_path_but_no_target_size(self, tmp_path): class TestIterLineDocs: + def test_snapshot_no_ref_key(self): + # Snapshot line docs carry git.commit but no git.ref_key (field removed). + docs = list(iter_line_docs("github", "acme", "widgets", "deadbeef", "a.txt", "one")) + _id, doc = docs[0] + assert doc["git"]["commit"] == "deadbeef" + assert "ref_key" not in doc["git"] + def test_line_numbering_starts_at_one(self): docs = list(iter_line_docs("github", "acme", "widgets", "deadbeef", "a.txt", "one\ntwo\nthree")) numbers = [d["line"]["number"] for _id, d in docs] @@ -162,6 +183,92 @@ def test_no_optional_fields_when_omitted(self): assert "attributes" not in d["file"] +class TestIncrementalDocs: + def test_ref_pattern_set_no_ref_no_ref_key(self, tmp_path): + # Incremental docs carry git.ref_pattern (identity) and git.ref_type but NOT git.ref or + # git.ref_key. git.ref lives only on the refs join doc (written wholesale each run); + # content docs omit it to avoid staleness when unchanged paths survive a tag promotion. + p = tmp_path / "a.txt" + p.write_text("hello") + _id, doc = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) + assert doc["git"]["ref_pattern"] == "main" + assert doc["git"]["ref_type"] == "branch" + assert "ref" not in doc["git"] + assert "ref_key" not in doc["git"] + + def test_no_commit_field(self, tmp_path): + p = tmp_path / "a.txt" + p.write_text("hello") + _id, doc = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) + assert "commit" not in doc["git"] + + def test_id_stable_across_commits(self, tmp_path): + # The whole point of ref-addressing: the id does not depend on the commit, only the + # ref_type+ref_pattern, so a modified file's doc overwrites in place rather than minting a new id. + p = tmp_path / "a.txt" + p.write_text("hello") + id1, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) + p.write_text("hello world -- content changed, same ref/path") + id2, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) + assert id1 == id2 + + def test_id_stable_across_tag_promotions(self, tmp_path): + # For delta-tag streams: promoting from deploy@1 to deploy@2 must not mint a new doc id. + # Content _id is keyed on ref_pattern (stream identity), not the concrete tag. + p = tmp_path / "a.txt" + p.write_text("hello") + id1, _ = build_incremental_file_doc("github", "acme", "widgets", "tag", "deploy@{major}", "a.txt", p) + id2, _ = build_incremental_file_doc("github", "acme", "widgets", "tag", "deploy@{major}", "a.txt", p) + assert id1 == id2, "doc id must be keyed on ref_pattern, not concrete ref" + + def test_delta_content_has_ref_pattern_not_ref(self, tmp_path): + # Content docs carry git.ref_pattern (stream identity) but NOT git.ref (concrete). + # After a tag-stream promotion, unchanged paths are NOT rewritten (delta only touches + # changed paths), so git.ref on content would be inconsistent/stale across a single ref. + # The concrete git.ref lives only on the refs join doc (one doc, rewritten each run). + p = tmp_path / "a.txt" + p.write_text("hello") + _id, doc = build_incremental_file_doc("github", "acme", "widgets", "tag", "deploy@{major}", "a.txt", p) + assert "ref" not in doc["git"], "git.ref must be absent from delta content docs" + assert doc["git"]["ref_pattern"] == "deploy@{major}" + + def test_id_differs_from_snapshot_id(self, tmp_path): + p = tmp_path / "a.txt" + p.write_text("hello") + snap_id, _ = build_file_doc("github", "acme", "widgets", "deadbeef", "a.txt", p) + incr_id, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "main", "a.txt", p) + assert snap_id != incr_id + + def test_branch_and_tag_same_name_have_distinct_ids(self, tmp_path): + # A same-named branch and tag in delta mode must produce distinct content ids. + p = tmp_path / "a.txt" + p.write_text("hello") + branch_id, _ = build_incremental_file_doc("github", "acme", "widgets", "branch", "deploy", "a.txt", p) + tag_id, _ = build_incremental_file_doc("github", "acme", "widgets", "tag", "deploy", "a.txt", p) + assert branch_id != tag_id + + def test_line_docs_ref_pattern_no_ref_no_commit_no_ref_key(self): + # Incremental line docs carry git.ref_pattern and git.ref_type; git.ref and git.commit + # are absent for the same reason as file docs (see test_delta_content_has_ref_pattern_not_ref). + docs = list(iter_incremental_line_docs("github", "acme", "widgets", "branch", "main", "a.txt", "one\ntwo")) + for _id, d in docs: + assert d["git"]["ref_pattern"] == "main" + assert d["git"]["ref_type"] == "branch" + assert "ref" not in d["git"] + assert "commit" not in d["git"] + assert "ref_key" not in d["git"] + + def test_worker_ctx_routes_to_incremental_builders(self, tmp_path): + (tmp_path / "a.txt").write_text("one\ntwo\n") + _set_worker_ctx_incremental("github", "acme", "widgets", tmp_path, ref_pattern="main") + actions = _build_one_file_actions("a.txt") + assert actions[0]["_source"]["git"]["ref_pattern"] == "main" + assert actions[0]["_source"]["git"]["ref_type"] == "branch" + assert "ref" not in actions[0]["_source"]["git"] + assert "commit" not in actions[0]["_source"]["git"] + assert "ref_key" not in actions[0]["_source"]["git"] + + class TestFileAttributes: def test_plain_file_has_no_attributes(self, tmp_path): p = tmp_path / "a.txt" diff --git a/tests/test_git_changes.py b/tests/test_git_changes.py new file mode 100644 index 0000000..ea47fe4 --- /dev/null +++ b/tests/test_git_changes.py @@ -0,0 +1,166 @@ +"""Tests for the NUL-safe Git diff planner in sourcerer.commands.index.git. + +Two layers: + 1. Integration against a real temporary Git repository for the common change kinds (add, + modify, delete, rename, type change) and awkward path bytes (spaces, tabs, Unicode). + 2. Pure-parser tests over crafted `-z` byte streams for copy/rename records and truncation, + which are hard to elicit deterministically from git itself. +""" + +# Standard packages +import os +import subprocess + +# Third-party packages +import pytest + +# App packages +from sourcerer.commands.index.git import ( + ChangePlan, + _parse_name_status_z, + base_commit_available, + plan_changes, +) + + +def _git(repo, *args, env=None): + subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, env=env) + + +def _commit_all(repo, message): + env = { + **os.environ, + "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@e", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@e", + } + _git(repo, "add", "-A", env=env) + _git(repo, "commit", "-m", message, env=env) + out = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + check=True, capture_output=True, text=True, env=env, + ) + return out.stdout.strip() + + +@pytest.fixture +def repo(tmp_path): + _git(tmp_path, "init") + _git(tmp_path, "config", "user.email", "t@e") + _git(tmp_path, "config", "user.name", "t") + return tmp_path + + +class TestBaseCommitAvailable: + def test_plan_changes_base_available_for_present_commit(self, repo): + (repo / "a.txt").write_text("hi\n") + sha = _commit_all(repo, "init") + assert base_commit_available(repo, sha) is True + + def test_plan_changes_base_unavailable_for_absent_commit(self, repo): + (repo / "a.txt").write_text("hi\n") + _commit_all(repo, "init") + assert base_commit_available(repo, "0" * 40) is False + + +class TestPlanChangesIntegration: + def test_plan_changes_base_missing_returns_flagged_plan(self, repo): + (repo / "a.txt").write_text("hi\n") + new = _commit_all(repo, "init") + plan = plan_changes(repo, "0" * 40, new) + assert plan.base_missing is True + assert plan.delete_paths == [] and plan.index_paths == [] + + def test_plan_changes_add_modify_delete(self, repo): + (repo / "keep.txt").write_text("keep\n") + (repo / "gone.txt").write_text("gone\n") + (repo / "mod.txt").write_text("v1\n") + old = _commit_all(repo, "init") + (repo / "gone.txt").unlink() + (repo / "mod.txt").write_text("incremental\n") + (repo / "new.txt").write_text("new\n") + new = _commit_all(repo, "change") + plan = plan_changes(repo, old, new) + assert set(plan.index_paths) == {"mod.txt", "new.txt"} + assert set(plan.delete_paths) == {"gone.txt", "mod.txt"} + + def test_plan_changes_rename(self, repo): + (repo / "old_name.txt").write_text("stable content here\nline two\n") + old = _commit_all(repo, "init") + _git(repo, "mv", "old_name.txt", "new_name.txt") + new = _commit_all(repo, "rename") + plan = plan_changes(repo, old, new) + assert "old_name.txt" in plan.delete_paths + assert "new_name.txt" in plan.index_paths + assert "old_name.txt" not in plan.index_paths + + def test_plan_changes_type_change_file_to_symlink(self, repo): + (repo / "target.txt").write_text("target\n") + (repo / "thing").write_text("regular\n") + old = _commit_all(repo, "init") + (repo / "thing").unlink() + os.symlink("target.txt", repo / "thing") + new = _commit_all(repo, "typechange") + plan = plan_changes(repo, old, new) + # A type change replaces the whole file: delete the stale docs and re-index. + assert "thing" in plan.delete_paths + assert "thing" in plan.index_paths + + def test_plan_changes_awkward_paths_preserved(self, repo): + (repo / "seed.txt").write_text("seed\n") + old = _commit_all(repo, "init") + weird = "dir with spaces/tab\tfile.txt" + unicode_path = "café/naïve.txt" + (repo / "dir with spaces").mkdir() + (repo / weird).write_text("x\n") + (repo / "café").mkdir() + (repo / unicode_path).write_text("y\n") + new = _commit_all(repo, "add weird paths") + plan = plan_changes(repo, old, new) + assert weird in plan.index_paths + assert unicode_path in plan.index_paths + + +class TestParseNameStatusZ: + def test_plan_changes_parser_copy_indexes_destination_only(self): + raw = b"C100\x00src.txt\x00dst.txt\x00" + delete, index = _parse_name_status_z(raw) + assert delete == [] + assert index == ["dst.txt"] + + def test_plan_changes_parser_rename_deletes_source_indexes_destination(self): + raw = b"R100\x00from.txt\x00to.txt\x00" + delete, index = _parse_name_status_z(raw) + assert delete == ["from.txt"] + assert index == ["to.txt"] + + def test_plan_changes_parser_mixed_stream(self): + raw = ( + b"A\x00added.txt\x00" + b"M\x00mod.txt\x00" + b"D\x00del.txt\x00" + b"R096\x00old.txt\x00new.txt\x00" + ) + delete, index = _parse_name_status_z(raw) + assert delete == ["mod.txt", "del.txt", "old.txt"] + assert index == ["added.txt", "mod.txt", "new.txt"] + + def test_plan_changes_parser_dedupe_preserves_order(self): + raw = b"M\x00a.txt\x00M\x00a.txt\x00" + delete, index = _parse_name_status_z(raw) + assert delete == ["a.txt"] + assert index == ["a.txt"] + + def test_plan_changes_parser_truncated_rename_record_ignored(self): + raw = b"R100\x00only_one_path.txt\x00" + delete, index = _parse_name_status_z(raw) + assert delete == [] and index == [] + + def test_plan_changes_parser_empty_stream(self): + assert _parse_name_status_z(b"") == ([], []) + + +class TestChangePlanDefaults: + def test_plan_changes_defaults(self): + plan = ChangePlan() + assert plan.delete_paths == [] and plan.index_paths == [] + assert plan.base_missing is False diff --git a/tests/test_incremental_index.py b/tests/test_incremental_index.py new file mode 100644 index 0000000..24cc7df --- /dev/null +++ b/tests/test_incremental_index.py @@ -0,0 +1,396 @@ +"""Tests for the incremental (ref-addressed) branch orchestration in +sourcerer.commands.index.command.index_incremental_branch_in_dir: the two-phase +indexing -> complete publication, full rebuild vs delta update selection, and failure handling. +Every ES call and every git/documents side effect is mocked/patched -- these are orchestration +tests, not an end-to-end index run (see specs/incremental-indexing.md Task 16 for that).""" + +# Standard packages +from unittest.mock import MagicMock, patch + +# App packages +from sourcerer.commands.index.command import index_incremental_branch_in_dir +from sourcerer.commands.index.git import ChangePlan +from sourcerer.progress import ProgressReporter, Unit + +OLD = "1111111111111111111111111111111111111111" +NEW = "2222222222222222222222222222222222222222" + + +def _patch_common(prior=None, plan=None, ref_dates_return=None): + """Patch every git/documents/markers side effect index_incremental_branch_in_dir calls, + returning the patcher context managers as a dict of MagicMocks keyed by name. + + ref_dates_return: dict for ref_dates mock; default {} (branches never call ref_dates). + For tag stream tests pass e.g. {("tag", "deploy@1788"): 1788, ("tag", "deploy@1787"): 1787}. + """ + patchers = { + "checkout_branch": patch("sourcerer.commands.index.command.checkout_branch"), + "checkout_ref": patch("sourcerer.commands.index.command.checkout_ref"), + "ref_dates": patch("sourcerer.commands.index.command.ref_dates", + return_value=ref_dates_return if ref_dates_return is not None else {}), + "resolve_commit": patch("sourcerer.commands.index.command.resolve_commit", return_value=NEW), + "commit_date": patch("sourcerer.commands.index.command.commit_date", return_value="2026-01-01T00:00:00+00:00"), + "read_incremental_ref": patch("sourcerer.commands.index.command.read_incremental_ref", return_value=prior), + "plan_changes": patch("sourcerer.commands.index.command.plan_changes", return_value=plan or ChangePlan()), + "delete_incremental_branch": patch("sourcerer.commands.index.command.delete_incremental_branch"), + "delete_incremental_paths": patch("sourcerer.commands.index.command.delete_incremental_paths"), + "index_incremental_paths": patch("sourcerer.commands.index.command.index_incremental_paths", return_value=(3, 30)), + "count_tracked_files": patch("sourcerer.commands.index.command.count_tracked_files", return_value=3), + "refresh_incremental_content": patch("sourcerer.commands.index.command.refresh_incremental_content"), + "count_incremental_branch_docs": patch( + "sourcerer.commands.index.command.count_incremental_branch_docs", return_value=(3, 30) + ), + "write_incremental_indexing": patch("sourcerer.commands.index.command.write_incremental_indexing"), + "write_incremental_ready": patch("sourcerer.commands.index.command.write_incremental_ready"), + "write_incremental_failed": patch("sourcerer.commands.index.command.write_incremental_failed"), + } + mocks = {name: p.start() for name, p in patchers.items()} + return patchers, mocks + + +def _stop(patchers): + for p in patchers.values(): + p.stop() + + +class TestIncrementalIndexFirstRun: + def test_first_index_does_full_rebuild(self): + patchers, mocks = _patch_common(prior=None) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + mode="delta") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + # Branch: uses checkout_branch, NOT checkout_ref. + mocks["checkout_branch"].assert_called_once() + mocks["checkout_ref"].assert_not_called() + mocks["delete_incremental_branch"].assert_called_once() + mocks["index_incremental_paths"].assert_called_once() + # rel_paths is None -> full tree walk. Signature: (es,host,org,repo,repo_dir,ref_pattern,rel_paths,...). + call_args = mocks["index_incremental_paths"].call_args + assert call_args[0][6] is None + mocks["delete_incremental_paths"].assert_not_called() + mocks["write_incremental_ready"].assert_called_once() + # write_incremental_ready(es, host, org, repo, ref_type, ref, commit, ...) + # ref_type at [4], ref at [5], commit (sha) at [6] + assert mocks["write_incremental_ready"].call_args[0][6] == NEW + finally: + _stop(patchers) + + +class TestIncrementalIndexDeltaRun: + def test_second_run_indexes_only_changed_paths(self): + prior = {"git": {"commit": OLD}} + plan = ChangePlan(delete_paths=["gone.txt"], index_paths=["new.txt"]) + patchers, mocks = _patch_common(prior=prior, plan=plan) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + mode="delta") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + mocks["delete_incremental_branch"].assert_not_called() + mocks["delete_incremental_paths"].assert_called_once() + # delete_incremental_paths(es, host, org, repo, ref_type, ref_pattern, paths, ...) + # ref_type at [4], ref_pattern at [5], paths at [6] + assert mocks["delete_incremental_paths"].call_args[0][6] == ["gone.txt"] + mocks["index_incremental_paths"].assert_called_once() + call_args = mocks["index_incremental_paths"].call_args + # index_incremental_paths(es, host, org, repo, repo_dir, ref_pattern, rel_paths, ...) + assert call_args[0][6] == ["new.txt"] + mocks["write_incremental_ready"].assert_called_once() + finally: + _stop(patchers) + + def test_missing_diff_base_triggers_full_rebuild(self): + prior = {"git": {"commit": OLD}} + plan = ChangePlan(base_missing=True) + patchers, mocks = _patch_common(prior=prior, plan=plan) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + mode="delta") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + mocks["delete_incremental_branch"].assert_called_once() + call_args = mocks["index_incremental_paths"].call_args + assert call_args[0][6] is None + finally: + _stop(patchers) + + def test_no_change_skips_entirely(self): + prior = {"git": {"commit": NEW}} # already at the new (checked-out) commit + patchers, mocks = _patch_common(prior=prior) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + mode="delta") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + mocks["index_incremental_paths"].assert_not_called() + mocks["write_incremental_indexing"].assert_not_called() + mocks["write_incremental_ready"].assert_not_called() + assert unit.status == "no-changes" + finally: + _stop(patchers) + + +class TestIncrementalIndexFailure: + def test_failed_run_does_not_advance_commit(self): + prior = {"git": {"commit": OLD}} + plan = ChangePlan(delete_paths=[], index_paths=["a.txt"]) + patchers, mocks = _patch_common(prior=prior, plan=plan) + mocks["index_incremental_paths"].side_effect = RuntimeError("bulk failed") + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + mode="delta") + try: + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + assert False, "expected RuntimeError to propagate" + except RuntimeError: + pass + mocks["write_incremental_ready"].assert_not_called() + mocks["write_incremental_failed"].assert_called_once() + # The completed pointer stays at OLD -- a failed run must not advance it. + assert mocks["write_incremental_failed"].call_args.kwargs["completed_commit"] == OLD + finally: + _stop(patchers) + + +class TestIncrementalIndexRoutingMigration: + """When index.level or index.suffix changes on an already-indexed branch the run must: + 1. NOT return no-changes even when the commit hasn't advanced. + 2. Do a full rebuild into the NEW routing. + 3. After the ready marker flips, delete the old-routing copy (write-new -> flip -> delete-old). + """ + + def _prior_at_routing(self, level="repo", suffix=None): + """A completed incremental join doc recorded at the given routing.""" + doc = {"git": {"commit": NEW}, "index_level": level} + if suffix is not None: + doc["index_suffix"] = suffix + return doc + + def test_suffix_change_forces_full_rebuild_and_old_copy_delete(self): + """repo -> repo^deploy: full rebuild at new routing, then delete at old routing.""" + prior = self._prior_at_routing(level="repo", suffix=None) + patchers, mocks = _patch_common(prior=prior) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + mode="delta", index_level="repo", index_suffix="deploy") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + # Full rebuild path: delete_incremental_branch called at new routing, full tree indexed. + assert mocks["delete_incremental_branch"].call_count == 2, ( + "Expected 2 calls to delete_incremental_branch: one for new routing (rebuild), " + "one for old routing (migration cleanup)" + ) + call_kwargs_list = [c.kwargs for c in mocks["delete_incremental_branch"].call_args_list] + # First call: full rebuild at new (repo^deploy) routing. + assert call_kwargs_list[0].get("index_level") == "repo" + assert call_kwargs_list[0].get("index_suffix") == "deploy" + # Second call: delete old (repo, no suffix) routing. + assert call_kwargs_list[1].get("index_level") == "repo" + assert call_kwargs_list[1].get("index_suffix") is None + # Ready marker was published. + mocks["write_incremental_ready"].assert_called_once() + finally: + _stop(patchers) + + def test_level_change_forces_full_rebuild_and_old_copy_delete(self): + """repo -> org level: full rebuild at org routing, then delete at repo routing.""" + prior = self._prior_at_routing(level="repo", suffix=None) + patchers, mocks = _patch_common(prior=prior) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + mode="delta", index_level="org", index_suffix=None) + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + assert mocks["delete_incremental_branch"].call_count == 2 + call_kwargs_list = [c.kwargs for c in mocks["delete_incremental_branch"].call_args_list] + assert call_kwargs_list[0].get("index_level") == "org" + assert call_kwargs_list[1].get("index_level") == "repo" + mocks["write_incremental_ready"].assert_called_once() + finally: + _stop(patchers) + + def test_no_changes_with_routing_change_still_migrates(self): + """Commit unchanged but routing changed: must NOT return no-changes; must migrate.""" + # prior already at NEW sha, but at old routing + prior = self._prior_at_routing(level="repo", suffix=None) + prior["git"]["commit"] = NEW # same commit as what resolve_commit returns + patchers, mocks = _patch_common(prior=prior) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + mode="delta", index_level="repo", index_suffix="v2") + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + # Must NOT skip even though old_sha == new_sha. + mocks["write_incremental_indexing"].assert_called_once() + mocks["write_incremental_ready"].assert_called_once() + # Old routing must be cleaned up. + assert mocks["delete_incremental_branch"].call_count == 2 + finally: + _stop(patchers) + + def test_same_routing_no_old_copy_delete(self): + """When routing is unchanged a delta run must not call delete_incremental_branch at all.""" + prior = self._prior_at_routing(level="repo", suffix=None) + plan = ChangePlan(delete_paths=[], index_paths=["changed.txt"]) + patchers, mocks = _patch_common(prior=prior, plan=plan) + try: + es = MagicMock() + unit = Unit(host="github", org="acme", repo="widgets", ref="main", kind="branch", + mode="delta", index_level="repo", index_suffix=None) + index_incremental_branch_in_dir(es, "github", "acme", "widgets", "/repo", "main", + reporter=ProgressReporter(), unit=unit) + # Delta run: no full rebuild (delete_incremental_branch not called), no extra delete. + mocks["delete_incremental_branch"].assert_not_called() + finally: + _stop(patchers) + + +class TestIncrementalIndexTagFirstRun: + """Delta-mode tag stream orchestration tests. + + A tag stream Unit carries `ref = ` (e.g. "deploy@{major}") as its stable + identity; ref_dates() resolves the newest concrete tag for checkout. All marker/content calls + receive the pattern; only checkout_ref receives the concrete tag name. + """ + + # Two fake matching tags: 1788 is newer than 1787. + TAG_DATES = {("tag", "deploy@1788000000"): 1788000000, ("tag", "deploy@1787000000"): 1787000000} + NEWEST_TAG = "deploy@1788000000" + PATTERN = "deploy@{major}" + + def test_tag_first_index_uses_checkout_ref_with_newest_tag(self): + """First run: checkout_ref called with the NEWEST concrete tag; all stored refs use pattern.""" + patchers, mocks = _patch_common(prior=None, ref_dates_return=self.TAG_DATES) + try: + es = MagicMock() + unit = Unit(host="github", org="elastic", repo="kibana", + ref=self.PATTERN, kind="tag", mode="delta") + index_incremental_branch_in_dir( + es, "github", "elastic", "kibana", "/repo", self.PATTERN, + reporter=ProgressReporter(), unit=unit, + ) + # Tag stream: checkout_ref called with the newest concrete tag, not the pattern. + mocks["checkout_ref"].assert_called_once_with("/repo", self.NEWEST_TAG) + mocks["checkout_branch"].assert_not_called() + mocks["delete_incremental_branch"].assert_called_once() + mocks["index_incremental_paths"].assert_called_once() + # Full tree walk: rel_paths arg is None. Signature: (..., ref_pattern, rel_paths, ...) + assert mocks["index_incremental_paths"].call_args[0][6] is None + mocks["write_incremental_ready"].assert_called_once() + # write_incremental_ready(es, host, org, repo, ref_type, ref, commit, ...) + assert mocks["write_incremental_ready"].call_args[0][6] == NEW + finally: + _stop(patchers) + + def test_tag_stream_concrete_ref_and_match_pattern(self): + """refs join doc: git.ref = concrete tag (newest match), git.ref_pattern = pattern. + Content docs keyed on ref_pattern (pattern); only the refs join-doc git.ref advances.""" + patchers, mocks = _patch_common(prior=None, ref_dates_return=self.TAG_DATES) + try: + es = MagicMock() + unit = Unit(host="github", org="elastic", repo="kibana", + ref=self.PATTERN, kind="tag", mode="delta") + index_incremental_branch_in_dir( + es, "github", "elastic", "kibana", "/repo", self.PATTERN, + reporter=ProgressReporter(), unit=unit, + ) + ready_args = mocks["write_incremental_ready"].call_args[0] + ready_kwargs = mocks["write_incremental_ready"].call_args[1] + # write_incremental_ready(es, host, org, repo, ref_type, ref, sha, ..., ref_pattern=) + assert ready_args[4] == "tag" + # git.ref payload = the CONCRETE newest tag. + assert ready_args[5] == self.NEWEST_TAG, ( + f"git.ref must be the concrete tag '{self.NEWEST_TAG}', got {ready_args[5]!r}" + ) + # git.ref_pattern = the stable PATTERN (stream identity). + assert ready_kwargs.get("ref_pattern") == self.PATTERN, ( + f"ref_pattern kwarg must be the pattern '{self.PATTERN}', got {ready_kwargs.get('ref_pattern')!r}" + ) + # unit.ref is updated to the concrete tag for reporter display. + assert unit.ref == self.NEWEST_TAG, ( + f"unit.ref must advance to concrete tag '{self.NEWEST_TAG}', got {unit.ref!r}" + ) + # Content calls: arg[5] = ref_pattern = pattern (stream identity). The concrete ref is + # NOT passed to index_incremental_paths -- content docs carry only git.ref_pattern; + # git.ref (concrete) lives only on the refs join doc (write_incremental_ready above). + # index_incremental_paths(es, host, org, repo, repo_dir, ref_pattern, rel_paths, ...) + ref_pattern_arg = mocks["index_incremental_paths"].call_args[0][5] + assert ref_pattern_arg == self.PATTERN, ( + f"index_incremental_paths ref_pattern (arg 5) must be pattern '{self.PATTERN}', got {ref_pattern_arg!r}" + ) + finally: + _stop(patchers) + + def test_tag_delta_run_indexes_only_changed_paths(self): + """Second run: delta diff applied, full rebuild skipped; still uses pattern identity.""" + prior = {"git": {"commit": OLD}} + plan = ChangePlan(delete_paths=["gone.ts"], index_paths=["new.ts"]) + patchers, mocks = _patch_common(prior=prior, plan=plan, ref_dates_return=self.TAG_DATES) + try: + es = MagicMock() + unit = Unit(host="github", org="elastic", repo="kibana", + ref=self.PATTERN, kind="tag", mode="delta") + index_incremental_branch_in_dir( + es, "github", "elastic", "kibana", "/repo", self.PATTERN, + reporter=ProgressReporter(), unit=unit, + ) + mocks["checkout_ref"].assert_called_once_with("/repo", self.NEWEST_TAG) + mocks["checkout_branch"].assert_not_called() + mocks["delete_incremental_branch"].assert_not_called() + mocks["delete_incremental_paths"].assert_called_once() + # delete_incremental_paths(es, host, org, repo, ref_type, ref_pattern, paths, ...) + assert mocks["delete_incremental_paths"].call_args[0][6] == ["gone.ts"] + mocks["index_incremental_paths"].assert_called_once() + # index_incremental_paths(es, host, org, repo, repo_dir, ref_pattern, rel_paths, ...) + assert mocks["index_incremental_paths"].call_args[0][6] == ["new.ts"] + mocks["write_incremental_ready"].assert_called_once() + finally: + _stop(patchers) + + def test_tag_missing_diff_base_triggers_full_rebuild(self): + """Newest tag's diff base gone → base_missing → full rebuild.""" + prior = {"git": {"commit": OLD}} + plan = ChangePlan(base_missing=True) + patchers, mocks = _patch_common(prior=prior, plan=plan, ref_dates_return=self.TAG_DATES) + try: + es = MagicMock() + unit = Unit(host="github", org="elastic", repo="kibana", + ref=self.PATTERN, kind="tag", mode="delta") + index_incremental_branch_in_dir( + es, "github", "elastic", "kibana", "/repo", self.PATTERN, + reporter=ProgressReporter(), unit=unit, + ) + mocks["checkout_ref"].assert_called_once_with("/repo", self.NEWEST_TAG) + mocks["delete_incremental_branch"].assert_called_once() + assert mocks["index_incremental_paths"].call_args[0][6] is None + finally: + _stop(patchers) + + def test_tag_no_matching_tags_in_clone_skips(self): + """If ref_dates returns no tags matching the pattern, the unit is skipped gracefully.""" + patchers, mocks = _patch_common(prior=None, ref_dates_return={}) + try: + es = MagicMock() + unit = Unit(host="github", org="elastic", repo="kibana", + ref=self.PATTERN, kind="tag", mode="delta") + index_incremental_branch_in_dir( + es, "github", "elastic", "kibana", "/repo", self.PATTERN, + reporter=ProgressReporter(), unit=unit, + ) + mocks["checkout_ref"].assert_not_called() + mocks["index_incremental_paths"].assert_not_called() + mocks["write_incremental_ready"].assert_not_called() + assert unit.status == "no-changes" + finally: + _stop(patchers) diff --git a/tests/test_index_orphans.py b/tests/test_index_orphans.py index c9129cb..f183ff0 100644 --- a/tests/test_index_orphans.py +++ b/tests/test_index_orphans.py @@ -43,11 +43,11 @@ class TestListSourcererIndices: def test_discovers_backing_indices_through_content_aliases(self): es = MagicMock() es.indices.get_alias.side_effect = [ - {"sourcerer-v2-files~github~acme~widgets": {}}, - {"sourcerer-v2-lines~github~acme~widgets": {}}, + {"sourcerer-v3-files~github~acme~widgets": {}}, + {"sourcerer-v3-lines~github~acme~widgets": {}}, ] names = list_sourcerer_indices(es) - assert names == ["sourcerer-v2-files~github~acme~widgets", "sourcerer-v2-lines~github~acme~widgets"] + assert names == ["sourcerer-v3-files~github~acme~widgets", "sourcerer-v3-lines~github~acme~widgets"] assert es.indices.get_alias.call_args_list[0].kwargs == {"name": FILES_ALIAS} assert es.indices.get_alias.call_args_list[1].kwargs == {"name": LINES_ALIAS} @@ -119,12 +119,12 @@ def fake_count(index): def test_returns_only_zero_doc_content_indices(self): counts = { - "sourcerer-v2-files~github~acme~widgets": 0, # empty -> returned - "sourcerer-v2-files~github~acme~widgets^deploy": 5, # non-empty -> skipped + "sourcerer-v3-files~github~acme~widgets": 0, # empty -> returned + "sourcerer-v3-files~github~acme~widgets^deploy": 5, # non-empty -> skipped } es = self._es_with_counts(counts) result = empty_content_indices(es, list(counts)) - assert result == ["sourcerer-v2-files~github~acme~widgets"] + assert result == ["sourcerer-v3-files~github~acme~widgets"] def test_non_sourcerer_index_never_considered(self): # An unrelated (even empty) index must not be counted or returned. @@ -134,11 +134,11 @@ def test_non_sourcerer_index_never_considered(self): es.count.assert_not_called() # guarded by parse_index_name before any count def test_refs_index_not_considered(self): - es = self._es_with_counts({"sourcerer-v2-refs": 0}) - assert empty_content_indices(es, ["sourcerer-v2-refs"]) == [] + es = self._es_with_counts({"sourcerer-v3-refs": 0}) + assert empty_content_indices(es, ["sourcerer-v3-refs"]) == [] es.count.assert_not_called() def test_index_that_vanished_is_skipped(self): # count raises NotFound (deleted between listing and counting) -> just skipped. es = self._es_with_counts({}) # every count -> NotFound - assert empty_content_indices(es, ["sourcerer-v2-files~github~acme~widgets"]) == [] + assert empty_content_indices(es, ["sourcerer-v3-files~github~acme~widgets"]) == [] diff --git a/tests/test_index_ref.py b/tests/test_index_ref.py index 143ee77..a33984b 100644 --- a/tests/test_index_ref.py +++ b/tests/test_index_ref.py @@ -66,7 +66,7 @@ def test_reuse_writes_marker_counts_not_index_repo(self): # unchanged), so the reuse probe targets exactly where this commit's content lives. mock_cp.assert_called_once_with( es, "github", "elastic", "myrepo", FULL_SHA, - at_index="sourcerer-v2-files~github~elastic~myrepo", + at_index="sourcerer-v3-files~github~elastic~myrepo", ) mock_index.assert_not_called() mock_wim.assert_not_called() # indexing marker only written for fresh ingest @@ -266,8 +266,8 @@ def track_delete(es_, host, org, repo, sha, index_names): i for i, c in enumerate(call_order) if isinstance(c, tuple) ) deleted_indices = mock_del.call_args.args[5] - assert "sourcerer-v2-files~github~elastic~myrepo" in deleted_indices - assert "sourcerer-v2-lines~github~elastic~myrepo" in deleted_indices + assert "sourcerer-v3-files~github~elastic~myrepo" in deleted_indices + assert "sourcerer-v3-lines~github~elastic~myrepo" in deleted_indices def test_unchanged_routing_does_not_delete(self): """Same routing as recorded -> normal ingest, no migration delete.""" diff --git a/tests/test_indices.py b/tests/test_indices.py index aeea7cd..ea63831 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -1,5 +1,5 @@ """Unit tests for the index-name builders in sourcerer.indices, including a round-trip -check against planner.parse_index_name (their documented inverse). v2 names carry a leading +check against planner.parse_index_name (their documented inverse). v3 names carry a leading git.host segment.""" # App packages @@ -15,54 +15,54 @@ def test_read_aliases(self): "sourcerer-refs", ) - def test_refs_index_is_v2(self): - assert REFS_INDEX == "sourcerer-v2-refs" + def test_refs_index_is_v3(self): + assert REFS_INDEX == "sourcerer-v3-refs" def test_files_index_shape(self): assert files_index("github", "elastic", "elasticsearch") == \ - "sourcerer-v2-files~github~elastic~elasticsearch" + "sourcerer-v3-files~github~elastic~elasticsearch" def test_lines_index_shape(self): assert lines_index("github", "elastic", "elasticsearch") == \ - "sourcerer-v2-lines~github~elastic~elasticsearch" + "sourcerer-v3-lines~github~elastic~elasticsearch" def test_host_org_repo_lowercased(self): assert files_index("GitHub", "Elastic", "ElasticSearch") == \ - "sourcerer-v2-files~github~elastic~elasticsearch" + "sourcerer-v3-files~github~elastic~elasticsearch" assert lines_index("MyGitea", "ACME", "Widgets") == \ - "sourcerer-v2-lines~mygitea~acme~widgets" + "sourcerer-v3-lines~mygitea~acme~widgets" def test_same_org_repo_distinct_hosts_differ(self): assert files_index("github", "acme", "w") != files_index("gitlab", "acme", "w") def test_level_host(self): - assert files_index("github", "elastic", "kibana", level="host") == "sourcerer-v2-files~github" + assert files_index("github", "elastic", "kibana", level="host") == "sourcerer-v3-files~github" def test_level_org(self): - assert files_index("github", "Elastic", "Kibana", level="org") == "sourcerer-v2-files~github~elastic" + assert files_index("github", "Elastic", "Kibana", level="org") == "sourcerer-v3-files~github~elastic" def test_level_commit_requires_commit(self): assert lines_index("github", "elastic", "kibana", commit="ABC123", level="commit") == \ - "sourcerer-v2-lines~github~elastic~kibana~abc123" + "sourcerer-v3-lines~github~elastic~kibana~abc123" import pytest with pytest.raises(ValueError): files_index("github", "elastic", "kibana", level="commit") # no commit def test_suffix_appended_with_caret(self): assert files_index("github", "elastic", "kibana", suffix="deploy") == \ - "sourcerer-v2-files~github~elastic~kibana^deploy" + "sourcerer-v3-files~github~elastic~kibana^deploy" def test_suffix_lowercased(self): assert files_index("github", "elastic", "kibana", suffix="Deploy") == \ - "sourcerer-v2-files~github~elastic~kibana^deploy" + "sourcerer-v3-files~github~elastic~kibana^deploy" def test_empty_suffix_is_no_suffix(self): assert files_index("github", "elastic", "kibana", suffix="") == \ - "sourcerer-v2-files~github~elastic~kibana" + "sourcerer-v3-files~github~elastic~kibana" def test_level_and_suffix_combine(self): assert files_index("github", "elastic", "kibana", commit="abc", level="commit", suffix="deploy") == \ - "sourcerer-v2-files~github~elastic~kibana~abc^deploy" + "sourcerer-v3-files~github~elastic~kibana~abc^deploy" class TestRoundTripWithParseIndexName: @@ -80,30 +80,30 @@ def test_refs_index_is_not_parsed(self): assert parse_index_name(REFS_INDEX) is None def test_host_only_granularity(self): - parsed = parse_index_name("sourcerer-v2-files~github") + parsed = parse_index_name("sourcerer-v3-files~github") assert (parsed.host, parsed.org, parsed.repo, parsed.commit) == ("github", None, None, None) def test_host_org_repo_commit_granularity(self): - parsed = parse_index_name("sourcerer-v2-files~github~acme~w~abc123") + parsed = parse_index_name("sourcerer-v3-files~github~acme~w~abc123") assert (parsed.host, parsed.org, parsed.repo, parsed.commit) == ("github", "acme", "w", "abc123") def test_too_many_segments_rejected(self): - assert parse_index_name("sourcerer-v2-files~a~b~c~d~e") is None + assert parse_index_name("sourcerer-v3-files~a~b~c~d~e") is None def test_empty_segment_rejected(self): - assert parse_index_name("sourcerer-v2-files~github~~repo") is None + assert parse_index_name("sourcerer-v3-files~github~~repo") is None def test_suffix_parsed_and_identity_suffix_blind(self): - parsed = parse_index_name("sourcerer-v2-files~github~acme~widgets^deploy") + parsed = parse_index_name("sourcerer-v3-files~github~acme~widgets^deploy") assert (parsed.host, parsed.org, parsed.repo, parsed.commit, parsed.suffix) == \ ("github", "acme", "widgets", None, "deploy") def test_commit_level_with_suffix(self): - parsed = parse_index_name("sourcerer-v2-lines~github~acme~w~abc123^deploy") + parsed = parse_index_name("sourcerer-v3-lines~github~acme~w~abc123^deploy") assert (parsed.repo, parsed.commit, parsed.suffix) == ("w", "abc123", "deploy") def test_trailing_caret_is_malformed(self): - assert parse_index_name("sourcerer-v2-files~github~acme~widgets^") is None + assert parse_index_name("sourcerer-v3-files~github~acme~widgets^") is None def test_round_trip_invariant_all_levels_and_suffix(self): """The builder <-> parser contract: parse_index_name(files_index(...)) recovers the inputs diff --git a/tests/test_markers.py b/tests/test_markers.py index f85136b..fb27391 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -11,10 +11,27 @@ # App packages from sourcerer.commands.index.markers import ( - build_ref_id, commit_prefix_indexed, commits_with_content, fully_indexed_counts, - markers_status_by_id, _needs_index, _parse_marker_started, pre_clone_skip, should_index, + build_ref_id, + commit_prefix_indexed, + commits_with_content, + count_incremental_branch_docs, + delete_incremental_branch, + delete_incremental_paths, + fully_indexed_counts, + markers_status_by_id, + _needs_index, + _parse_marker_started, + pre_clone_skip, + read_incremental_ref, + should_index, + write_incremental_failed, + write_incremental_indexing, + write_incremental_ready, + write_indexing_marker, + write_ref_marker, ) -from sourcerer.indices import FILES_ALIAS, REFS_ALIAS +from sourcerer.indices import FILES_ALIAS, REFS_ALIAS, REFS_INDEX +from sourcerer.utils import build_ref_key FULL_SHA = "cfefb3b2378ccbadefa7c8f4f9e21b3a1d2e5f60" @@ -376,3 +393,318 @@ def test_host_changes_result(self): assert fully_indexed_counts(es, "gitlab", "acme", "widgets", FULL_SHA) == (7, 900) call = es.search.call_args.kwargs assert {"term": {"git.host": "gitlab"}} in call["query"]["bool"]["filter"] + + +OLD = "1111111111111111111111111111111111111111" +NEW = "2222222222222222222222222222222222222222" + + +def _indexed_doc(es): + return es.index.call_args.kwargs["document"] + + +class TestWriteRefMarker: + """write_ref_marker is the single refs doc per snapshot source. git.ref_key has + been removed; snapshot refs are identified by (host, org, repo, ref, commit) on the marker.""" + + def test_marker_carries_commit_no_ref_key(self): + es = MagicMock() + write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + files_count=10, lines_count=200) + doc = es.index.call_args.kwargs["document"] + assert doc["git"]["commit"] == OLD + assert "ref_key" not in doc["git"] + assert doc["mode"] == "snapshot" + + def test_marker_id_is_hashed_not_the_commit(self): + # _id is build_ref_id (BLAKE2b hash) -- one per (ref, commit), NOT the bare commit SHA. + es = MagicMock() + write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + files_count=10, lines_count=200) + call = es.index.call_args.kwargs + assert call["index"] == REFS_INDEX + assert call["id"] != OLD # hashed, not the bare commit + assert call["id"] == build_ref_id("github", "acme", "widgets", "tag", "v1.0.0", OLD) + + def test_default_write_does_not_refresh(self): + es = MagicMock() + write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + files_count=1, lines_count=1) + assert es.index.call_args.kwargs.get("refresh") is False + + def test_refresh_true_is_propagated(self): + # write_ref_marker accepts refresh=True for callers that need the doc visible before + # the next gate runs. + es = MagicMock() + write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + files_count=1, lines_count=1, refresh=True) + assert es.index.call_args.kwargs["refresh"] is True + + def test_marker_status_complete(self): + es = MagicMock() + write_ref_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None, + files_count=5, lines_count=100) + assert es.index.call_args.kwargs["document"]["status"] == "complete" + + def test_complete_marker_carries_ref_pattern(self): + # write_ref_marker (status:complete) must populate git.ref_pattern. When no ref_pattern + # arg is given (branch == pattern), the fallback is git.ref. + es = MagicMock() + write_ref_marker(es, "github", "acme", "widgets", "branch", "main", OLD, None, + files_count=1, lines_count=10) + doc = es.index.call_args.kwargs["document"] + assert doc["git"]["ref_pattern"] == "main" + + def test_complete_marker_explicit_ref_pattern_differs_from_ref(self): + # When a versioned snapshot tag has a pattern like "zentity-{major}.{minor}.{patch}", + # git.ref holds the concrete tag and git.ref_pattern holds the raw match pattern. + es = MagicMock() + write_ref_marker(es, "github", "zentity-io", "zentity", "tag", "zentity-1.7.0", OLD, None, + files_count=10, lines_count=200, + ref_pattern="zentity-{major}.{minor}.{patch}") + doc = es.index.call_args.kwargs["document"] + assert doc["git"]["ref"] == "zentity-1.7.0" + assert doc["git"]["ref_pattern"] == "zentity-{major}.{minor}.{patch}" + # _id still keys on build_ref_id(concrete ref + commit), not the pattern, + # so sibling tags sharing a ref_pattern get distinct docs. + from sourcerer.commands.index.markers import build_ref_id + expected_id = build_ref_id("github", "zentity-io", "zentity", "tag", "zentity-1.7.0", OLD) + assert es.index.call_args.kwargs["id"] == expected_id + + +class TestWriteIndexingMarker: + """write_indexing_marker is the in-progress (status:'indexing') snapshot writer. It must + populate git.ref_pattern from the very first write, so the field is never NULL in a snapshot + refs doc -- even while the ref is mid-index (before write_ref_marker's 'complete' overwrite).""" + + def test_indexing_marker_carries_ref_pattern(self): + # Fallback: when no ref_pattern arg is passed, git.ref_pattern == git.ref. + es = MagicMock() + write_indexing_marker(es, "github", "acme", "widgets", "branch", "main", OLD, None) + doc = es.index.call_args.kwargs["document"] + assert doc["git"]["ref_pattern"] == "main" + assert doc["git"]["ref"] == "main" + assert doc["git"]["ref_pattern"] == doc["git"]["ref"] + + def test_indexing_marker_explicit_ref_pattern_differs_from_ref(self): + # When a versioned snapshot tag has a pattern, the in-progress marker also carries it. + es = MagicMock() + write_indexing_marker(es, "github", "zentity-io", "zentity", "tag", "zentity-1.7.0", OLD, None, + ref_pattern="zentity-{major}.{minor}.{patch}") + doc = es.index.call_args.kwargs["document"] + assert doc["git"]["ref"] == "zentity-1.7.0" + assert doc["git"]["ref_pattern"] == "zentity-{major}.{minor}.{patch}" + + def test_indexing_marker_status_is_indexing(self): + es = MagicMock() + write_indexing_marker(es, "github", "acme", "widgets", "tag", "v1.0.0", OLD, None) + doc = es.index.call_args.kwargs["document"] + assert doc["status"] == "indexing" + assert doc["mode"] == "snapshot" + + def test_indexing_marker_id_matches_write_ref_marker_id(self): + # Both writers must use the same ref_id so write_ref_marker overwrites in place. + es = MagicMock() + write_indexing_marker(es, "github", "acme", "widgets", "tag", "v2.0.0", OLD, None) + indexing_id = es.index.call_args.kwargs["id"] + write_ref_marker(es, "github", "acme", "widgets", "tag", "v2.0.0", OLD, None, + files_count=1, lines_count=1) + complete_id = es.index.call_args.kwargs["id"] + assert indexing_id == complete_id + + +class TestIncrementalRefKeyIdentity: + def test_id_is_ref_key_not_a_hash(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", OLD, NEW) + assert es.index.call_args.kwargs["id"] == build_ref_key("github", "acme", "widgets", "branch", "main") + + def test_stable_across_calls_commit_independent(self): + a = build_ref_key("github", "acme", "widgets", "branch", "main") + b = build_ref_key("github", "acme", "widgets", "branch", "main") + assert a == b # one document per ref, no commit folded in + + def test_branch_and_tag_same_name_have_distinct_keys(self): + branch_key = build_ref_key("github", "acme", "widgets", "branch", "deploy") + tag_key = build_ref_key("github", "acme", "widgets", "tag", "deploy") + assert branch_key != tag_key # ref_type distinguishes them + + +class TestWriteIncrementalIndexing: + def test_incremental_marker_preserves_completed_commit_and_exposes_target(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", + completed_commit=OLD, commit_target=NEW) + doc = _indexed_doc(es) + assert doc["status"] == "indexing" + assert doc["git"]["commit"] == OLD # completed pointer unchanged + assert doc["git"]["commit_target"] == NEW + assert doc["mode"] == "delta" + assert doc["git"]["ref_type"] == "branch" + assert es.index.call_args.kwargs["index"] == REFS_INDEX + + def test_incremental_marker_first_index_has_no_completed_commit(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", + completed_commit=None, commit_target=NEW) + assert _indexed_doc(es)["git"]["commit"] is None + + def test_incremental_marker_carries_prior_counts(self): + es = MagicMock() + prior = {"files_count": 12, "lines_count": 340, "git": {"commit_date": "2026-01-01T00:00:00+00:00"}} + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", OLD, NEW, prior=prior) + doc = _indexed_doc(es) + assert doc["files_count"] == 12 and doc["lines_count"] == 340 + assert doc["git"]["commit_date"] == "2026-01-01T00:00:00+00:00" + + def test_incremental_indexing_carries_routing(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", OLD, NEW, + index_level="commit", index_suffix="s1") + doc = _indexed_doc(es) + assert doc["index_level"] == "commit" + assert doc["index_suffix"] == "s1" + + def test_incremental_indexing_default_routing(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "branch", "main", OLD, NEW) + doc = _indexed_doc(es) + assert doc["index_level"] == "repo" + assert doc["index_suffix"] is None + + def test_tag_ref_type_propagates_to_doc(self): + es = MagicMock() + write_incremental_indexing(es, "github", "acme", "widgets", "tag", "deploy@1", + completed_commit=None, commit_target=NEW) + doc = _indexed_doc(es) + assert doc["git"]["ref_type"] == "tag" + assert doc["git"]["ref"] == "deploy@1" + # Join doc id must differ from a same-named branch's + branch_id = build_ref_key("github", "acme", "widgets", "branch", "deploy@1") + tag_id = build_ref_key("github", "acme", "widgets", "tag", "deploy@1") + assert es.index.call_args.kwargs["id"] == tag_id + assert tag_id != branch_id + + +class TestWriteIncrementalReady: + def test_incremental_marker_advances_commit_and_clears_target_and_error(self): + es = MagicMock() + write_incremental_ready(es, "github", "acme", "widgets", "branch", "main", commit=NEW, + commit_date_iso="2026-02-02T00:00:00+00:00", + files_count=5, lines_count=99) + doc = _indexed_doc(es) + assert doc["status"] == "complete" + assert doc["git"]["commit"] == NEW # advances only after a successful run + assert doc["git"]["commit_target"] is None + assert doc["files_count"] == 5 and doc["lines_count"] == 99 + assert es.index.call_args.kwargs["refresh"] is True # publication boundary + + def test_incremental_ready_carries_routing(self): + es = MagicMock() + write_incremental_ready(es, "github", "acme", "widgets", "branch", "main", commit=NEW, + commit_date_iso=None, files_count=1, lines_count=1, + index_level="commit", index_suffix="s1") + doc = _indexed_doc(es) + assert doc["index_level"] == "commit" + assert doc["index_suffix"] == "s1" + + +class TestWriteIncrementalFailed: + def test_incremental_failed_sets_status_failed_and_retains_old_pointer(self): + es = MagicMock() + write_incremental_failed(es, "github", "acme", "widgets", "branch", "main", + completed_commit=OLD, commit_target=NEW, error="boom") + doc = _indexed_doc(es) + assert doc["status"] == "failed" + assert doc["git"]["commit"] == OLD # completed pointer unchanged + assert doc["git"]["commit_target"] == NEW + assert "error" not in doc + assert "failed_at" not in doc + + def test_incremental_failed_carries_routing(self): + es = MagicMock() + write_incremental_failed(es, "github", "acme", "widgets", "branch", "main", OLD, NEW, error="boom", + index_level="commit", index_suffix="s1") + doc = _indexed_doc(es) + assert doc["index_level"] == "commit" + assert doc["index_suffix"] == "s1" + + +class TestReadIncrementalRef: + def test_returns_source(self): + es = MagicMock() + es.get.return_value = {"_source": {"status": "ready", "git": {"commit": NEW}}} + assert read_incremental_ref(es, "github", "acme", "widgets", "branch", "main") == ( + {"status": "ready", "git": {"commit": NEW}} + ) + assert es.get.call_args.kwargs["id"] == build_ref_key("github", "acme", "widgets", "branch", "main") + + def test_missing_returns_none(self): + es = MagicMock() + es.get.side_effect = _not_found() + assert read_incremental_ref(es, "github", "acme", "widgets", "branch", "main") is None + + +class TestDeleteIncrementalPaths: + def test_empty_paths_is_noop(self): + es = MagicMock() + delete_incremental_paths(es, "github", "acme", "widgets", "branch", "main", []) + es.delete_by_query.assert_not_called() + + def test_scoped_to_exact_ref_key_and_paths(self): + es = MagicMock() + delete_incremental_paths(es, "github", "acme", "widgets", "branch", "main", ["a.txt", "b.txt"]) + assert es.delete_by_query.call_count == 2 # files + lines indices + for call in es.delete_by_query.call_args_list: + query = call.kwargs["query"] + filt = query["bool"]["filter"] + assert {"term": {"git.host": "github"}} in filt + assert {"term": {"git.org": "acme"}} in filt + assert {"term": {"git.repo": "widgets"}} in filt + assert {"term": {"git.ref_type": "branch"}} in filt + assert {"term": {"git.ref_pattern": "main"}} in filt + assert {"terms": {"file.path": ["a.txt", "b.txt"]}} in filt + + def test_missing_index_is_ignored(self): + es = MagicMock() + es.delete_by_query.side_effect = _not_found() + delete_incremental_paths(es, "github", "acme", "widgets", "branch", "main", ["a.txt"]) # no raise + + +class TestDeleteIncrementalBranch: + def test_scoped_to_exact_ref_key_only(self): + es = MagicMock() + delete_incremental_branch(es, "github", "acme", "widgets", "main") + assert es.delete_by_query.call_count == 2 + for call in es.delete_by_query.call_args_list: + query = call.kwargs["query"] + filt = query["bool"]["filter"] + assert {"term": {"git.host": "github"}} in filt + assert {"term": {"git.org": "acme"}} in filt + assert {"term": {"git.repo": "widgets"}} in filt + assert {"term": {"git.ref_type": "branch"}} in filt + assert {"term": {"git.ref_pattern": "main"}} in filt + assert not any("ref_key" in str(f) for f in filt) + + def test_isolated_from_another_branch(self): + # Two incremental branches indexed; deleting one's docs must never scope to the other's + # (host,org,repo,ref) quadruple -- asserted here at the query-construction level. + es_a = MagicMock() + es_b = MagicMock() + delete_incremental_branch(es_a, "github", "acme", "widgets", "main") + delete_incremental_branch(es_b, "github", "acme", "widgets", "dev") + query_a = es_a.delete_by_query.call_args_list[0].kwargs["query"] + query_b = es_b.delete_by_query.call_args_list[0].kwargs["query"] + assert query_a != query_b + + +class TestCountIncrementalBranchDocs: + def test_missing_index_returns_zero(self): + es = MagicMock() + es.count.side_effect = _not_found() + assert count_incremental_branch_docs(es, "github", "acme", "widgets", "main") == (0, 0) + + def test_returns_counts(self): + es = MagicMock() + es.count.return_value = {"count": 5} + assert count_incremental_branch_docs(es, "github", "acme", "widgets", "main") == (5, 5) diff --git a/tests/test_planner_orphans.py b/tests/test_planner_orphans.py index 8b21268..22a2f12 100644 --- a/tests/test_planner_orphans.py +++ b/tests/test_planner_orphans.py @@ -1,6 +1,6 @@ """Unit tests for the pure orphan-detection helpers in sourcerer.planner. No Elasticsearch here -- every case is expressed as plain index-name lists and (host, org, repo, commit) tuple -sets. v2 index names carry a leading host segment; repo tuples are keyed (host, org, repo).""" +sets. v3 index names carry a leading host segment; repo tuples are keyed (host, org, repo).""" # App packages from sourcerer.planner import ( @@ -15,29 +15,29 @@ class TestParseIndexName: def test_host_org(self): - assert parse_index_name("sourcerer-v2-lines~github~acme~widgets") == ParsedIndex( + assert parse_index_name("sourcerer-v3-lines~github~acme~widgets") == ParsedIndex( kind="lines", host="github", org="acme", repo="widgets", commit=None, - name="sourcerer-v2-lines~github~acme~widgets", + name="sourcerer-v3-lines~github~acme~widgets", ) def test_host_org_repo_commit(self): - parsed = parse_index_name("sourcerer-v2-files~github~acme~widgets~deadbeef") + parsed = parse_index_name("sourcerer-v3-files~github~acme~widgets~deadbeef") assert parsed == ParsedIndex( kind="files", host="github", org="acme", repo="widgets", commit="deadbeef", - name="sourcerer-v2-files~github~acme~widgets~deadbeef", + name="sourcerer-v3-files~github~acme~widgets~deadbeef", ) def test_refs_index_is_not_a_files_or_lines_index(self): - assert parse_index_name("sourcerer-v2-refs") is None + assert parse_index_name("sourcerer-v3-refs") is None def test_unrelated_index_returns_none(self): assert parse_index_name("kibana_sample_data_ecommerce") is None def test_too_many_segments_returns_none(self): - assert parse_index_name("sourcerer-v2-files~a~b~c~d~e") is None + assert parse_index_name("sourcerer-v3-files~a~b~c~d~e") is None def test_empty_segment_returns_none(self): - assert parse_index_name("sourcerer-v2-files~github~acme~~deadbeef") is None + assert parse_index_name("sourcerer-v3-files~github~acme~~deadbeef") is None def test_custom_prefixes(self): parsed = parse_index_name("myprefix-files~github~acme~widgets", files_prefix="myprefix-files") @@ -46,12 +46,12 @@ def test_custom_prefixes(self): class TestOrphanIndices: def test_host_org_repo_index_with_no_ref_repo_is_orphaned(self): - names = ["sourcerer-v2-files~github~acme~widgets"] + names = ["sourcerer-v3-files~github~acme~widgets"] result = orphan_indices(names, ref_orgs=set(), ref_repos=set(), ref_commits=set()) - assert result == ["sourcerer-v2-files~github~acme~widgets"] + assert result == ["sourcerer-v3-files~github~acme~widgets"] def test_index_with_ref_repo_is_not_orphaned(self): - names = ["sourcerer-v2-files~github~acme~widgets", "sourcerer-v2-lines~github~acme~widgets"] + names = ["sourcerer-v3-files~github~acme~widgets", "sourcerer-v3-lines~github~acme~widgets"] result = orphan_indices( names, ref_orgs={("github", "acme")}, ref_repos={("github", "acme", "widgets")}, ref_commits=set() ) @@ -61,33 +61,33 @@ def test_same_repo_distinct_hosts_judged_independently(self): # github/acme/widgets is in refs; gitlab/acme/widgets is not -> only the gitlab index # is an orphan. names = [ - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-files~gitlab~acme~widgets", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~gitlab~acme~widgets", ] result = orphan_indices( names, ref_orgs={("github", "acme")}, ref_repos={("github", "acme", "widgets")}, ref_commits=set() ) - assert result == ["sourcerer-v2-files~gitlab~acme~widgets"] + assert result == ["sourcerer-v3-files~gitlab~acme~widgets"] def test_org_level_orphan_subsumes_repo_and_commit_level_of_the_same_kind(self): names = [ - "sourcerer-v2-files~github~acme", - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-files~github~acme~widgets~deadbeef", + "sourcerer-v3-files~github~acme", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~github~acme~widgets~deadbeef", ] result = orphan_indices(names, ref_orgs=set(), ref_repos=set(), ref_commits=set()) - assert result == ["sourcerer-v2-files~github~acme"] + assert result == ["sourcerer-v3-files~github~acme"] def test_repo_level_orphan_subsumes_commit_level(self): names = [ - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-files~github~acme~widgets~deadbeef", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~github~acme~widgets~deadbeef", ] result = orphan_indices(names, ref_orgs={("github", "acme")}, ref_repos=set(), ref_commits=set()) - assert result == ["sourcerer-v2-files~github~acme~widgets"] + assert result == ["sourcerer-v3-files~github~acme~widgets"] def test_commit_level_not_orphaned_when_ref_commit_present(self): - names = ["sourcerer-v2-files~github~acme~widgets~deadbeef"] + names = ["sourcerer-v3-files~github~acme~widgets~deadbeef"] result = orphan_indices( names, ref_orgs={("github", "acme")}, @@ -98,15 +98,15 @@ def test_commit_level_not_orphaned_when_ref_commit_present(self): def test_subsumption_does_not_cross_files_and_lines(self): names = [ - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-lines~github~acme~widgets~aaa", - "sourcerer-v2-lines~github~acme~widgets~bbb", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-lines~github~acme~widgets~aaa", + "sourcerer-v3-lines~github~acme~widgets~bbb", ] result = orphan_indices(names, ref_orgs=set(), ref_repos=set(), ref_commits=set()) assert set(result) == set(names) def test_unparseable_names_are_ignored(self): - names = ["sourcerer-v2-refs", "some-other-index"] + names = ["sourcerer-v3-refs", "some-other-index"] assert orphan_indices(names, ref_orgs=set(), ref_repos=set(), ref_commits=set()) == [] @@ -138,7 +138,7 @@ def test_skip_repos_excludes_class_a_repos(self): class TestPlanOrphans: def test_no_orphans(self): - names = ["sourcerer-v2-files~github~acme~widgets", "sourcerer-v2-lines~github~acme~widgets"] + names = ["sourcerer-v3-files~github~acme~widgets", "sourcerer-v3-lines~github~acme~widgets"] ref_tuples = {("github", "acme", "widgets", "aaa")} content_tuples = {("github", "acme", "widgets", "aaa")} plan = plan_orphans(names, ref_tuples, content_tuples) @@ -147,16 +147,16 @@ def test_no_orphans(self): assert plan.orphan_marker_commits == {} def test_class_a_subsumes_class_b_for_the_same_repo(self): - names = ["sourcerer-v2-files~github~acme~widgets"] + names = ["sourcerer-v3-files~github~acme~widgets"] ref_tuples: set[tuple[str, str, str, str]] = set() content_tuples = {("github", "acme", "widgets", "bbb")} plan = plan_orphans(names, ref_tuples, content_tuples) - assert plan.orphan_index_names == ["sourcerer-v2-files~github~acme~widgets"] + assert plan.orphan_index_names == ["sourcerer-v3-files~github~acme~widgets"] assert plan.orphan_content == {} assert plan.orphan_marker_commits == {} def test_class_b_and_c_fire_independently_when_index_present(self): - names = ["sourcerer-v2-files~github~acme~widgets"] + names = ["sourcerer-v3-files~github~acme~widgets"] ref_tuples = {("github", "acme", "widgets", "aaa"), ("github", "acme", "widgets", "bbb")} content_tuples = {("github", "acme", "widgets", "aaa"), ("github", "acme", "widgets", "ccc")} plan = plan_orphans(names, ref_tuples, content_tuples) @@ -171,13 +171,13 @@ def test_same_repo_two_hosts_do_not_cross_contaminate(self): # must be judged independently: github's marker orphan must not be cancelled by gitlab's # content just because org/repo match. names = [ - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-files~gitlab~acme~widgets", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~gitlab~acme~widgets", ] ref_tuples = {("github", "acme", "widgets", "aaa")} content_tuples = {("gitlab", "acme", "widgets", "bbb")} plan = plan_orphans(names, ref_tuples, content_tuples) - assert plan.orphan_index_names == ["sourcerer-v2-files~gitlab~acme~widgets"] + assert plan.orphan_index_names == ["sourcerer-v3-files~gitlab~acme~widgets"] assert plan.orphan_content == {} # gitlab content subsumed by the Class-A index DELETE assert plan.orphan_marker_commits == {("github", "acme", "widgets"): {"aaa"}} @@ -190,53 +190,127 @@ def test_content_at_unintended_index_is_stale(self): from sourcerer.planner import orphan_stale_content ct = ("github", "acme", "widgets", "abc") content_by_index = { - "sourcerer-v2-files~github~acme~widgets": {ct}, # old copy, left by a migration - "sourcerer-v2-files~github~acme~widgets^deploy": {ct}, # new (intended) copy + "sourcerer-v3-files~github~acme~widgets": {ct}, # old copy, left by a migration + "sourcerer-v3-files~github~acme~widgets^deploy": {ct}, # new (intended) copy } - intended = {ct: {"sourcerer-v2-files~github~acme~widgets^deploy"}} + intended = {ct: {"sourcerer-v3-files~github~acme~widgets^deploy"}} stale = orphan_stale_content(content_by_index, intended, skip_indices=set()) - assert stale == {"sourcerer-v2-files~github~acme~widgets": {"abc"}} + assert stale == {"sourcerer-v3-files~github~acme~widgets": {"abc"}} def test_intended_index_not_flagged(self): from sourcerer.planner import orphan_stale_content ct = ("github", "acme", "widgets", "abc") - content_by_index = {"sourcerer-v2-files~github~acme~widgets^deploy": {ct}} - intended = {ct: {"sourcerer-v2-files~github~acme~widgets^deploy"}} + content_by_index = {"sourcerer-v3-files~github~acme~widgets^deploy": {ct}} + intended = {ct: {"sourcerer-v3-files~github~acme~widgets^deploy"}} assert orphan_stale_content(content_by_index, intended, set()) == {} def test_commit_without_marker_is_not_class_d(self): """No marker at all -> Class B territory (orphan_content), not stale-location.""" from sourcerer.planner import orphan_stale_content ct = ("github", "acme", "widgets", "abc") - content_by_index = {"sourcerer-v2-files~github~acme~widgets": {ct}} + content_by_index = {"sourcerer-v3-files~github~acme~widgets": {ct}} assert orphan_stale_content(content_by_index, {}, set()) == {} def test_skip_indices_excluded(self): from sourcerer.planner import orphan_stale_content ct = ("github", "acme", "widgets", "abc") - content_by_index = {"sourcerer-v2-files~github~acme~widgets": {ct}} - intended = {ct: {"sourcerer-v2-files~github~acme~widgets^deploy"}} + content_by_index = {"sourcerer-v3-files~github~acme~widgets": {ct}} + intended = {ct: {"sourcerer-v3-files~github~acme~widgets^deploy"}} # index is already going away via a Class-A whole-index DELETE assert orphan_stale_content(content_by_index, intended, - {"sourcerer-v2-files~github~acme~widgets"}) == {} + {"sourcerer-v3-files~github~acme~widgets"}) == {} def test_plan_orphans_wires_class_d(self): ct = ("github", "acme", "widgets", "abc") names = [ - "sourcerer-v2-files~github~acme~widgets", - "sourcerer-v2-files~github~acme~widgets^deploy", + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~github~acme~widgets^deploy", ] ref_tuples = {ct} content_tuples = {ct} content_by_index = { - "sourcerer-v2-files~github~acme~widgets": {ct}, - "sourcerer-v2-files~github~acme~widgets^deploy": {ct}, + "sourcerer-v3-files~github~acme~widgets": {ct}, + "sourcerer-v3-files~github~acme~widgets^deploy": {ct}, } - intended = {ct: {"sourcerer-v2-files~github~acme~widgets^deploy"}} + intended = {ct: {"sourcerer-v3-files~github~acme~widgets^deploy"}} plan = plan_orphans(names, ref_tuples, content_tuples, content_by_index_commit=content_by_index, intended_index_by_commit=intended) - assert plan.orphan_stale == {"sourcerer-v2-files~github~acme~widgets": {"abc"}} + assert plan.orphan_stale == {"sourcerer-v3-files~github~acme~widgets": {"abc"}} + + +class TestOrphanStaleIncrementalContent: + """Class D-I: incremental (ref-addressed, commit-less) content in an index that the branch's + join doc no longer intends. Mirrors Class D but keyed on (host, org, repo, ref) tuples.""" + + def test_incremental_content_at_unintended_index_is_stale(self): + from sourcerer.planner import orphan_stale_incremental_content + rt = ("github", "acme", "widgets", "main") + content_by_index = { + "sourcerer-v3-files~github~acme~widgets": {rt}, # old copy from a suffix migration + "sourcerer-v3-files~github~acme~widgets^deploy": {rt}, # new (intended) copy + } + intended = {rt: {"sourcerer-v3-files~github~acme~widgets^deploy"}} + stale = orphan_stale_incremental_content(content_by_index, intended, skip_indices=set()) + assert stale == {"sourcerer-v3-files~github~acme~widgets": {rt}} + + def test_intended_index_not_flagged(self): + from sourcerer.planner import orphan_stale_incremental_content + rt = ("github", "acme", "widgets", "main") + content_by_index = {"sourcerer-v3-files~github~acme~widgets^deploy": {rt}} + intended = {rt: {"sourcerer-v3-files~github~acme~widgets^deploy"}} + assert orphan_stale_incremental_content(content_by_index, intended, set()) == {} + + def test_ref_without_join_doc_is_not_class_di(self): + """No join doc for this ref -> different sweep, not stale-location.""" + from sourcerer.planner import orphan_stale_incremental_content + rt = ("github", "acme", "widgets", "main") + content_by_index = {"sourcerer-v3-files~github~acme~widgets": {rt}} + assert orphan_stale_incremental_content(content_by_index, {}, set()) == {} + + def test_skip_indices_excluded(self): + from sourcerer.planner import orphan_stale_incremental_content + rt = ("github", "acme", "widgets", "main") + content_by_index = {"sourcerer-v3-files~github~acme~widgets": {rt}} + intended = {rt: {"sourcerer-v3-files~github~acme~widgets^deploy"}} + # Index is already going away via a Class-A whole-index DELETE. + assert orphan_stale_incremental_content( + content_by_index, intended, {"sourcerer-v3-files~github~acme~widgets"} + ) == {} + + def test_plan_orphans_wires_class_di(self): + """plan_orphans exposes orphan_stale_incremental when location data is supplied.""" + rt = ("github", "acme", "widgets", "main") + ct = ("github", "acme", "widgets", "abc") + names = [ + "sourcerer-v3-files~github~acme~widgets", + "sourcerer-v3-files~github~acme~widgets^deploy", + ] + # One snapshot commit ref present so Class A/B/C don't fire on the identity. + ref_tuples = {ct} + content_tuples = {ct} + incremental_content_by_index = { + "sourcerer-v3-files~github~acme~widgets": {rt}, # stale copy + "sourcerer-v3-files~github~acme~widgets^deploy": {rt}, # intended + } + intended_incremental_by_ref = {rt: {"sourcerer-v3-files~github~acme~widgets^deploy"}} + plan = plan_orphans( + names, ref_tuples, content_tuples, + incremental_content_by_index=incremental_content_by_index, + intended_incremental_index_by_ref=intended_incremental_by_ref, + ) + assert plan.orphan_stale_incremental == { + "sourcerer-v3-files~github~acme~widgets": {rt}, + } + + def test_plan_orphans_class_di_empty_when_not_supplied(self): + """Back-compat: callers that omit incremental location data get an empty Class D-I.""" + ct = ("github", "acme", "widgets", "abc") + plan = plan_orphans( + ["sourcerer-v3-files~github~acme~widgets"], + {ct}, {ct}, + ) + assert plan.orphan_stale_incremental == {} class TestEmptyIndexSweep: @@ -246,25 +320,25 @@ class TestEmptyIndexSweep: def test_empty_index_included_even_when_identity_has_markers(self): ct = ("github", "acme", "widgets", "abc") names = [ - "sourcerer-v2-files~github~acme~widgets^a", # drained by a suffix a->b migration - "sourcerer-v2-files~github~acme~widgets^b", # now holds the content + "sourcerer-v3-files~github~acme~widgets^a", # drained by a suffix a->b migration + "sourcerer-v3-files~github~acme~widgets^b", # now holds the content ] # Identity (github, acme, widgets) still has markers (they point at ^b), so ^a is NOT a # Class-A orphan -- but it's empty, so Class E must catch it. plan = plan_orphans( names, ref_commit_tuples={ct}, content_commit_tuples={ct}, - empty_index_names=["sourcerer-v2-files~github~acme~widgets^a"], + empty_index_names=["sourcerer-v3-files~github~acme~widgets^a"], ) - assert "sourcerer-v2-files~github~acme~widgets^a" in plan.empty_index_names + assert "sourcerer-v3-files~github~acme~widgets^a" in plan.empty_index_names # ^a is not a Class-A orphan (identity is backed by refs). - assert "sourcerer-v2-files~github~acme~widgets^a" not in plan.orphan_index_names + assert "sourcerer-v3-files~github~acme~widgets^a" not in plan.orphan_index_names def test_empty_index_deduped_against_class_a(self): # An index that is BOTH empty AND identity-orphaned appears only under Class A (one DELETE). - names = ["sourcerer-v2-files~github~ghostorg~gone"] + names = ["sourcerer-v3-files~github~ghostorg~gone"] plan = plan_orphans( names, ref_commit_tuples=set(), content_commit_tuples=set(), - empty_index_names=["sourcerer-v2-files~github~ghostorg~gone"], + empty_index_names=["sourcerer-v3-files~github~ghostorg~gone"], ) - assert "sourcerer-v2-files~github~ghostorg~gone" in plan.orphan_index_names + assert "sourcerer-v3-files~github~ghostorg~gone" in plan.orphan_index_names assert plan.empty_index_names == [] diff --git a/tests/test_prune_deletions.py b/tests/test_prune_deletions.py index 84421ac..ebd463b 100644 --- a/tests/test_prune_deletions.py +++ b/tests/test_prune_deletions.py @@ -2,7 +2,7 @@ delete_index and execute_orphan_deletions (the orphan sweep), and execute_deletions (retention). Every ES call is mocked -- these assert the shape of the requests (wildcard-free deletes, query filters, single combined refs query), not against a real cluster. Repo tuples -are keyed (host, org, repo); index names are v2 and carry a leading host segment.""" +are keyed (host, org, repo); index names are v3 and carry a leading host segment.""" # Standard packages from unittest.mock import MagicMock @@ -25,26 +25,26 @@ def _not_found() -> NotFoundError: class TestDeleteIndex: def test_deletes_by_exact_name_no_wildcard(self): es = MagicMock() - assert delete_index(es, "sourcerer-v2-files~github~acme~widgets") is True - es.indices.delete.assert_called_once_with(index="sourcerer-v2-files~github~acme~widgets") + assert delete_index(es, "sourcerer-v3-files~github~acme~widgets") is True + es.indices.delete.assert_called_once_with(index="sourcerer-v3-files~github~acme~widgets") def test_missing_index_returns_false_not_raise(self): es = MagicMock() es.indices.delete.side_effect = _not_found() - assert delete_index(es, "sourcerer-v2-files~github~acme~widgets") is False + assert delete_index(es, "sourcerer-v3-files~github~acme~widgets") is False class TestExecuteOrphanDeletions: def test_deletes_indices_first(self): es = MagicMock() plan = OrphanPlan( - orphan_index_names=["sourcerer-v2-files~github~ghostorg"], + orphan_index_names=["sourcerer-v3-files~github~ghostorg"], orphan_content={}, orphan_marker_commits={}, ) indices_deleted, content_dropped, markers_dropped, stale_dropped, empty_deleted = execute_orphan_deletions(es, plan) assert (indices_deleted, content_dropped, markers_dropped, stale_dropped, empty_deleted) == (1, 0, 0, 0, 0) - es.indices.delete.assert_called_once_with(index="sourcerer-v2-files~github~ghostorg") + es.indices.delete.assert_called_once_with(index="sourcerer-v3-files~github~ghostorg") es.delete_by_query.assert_not_called() def test_content_delete_by_query_targets_both_content_indices_with_terms_and_host_filter(self): @@ -98,12 +98,12 @@ def test_empty_indices_deleted_whole(self): es = MagicMock() plan = OrphanPlan( orphan_index_names=[], orphan_content={}, orphan_marker_commits={}, - empty_index_names=["sourcerer-v2-files~github~acme~widgets^olddeploy"], + empty_index_names=["sourcerer-v3-files~github~acme~widgets^olddeploy"], ) _, _, _, _, empty_deleted = execute_orphan_deletions(es, plan) assert empty_deleted == 1 es.indices.delete.assert_called_once_with( - index="sourcerer-v2-files~github~acme~widgets^olddeploy") + index="sourcerer-v3-files~github~acme~widgets^olddeploy") es.delete_by_query.assert_not_called() diff --git a/tests/test_selection.py b/tests/test_selection.py new file mode 100644 index 0000000..fe97e12 --- /dev/null +++ b/tests/test_selection.py @@ -0,0 +1,164 @@ +"""Tests for _resolve_entry's delta-mode tag stream collapsing. + +A delta-mode tag selector whose match pattern covers multiple remote tags must emit exactly ONE +stream Unit per raw pattern (not one Unit per matching tag name). The Unit's ref is the literal +match pattern string; the concrete newest tag is resolved post-clone via ref_dates. +""" + +from unittest.mock import patch + +import yaml + +from sourcerer.commands.index.selection import _resolve_entry +from sourcerer.config import parse_config + + +def _resolve(source_yaml: str, remote_tags: dict[str, str], remote_branches: dict[str, str] | None = None): + """Run _resolve_entry with mocked ls-remote returning the given ref maps.""" + raw = yaml.safe_load(f""" +hosts: + - id: github + type: github + base_url: https://github.com +sources: +{source_yaml} +""") + cfg = parse_config(raw) + repo_cfg = cfg.repos[0] + host = cfg.hosts[repo_cfg.host] + + def _fake_list_remote_refs(url, kind): + if kind == "tags": + return remote_tags + return remote_branches or {} + + with patch("sourcerer.commands.index.selection.list_remote_refs", side_effect=_fake_list_remote_refs): + return _resolve_entry(repo_cfg, host) + + +class TestDeltaTagStreamSelection: + """Delta-mode tag selectors collapse to one stream Unit per pattern.""" + + MANY_TAGS = {f"deploy@{i}": f"sha{i}" for i in range(1, 10)} + + def test_delta_tag_glob_emits_one_unit_per_pattern(self): + """deploy@* matching 9 tags → 1 stream Unit, ref == pattern.""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@*" + mode: delta +""", remote_tags=self.MANY_TAGS) + assert len(units) == 1 + assert units[0].ref == "deploy@*" + assert units[0].kind == "tag" + assert units[0].mode == "delta" + assert units[0].remote_sha is None # resolved post-clone, not from ls-remote + + def test_delta_tag_stream_unit_carries_match_pattern(self): + """Stream unit has ref_pattern == pattern (stream identity, stable across tag promotions).""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@{major}" + mode: delta +""", remote_tags=self.MANY_TAGS) + assert len(units) == 1 + u = units[0] + assert u.ref == "deploy@{major}" + assert u.ref_pattern == "deploy@{major}", ( + f"Stream unit.ref_pattern must equal the pattern, got {u.ref_pattern!r}" + ) + + def test_delta_tag_version_pattern_emits_one_unit(self): + """deploy@{major} matching 9 deploy@N tags → 1 stream Unit.""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@{major}" + mode: delta +""", remote_tags=self.MANY_TAGS) + assert len(units) == 1 + assert units[0].ref == "deploy@{major}" + + def test_delta_tag_multi_pattern_emits_one_unit_per_pattern(self): + """Two raw patterns → two stream Units with distinct identities.""" + tags = {**{f"deploy@{i}": f"sha{i}" for i in range(1, 5)}, + **{f"release@{i}": f"rsha{i}" for i in range(1, 4)}} + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: ["deploy@*", "release@*"] + mode: delta +""", remote_tags=tags) + assert len(units) == 2 + refs = {u.ref for u in units} + assert refs == {"deploy@*", "release@*"} + + def test_delta_tag_pattern_with_no_matches_emits_nothing(self): + """If no remote tag matches the pattern, no stream Unit is emitted.""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "release@*" + mode: delta +""", remote_tags=self.MANY_TAGS) # only deploy@* tags, no release@* + assert units == [] + + def test_snapshot_tag_still_emits_one_unit_per_tag(self): + """snapshot mode is unchanged: one Unit per matching tag name.""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@*" +""", remote_tags={f"deploy@{i}": f"sha{i}" for i in range(1, 4)}) + assert len(units) == 3 + assert all(u.ref.startswith("deploy@") and not u.ref.endswith("*") for u in units) + assert all(u.mode == "snapshot" for u in units) + + def test_delta_branch_still_emits_one_unit_per_branch(self): + """delta mode on branches (not tags) is unchanged: one Unit per matching branch.""" + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: branch} + match: "main" + mode: delta +""", remote_tags={}, remote_branches={"main": "abc123"}) + assert len(units) == 1 + assert units[0].ref == "main" + assert units[0].remote_sha == "abc123" + + def test_snapshot_tag_units_use_raw_match_pattern(self): + """Snapshot tag units carry the raw match pattern in ref_pattern, not the concrete tag.""" + snapshot_units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@*" +""", remote_tags={f"deploy@{i}": f"sha{i}" for i in range(1, 4)}) + for u in snapshot_units: + assert u.ref_pattern == "deploy@*", ( + f"snapshot tag unit.ref_pattern must be the raw pattern, got {u.ref_pattern!r}" + ) + assert u.ref != u.ref_pattern, ( + f"snapshot tag unit.ref (concrete) should differ from ref_pattern (pattern), got ref={u.ref!r}" + ) + + def test_delta_branch_units_have_ref_pattern_equal_to_ref(self): + """Delta branch units: branch name is its own pattern, so ref_pattern == ref.""" + # Delta branch: ref_pattern == branch name (literal match, no version components). + branch_units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: branch} + match: "main" + mode: delta +""", remote_tags={}, remote_branches={"main": "abc123"}) + for u in branch_units: + assert u.ref_pattern == u.ref, f"branch unit.ref_pattern must == unit.ref, got ref_pattern={u.ref_pattern!r} ref={u.ref!r}" + + def test_two_delta_tag_sources_on_same_repo_produce_distinct_streams(self): + """Two delta-tag selectors on the same repo with different patterns are two streams.""" + tags = {**{f"deploy@{i}": f"sha{i}" for i in range(1, 4)}, + **{f"v1.{i}.0": f"vsha{i}" for i in range(1, 4)}} + units = _resolve(""" + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "deploy@*" + mode: delta + - git: {host: github, org: elastic, repo: kibana, ref_type: tag} + match: "v{major}.{minor}.{patch}" + mode: delta +""", remote_tags=tags) + assert len(units) == 2 + refs = {u.ref for u in units} + assert "deploy@*" in refs + assert "v{major}.{minor}.{patch}" in refs diff --git a/tests/test_setup.py b/tests/test_setup.py index e34a81d..845a9b0 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -45,48 +45,48 @@ def test_multiple_categories_returned_as_set(self): class TestLoadIndexTemplates: def test_loads_template_and_applies_its_alias_to_existing_indices(self, tmp_path): - (tmp_path / "sourcerer-v2-files.json").write_text(json.dumps({ - "index_patterns": ["sourcerer-v2-files*"], + (tmp_path / "sourcerer-v3-files.json").write_text(json.dumps({ + "index_patterns": ["sourcerer-v3-files*"], "template": {"aliases": {"sourcerer-files": {}}}, })) es = MagicMock() loaded = load_index_templates(es, tmp_path) - assert loaded == ["sourcerer-v2-files"] + assert loaded == ["sourcerer-v3-files"] es.indices.put_index_template.assert_called_once_with( - name="sourcerer-v2-files", - index_patterns=["sourcerer-v2-files*"], + name="sourcerer-v3-files", + index_patterns=["sourcerer-v3-files*"], template={"aliases": {"sourcerer-files": {}}}, _meta=None, ) es.indices.update_aliases.assert_called_once_with(actions=[{ "add": { "alias": "sourcerer-files", - "index": "sourcerer-v2-files*", + "index": "sourcerer-v3-files*", }, }]) def test_ignores_missing_indices_when_applying_template_alias(self, tmp_path): - (tmp_path / "sourcerer-v2-files.json").write_text(json.dumps({ - "index_patterns": ["sourcerer-v2-files*"], + (tmp_path / "sourcerer-v3-files.json").write_text(json.dumps({ + "index_patterns": ["sourcerer-v3-files*"], "template": {"aliases": {"sourcerer-files": {}}}, })) es = MagicMock() es.indices.update_aliases.side_effect = _not_found() - assert load_index_templates(es, tmp_path) == ["sourcerer-v2-files"] + assert load_index_templates(es, tmp_path) == ["sourcerer-v3-files"] assert es.indices.method_calls == [ call.put_index_template( - name="sourcerer-v2-files", - index_patterns=["sourcerer-v2-files*"], + name="sourcerer-v3-files", + index_patterns=["sourcerer-v3-files*"], template={"aliases": {"sourcerer-files": {}}}, _meta=None, ), call.update_aliases(actions=[{ "add": { "alias": "sourcerer-files", - "index": "sourcerer-v2-files*", + "index": "sourcerer-v3-files*", }, }]), ] diff --git a/tests/test_uniqueness_gate.py b/tests/test_uniqueness_gate.py new file mode 100644 index 0000000..a73574d --- /dev/null +++ b/tests/test_uniqueness_gate.py @@ -0,0 +1,194 @@ +"""Tests for the post-index join-uniqueness gate: sourcerer.queries.check_join_uniqueness +and its command.py wiring in _run_uniqueness_gate. Every ES call is mocked. + +The gate is split by content shape (no mode on content docs): + - Snapshot (git.commit IS NOT NULL): each commit must have ≥1 complete refs doc. + - Incremental (git.ref_pattern IS NOT NULL): each (ref_type, ref_pattern) pair must have EXACTLY + ONE incremental join doc (keyed by git.ref_pattern; one join doc per stream identity). +""" + +# Standard packages +from unittest.mock import MagicMock + +# Third-party packages +from elastic_transport import ApiResponseMeta, HttpHeaders +from elasticsearch import NotFoundError + +# App packages +from sourcerer.commands.index.command import _run_uniqueness_gate +from sourcerer.queries import check_join_uniqueness + + +def _not_found() -> NotFoundError: + meta = ApiResponseMeta(status=404, http_version="1.1", headers=HttpHeaders({}), duration=0.0, node=None) + return NotFoundError("index_not_found_exception", meta, None) + + +def _commit_enum_resp(commits: list[str]) -> dict: + """Response for _enumerate_content_field(git.commit) — composite agg 'keys'.""" + return {"aggregations": {"keys": {"buckets": [{"key": {"val": c}} for c in commits]}}} + + +def _ref_enum_resp(ref_pairs: list[tuple[str, str]]) -> dict: + """Response for _enumerate_incremental_content_ref_pairs — composite agg 'pairs'. + Each pair is (ref_type, ref_pattern); content docs carry git.ref_pattern = the stream identity.""" + return {"aggregations": {"pairs": {"buckets": [ + {"key": {"ref_type": rt, "ref_pattern": r}} for rt, r in ref_pairs + ]}}} + + +def _snapshot_presence_resp(found_commits: list[str]) -> dict: + """Response for snapshot join-doc presence check — terms agg 'commits'.""" + return {"aggregations": {"commits": {"buckets": [ + {"key": c, "doc_count": 1} for c in found_commits + ]}}} + + +def _incremental_unique_resp(pair_counts: dict[tuple[str, str], int]) -> dict: + """Response for incremental uniqueness check — composite agg 'ref_pairs' on git.ref_pattern. + pair_counts is {(ref_type, ref_pattern): doc_count}.""" + return {"aggregations": {"ref_pairs": {"buckets": [ + {"key": {"ref_type": rt, "ref_pattern": r}, "doc_count": cnt} + for (rt, r), cnt in pair_counts.items() + ]}}} + + +class TestCheckJoinUniqueness: + """Tests for check_join_uniqueness: the combined snapshot + incremental gate.""" + + def _make_es( + self, + snapshot_commits: list[str] = (), + snapshot_found: list[str] = (), + incremental_ref_pairs: list[tuple[str, str]] = (), + incremental_pair_counts: dict[tuple[str, str], int] | None = None, + ) -> MagicMock: + """Build a mock ES with side_effects matching the exact call order of check_join_uniqueness: + + 1. _enumerate_content_field(git.commit): 1 search per index (FILES, LINES) + → composite agg 'keys', each bucket {'key': {'val': }} + 2. if commits non-empty → snapshot presence check (1 search on sourcerer-refs) + → terms agg 'commits' + 3. _enumerate_incremental_content_ref_pairs: 1 search per index (FILES, LINES) + → composite agg 'pairs', each bucket {'key': {'ref_type': ..., 'ref_pattern': ...}} + 4. if ref_pairs non-empty → incremental uniqueness check (1 search on sourcerer-refs) + → composite agg 'ref_pairs' (on git.ref_pattern), each bucket + {'key': {'ref_type': ..., 'ref_pattern': ...}, 'doc_count': N} + + The composite agg loop breaks on the first empty page, so exactly one search per index. + """ + es = MagicMock() + side_effects: list[dict] = [] + # (1) enumerate git.commit: FILES then LINES + side_effects.append(_commit_enum_resp(list(snapshot_commits))) # FILES + side_effects.append(_commit_enum_resp(list(snapshot_commits))) # LINES + # (2) snapshot join-doc presence check (only if commits found) + if snapshot_commits: + side_effects.append(_snapshot_presence_resp(list(snapshot_found))) + # (3) enumerate incremental (ref_type, ref) pairs: FILES then LINES + side_effects.append(_ref_enum_resp(list(incremental_ref_pairs))) # FILES + side_effects.append(_ref_enum_resp(list(incremental_ref_pairs))) # LINES + # (4) incremental uniqueness check on refs-side `match` (only if pairs found) + if incremental_ref_pairs: + counts = incremental_pair_counts or {} + side_effects.append(_incremental_unique_resp(counts)) + es.search.side_effect = side_effects + return es + + def test_clean_repo_no_content(self): + """No content at all → gate passes.""" + es = MagicMock() + # All composite enumeration calls return empty buckets. + # Steps 1+3: commit enum (keys agg, empty) and ref-pair enum (pairs agg, empty). + # We return a response that satisfies both agg names by making the mock return + # the correct format for each call in sequence. + es.search.side_effect = [ + _commit_enum_resp([]), # step 1a FILES git.commit + _commit_enum_resp([]), # step 1b LINES git.commit + _ref_enum_resp([]), # step 3a FILES ref pairs + _ref_enum_resp([]), # step 3b LINES ref pairs + ] + assert check_join_uniqueness(es, "github", "acme", "widgets") == [] + + def test_clean_snapshot_all_present(self): + """Snapshot commits all have a complete refs doc → clean.""" + es = self._make_es( + snapshot_commits=["aaa"], + snapshot_found=["aaa"], + ) + assert check_join_uniqueness(es, "github", "acme", "widgets") == [] + + def test_snapshot_missing_refs_doc_is_offending(self): + """A snapshot commit with no complete refs doc is reported.""" + es = self._make_es( + snapshot_commits=["aaa"], + snapshot_found=[], # no complete refs doc found + ) + result = check_join_uniqueness(es, "github", "acme", "widgets") + assert "aaa" in result + + def test_clean_incremental_exactly_one_join_doc(self): + """Incremental ref with exactly one join doc → clean.""" + es = self._make_es( + incremental_ref_pairs=[("branch", "main")], + incremental_pair_counts={("branch", "main"): 1}, + ) + assert check_join_uniqueness(es, "github", "acme", "widgets") == [] + + def test_clean_incremental_tag_stream_one_join_doc(self): + """A delta-tag stream (pattern as identity) with one join doc → clean.""" + es = self._make_es( + incremental_ref_pairs=[("tag", "deploy@{major}")], + incremental_pair_counts={("tag", "deploy@{major}"): 1}, + ) + assert check_join_uniqueness(es, "github", "acme", "widgets") == [] + + def test_incremental_missing_join_doc_is_offending(self): + """An incremental ref with no join doc is reported.""" + es = self._make_es( + incremental_ref_pairs=[("branch", "main")], + incremental_pair_counts={}, # zero docs found + ) + result = check_join_uniqueness(es, "github", "acme", "widgets") + assert "branch/main" in result + + def test_incremental_duplicate_join_doc_is_offending(self): + """An incremental ref with more than one join doc (e.g. stale snapshot marker) is reported.""" + es = self._make_es( + incremental_ref_pairs=[("branch", "main")], + incremental_pair_counts={("branch", "main"): 2}, # two docs — fan-out! + ) + result = check_join_uniqueness(es, "github", "acme", "widgets") + assert "branch/main" in result + + def test_missing_index_contributes_nothing(self): + es = MagicMock() + es.search.side_effect = _not_found() + assert check_join_uniqueness(es, "github", "acme", "widgets") == [] + + +class TestRunUniquenessGate: + def test_passes_silently_when_clean(self): + es = MagicMock() + # No content: all enumerations return empty pages. + es.search.side_effect = [ + _commit_enum_resp([]), # step 1a + _commit_enum_resp([]), # step 1b + _ref_enum_resp([]), # step 3a + _ref_enum_resp([]), # step 3b + ] + assert _run_uniqueness_gate(es, "github", "acme", "widgets") is True + + def test_fails_and_reports_on_snapshot_violation(self, capsys): + es = MagicMock() + # Snapshot commit "aaa" exists in content but has no complete refs doc. + es.search.side_effect = [ + _commit_enum_resp(["aaa"]), # step 1a: FILES git.commit + _commit_enum_resp(["aaa"]), # step 1b: LINES git.commit + _snapshot_presence_resp([]), # step 2: no complete refs doc for "aaa" + _ref_enum_resp([]), # step 3a: no incremental refs + _ref_enum_resp([]), # step 3b + ] + assert _run_uniqueness_gate(es, "github", "acme", "widgets") is False + captured = capsys.readouterr() + assert "aaa" in captured.err diff --git a/tests/test_utils.py b/tests/test_utils.py index debadce..5e6f851 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch # App packages -from sourcerer.utils import make_doc_id, make_client +from sourcerer.utils import make_doc_id, make_client, build_ref_key class TestMakeDocId: @@ -33,6 +33,35 @@ def test_non_utf8_parts_round_trip_via_surrogateescape(self): assert make_doc_id("acme", "widgets", weird) == make_doc_id("acme", "widgets", weird) +class TestBuildRefKey: + def test_ref_key_incremental_shape(self): + assert build_ref_key("github", "elastic", "sourcerer", "branch", "main") == ( + "github~elastic~sourcerer~branch~main" + ) + + def test_ref_key_lowercases_host_org_repo_preserves_ref_case(self): + assert build_ref_key("GitHub", "Elastic", "Sourcerer", "branch", "Feature/Mixed-Case") == ( + "github~elastic~sourcerer~branch~Feature/Mixed-Case" + ) + + def test_ref_key_tag_stream_keys_on_pattern(self): + """A delta-tag stream's ref key uses the match pattern, not the concrete tag.""" + pattern = "deploy@{major}" + concrete = "deploy@1788000000" + assert build_ref_key("github", "elastic", "kibana", "tag", pattern) != ( + build_ref_key("github", "elastic", "kibana", "tag", concrete) + ) + # The stable identity (pattern) produces a consistent key. + assert build_ref_key("github", "elastic", "kibana", "tag", pattern) == ( + build_ref_key("github", "elastic", "kibana", "tag", pattern) + ) + + def test_ref_key_deterministic(self): + assert build_ref_key("github", "acme", "widgets", "branch", "main") == build_ref_key( + "github", "acme", "widgets", "branch", "main" + ) + + class TestMakeClient: """make_client TLS behaviour — Elasticsearch constructor is patched, no real connection.""" diff --git a/uv.lock b/uv.lock index d4db3af..22b6d77 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -1667,7 +1667,7 @@ wheels = [ [[package]] name = "sourcerer" -version = "2.5.0" +version = "3.0.0" source = { editable = "." } dependencies = [ { name = "click" },