diff --git a/.claude/skills/forge/SKILL.md b/.claude/skills/forge/SKILL.md index 54618c7f9..43b3ec82d 100644 --- a/.claude/skills/forge/SKILL.md +++ b/.claude/skills/forge/SKILL.md @@ -22,6 +22,132 @@ Forge concept commands decouple codev from direct `gh` CLI calls. Each GitHub op | `pr-view` | `CODEV_PR_NUMBER`, `CODEV_INCLUDE_COMMENTS` (optional) | View PR details (JSON or text) | | `pr-diff` | `CODEV_PR_NUMBER`, `CODEV_DIFF_NAME_ONLY` (optional) | Get PR diff | | `auth-status` | — | Check forge authentication status | +| `ci-runs` | `CODEV_BRANCH_NAME`, `CODEV_CI_STATUS`, `CODEV_CI_WORKFLOW`, `CODEV_CI_LIMIT` (all optional) | List workflow runs (no log bytes) | +| `ci-run-view` | `CODEV_CI_RUN_ID` | One run plus per-job status (no log bytes) | +| `ci-failures` | `CODEV_CI_RUN_ID`, `CODEV_CI_JOB_ID` (optional) | The failing job's assertion, extracted and capped | +| `ci-run-log` | `CODEV_CI_RUN_ID`, `CODEV_CI_JOB_ID` (opt), and exactly one of `CODEV_CI_LOG_TAIL` / `CODEV_CI_LOG_HEAD` / `CODEV_CI_LOG_GREP` | A raw log window | + +## Running a concept + +```bash +codev forge # CODEV_* environment is passed through +CODEV_CI_RUN_ID=32515040122 codev forge ci-failures | jq +``` + +**Never call a concept script by its path.** `packages/codev/scripts/forge/github/ci-failures.sh` +bypasses resolution — it skips the `.codev/config.json` lookup, the provider +preset, and any per-repo override — so a project that overrides that concept +gets GitHub's script against its own forge and never learns why. + +`codev forge` is a thin dispatcher over the same resolver every other caller +uses: it prints the script's stdout verbatim (envelope included on the failure +path) and exits with the script's own exit code. Its own exit codes are `2` for +an unknown concept name (it lists the valid ones) and `3` for a concept +disabled for this provider, which it names rather than printing nothing. + +## CI concepts + +Four concepts, **tiered so the cheap question stays cheap**. A builder asks about +CI at four moments and they cost very different amounts: + +| Question | Concept | Reads a log? | +|---|---|---| +| Did my push pass? | `ci-runs` | No | +| Is it still running, and which job is pending? | `ci-runs`, `ci-run-view` | No | +| It failed — why? | `ci-failures` | Yes, one job | +| Is this mine or pre-existing? | `ci-runs` with `CODEV_CI_WORKFLOW` | No | +| Extraction gave up — show me the log | `ci-run-log` | Yes, one window | + +`ci-run-log` is a separate concept rather than a flag on `ci-failures` on +purpose: a window parameter on the main call gets passed by habit, and then +every status question drags a log again. + +### The response envelope + +Every ci-* concept prints ONE JSON object on stdout — **on success and on +failure**. Errors are values, not absences, because `executeForgeCommand` +flattens every failure mode to `null`: + +```json +{ "ok": false, "error": "timeout", "seconds": 60, + "detail": "GET repos/o/r/actions/tasks did not return within 60s", + "remedy": "raise CODEV_FORGE_TIMEOUT" } +``` + +`error` is one of `timeout`, `not-found`, `unsupported-server`, `forge-error`, +`bad-input`. Use `executeForgeCommandDetailed()` (not `executeForgeCommand`) when +you need to tell a timeout from a failure — it returns `{ok, data, stdout, +stderr, exitCode, timedOut, unavailable, durationMs}` and keeps stdout on the +failure path. + +**Any response carrying log text also carries `logLines`, `returnedLines` and +`truncated`.** A trimmed answer must never read as a whole one. + +### `ci-failures`, and what it does when it cannot tell + +Extraction runs a ladder and names the rung that fired in `matchedBy`: +`vitest`, `go-test`, `tsc`, `runner-marker`, `first-error`. When nothing +matches it does **not** fall back to the last N lines: + +```json +{ "extracted": false, "reason": "no recognized failure pattern", + "failures": [{ "jobId": 11952749, "jobName": "test-unit", "logLines": 1599 }], + "next": "ci-run-log CODEV_CI_RUN_ID=6554924 CODEV_CI_JOB_ID=11952749 CODEV_CI_LOG_TAIL=80" } +``` + +A builder handed 50 arbitrary lines treats them as the diagnosis; one told +extraction failed reads the log with the targeted call the response hands it. + +### `ci-run-log` windows + +Exactly one of `CODEV_CI_LOG_TAIL=N`, `CODEV_CI_LOG_HEAD=N`, or +`CODEV_CI_LOG_GREP=` (with `CODEV_CI_LOG_CONTEXT`, default 3). Zero or two +is exit 2 — there is deliberately no default. The response carries `from` and +`to` as line numbers into the full log; in grep mode `contiguous` is false and +`matchLines` lists exactly which lines matched. + +### Run ids + +`CODEV_CI_RUN_ID` is always the **`id`** field from `ci-runs`, never `number`. +On Forgejo the two are separate id spaces and **both resolve on the same +route**, so passing the number silently answers about a different, real run. + +### Provider notes + +| | GitHub | Forgejo/Gitea | +|---|---|---| +| Runs | `gh run list` | `GET actions/runs` (always with `page=`) | +| Jobs | `gh run view --json jobs` | `GET actions/runs/{id}/jobs` (Forgejo ≥16) or a scan of `actions/tasks` (Forgejo 15) | +| Log | `gh api repos/{owner}/{repo}/actions/jobs/{id}/logs` | `GET actions/jobs/{id}/logs` (**Forgejo ≥16 only**) | +| Per-step data | yes | no — `failedSteps` is always `[]` | +| `conclusion` | yes | always `null`; `status` carries it | + +**`gh run view --log-failed` is not used**, despite selecting failed steps in +principle. Measured on run 32515040122 of this repository it returned 2528 +lines / 293 KB with every line tagged `UNKNOWN STEP`: it selects the failing +JOB, and its filename-to-step mapping had missed. The per-job log endpoint is +used on both providers instead. + +**Forgejo below 16.0 has no Actions log API at all.** `ci-failures` and +`ci-run-log` return `error: "unsupported-server"` there, naming the version +found and the version needed and still listing the failing job names; +`ci-runs` and `ci-run-view` keep working. An old server must never be +mistaken for a run with no failures. + +**Forgejo records a `pull_request` run's branch as `#`**, not as a +branch name, and ignores `branch=` on the query. `ci-runs` resolves +`CODEV_BRANCH_NAME` to its PR ref first and filters client-side, so a branch +with no PR gets a `note` saying only push runs can match. + +**`gitlab` and `linear` have all four concepts explicitly disabled**, not merely +unimplemented — otherwise they would fall through to the GitHub default and run +`gh` against whatever remote it resolved. + +### Log cache + +A terminal job's log is immutable, so both log concepts cache it under +`$TMPDIR/codev-ci-logs///.log`. An in-progress job is +never cached or read from cache. `CODEV_CI_NO_CACHE=1` bypasses it. ## Configuration @@ -88,6 +214,13 @@ so a PR targeting an integration branch needs that variable. | `CODEV_FORGE_MERGED_DAYS` | 7 | `recently-merged` window when `CODEV_SINCE_DATE` is unset | | `CODEV_FORGE_MERGED_MAX` | 300 | merged PRs `recently-merged` will resolve before refusing | | `CODEV_FORGE_SEARCH_MAX` | 10 | PRs `pr-search` resolves for one issue number | +| `CODEV_CI_LIMIT` | 20 | runs `ci-runs` returns | +| `CODEV_CI_MAX_PAGES` | 4 | pages `ci-runs` walks while filtering client-side | +| `CODEV_CI_TASKS_MAX_PAGES` | 20 | pages the Forgejo-15 task scan walks (it stops early once past the run) | +| `CODEV_CI_MAX_STEP_BYTES` | 2048 | cap on one extracted failure | +| `CODEV_CI_MAX_BYTES` | 8192 | cap on a whole ci-* response | +| `CODEV_CI_NO_CACHE` | unset | `1` disables the log cache | +| `CODEV_CI_CACHE_MAX_MB` | 32 | largest log that will be cached | ### Exit statuses diff --git a/.codex/skills/forge/SKILL.md b/.codex/skills/forge/SKILL.md index 54618c7f9..43b3ec82d 100644 --- a/.codex/skills/forge/SKILL.md +++ b/.codex/skills/forge/SKILL.md @@ -22,6 +22,132 @@ Forge concept commands decouple codev from direct `gh` CLI calls. Each GitHub op | `pr-view` | `CODEV_PR_NUMBER`, `CODEV_INCLUDE_COMMENTS` (optional) | View PR details (JSON or text) | | `pr-diff` | `CODEV_PR_NUMBER`, `CODEV_DIFF_NAME_ONLY` (optional) | Get PR diff | | `auth-status` | — | Check forge authentication status | +| `ci-runs` | `CODEV_BRANCH_NAME`, `CODEV_CI_STATUS`, `CODEV_CI_WORKFLOW`, `CODEV_CI_LIMIT` (all optional) | List workflow runs (no log bytes) | +| `ci-run-view` | `CODEV_CI_RUN_ID` | One run plus per-job status (no log bytes) | +| `ci-failures` | `CODEV_CI_RUN_ID`, `CODEV_CI_JOB_ID` (optional) | The failing job's assertion, extracted and capped | +| `ci-run-log` | `CODEV_CI_RUN_ID`, `CODEV_CI_JOB_ID` (opt), and exactly one of `CODEV_CI_LOG_TAIL` / `CODEV_CI_LOG_HEAD` / `CODEV_CI_LOG_GREP` | A raw log window | + +## Running a concept + +```bash +codev forge # CODEV_* environment is passed through +CODEV_CI_RUN_ID=32515040122 codev forge ci-failures | jq +``` + +**Never call a concept script by its path.** `packages/codev/scripts/forge/github/ci-failures.sh` +bypasses resolution — it skips the `.codev/config.json` lookup, the provider +preset, and any per-repo override — so a project that overrides that concept +gets GitHub's script against its own forge and never learns why. + +`codev forge` is a thin dispatcher over the same resolver every other caller +uses: it prints the script's stdout verbatim (envelope included on the failure +path) and exits with the script's own exit code. Its own exit codes are `2` for +an unknown concept name (it lists the valid ones) and `3` for a concept +disabled for this provider, which it names rather than printing nothing. + +## CI concepts + +Four concepts, **tiered so the cheap question stays cheap**. A builder asks about +CI at four moments and they cost very different amounts: + +| Question | Concept | Reads a log? | +|---|---|---| +| Did my push pass? | `ci-runs` | No | +| Is it still running, and which job is pending? | `ci-runs`, `ci-run-view` | No | +| It failed — why? | `ci-failures` | Yes, one job | +| Is this mine or pre-existing? | `ci-runs` with `CODEV_CI_WORKFLOW` | No | +| Extraction gave up — show me the log | `ci-run-log` | Yes, one window | + +`ci-run-log` is a separate concept rather than a flag on `ci-failures` on +purpose: a window parameter on the main call gets passed by habit, and then +every status question drags a log again. + +### The response envelope + +Every ci-* concept prints ONE JSON object on stdout — **on success and on +failure**. Errors are values, not absences, because `executeForgeCommand` +flattens every failure mode to `null`: + +```json +{ "ok": false, "error": "timeout", "seconds": 60, + "detail": "GET repos/o/r/actions/tasks did not return within 60s", + "remedy": "raise CODEV_FORGE_TIMEOUT" } +``` + +`error` is one of `timeout`, `not-found`, `unsupported-server`, `forge-error`, +`bad-input`. Use `executeForgeCommandDetailed()` (not `executeForgeCommand`) when +you need to tell a timeout from a failure — it returns `{ok, data, stdout, +stderr, exitCode, timedOut, unavailable, durationMs}` and keeps stdout on the +failure path. + +**Any response carrying log text also carries `logLines`, `returnedLines` and +`truncated`.** A trimmed answer must never read as a whole one. + +### `ci-failures`, and what it does when it cannot tell + +Extraction runs a ladder and names the rung that fired in `matchedBy`: +`vitest`, `go-test`, `tsc`, `runner-marker`, `first-error`. When nothing +matches it does **not** fall back to the last N lines: + +```json +{ "extracted": false, "reason": "no recognized failure pattern", + "failures": [{ "jobId": 11952749, "jobName": "test-unit", "logLines": 1599 }], + "next": "ci-run-log CODEV_CI_RUN_ID=6554924 CODEV_CI_JOB_ID=11952749 CODEV_CI_LOG_TAIL=80" } +``` + +A builder handed 50 arbitrary lines treats them as the diagnosis; one told +extraction failed reads the log with the targeted call the response hands it. + +### `ci-run-log` windows + +Exactly one of `CODEV_CI_LOG_TAIL=N`, `CODEV_CI_LOG_HEAD=N`, or +`CODEV_CI_LOG_GREP=` (with `CODEV_CI_LOG_CONTEXT`, default 3). Zero or two +is exit 2 — there is deliberately no default. The response carries `from` and +`to` as line numbers into the full log; in grep mode `contiguous` is false and +`matchLines` lists exactly which lines matched. + +### Run ids + +`CODEV_CI_RUN_ID` is always the **`id`** field from `ci-runs`, never `number`. +On Forgejo the two are separate id spaces and **both resolve on the same +route**, so passing the number silently answers about a different, real run. + +### Provider notes + +| | GitHub | Forgejo/Gitea | +|---|---|---| +| Runs | `gh run list` | `GET actions/runs` (always with `page=`) | +| Jobs | `gh run view --json jobs` | `GET actions/runs/{id}/jobs` (Forgejo ≥16) or a scan of `actions/tasks` (Forgejo 15) | +| Log | `gh api repos/{owner}/{repo}/actions/jobs/{id}/logs` | `GET actions/jobs/{id}/logs` (**Forgejo ≥16 only**) | +| Per-step data | yes | no — `failedSteps` is always `[]` | +| `conclusion` | yes | always `null`; `status` carries it | + +**`gh run view --log-failed` is not used**, despite selecting failed steps in +principle. Measured on run 32515040122 of this repository it returned 2528 +lines / 293 KB with every line tagged `UNKNOWN STEP`: it selects the failing +JOB, and its filename-to-step mapping had missed. The per-job log endpoint is +used on both providers instead. + +**Forgejo below 16.0 has no Actions log API at all.** `ci-failures` and +`ci-run-log` return `error: "unsupported-server"` there, naming the version +found and the version needed and still listing the failing job names; +`ci-runs` and `ci-run-view` keep working. An old server must never be +mistaken for a run with no failures. + +**Forgejo records a `pull_request` run's branch as `#`**, not as a +branch name, and ignores `branch=` on the query. `ci-runs` resolves +`CODEV_BRANCH_NAME` to its PR ref first and filters client-side, so a branch +with no PR gets a `note` saying only push runs can match. + +**`gitlab` and `linear` have all four concepts explicitly disabled**, not merely +unimplemented — otherwise they would fall through to the GitHub default and run +`gh` against whatever remote it resolved. + +### Log cache + +A terminal job's log is immutable, so both log concepts cache it under +`$TMPDIR/codev-ci-logs///.log`. An in-progress job is +never cached or read from cache. `CODEV_CI_NO_CACHE=1` bypasses it. ## Configuration @@ -88,6 +214,13 @@ so a PR targeting an integration branch needs that variable. | `CODEV_FORGE_MERGED_DAYS` | 7 | `recently-merged` window when `CODEV_SINCE_DATE` is unset | | `CODEV_FORGE_MERGED_MAX` | 300 | merged PRs `recently-merged` will resolve before refusing | | `CODEV_FORGE_SEARCH_MAX` | 10 | PRs `pr-search` resolves for one issue number | +| `CODEV_CI_LIMIT` | 20 | runs `ci-runs` returns | +| `CODEV_CI_MAX_PAGES` | 4 | pages `ci-runs` walks while filtering client-side | +| `CODEV_CI_TASKS_MAX_PAGES` | 20 | pages the Forgejo-15 task scan walks (it stops early once past the run) | +| `CODEV_CI_MAX_STEP_BYTES` | 2048 | cap on one extracted failure | +| `CODEV_CI_MAX_BYTES` | 8192 | cap on a whole ci-* response | +| `CODEV_CI_NO_CACHE` | unset | `1` disables the log cache | +| `CODEV_CI_CACHE_MAX_MB` | 32 | largest log that will be cached | ### Exit statuses diff --git a/codev/plans/13-ci-forge-concepts.md b/codev/plans/13-ci-forge-concepts.md new file mode 100644 index 000000000..1f56d88e8 --- /dev/null +++ b/codev/plans/13-ci-forge-concepts.md @@ -0,0 +1,294 @@ +# PIR Plan: CI concepts for the forge layer — `ci-runs`, `ci-run-view`, `ci-failures`, `ci-run-log` + +Issue: #13 · Branch: `builder/pir-13` · Verification targets: `pseudoseed/codev` on GitHub (`gh` 2.87.0), `~/dev/entriq` (live Forgejo **15.0.2**, `tea` 0.14.2), and `codeberg.org/forgejo/forgejo` (live Forgejo **16.0.0-dev**, public, unauthenticated). + +## Understanding + +A builder asking "why did CI fail" currently shells out and pastes log text into its context. `KNOWN_CONCEPTS` (`packages/codev/src/lib/forge.ts:65-70`) has 18 concepts and none of them touch workflow runs. The issue and its two comments ask for four concepts, tiered so that only one of them ever fetches a log: + +| Question | Concept | Fetches a log? | +|---|---|---| +| Did my push pass? | `ci-runs` | No | +| Is it still running / which job is pending? | `ci-runs`, `ci-run-view` | No | +| It failed — why? | `ci-failures` | Yes, one job | +| Is this mine or pre-existing? | `ci-runs` (same workflow, other commits) | No | +| Extraction gave up — show me the log | `ci-run-log` (deliberate, separate) | Yes, windowed | + +I spent the investigation measuring the two providers rather than reasoning about them, because the issue's premises about both turned out to be **wrong in ways that change the design**. + +--- + +## What is actually true, measured 2026-08-21 + +### 1. `gh run view --log-failed` does NOT narrow to the failing step + +The issue says GitHub "already does the extraction for you." It does not, at least not here. Against a real failing run in this repository: + +``` +gh run view 32515040122 --log-failed → 2528 lines, 293 KB, 2.0 s +``` + +Every one of those 2528 lines is labelled `UNKNOWN STEP`: + +``` +Unit Tests UNKNOWN STEP 2026-08-21T18:47:09.5820646Z Current runner version: '2.336.0' +``` + +`--log-failed` selected the failing **job** and returned all of it. `gh` maps log files to steps by name and falls back to `UNKNOWN STEP` when that mapping fails, which is what happened. **293 KB is precisely the context bomb this issue exists to prevent**, so codev must do its own extraction on GitHub too — the same extractor both providers use. What `--log-failed` still buys is job selection: it is one call, and it never returns a passing job's log. + +### 2. The naive extraction heuristics both pick the wrong line on that same log + +The real failure is at line 2471 of 2528: + +``` +2471: FAIL src/commands/consult/__tests__/agy-auth-cache.test.ts > gemini lane burst behaviour (#1077 regression) +2472: AssertionError: expected null to be 'unauth' // Object.is equality +2491: Test Files 1 failed | 276 passed +2497: ##[error]AssertionError: expected null to be 'unauth' // Object.is equality +``` + +- **"First line matching an error pattern"** returns line 1257: `[artifact-canvas] Error: host blew up` — a fixture string printed by a **passing** test. A builder handed that would go debugging a host crash that never happened. +- **"Last N lines"** happens to work on this log and would not on the next one; the run's tail here is git-submodule cleanup and a Node deprecation warning. +- **`Test Files …`** appears **five** times, the first four all `passed` (lines 296, 317, 1450, 1477). Matching the first is wrong; the summary that matters is the one containing `failed`. +- Every payload line is wrapped in ANSI SGR codes — the raw bytes are `\e[41m\e[1m FAIL \e[22m\e[49m src/…`. **A matcher that does not strip ANSI first matches nothing at all.** This is the single most likely way for an extractor to silently return `extracted: false` on a log that plainly contains the answer. +- GitHub emits `##[error]` at the true failure (2 occurrences, both real). On GitHub that marker is the highest-precision signal available and costs nothing. + +### 3. Forgejo 15.0.2 has **no Actions log API at all**, and `tea actions` is broken against it + +`tea actions runs list --status failure --output json` works but is **lossy** — `workflow`, `branch`, `started` and `duration` all come back empty strings: + +```json +{ "id": "11130", "status": "failure", "workflow": "", "branch": "", "event": "pull_request", "started": "", "duration": "" } +``` + +The other two subcommands from the issue do not work at all against this server: + +``` +tea actions runs view 11130 → Error: failed to get jobs: unknown API error: 404 + GET /api/v1/repos/pseudoseed/entriq/actions/runs/11130/jobs +tea actions runs logs 6881 --job 40084 + → Error: failed to get logs for job 40084: unknown API error: 404 + GET /api/v1/repos/pseudoseed/entriq/actions/jobs/40084/logs +``` + +Both routes 404 because **they were added in Forgejo v16.0** (released 2026-07-16); `git.pseudoseed.com` reports `15.0.2+gitea-1.22.0`. I probed every plausible alternative on 15.0.2 — `actions/runs/{id}/jobs`, `actions/runs/{n}/jobs/{j}/logs`, `actions/tasks/{id}`, `actions/tasks/{id}/logs`, `actions/jobs/{id}`, `actions/artifacts`, `actions/workflows` — all 404. The web UI's own log route (`/{owner}/{repo}/actions/runs/{n}/jobs/{j}/logs`) exists but is **session-only**: it rejects both `Authorization: token` and HTTP basic auth with the API token (the API accepts the same token fine — verified as a control). There is no token-reachable path to a log on 15.0.2. + +So on this fork's own reference Forgejo, `ci-failures` and `ci-run-log` **cannot work**. That is a fact about the server, not a gap in the implementation, and the only correct response is to say so by name — see "the version gate" below. + +### 4. What Forgejo 15.0.2 *does* have, and what it costs + +| Endpoint | What it returns | Measured | +|---|---|---| +| `actions/runs?page=1&limit=N` | runs, each embedding a **full repository object** | 0.30 s / 3.8 KB at N=1; 0.33 s / **892 KB** at N=50 | +| `actions/runs/{id}` | one run | 0.3 s | +| `actions/tasks?page=1&limit=N` | **jobs**, GitHub-shaped (`id`, `name`, `head_branch`, `head_sha`, `run_number`, `status`, `workflow_id`, `url`) | 0.22 s / 507 B at N=1; 0.30 s / 24 KB at N=50 | + +Unlike `/pulls` in #12, these are priced **per request, not per item** — 50 runs cost the same 0.3 s as one. The cost is bytes, not seconds: `actions/runs` is 17.8 KB per run because of the embedded repo object, so it must be reduced with `jq` immediately. `actions/tasks` is 482 B per job and is the better list to build on. + +Three query-parameter facts, all measured, all footguns: + +- **`limit` is ignored unless `page` is also present.** `actions/runs?limit=3` returned **all 6922 runs**; `actions/runs?page=1&limit=3` returned 3. That is a #12-class hazard sitting in a default. +- **`status=` is honoured server-side** (`status=failure` → `total_count` 1541 of 6922). +- **`branch=` and `event=` are silently ignored** on both endpoints. Filtering by branch must happen client-side. +- `actions/tasks` caps at 50 per page regardless of `limit`. + +And the branch itself is not what you would expect: for `pull_request` runs, Forgejo reports `head_branch` / `prettyref` as **`#3847`** — the PR number, not the branch name. Only `push`/`schedule` runs carry a real branch. In the first 100 tasks on entriq: `#3869`×32, `#3865`×10, `main`×7, `v1.0.230`×1. So `CODEV_BRANCH_NAME=builder/pir-13` matches **nothing** on a repo that runs CI on pull requests unless the branch is first resolved to its PR number. + +Finally, Forgejo has **two id spaces** and both are valid inputs to the same route: run `id` 11130 has `index_in_repo` 6881, and `actions/runs/6881` resolves to a *different* run (a real one, whose own index is 4258). The web URL shows the index. A concept that guessed would confidently answer about the wrong run. + +### 5. Forgejo 16 does have it — verified live, unauthenticated + +`codeberg.org` runs `16.0.0-dev-694` and `forgejo/forgejo` is public with real Forgejo Actions history, so the v16 code path can be verified for free: + +| Call | Result | +|---|---| +| `GET repos/forgejo/forgejo/actions/runs/6554924/jobs` | JSON array of jobs: `id`, `task_id`, `name`, `status`, `needs`, `runs_on` | +| `GET repos/forgejo/forgejo/actions/jobs/11952749/logs` | **200**, `text/plain`, 142 KB, 1599 lines, 1.02 s, `accept-ranges: bytes` | +| the same with `?step=3` | **byte-identical** — `?step=` is not honoured on this build | +| `GET repos/forgejo/forgejo/actions/runs/6554924/logs` | 200, `application/zip`, 522 KB, every job's log | + +Two things to carry into the design: the log endpoint takes the **job `id`** (11952749), *not* the `task_id` (8848703) that `actions/tasks` lists — passing the task id returns `{"message":"resource does not exist"}`, which is exactly the sort of 404 that reads as "no logs" if it is not distinguished. And `accept-ranges: bytes` means a tail can be fetched as a byte range instead of downloading 142 KB to print 50 lines. + +--- + +## Proposed Change + +Four concepts, one shared extraction contract, and a hard rule that **every response is a JSON envelope that says how much of the truth it contains**. + +### The envelope + +Every ci-* concept prints one JSON object on stdout, on success *and* on failure, so a caller always gets something structured and never has to interpret an empty string: + +```jsonc +// ci-failures, extraction succeeded +{ + "ok": true, "provider": "github", "runId": 32515040122, + "failures": [{ + "jobId": 96874679182, "jobName": "Unit Tests", + "stepName": "Run unit tests with coverage", "stepNumber": 15, + "matchedBy": "vitest", // which rung of the ladder fired + "text": "FAIL src/…/agy-auth-cache.test.ts > gemini lane burst…\nAssertionError: expected null to be 'unauth'…", + "logLines": 2528, "returnedLines": 24, "truncated": false + }], + "jobsFailed": 1, "truncated": false +} + +// ci-failures, extraction gave up — a HANDOFF, not a dead end (issue comment 2) +{ + "ok": true, "provider": "gitea", "runId": 11130, + "extracted": false, "reason": "no recognized failure pattern", + "failures": [{ "jobId": 11952749, "jobName": "test-unit", "logLines": 1599 }], + "next": "ci-run-log CODEV_CI_RUN_ID=11130 CODEV_CI_JOB_ID=11952749 CODEV_CI_LOG_TAIL=80" +} + +// any concept, transport failure — a timeout is NEVER an empty result +{ "ok": false, "error": "timeout", "seconds": 60, + "detail": "GET repos/pseudoseed/entriq/actions/tasks did not return within 60s", + "remedy": "raise CODEV_FORGE_TIMEOUT" } + +// gitea, server too old +{ "ok": false, "error": "unsupported-server", "serverVersion": "15.0.2+gitea-1.22.0", + "needs": ">=16.0", + "detail": "this Forgejo has no Actions job-log API (added in Forgejo 16.0); ci-runs and ci-run-view still work" } +``` + +`logLines` / `returnedLines` / `truncated` are on **every** response that returns log text, per the issue. Never a bare array, never a bare string. + +**Exit statuses** keep the #12 contract — `0` answered, `1` could not answer, `2` bad input — with one addition specific to these concepts: **the JSON envelope is printed on stdout even when the exit status is non-zero**, because "the concept failed" and "the concept failed *because the API timed out at 60 s*" must not be the same observation. stderr still carries the one-line human message. + +### `ci-runs` — the cheap question + +Inputs: `CODEV_BRANCH_NAME` (opt), `CODEV_CI_STATUS` (opt: `success|failure|pending|queued|in_progress|skipped|canceled`), `CODEV_CI_LIMIT` (opt, default 20), `CODEV_CI_WORKFLOW` (opt — the "is it mine or flaky" filter). + +Output: `{ ok, provider, runs: [{ id, number, name, workflow, status, conclusion, branch, sha, event, url, createdAt }], truncated }`. + +- **github**: one `gh run list --json …` call (measured 0.7 s), `--branch`/`--status`/`--workflow`/`--limit` passed through. +- **gitea**: `actions/runs?page=1&limit=N&status=…`, always with `page=1` (see the `limit` footgun), reduced by `jq` on the way out. `status` server-side; `branch` **client-side**, and when `CODEV_BRANCH_NAME` is set and the repo runs CI on pull requests, the branch is first resolved to its PR number with the base/head lookup #12 already built (`gitea_default_branch` + `pulls/{base}/{head}`, ~1 s), then matched against `#N` as well as the literal branch. `conclusion` is emitted as `null` — Forgejo has no such field; `status` carries `failure`/`success` — and this asymmetry is documented rather than faked. +- The client-side branch filter walks at most `CODEV_CI_MAX_PAGES` (default 4 → 200 runs) and sets `truncated: true` with a stderr line if it stops early. It never returns a short list that reads as a complete one. + +### `ci-run-view` — status per job, still no log + +Input: `CODEV_CI_RUN_ID` (required). Output: the run plus `jobs: [{ id, name, status, conclusion, startedAt, completedAt, failedSteps: [{ name, number, conclusion }] }]`. + +- **github**: `gh run view --json jobs,status,conclusion,headBranch,headSha,workflowName,url,displayTitle` — 1.3 s, and it already carries per-step conclusions, so `failedSteps` is a `jq` filter over data we already paid for. +- **gitea, Forgejo ≥16**: `actions/runs/{id}` + `actions/runs/{id}/jobs`. Job `id` (not `task_id`) is what the output carries, because that is what the log endpoint takes. +- **gitea, Forgejo 15**: `actions/runs/{id}` gives `index_in_repo`; jobs are recovered by scanning `actions/tasks` for `run_number == index_in_repo`, bounded to `CODEV_CI_MAX_PAGES`. The response says which route answered (`"jobSource": "runs/jobs" | "tasks-scan"`) so a reader knows whether it is seeing the server's own grouping or ours. Forgejo has no per-step data on either version, so `failedSteps` is `[]` there — stated in the output, not silently omitted. + +`CODEV_CI_RUN_ID` is always the **API id** from `ci-runs`'s `id` field, never the URL number. `ci-runs` emits both `id` and `number` and the docs say which to pass; the two-id-space ambiguity above is why the concept refuses to guess. + +### `ci-failures` — the one that matters + +Input: `CODEV_CI_RUN_ID` (required), `CODEV_CI_JOB_ID` (opt, to pin one job). + +1. Find the failing jobs (from `ci-run-view`'s data, one call). +2. Fetch **only the first failing job's** log — `gh run view --log-failed` on GitHub (job-scoped, one call), `actions/jobs/{id}/logs` on Forgejo ≥16. "Prefer the FIRST failing step in the first failing job" per the issue; later failures are usually downstream. Other failing jobs are *listed* (name + id) but not fetched, and the response says so. +3. Strip ANSI, strip the `job\tstep\t` prefix and the leading RFC3339 timestamp, then run the ladder. +4. Cap: **2 KB per failing step, 8 KB per response**, `truncated: true` when either bites. + +**The extraction ladder** (first rung that matches wins, and the response names the rung in `matchedBy`): + +| Rung | Matches | Returns | +|---|---|---| +| 1. `runner-marker` | `##[error]` (GitHub) | the marker line plus 3 lines of leading context | +| 2. `vitest` / `jest` | `FAIL > ` and/or `AssertionError:` / `expected … to …`, anchored by the `Test Files … failed` summary — **the one containing `failed`**, not the first | the `Failed Tests` block from the first `FAIL` to the summary | +| 3. `tsc` | `error TS####:` | the first such line plus following lines of the same diagnostic | +| 4. `first-error` | first `^Error:` / `error:` / `Exception` **after the last passing-suite boundary**, never the first in the file | the line plus 3 either side | +| 5. give up | — | `extracted: false` + `jobId`, `jobName`, `logLines`, and a ready-to-run `next` | + +Rung 4 is deliberately weaker than "first match in the file": on the real log above, the unqualified version returns a passing test's fixture string from line 1257. The rule is only allowed to fire after the last recognisable "N passed" boundary, and if there is no such boundary it falls to rung 5 rather than guessing. **Rung 5 never returns arbitrary lines.** A builder that receives 50 unexplained lines treats them as the diagnosis; one told extraction failed goes and looks, which is correct and cheaper. + +### `ci-run-log` — the deliberate escape hatch + +Input: `CODEV_CI_RUN_ID`, `CODEV_CI_JOB_ID` (opt — defaults to the first failing job, else the only job), and **exactly one** window: `CODEV_CI_LOG_TAIL=N`, `CODEV_CI_LOG_HEAD=N`, or `CODEV_CI_LOG_GREP=` with `CODEV_CI_LOG_CONTEXT` (default 3). Zero windows or more than one is exit 2 with a named message — not a silent default, because a defaulted window is how this becomes "tail by habit," which the issue's second comment exists to prevent. + +Output: `{ ok, runId, jobId, jobName, logLines, returnedLines, from, to, truncated, lines: [...] }`. Same 8 KB response cap. `from`/`to` are 1-based line numbers into the full log, so a builder always knows where it is standing. + +It is a separate concept, not a flag on `ci-failures`, for exactly the reason the issue gives. + +### Caching + +A completed run's log is immutable. Both log-fetching concepts cache the **raw** log to `${TMPDIR}/codev-ci-logs///.log`, and read from it when present, **only when the job's status is terminal** (`success|failure|skipped|canceled`). An in-progress job is never cached and never read from cache. `CODEV_CI_NO_CACHE=1` bypasses; entries over `CODEV_CI_CACHE_MAX_MB` (default 32) are not written. This makes the realistic sequence — `ci-failures`, then `ci-run-log … TAIL`, then `ci-run-log … GREP` — cost exactly one download. + +### Timeouts, and making a timeout say "timeout" + +Three layers, because the current stack loses the distinction at every one of them: + +1. **Scripts.** gitea routes through `gitea_api` (#12's watchdog, `CODEV_FORGE_TIMEOUT`, default 60 s). GitHub has no equivalent today: `github/*.sh` call `gh` bare. I will extract #12's `gitea_timeout` into `scripts/forge/_timeout.sh`, source it from both providers, and wrap every `gh` call in the new scripts. Existing github scripts are left alone — retrofitting them is a separate change. +2. **Envelope.** Timeout ⇒ `{"ok":false,"error":"timeout","seconds":N,…}` on stdout, message on stderr, exit 1. +3. **Dispatcher.** `executeForgeCommand` returns `null` for every failure mode (`forge.ts:390-403`), so a timeout, a crash and unparseable output are one value. I will add `executeForgeCommandDetailed()` returning `{ ok, data, stdout, stderr, exitCode, timedOut, durationMs }` — non-destructive, existing callers untouched — and have it set `timedOut` from `err.killed`/`err.signal`, which #12 verified fires reliably at the Node level. Without this the shell layer can be as honest as it likes and the TS layer still flattens it to `null`. + +### gitlab and linear must be *disabled*, not absent + +`buildPresetFromScripts` only sets keys for concepts that have a script, and `getForgeCommand` falls back to the **github default** for anything unset (`forge.ts:280-296`). So adding four concepts without touching those presets would make a GitLab repo silently run `gh run list` — the exact silent-fallthrough class #1455 closed. All four go in the disabled list for `gitlab` and `linear`, so they resolve as `disabled` and `describeUnavailableConcept` names the provider. + +### Commits + +1. `_timeout.sh` extraction + `executeForgeCommandDetailed` + `KNOWN_CONCEPTS` + preset disables (github/gitea unblocked, gitlab/linear explicitly null). +2. GitHub scripts: `ci-runs`, `ci-run-view`, `ci-failures`, `ci-run-log` + the shared extractor. +3. Gitea scripts: the same four, with the version gate and the 15-vs-16 job-source fallback. +4. Tests, both `SKILL.md` twins, `arch.md` / `lessons-learned.md` routing. + +--- + +## Files to Change + +- `packages/codev/src/lib/forge.ts:65-70` — add `ci-runs`, `ci-run-view`, `ci-failures`, `ci-run-log` to `KNOWN_CONCEPTS`. +- `packages/codev/src/lib/forge.ts:126-135` — add all four to the disabled lists for `gitlab` and `linear`; leave `gitea` enabled. +- `packages/codev/src/lib/forge.ts:390-403` — new `executeForgeCommandDetailed()` beside `executeForgeCommand`. +- `packages/codev/src/lib/forge-contracts.ts` — `CiRunItem`, `CiRunViewResult`, `CiFailuresResult`, `CiRunLogResult`, and the shared `CiError`. +- `packages/codev/scripts/forge/_timeout.sh` — **new**, `gitea_timeout` lifted verbatim (it is provider-neutral) with its comments; `gitea/_lib.sh` sources it instead of defining it. +- `packages/codev/scripts/forge/_ci-extract.sh` — **new**, the ANSI-stripping extraction ladder, shared by both providers so they cannot drift. +- `packages/codev/scripts/forge/github/{ci-runs,ci-run-view,ci-failures,ci-run-log}.sh` — **new** (mode 755; `bugfix-693-forge-exec-bit.test.ts` pins the bit). +- `packages/codev/scripts/forge/gitea/{ci-runs,ci-run-view,ci-failures,ci-run-log}.sh` — **new**. +- `packages/codev/src/__tests__/pir-13-ci-concepts.test.ts` — **new**. +- `.claude/skills/forge/SKILL.md` and `.codex/skills/forge/SKILL.md` — the concept table, the env-var table, the envelope, the Forgejo-16 requirement. Byte-identical; the twin test from #12 covers it. +- `codev/resources/arch.md` (§ Integration Points → Forge Concept Commands) — the measured Forgejo facts. `codev/resources/lessons-learned.md` — the extraction lesson. Hot-tier promotion only if it displaces something; likely not. +- `codev/reviews/13-ci-forge-concepts.md` — at review time. + +No `codev-skeleton/` mirror: forge scripts and `SKILL.md` are single-source (established in #12). + +--- + +## Risks & Alternatives Considered + +- **Risk: `ci-failures` cannot work on the fork's own reference Forgejo.** entriq is 15.0.2 and the log API is 16.0+. Mitigation: the version gate returns `error: "unsupported-server"` naming both versions, `ci-runs`/`ci-run-view` keep working there, and the v16 path is verified against Codeberg. This is loud degradation, which is what the issue asks for from `gitlab`; it would be dishonest to hide it for gitea. **If `git.pseudoseed.com` is upgraded to Forgejo 16, the same code starts working with no change.** +- **Risk: the extractor is tuned to the two logs I have.** Mitigation: both are captured as fixtures (a 2528-line GitHub vitest failure and a 1599-line Forgejo `test-unit` failure), the ladder names its rung in `matchedBy`, and rung 5 is a first-class outcome rather than a fallback into guessing. When it is wrong it says so. +- **Risk: `##[error]` is GitHub-only.** Forgejo runners do not emit it. Rungs 2-4 carry Forgejo, which is why the ladder is shared rather than per-provider. +- **Alternative rejected: parse `gh run view --log-failed`'s step column.** It is `UNKNOWN STEP` on the very first run I tested. Step attribution comes from `--json jobs` instead, which is structured and reliable. +- **Alternative rejected: a `tail` flag on `ci-failures`.** The issue's second comment gives the reason and I agree with it: a flag gets passed by habit and every status question starts dragging a log. +- **Alternative rejected: implementing via `tea actions …` subcommands.** They are broken against 15.0.2 and lossy where they work. `tea api` is the load-bearing surface, as #12 established. +- **Alternative rejected: caching under `.codev/`.** Logs are large and disposable; `$TMPDIR` avoids polluting a repo and inherits OS cleanup. +- **Risk: response caps hide the answer.** Mitigation: `truncated` is always present, and `ci-run-log` exists precisely so the next call is targeted rather than blind. + +--- + +## Test Plan + +**Unit** (`pir-13-ci-concepts.test.ts`, stubbing `gh`/`tea` on `PATH`, the #12 pattern): + +1. Extraction against the **real captured** 2528-line GitHub log: returns the `agy-auth-cache` assertion, and specifically **not** `[artifact-canvas] Error: host blew up` (line 1257) and **not** the four passing `Test Files` summaries. +2. ANSI-wrapped `FAIL`/`AssertionError` still match (the raw fixture bytes carry the escapes). +3. A log with no recognisable failure ⇒ `extracted: false` with `jobId`/`jobName`/`logLines`/`next`, and **zero** log lines in the payload. +4. Caps: a >2 KB step extract sets `truncated: true`; `returnedLines < logLines`. +5. `ci-run-log`: zero windows ⇒ exit 2; two windows ⇒ exit 2; tail/head/grep return correct `from`/`to`; grep honours `CODEV_CI_LOG_CONTEXT`. +6. A stub that sleeps past `CODEV_FORGE_TIMEOUT` ⇒ `{"ok":false,"error":"timeout"}` on stdout, non-zero exit, and `executeForgeCommandDetailed().timedOut === true`. +7. gitea `ci-runs` sends `page=1` (the `limit`-ignored footgun) and never calls a bare `actions/runs?limit=`. +8. gitea `ci-runs` with `CODEV_BRANCH_NAME` matches a `#` `head_branch`, not just a literal branch. +9. `ci-failures` on a Forgejo-15 stub ⇒ `error: "unsupported-server"` with both versions named; `ci-run-view` on the same stub still answers via the tasks scan and reports `jobSource: "tasks-scan"`. +10. `resolveAllConcepts` reports all four as `disabled` for `gitlab` and `linear`, and as `preset` for `gitea`. +11. Exec bit on all eight new scripts; `SKILL.md` twins byte-identical. + +**Live, through the real dispatcher** (config load → preset → env → script → JSON parse), with timings reported in the review the way #12 did: + +- **GitHub / `pseudoseed/codev`**: `ci-runs` (all, by branch, `--status failure`), `ci-run-view` on run 32515040122, `ci-failures` on the same run (must return the assertion, not 293 KB), `ci-run-log` tail/head/grep, and a second `ci-failures` proving the cache makes it free. +- **Forgejo 15.0.2 / `~/dev/entriq`**, bare `gitea` preset, no overrides: `ci-runs` unfiltered, `--status failure`, and by branch via the PR-number mapping; `ci-run-view` on run 11130 via the tasks scan; `ci-failures` and `ci-run-log` returning the named `unsupported-server` envelope. entriq's config is read-only for this — nothing is written to it. +- **Forgejo 16 / `codeberg.org/forgejo/forgejo`**: the v16 log path end-to-end on run 6554924 / job 11952749. See the open question below about how deep this can go. + +**Manual, at the `dev-approval` gate**: `codev doctor` in this worktree and in entriq shows the four concepts with the right source per provider; a deliberately broken `CODEV_CI_RUN_ID` produces a named error rather than an empty object. + +--- + +## Open Questions for the Architect + +1. **Codeberg token.** The gitea v16 log path is verified at HTTP level today (unauthenticated curl against `codeberg.org`, timings above). Driving it through the **real dispatcher** needs `tea` to have a login for codeberg, and `tea` requires a token to add one. If you have a Codeberg read-only token, that lane becomes a full end-to-end verification instead of an HTTP-level one. If not, I ship with the HTTP-level verification plus fixture tests and say exactly that in the review. +2. **Is a Forgejo 16 upgrade for `git.pseudoseed.com` on the cards?** It does not change the code — the same scripts light up — but it changes whether `ci-failures` is usable on entriq the day this merges, and I would rather the review state the real situation than a hopeful one. diff --git a/codev/projects/13-add-ci-concepts-to-the-forge-l/status.yaml b/codev/projects/13-add-ci-concepts-to-the-forge-l/status.yaml new file mode 100644 index 000000000..a1dbc0741 --- /dev/null +++ b/codev/projects/13-add-ci-concepts-to-the-forge-l/status.yaml @@ -0,0 +1,30 @@ +id: '13' +title: add-ci-concepts-to-the-forge-l +protocol: pir +phase: review +plan_phases: [] +current_plan_phase: null +gates: + plan-approval: + status: approved + requested_at: '2026-08-21T20:58:03.660Z' + approved_at: '2026-08-21T20:58:47.286Z' + dev-approval: + status: approved + requested_at: '2026-08-21T22:38:21.754Z' + approved_at: '2026-08-21T22:39:29.289Z' + pr: + status: approved + requested_at: '2026-08-21T23:45:10.673Z' + approved_at: '2026-08-21T23:55:10.108Z' +iteration: 1 +build_complete: false +history: [] +started_at: '2026-08-21T20:35:23.798Z' +updated_at: '2026-08-21T23:55:10.109Z' +pr_history: + - phase: review + pr_number: 23 + branch: builder/pir-13 + created_at: '2026-08-21T22:50:04.111Z' +pr_ready_for_human: false diff --git a/codev/resources/arch.md b/codev/resources/arch.md index 5a54d570e..e5cbf0211 100644 --- a/codev/resources/arch.md +++ b/codev/resources/arch.md @@ -2086,7 +2086,7 @@ All interactions with the repository hosting platform (GitHub by default) are ro **Configuration**: `.codev/config.json` `forge` section maps concept names to shell commands. Set to `null` to disable a concept. Omit to use the default (`gh`-based) command. -**18 concepts**: `issue-view`, `pr-list`, `issue-list`, `issue-search`, `issue-comment`, `pr-exists`, `recently-closed`, `recently-merged`, `user-identity`, `team-activity`, `on-it-timestamps`, `pr-create`, `pr-merge`, `pr-search`, `pr-view`, `pr-diff`, `auth-status`, `repo-archive`. +**22 concepts**: `issue-view`, `pr-list`, `issue-list`, `issue-search`, `issue-comment`, `pr-exists`, `recently-closed`, `recently-merged`, `user-identity`, `team-activity`, `on-it-timestamps`, `pr-create`, `pr-merge`, `pr-search`, `pr-view`, `pr-diff`, `auth-status`, `repo-archive`, `ci-runs`, `ci-run-view`, `ci-failures`, `ci-run-log`. **Environment variables**: Each concept receives `CODEV_*` env vars (e.g., `CODEV_ISSUE_NUMBER`, `CODEV_PR_NUMBER`) that the command uses to parameterize its output. @@ -2101,6 +2101,27 @@ All interactions with the repository hosting platform (GitHub by default) are ro - Gitea's `/repos/{o}/{r}/pulls` list is priced **per returned PR object**, not per request: 0.78s at `limit=1`, 32.8s at `limit=50`. Paging it is linear in total PRs regardless of page size, so on a 1599-PR repo a full walk costs ~17 minutes. Anything answerable by a targeted endpoint must not use it. The cheap index for the same rows is `/issues?type=pulls` (~1.8s per 50), which carries `pull_request.merged_at` but no head/base refs. - `head.ref` is rewritten to `refs/pull/N/head` once a merged PR's source branch is deleted — the normal state of every merged PR — but **`head.label` retains the original branch name**, and `GET /pulls/{base}/{head}` matches on the stored head branch. Branch→PR lookup is therefore possible after branch deletion, which the earlier `head.ref` scan had documented as impossible. +**Invoking a concept**: `codev forge ` (`src/commands/forge.ts`), a thin wrapper over `executeForgeCommandDetailed` that passes the ambient `CODEV_*` through, prints stdout verbatim and exits with the script's own code. **Naming a concept script by path bypasses resolution** — the config lookup, the provider preset and any per-repo override — so a project that overrides a concept gets the github default against its own forge. Its own exit codes: `2` unknown concept (lists the valid ones), `3` disabled for this provider (named). + +**`executeForgeCommandDetailed(concept, env, options)`** exists because `executeForgeCommand` collapses every failure mode to `null`: a timeout, a non-zero exit, unparseable output and a disabled concept arrive identically. It returns `{ok, data, stdout, stderr, exitCode, timedOut, unavailable, durationMs}` and **keeps stdout on the failure path**. Use it whenever "could not answer" and "answered no" must not be the same value. + +**CI concepts (PIR #13)** — `ci-runs`, `ci-run-view`, `ci-failures`, `ci-run-log`, tiered so the cheap question stays cheap: only the last two ever read log bytes. Their contract adds two rules to the exit-status one above: + +- **Errors are values.** Every ci-* concept prints one JSON object on stdout on success *and* failure, so the class of failure (`timeout` | `not-found` | `unsupported-server` | `forge-error` | `bad-input`) survives a non-zero exit. +- **Anything carrying log text also carries `logLines`, `returnedLines`, `truncated`.** When extraction recognises nothing it returns `extracted: false` with the job identity and a ready-to-run `next`, and **no log lines at all** — never a fallback slice, which a reader treats as a diagnosis. + +Shared implementation, so the two providers cannot drift: `scripts/forge/_ci-extract.sh` (the extraction ladder — ANSI stripping, then vitest/jest → go test → tsc → the runner's `##[error]` marker → a line-**anchored** first error → refusal), `_ci-lib.sh` (envelope, caps, `$TMPDIR` log cache keyed by job id and written only for terminal jobs, window parsing), `_timeout.sh` (#12's watchdog, now used by both providers). + +**CI behavior verified live against GitHub (gh 2.87.0), Forgejo 15.0.2 and Forgejo 16.0.0-dev** (PIR #13). Each of these looks like something else and is not: + +- **`gh run view --log-failed` does not return an extract, and its step attribution is unreliable.** Measured on two runs of this repository: on 32515040122 all 2528 lines came back tagged `UNKNOWN STEP` (gh maps log files to steps by name and falls back when that misses); on 32536232930 the same command attributed all 1193 lines to the failing step correctly. **Either way the output is a whole job or a whole step** — 293 KB and 108 KB respectively — never the assertion. So codev extracts on both providers, and takes the failing step NAME from `gh run view --json jobs`, which is structured and does not depend on that mapping. +- **Forgejo has no Actions job-log API before 16.0** (released 2026-07-16). On 15.x there is no token-reachable log by any route: `tea actions runs view` / `runs logs` both 404, and the web UI's log route is session-only, rejecting an API token and basic auth alike. `ci-failures` / `ci-run-log` return `unsupported-server` there naming both versions; `ci-runs` / `ci-run-view` keep working. +- **Forgejo ignores `limit` unless `page` is also sent** — `actions/runs?limit=3` returned all 6922 runs. `status=` filters server-side; **`branch=` and `event=` are silently ignored**. +- **A `pull_request` run records `head_branch` as `#`**, not a branch name, so branch filtering resolves the branch to its PR first (the #12 base/head lookup) and filters client-side. +- **Run `id` and `index_in_repo` are two id spaces and both resolve on `/actions/runs/{x}`**, to different real runs. Concepts take `id` only and refuse a non-numeric value rather than guess. +- **Forgejo rejects `status=canceled`** — the spelling its own `tea` CLI documents — and accepts `cancelled`; GitHub wants `cancelled` too. +- `actions/runs` costs ~17.8 KB **per run** (an embedded repository object) at ~0.3s per page; `actions/tasks` is 482 B per job and is the cheaper index, but its ids are TASK ids, which the log API does not accept. + ### Two remote-command paths into an editor surface (Spec 1401) Tower has **two** ways for an external controller to drive an editor, and picking the wrong one diff --git a/codev/resources/lessons-critical.md b/codev/resources/lessons-critical.md index cdad4cac1..0f4e3a8c8 100644 --- a/codev/resources/lessons-critical.md +++ b/codev/resources/lessons-critical.md @@ -9,7 +9,7 @@ MAINTAIN polices the cap and keeps the map in sync with lessons-learned.md's sec - Trust the protocol — never skip CMAP/consultation; it catches security, design, and protocol issues solo review misses. - Check for existing work (PRs, git history) before building from scratch. - "It compiled" / "tests pass" is not "it works" — verify the real user path end-to-end before calling it done. -- A truncated result is indistinguishable from a complete one once emitted — give "I stopped early" its own signal and emit nothing, never a partial answer that reads as whole. +- "I could not tell" must never be spelled the same way as "no". A truncation, an unreachable API, and a server too old to answer each need their own signal and must emit nothing else — a partial or empty answer reads as a complete, negative one. - Single source of truth beats distributed state — consolidate duplicates rather than syncing them. - After any rename or framework change, grep the whole repo across BOTH codev/ and codev-skeleton/ before claiming "all fixed." - When stuck (2 failed hypotheses or ~30 min), get an outside model's perspective instead of guessing. diff --git a/codev/resources/lessons-learned.md b/codev/resources/lessons-learned.md index 263584b1e..be0c84b78 100644 --- a/codev/resources/lessons-learned.md +++ b/codev/resources/lessons-learned.md @@ -304,6 +304,10 @@ just its inputs. - [From 0755] Vestigial production code can survive for unknown durations. `setArchitect` was orphaned (only called from tests) for an unknown period; the local `architect` table it wrote to was effectively dead state. When a feature touches a long-lived API, run a "who calls this in production?" grep during planning, not after the implementation has diverged. Reviewers caught it in iter-1; planning would have caught it earlier. - [From 0755] When a plan references specific migration version numbers, verify against the current schema before commit -- or reference migrations by purpose ("the next available after issue_number widening") rather than fixed numbers. The plan said v5 local + v5 global; the actual code needed v9 + v13 because the project had already advanced past those. - [From 1313] Trace a contract change end-to-end before calling it specified. A send-outcome change (`delivered` vs `held`+reason) was specified server-side but not client-side (`packages/core/src/tower-client.ts` + `commands/send.ts`, on BOTH the single-send and `--all` paths), and drew repeat REQUEST_CHANGES across the plan and Phase 4. Name every layer the contract crosses (wire → client → each CLI path) in the plan deliverable so the client surfacing isn't discovered at review time. +- [From #13] **When an issue tells you a tool already does the work, measure it before building on that.** #13 asserted that `gh run view --log-failed` extracts the failing step and told the implementer not to duplicate it. It does not: it selects the failing JOB and returned 2528 lines / 293 KB with every line tagged `UNKNOWN STEP`. Two people had read that output that day without registering what it meant. The whole design rests on that premise, so a two-minute check was worth more than the paragraph asserting it. +- [From #13] **Extraction must recognise, not truncate — and it must strip ANSI before it can recognise anything.** The payload line in a real runner log is `ESC[41mESC[1m FAIL ESC[22m…`, so a matcher that skips cleaning reports "no recognized failure" on a log that plainly contains one. And the naive alternatives both pick the wrong line on the same log: the first "Error:" is a *passing* test's fixture string 1214 lines above the real failure, and the last 25 lines of the Forgejo log are git credential cleanup. Anchor error patterns at the START of the line — that alone kills the observed false positive, and it holds in logs with no test summary to measure against. +- [From #13] **A CLI's `--help` is not the API's vocabulary.** `tea actions runs list --help` documents `canceled`; the Forgejo API answers that with `{"message":"unknown status: canceled"}` and accepts `cancelled` (2240 runs). A filter spelled from the help text returns an empty list that nobody questions. +- [From #13] **Ask which id space an endpoint means.** Forgejo run `id` and `index_in_repo` BOTH resolve on `/actions/runs/{x}`, to different real runs, and the web URL shows the second. Its jobs list carries `id` and `task_id` and the log API accepts only the first. Where two plausible ids resolve on the same route, a concept must take one by name and refuse the other rather than guess. ## Testing @@ -457,6 +461,13 @@ Measure accumulated CPU time, or just let it finish, before concluding a process a fresh database. Forget the bump and a fresh install records the older version and re-runs the new block on its next open. Nothing breaks loudly — the columns arrive from the schema anyway — so it survives review. Pin the constant to the highest migration block in a test. +- [From #13] **Under `set -e`, a helper that returns non-zero decides its caller's exit status.** `ci_fail` printed an error envelope and returned 1; every call site then had `exit 2` for a bad input on the next line, and none of them ever ran — `set -e` killed the script at the helper. Reporting and deciding the exit status are separate jobs; a reporting helper should return 0. +- [From #13] **A shell helper that "sets a global" sets nothing when the caller captures it.** Two helpers assigned `CI_TRUNCATED` / `GITEA_JOB_SOURCE` for their callers to read, and every caller ran them inside `$( )` — a subshell, so the values were discarded and jq got an empty `--argjson`. Return the data, or write it to a file the caller reads. +- [From #13] **A test whose ceiling sits below its own cost reads as flaky.** `spec-1280-measurement-instrument.test.ts` capped tests at 60s that cost 80-100s (the instrument is ~25-30s per invocation and several tests invoke it two or three times). A *different* pair failed each full-suite run, which looks like flakiness and invites `.skip`. Measure the work, then set the ceiling above it — that is fixing the bug rather than hiding it. +- [From #13] **A test that overrides the defaults cannot detect the defaults being wrong.** The timeout test forced `CODEV_FORGE_TIMEOUT=2` and passed, proving the watchdog fires when told to — while at the real defaults the outer 30s ceiling killed the command before the inner 60s watchdog could name the endpoint, so the whole point of the inner watchdog never fired. Where two ceilings, retries or precedence rules compose, pin the ORDERING at the values that actually ship. +- [From #13] **`sed -n "1,0p"` is tolerated by BSD sed and rejected by GNU sed.** An empty input built that reversed range, which was invisible on the macOS box it was written on and would have aborted the script under `set -e` on Linux, where CI runs. Guard the empty case explicitly rather than trusting a range expression to degrade. +- [From #13] **macOS `mktemp -d` falls back when `TMPDIR` is unusable; GNU `mktemp` fails.** A test harness pointing `TMPDIR` at a directory it never created passed locally and failed 31 tests on the Linux runner — and worse, the failure surfaced as a *wrong answer*: the temp dir failed inside the timeout watchdog, the forge CLI never ran, the wrapper returned 1, and the concept reported "run could not be read". Check `[ -d "$TMPDIR" ]` explicitly so both platforms agree, and give "could not set up" a different status from "the command failed". +- [From #13] **A CI tolerance guard that greps the whole log for a word can never fire.** `test.yml` tolerates a known vitest worker-teardown crash only when the output contains no "failed" — but the runner echoes the guard's own script into the log, and ordinary test names (`clear-failed`, "reports a failed Tower send") match too. The escape hatch had never once been reachable, so every worker crash was a hard failure. Scope such a check to the summary line it means, not to the whole transcript. ## UI/UX diff --git a/codev/reviews/13-ci-forge-concepts.md b/codev/reviews/13-ci-forge-concepts.md new file mode 100644 index 000000000..6bea30bc5 --- /dev/null +++ b/codev/reviews/13-ci-forge-concepts.md @@ -0,0 +1,376 @@ +# PIR Review: CI concepts for the forge layer + +Fixes #13 + +## Summary + +Adds four CI concepts to the forge layer — `ci-runs`, `ci-run-view`, `ci-failures`, `ci-run-log` — for both `github` and `gitea`, tiered so that only the last two ever read log bytes. A builder asking why CI failed now gets the failing job, the failing step and the assertion instead of a log: on the reference run, **293 KB becomes a 1.2 KB response**. Also adds `codev forge `, because naming a concept script by path bypasses resolution and would silently ignore a repo's own overrides. + +## The record this PR corrects + +**Issue #13 says `gh run view --log-failed` "already returns only failed steps" and instructs the implementer not to re-derive that. It does not, and the instruction is wrong.** Measured on run `32515040122` of this repository: + +``` +gh run view 32515040122 --log-failed → 2528 lines, 293 KB, 2.0 s +Unit Tests UNKNOWN STEP 2026-08-21T18:47:09.5820646Z Current runner version: '2.336.0' +``` + +Every one of those 2528 lines is tagged `UNKNOWN STEP`. `--log-failed` selects the failing **job** and returns all of it; `gh` maps log files to steps by name and falls back to `UNKNOWN STEP` when that mapping misses. The architect independently reproduced this on run `32448538074`: 919 lines, all 919 tagged `UNKNOWN STEP` — and had read that same output earlier the same day while diagnosing #6 without registering what it meant. + +**One precision, raised by the claude lane and verified:** that attribution is *unreliable*, not always absent. On run `32536232930` the same command attributed all 1193 lines to the failing step correctly. It changes nothing about the design — attributed or not, what comes back is **a whole job or a whole step**, 293 KB and 108 KB respectively, and never the assertion — but "always UNKNOWN STEP" would have been an overstatement, so it is not claimed here or in `arch.md`. + +So codev extracts on **both** providers, and neither uses `--log-failed`. Both fetch `actions/jobs/{id}/logs` — one job, no invented step column, the same shape Forgejo 16 serves — which is also why they share one cache and one extractor. The failing step *name* comes from `gh run view --json jobs`, which is structured and reliable. + +Anyone reading #13 later should read this section instead of its "Provider notes". + +## What the two forges actually do + +Everything below was measured against live instances on 2026-08-21, not reasoned about. + +**Forgejo has no Actions job-log API before 16.0** (released 2026-07-16). `git.pseudoseed.com` reports `15.0.2+gitea-1.22.0`, and there `tea actions runs view` and `tea actions runs logs` both 404 — they call `/actions/runs/{id}/jobs` and `/actions/jobs/{id}/logs`, neither of which exists. Every alternative route was probed. The web UI's own log route exists but is session-only: it rejects `Authorization: token` and HTTP basic auth alike, while the API accepts the same token (verified as a control). There is no token-reachable log on 15.x by any path. + +`tea actions runs list --output json` is also lossy where it does work — `workflow`, `branch`, `started` and `duration` all come back as empty strings — so these scripts go through `tea api`, as #12 established. + +Four query-parameter facts, all footguns: + +| | | +|---|---| +| `limit` is ignored unless `page` is also sent | `actions/runs?limit=3` returned **all 6922 runs** | +| `status=` filters server-side | works, and is used | +| `branch=` and `event=` are silently ignored | branch filtering is client-side | +| `status=canceled` is rejected | `{"message":"unknown status: canceled"}`; `cancelled` returns 2240 runs — the opposite of what `tea`'s own `--help` documents | + +And two shape facts: + +- **A `pull_request` run records `head_branch` as `#3847`** — the PR number, not a branch. In the first 100 tasks on the reference repo: `#3869` ×32, `#3865` ×10, `main` ×7, `v1.0.230` ×1. So `CODEV_BRANCH_NAME=builder/x` matches *nothing* on a repo that runs CI on pull requests unless the branch is resolved to its PR first, which `ci-runs` does with #12's base/head lookup. +- **Run `id` and `index_in_repo` are two id spaces, and both resolve on `/actions/runs/{x}`** to different real runs. The web URL shows the second. `ci-runs` emits both, the log concepts take `id` only, and a non-numeric value is refused rather than guessed. + +On Forgejo 16 (verified on codeberg.org, `16.0.0-dev-694`): `actions/runs/{id}/jobs` returns jobs carrying both `id` and `task_id`, and **the log endpoint accepts the job `id`, not the `task_id`** — passing the task id returns `{"message":"resource does not exist"}`, which is exactly the 404 that reads as "no logs" if it is not distinguished. `actions/jobs/{id}/logs` served 142 KB / 1599 lines in 1.02 s as `text/plain` with `accept-ranges: bytes`; `?step=N` is accepted and ignored. + +## Design + +### Tiering + +| Question | Concept | Reads a log? | +|---|---|---| +| Did my push pass? | `ci-runs` | No | +| Is it still running, which job is pending? | `ci-runs`, `ci-run-view` | No | +| It failed — why? | `ci-failures` | Yes, one job | +| Is this mine or pre-existing? | `ci-runs` + `CODEV_CI_WORKFLOW` | No | +| Extraction gave up — show me | `ci-run-log` | Yes, one window | + +`ci-run-log` is a separate concept rather than a flag, per the issue's second comment: a window parameter on the main call gets passed by habit, and then every status question drags a log again. + +### The extraction ladder, and the three traps it was built against + +All three are from the real captured logs, and each is a test: + +1. **ANSI.** The payload line is `ESC[41mESC[1m FAIL ESC[22mESC[49m src/…`, so a matcher that does not clean first matches **nothing** and reports "no recognized failure" on a log that plainly contains one. Cleaning is not cosmetic. +2. **"First line matching an error pattern" returns line 1257 of 2528**: `[artifact-canvas] Error: host blew up` — a fixture string printed by a *passing* test. The real failure is at 2471. The ladder's generic rung therefore anchors patterns at the **start** of the line; that decoy's `Error:` is mid-line and cannot match, and anchoring holds even in logs with no test summary to measure against. +3. **`Test Files` appears four times before the failing summary**, three of them saying `passed` and one a shell line echoing `grep -q "Test Files.*passed"`. Any rule taking the first match reports a passing suite as the failure. + +Rungs, in order, with the one that fired named in `matchedBy`: `vitest`/jest → `go-test` → `tsc` → the runner's `##[error]` marker → line-anchored `first-error` → refusal. Runner recognition sits above the `##[error]` marker deliberately (and as issue #13's own priority order asks): the marker returns one sentence, the vitest rung returns the whole Failed Tests block — test name, assertion, expected/received, file:line. + +**Deviation from the approved plan, stated plainly.** The plan said the generic rung would fire only *after* a passing-suite boundary and otherwise fall through to refusal. Implemented, it fires with anchoring always and the boundary as a preference. Anchoring is what actually kills the observed false positive; refusing whenever no test summary exists would have returned `extracted: false` for the whole class of install/setup/compile failures and bought no safety. + +### Refusal is a handoff + +```json +{ "extracted": false, "reason": "no recognized failure pattern", + "failures": [{ "jobId": 11952749, "jobName": "test-unit", "logLines": 1599 }], + "next": "ci-run-log CODEV_CI_RUN_ID=6554924 CODEV_CI_JOB_ID=11952749 CODEV_CI_LOG_TAIL=80" } +``` + +No log lines at all. A builder handed 50 arbitrary lines treats them as the diagnosis and reasons from noise; one told extraction failed reads the log with the call the response already handed it. + +### Errors are values + +Every ci-* concept prints one JSON object on stdout on success **and** failure, so `timeout` / `not-found` / `unsupported-server` / `forge-error` / `bad-input` stay distinguishable after `executeForgeCommand` has flattened everything else to `null`. `executeForgeCommandDetailed` is added for callers that need the distinction in TypeScript: it returns `{ok, data, stdout, stderr, exitCode, timedOut, unavailable, durationMs}` and keeps stdout on the failure path. + +On a Forgejo below 16 the response is `unsupported-server`, naming the version found and the version needed **and still listing the failing job names it could determine** — never an empty `failures` array. It calls that list `failingJobs`, not `failures`, and the difference is deliberate: a `failures` entry carries an extract (`matchedBy`, `text`, `from`/`to`, `returnedLines`), and these have none — only a name and an id. Reusing the key would make an unsupported server shaped like a successful extraction with the details missing, which is a smaller version of the same lie the envelope exists to prevent. (Raised as an inconsistency by the claude lane; kept, with the reason stated here.) "Your CI is fine" and "I cannot see your CI at all" are opposite facts and must not be the same observation. + +### `codev forge ` + +Added at the architect's direction at the dev-approval gate, and the reasoning is correctness rather than convenience: calling `packages/codev/scripts/forge/github/ci-failures.sh` by path **bypasses resolution** — the config lookup, the provider preset, and any per-repo override — so a repo that overrides a concept gets the github default against its own forge. The reference Forgejo repo carried three such overrides until #12 shipped. It delegates to `executeForgeCommandDetailed`, prints stdout verbatim and exits with the script's code; its own additions are exit 2 for an unknown concept (listing the valid ones) and exit 3 for a concept disabled for the provider, named. + +## Files Changed + +- `packages/codev/scripts/forge/_ci-extract.sh` (+202 / -0) — the extraction ladder +- `packages/codev/scripts/forge/_ci-lib.sh` (+399 / -0) — envelope, caps, cache, windows, id validation +- `packages/codev/scripts/forge/_timeout.sh` (+100 / -0) — #12's watchdog, now shared by both providers +- `packages/codev/scripts/forge/gitea/_lib.sh` (+8 / -79) — sources the shared watchdog +- `packages/codev/scripts/forge/gitea/_ci.sh` (+229 / -0) +- `packages/codev/scripts/forge/gitea/{ci-runs,ci-run-view,ci-failures,ci-run-log}.sh` (+468 / -0) +- `packages/codev/scripts/forge/github/_lib.sh` (+55 / -0) +- `packages/codev/scripts/forge/github/{ci-runs,ci-run-view,ci-failures,ci-run-log}.sh` (+407 / -0) +- `packages/codev/src/lib/forge.ts` (+119 / -11) — registration, gitlab/linear disabled, `executeForgeCommandDetailed` +- `packages/codev/src/lib/forge-contracts.ts` (+181 / -0) +- `packages/codev/src/commands/forge.ts` (+97 / -0) — `codev forge ` +- `packages/codev/src/cli.ts` (+19 / -0) +- `packages/codev/src/__tests__/pir-13-ci-concepts.test.ts` (+976 / -0) +- `packages/codev/src/__tests__/fixtures/pir-13/{github-vitest-failure,forgejo-go-failure,github-vitest-worker-crash}.log.gz` (3 files, 98 KB) — the third is this branch's own red CI run, which produced the capture-block decoy +- `packages/codev/src/__tests__/forge.test.ts` (+10 / -2) — concept count 18 → 22 +- `packages/codev/src/__tests__/spec-1280-measurement-instrument.test.ts` (+18 / -25) — timeout ceiling, see Flaky Tests +- `.claude/skills/forge/SKILL.md`, `.codex/skills/forge/SKILL.md` (+133 / -0 each, byte-identical twins) +- `codev/resources/arch.md`, `codev/resources/lessons-critical.md`, `codev/resources/lessons-learned.md` +- `codev/plans/13-ci-forge-concepts.md`, `codev/reviews/13-ci-forge-concepts.md`, `codev/state/pir-13_thread.md` + +## Commits + +- `700aefc63` feat(forge): CI concept plumbing — shared timeout, extraction ladder, envelope +- `d4605f6b2` feat(forge): the four CI concepts for GitHub +- `1fd18d491` feat(forge): the four CI concepts for Gitea/Forgejo +- `346243ef5` test(forge): pin the CI concepts against two real captured logs +- `5cd31ef16` fix(forge): send status=cancelled, the spelling both forges actually accept +- `3e552014e` fix(forge): a CLI that exits 0 with non-JSON still gets an envelope +- `494a17352` fix(forge): gitea ci-runs reports truncation when it hits the page ceiling +- `8e26a661a` fix(forge): reject a non-numeric run or job id before it reaches a URL +- `067f7b179` test(forge): the concept count is 22, not 18 +- `9893db9cd` test: give the prompt-surface instrument a ceiling above its own cost +- `3334bf9e8` feat(cli): codev forge — run a concept through the real resolver +- `5f57f107e` Review + retrospective +- `79fb7b664` docs: record the review-lane coverage gap and porch's wrong remedy +- `8a57b262c` fix: the two defects the claude review lane found +- `9a5a19bac` docs: test counts after the review-lane fixes (5572 passed, 67 new) +- `5a61226d6` fix: an unusable TMPDIR must not be reported as a missing run +- `b1f8c7fce` fix(forge): the extractor pointed at a passing test + +## Test Results + +- `npm run build`: ✓ pass +- `npm test`: ✓ pass — 5622 passed, 0 failed, 48 skipped (5670), after merging `origin/main`. **70 new tests** in `pir-13-ci-concepts.test.ts`, plus the two concept-count assertions updated in `forge.test.ts`. + +### Branch CI: red, and NOT because of a failing test + +**Read this before reading `npm test: ✓ pass` above.** That line is the local suite; the branch's own CI is a separate claim, and for several commits it was **red** while this file said nothing about it. The claude review lane caught that. + +**CI is green at HEAD** (`b1f8c7fc`: `Tests` ✓, `CLI Integration Tests` ✓), which confirms the diagnosis below — the red was an intermittent worker-teardown crash, not a failing test. The guard defect that turned that flake into a hard failure is still there for whoever hits it next. + +The last red run (`32536232930`) reports: + +``` +Test Files 280 passed | 3 skipped (284) +⎯⎯ Unhandled Errors ⎯⎯ +Error: [vitest-pool]: Worker forks emitted error. +Caused by: Error: Worker exited unexpectedly +``` + +**No test failed.** A vitest worker fork died during teardown, so one file went unreported. `.github/workflows/test.yml` already knows about this and tries to tolerate it — its own comment says *"Vitest forks pool has a known issue where the worker process crashes during cleanup after all tests pass"* — with: + +```sh +if grep -q "Test Files.*passed" /tmp/vitest-output.txt && ! grep -q "failed" /tmp/vitest-output.txt; then + echo "::warning::Vitest worker crashed during cleanup but all tests passed" +``` + +**That guard cannot fire.** `grep -q "failed"` runs over the whole captured output, and "failed" appears in it six times on this run — none of them a failing test: + +| line | what it is | +|---|---| +| 1475 | **the guard's own script**, echoed into the log by the Actions runner | +| 1657, 1663, 1664 | `spec-1470` test names and stdout — `reentry-failed`, `clear-failed`, "reports a **failed** Tower send" | +| 2202, 2205 | `git fetch … failed` warnings printed by consult tests | + +So *any* worker crash in this repository is a hard CI failure regardless of the test results, and the tolerance the workflow author wrote has never been reachable — which is why a flake that later cleared on a re-run cost this PR two red runs and a diagnosis. That is a defect in `test.yml`, not in this diff, and it is **not fixed here** — fixing another team's CI gate to turn an unrelated red green is the scope creep the review phase warns against. It is reported to the architect with this evidence. + +What this PR *did* cause, and has fixed, is the earlier red: the `TMPDIR` harness bug above, which failed 31 tests on Linux while passing locally. + +### Verification coverage — three tiers, and they are not the same + +**Do not read the third as if it were the first two.** + +**1. GitHub — live, end to end through the real dispatcher** (config load → preset → env → script → JSON parse), against `pseudoseed/codev`: + +| Call | Time | Result | +|---|---|---| +| `ci-runs` (limit 3) | 987 ms | 3 runs, `truncated: true` | +| `ci-runs --status failure` | 695 ms | 3 runs | +| `ci-runs --branch builder/pir-12` | 875 ms | 3 runs | +| `ci-run-view` run 32515040122 | 1464 ms | 5 jobs, 1 failing, failing step named | +| `ci-failures` (cold) | 2503 ms | `vitest`, **23 of 2528 lines**, 1.2 KB response | +| `ci-failures` (cached) | 1323 ms | identical answer, no download | +| `ci-run-log` tail 10 | 1215 ms | lines 2519–2528 | +| `ci-run-log` head 5 | 1232 ms | lines 1–5 | +| `ci-run-log` grep AssertionError | 1190 ms | 14 lines, matched at 2472 and 2497 | +| `ci-run-log` with no window | 29 ms | `bad-input` — refused before spending an API call | + +Also verified against a second real run (`32448538074`, the architect's): 919 lines → 20, `matchedBy: vitest`. + +**2. Forgejo 15.0.2 — live, end to end through the real dispatcher**, against `~/dev/entriq` on the bare `gitea` preset with no overrides: + +| Call | Time | Result | +|---|---|---| +| `ci-runs` (limit 3) | 726 ms | 3 runs | +| `ci-runs --status failure` | 407 ms | 3 runs | +| `ci-runs --branch builder/air-364` | 2255 ms | 2 runs, matched via PR ref `#3855` | +| `ci-run-view` run 11130 | 3269 ms | **10 jobs via `tasks-scan`**, 1 failing | +| `ci-failures` run 11130 | 3737 ms | `unsupported-server`, both versions named, failing job still listed | +| `ci-run-log` run 11130 | 3903 ms | `unsupported-server` | + +`codev doctor` in that repo resolves all four concepts to `tea`. **entriq was read-only** — nothing was written to it. Its working tree carries an unrelated uncommitted `.codev/config.json` edit that predates this session (13:34 MDT, deleting the three overrides #12 made redundant). + +**3. Forgejo 16 — HTTP level only. NOT driven through the dispatcher against a live v16 server.** The v16 routes were verified with unauthenticated `curl` against `codeberg.org/forgejo/forgejo` (jobs list, job log 200/`text/plain`/142 KB/1.02 s, run-log zip), and the code path is covered by unit tests driving a stubbed `tea` that serves the **real captured codeberg job log**. Driving it through the dispatcher needs `tea` to hold a Codeberg login, which the architect deferred. `git.pseudoseed.com` was still on 15.0.2 at the time of writing; the owner has filed the Forgejo 16 upgrade, and if it lands the same code lights up with no change. + +**4. Provider degradation**: `resolveAllConcepts` reports all four concepts as `disabled` for `gitlab` and `linear` — not merely absent, which would fall through to the github default and run `gh` against whatever remote it resolved. + +## ⚠ Review lane coverage — read this before trusting the review depth + +**The rotation this review ran under was `["gemini", "codex", "claude", "opencode"]`**, the four-lane list in effect at the time (2026-08-21, ~16:50–17:30 MDT). The owner removed `gemini` from the rotation shortly afterwards — `["codex", "claude", "opencode"]` — so a later reader will see three names where this table has four. The table records what actually ran, not the current config. + +The first pass had only ONE lane available and the PR was **held** rather than merged on it; the `opencode` lane (PR #24) was merged and installed mid-flight, which is what made a second live reviewer possible. + +| Lane | Verdict | Notes | +|---|---|---| +| **claude** (`claude-opus-5`) | **APPROVE**, HIGH | Ran twice more than required. First pass found the timeout inversion and the `sed -n "1,0p"` portability bug; second pass, on the corrected code, found the extractor pointing at a passing test and this file claiming green CI while the branch was red. Third pass, after those fixes: APPROVE, "no blocking issues", shellcheck and tsc clean, review claims verified independently. | +| **opencode** (`xai/grok-4.6`) | **APPROVE**, HIGH | No key issues. Checked the three documented deviations rather than the code alone. | +| **codex** (gpt-5.6-sol) | **NEVER RAN** | Provider quota, refused in seconds: *"You've hit your usage limit… try again at Aug 27th, 2026 4:01 PM."* Same quota blocked #2, #4, #11 and #12 the same day. | +| **gemini** (agy) | **NEVER RAN** | Provider quota, **not** the reason porch reported — see below. Resets ~2026-08-28. Removed from the rotation by the owner after this review ran. | + +Every finding from every lane was reproduced before being acted on, and none was argued down. The four that changed the code are described in **What the lanes found** below. + +**Porch's own gate summary will say otherwise, and it is wrong.** Both lane files carry `VERDICT: SKIPPED`, which `parseVerdict` (`porch/verdict.ts`) does not recognise — it knows only `APPROVE`, `REQUEST_CHANGES`, `COMMENT` — so it falls through to the "treat as COMMENT" default, and `allApprove` counts `COMMENT` as approval. Porch will print **"All reviewers approved!"** over two reviewers that read nothing. That is **#20**, filed by PIR #12; it is porch behaviour, not anything in this diff, and it is not fixed here. Read this table, not the summary line. + +**The lane files themselves are gitignored** (`.gitignore:65`, `codev/projects/*/*.txt`), which is why the evidence is restated here, where it survives the merge. + +### Porch reported a remedy that cannot work + +The gemini skip notice read: + +> The Gemini (Antigravity `agy`) reviewer was skipped: agy exited with code 1. This is a non-blocking skip; the remaining reviewers still apply. To enable the Gemini lane, install the CLI (https://antigravity.google/cli/install.sh) and run `agy` once to sign in. + +agy **is** installed and authenticated. Probed directly, verbatim: + +``` +$ agy --version +1.1.17 +$ which agy +/Users/chris/.local/bin/agy +$ echo hello | agy -p "reply with the single word OK" +Error: Individual quota reached. Please upgrade your subscription to increase your limits. Resets in 157h50m8s. +rc=1 +``` + +Reinstalling and signing in again would have changed nothing and cost whoever followed the advice their afternoon. A confidently printed remedy that cannot work is the same defect class as #21, where the stuck-mailbox alert names a command that cannot clear a composer. The architect is filing it separately. + +### What the lanes found + +**First pass — claude, COMMENT/HIGH.** Two real defects, both in precisely the class the absent lanes exist to catch. + +**1. The timeout layering was inverted, and my own test hid it.** `executeForgeCommandDetailed` defaults to a 30s ceiling; the scripts default to a 60s `CODEV_FORGE_TIMEOUT`. **At the defaults the outer kill fires first**, so a stalled forge arrived as a generic Node kill and the script's *named* timeout envelope — the entire point of the inner watchdog, and the thing #17 and #8 are about — never printed. The comment in `forge.ts` asserted the opposite ordering. + +The reason it survived to review is worth more than the fix: **the timeout test forced `CODEV_FORGE_TIMEOUT=2`, and a test that overrides the defaults cannot detect the defaults being wrong.** It proved the watchdog works when you tell it to; it could not prove that the watchdog is the ceiling that fires. The correction is a test that pins the **ordering at defaults** — it lets the real 60s-vs-30s relationship decide, and fails if the outer ceiling ever eats the inner one again. + +**The inversion predates this PR.** #12 gave the gitea scripts a 60s watchdog under this same 30s ceiling; it was inherited here, not introduced. Correcting it globally would change the timeout behaviour of every concept and every caller, so what this PR does is narrower: `codev forge` sets its ceiling to the script watchdog plus 30s, and `forge.ts` now documents the real ordering instead of the intended one. The general case is left to a caller passing `timeoutMs`. + +**2. `sed -n "1,0p"` on an empty job log.** head/tail built a reversed range whenever a log came back empty. **BSD sed tolerates it and GNU sed rejects it** — so this was invisible on the macOS box it was written on, and on Linux, *which is where CI runs*, the script would have aborted under `set -e` with nothing at all on stdout: the one shape these concepts promised never to produce. It is the textbook case for why the absent lanes matter, found by the lane that ran on the platform where it cannot bite. An empty log is now an answer (`logLines: 0`, empty window, `truncated: false`), pinned for all three window modes plus an assertion against the reversed range itself. + +A third finding was cosmetic (a misindented `exit` and a trailing space in `gitea/ci-runs.sh`), fixed. + +**Second pass — claude, REQUEST_CHANGES/HIGH**, on the code after those fixes. Three findings, all correct: + +- **This file claimed CI was green when the branch was red.** Addressed in the CI section above: disclosed, diagnosed, and the guard defect reported rather than fixed. +- **`ci-failures` pointed at a passing test** — the capture-block decoy, above. The best finding of the review, because it was found by *running this PR's tool against this PR's own failing CI*. +- **"How to Test Locally" gave commands that do not work here.** The globally installed `codev` predates this PR and has no `forge` subcommand; the worktree build predated the `opencode` lane and rejected the workspace config. `main` is now merged (fixing the second) and the instructions build the branch and invoke its own CLI (fixing the first). + +**Third pass — claude, APPROVE/HIGH.** No blocking issues; four minor notes, all documentation accuracy, all applied: the fixture count (2 → 3), the commit list, the `--log-failed` overstatement corrected above and in `arch.md`, and the reason `failingJobs` is deliberately not spelled `failures`. + +A further note was noted rather than requested: the `Ci*` contracts in `forge-contracts.ts` are documentation-only, with no conformance test tying them to actual script output. **Deliberately not done here.** Adding it for the four CI contracts alone would leave the other eighteen forge contracts untested while implying they were covered — worse than uniformly untested. The architect is filing it as its own issue across all forge contracts. + +### What CI on this PR found that the local suite could not + +The branch's own CI went red while `npm test` was green locally, and the cause is the same platform split the review lane had just warned about — this time with a **wrong answer** at the end of it rather than a crash. + +**The trigger was a test-harness bug**: the harness pointed `TMPDIR` at a directory it never created. macOS `mktemp -d` ignores an unusable `TMPDIR` and falls back to the system temp dir; GNU `mktemp` honours it strictly and fails. So 31 tests passed on the Mac and failed on the Linux runner. + +**What the failure exposed is the part worth reading.** Every CI concept needs a temp dir, and the watchdog in `_timeout.sh` needs one for *every single call*. With `TMPDIR` unusable, `mktemp -d` failed inside the watchdog, `gh` was never run, and the wrapper returned 1 — indistinguishable from the wrapped command failing. The concept then answered: + +```json +{"ok":false,"error":"not-found","detail":"run 32515040122 could not be read (gh exit 1); pass the `id` from ci-runs, not the run `number`"} +``` + +A temp-directory problem wearing the face of a missing run, complete with confident advice about which id to pass. That is the same rule this PR's hot-tier lesson is about, arriving one more time: **"I could not tell" must never be spelled the same way as "no."** + +Three fixes, and the diagnosis was made in an `ubuntu:24.04` container rather than by round-tripping CI: + +1. `forge_timeout` returns **125** when it cannot create a temp dir, not 1 — the command never ran, so it must not share a status with the command failing. +2. Every CI concept runs `ci_require_tmpdir` before it touches a forge, which fails by name with `forge-error` and makes **no forge call at all**. +3. The check tests `[ -d "$TMPDIR" ]` explicitly rather than leaning on `mktemp`, so macOS and Linux behave **identically**. Relying on `mktemp` alone would have preserved the split — and the log cache, which reads `${TMPDIR:-/tmp}` directly, silently does nothing on macOS in that state anyway. + +The harness bug is fixed, and the misdiagnosis is pinned by a test that passes on both platforms. + +### Why this PR is held rather than merged on one lane + +One of three would be the thinnest coverage of the day, on the diff least suited to it: five POSIX `sh` scripts, a hand-rolled awk extractor and a pile of jq, where the two absent lanes are the ones that most often catch quoting and portability defects — and where two of the three bugs found during implementation were exactly that class. PR #24 (issue #22) adds an **opencode** consult lane on an account unrelated to either exhausted quota; the architect is merging it and re-running this review phase with that lane available, rather than waiting for the Aug 27/28 quota resets or merging thin. **This section is rewritten with the real verdicts once that re-run completes.** + +## Architecture Updates + +**COLD — `codev/resources/arch.md`**, § Integration Points → Forge Concept Commands. Concept count 18 → 22, plus four additions, all current-state reference rather than changelog: + +1. **How to invoke a concept**, and that naming a script by path bypasses resolution — the defect `codev forge` exists to close. +2. **`executeForgeCommandDetailed`** and when the `null`-flattening of `executeForgeCommand` is not good enough. +3. **The ci-* contract**: errors as values on stdout, `logLines`/`returnedLines`/`truncated` on anything carrying log text, refusal carrying no log lines, and where the shared implementation lives. +4. **The measured CI behaviour of both forges** — the `--log-failed` correction, the Forgejo 16 log-API floor, the four query-parameter footguns, the `#` branch labelling, the two id spaces, the `cancelled` spelling, and the per-endpoint costs. + +**Nothing promoted to `arch-critical.md`.** It is at its cap of ten, and all of this matters only when writing or calling a forge concept — the SKILL.md and the cold map carry it to whoever needs it. + +## Lessons Learned Updates + +**HOT — `codev/resources/lessons-critical.md`**, one entry rewritten in place rather than a displacement, because the new instance is the same lesson arriving through a wider door. The existing entry covered truncation only: + +> ~~A truncated result is indistinguishable from a complete one once emitted — give "I stopped early" its own signal and emit nothing, never a partial answer that reads as whole.~~ + +is now + +> **"I could not tell" must never be spelled the same way as "no". A truncation, an unreachable API, and a server too old to answer each need their own signal and must emit nothing else — a partial or empty answer reads as a complete, negative one.** + +The architect counted this as the seventh arrival of the same rule in one day. This PR met it three times: an old Forgejo that would have looked like a green run, a page ceiling that reported `truncated: false`, and a CLI exiting 0 with non-JSON that produced no stdout at all. The file stays at ten entries. + +**COLD — `codev/resources/lessons-learned.md`**, seven entries across Process and Testing: + +- Measure a tool before building on the claim that it already does the work (`--log-failed`). +- Extraction must recognise, not truncate — and must strip ANSI before it can recognise anything; anchor generic error patterns at the start of the line. +- A CLI's `--help` is not the API's vocabulary (`canceled` vs `cancelled`). +- Ask which id space an endpoint means when two plausible ids both resolve. +- Under `set -e`, a helper returning non-zero decides its caller's exit status. +- A shell helper that "sets a global" sets nothing when the caller captures it in `$( )`. +- A test whose ceiling sits below its own cost reads as flaky. + +## Things to Look At During PR Review + +- **`_ci-extract.sh`, the awk program.** It is the piece most likely to be subtly wrong on a runner nobody here uses. The ladder is ordered and each rung names itself in `matchedBy`, so a wrong answer is at least attributable — but a new runner format falls to `first-error` or to refusal, and the refusal path is the safe one by design. +- **`ci_clean_log`'s sed pipeline.** ANSI/OSC stripping in POSIX `sed` with a literal ESC. It is line-count preserving, which is what makes `from`/`to` usable as line numbers into the raw log; a change that drops or adds a line silently breaks the `ci-run-log` handoff. +- **The Forgejo-15 `tasks-scan` fallback** (`gitea/_ci.sh`). It pages `actions/tasks` filtering on `run_number`, stops the moment it walks past the run, and reports truncation only when it ran out of allowance *before* reaching it. The early stop is what keeps a recent run at one page; the ceiling is a separate, higher knob (`CODEV_CI_TASKS_MAX_PAGES`, default 20) because a page of 50 tasks spans only ~6 runs. +- **`executeForgeCommandDetailed`'s `timedOut`** is derived from `err.killed && err.signal`, not from an exit code — a killed process can still exit with a status, which is the exact confusion #12 documented in `gitea_timeout`. +- **The gzipped fixtures.** They are verbatim captures; the tests assert against traps that only exist because the bytes are real. Regenerating or normalising them would quietly delete the coverage. + +## How to Test Locally + +- **View diff**: VSCode sidebar → right-click builder `pir-13` → **Review Diff** +- **Build the branch first, and invoke ITS cli** — the globally installed `codev` predates this + PR and has no `forge` subcommand (`error: unknown command 'forge'`): + ```bash + pnpm --filter @cluesmith/codev build + CODEV_CI_RUN_ID=32515040122 node packages/codev/dist/cli.js forge ci-failures | jq + # 23 lines out of 2528: the AssertionError, the test file and line, the step name + ``` + Substitute `node packages/codev/dist/cli.js` for `codev` in every command below. After this + merges and you reinstall globally, plain `codev forge …` works. +- **The windows**, and that the second call is free (cached): + ```bash + CODEV_CI_RUN_ID=32515040122 CODEV_CI_LOG_GREP=AssertionError node packages/codev/dist/cli.js forge ci-run-log | jq '{from,to,matches,matchLines}' + CODEV_CI_RUN_ID=32515040122 node packages/codev/dist/cli.js forge ci-run-log # refuses: no window + ``` +- **Loud degradation**, from a Forgejo repo (`~/dev/entriq`): + ```bash + CLI=/path/to/this/worktree/packages/codev/dist/cli.js + CODEV_CI_RUN_ID=11130 node $CLI forge ci-run-view | jq '{jobSource, jobs: (.jobs|length)}' # works + CODEV_CI_RUN_ID=11130 node $CLI forge ci-failures | jq '{error, serverVersion, needs}' # unsupported-server + node $CLI forge team-activity # named, exit 3 + node $CLI forge ci-failure # unknown, lists valid, exit 2 + ``` +- **`codev doctor`** in both repos: four new concepts, `gh` under github, `tea` under gitea. + +## Flaky Tests + +None skipped. One pre-existing flake was **fixed rather than skipped**: + +`packages/codev/src/__tests__/spec-1280-measurement-instrument.test.ts` capped every test at 60 s inline, while `scripts/measure-prompt-surface.sh` costs 25–30 s per invocation and several of those tests invoke it two or three times (two locales, two runs for determinism, a fixture plus the live repo). Under full-suite load a *different* pair failed each run — the signature of a ceiling set below the work, not of a defect. The file passes 24/24 in isolation before and after. Raised to 240 s via one named constant (`INSTRUMENT_TIMEOUT_MS`), which is the same coverage given room; `.skip` would have bought a green run by deleting the check. diff --git a/codev/state/pir-13_thread.md b/codev/state/pir-13_thread.md new file mode 100644 index 000000000..86faaf076 --- /dev/null +++ b/codev/state/pir-13_thread.md @@ -0,0 +1,84 @@ +# pir-13 thread — CI concepts for the forge layer (#13) + +## PLAN phase + +Investigation was mostly measurement, and it overturned three premises the issue rests on. + +1. **`gh run view --log-failed` does not narrow to the failing step.** On run 32518... (32515040122, + this repo, `Tests`, branch `builder/air-14`) it returned 2528 lines / 293 KB with every line + labelled `UNKNOWN STEP`. Job-scoped, not step-scoped. Codev has to extract on GitHub too. +2. **Forgejo 15.0.2 has no Actions log API at all.** `tea actions runs view` and `tea actions runs + logs` both 404 against git.pseudoseed.com — they call `/actions/runs/{id}/jobs` and + `/actions/jobs/{id}/logs`, which landed in **Forgejo 16.0** (2026-07-16). Probed every + alternative route; the web UI log route exists but is session-only (rejects token and basic auth, + while the API accepts the same token). So `ci-failures`/`ci-run-log` cannot work on entriq today. +3. **Forgejo 16 verified live and free** on codeberg.org (16.0.0-dev, `forgejo/forgejo` public): + `runs/{id}/jobs` ✓, `actions/jobs/{jobId}/logs` → 200 text/plain 142 KB / 1599 lines / 1.0 s, + `accept-ranges: bytes`, `?step=` ignored. Note the log route takes the **job id**, not the + `task_id` that `actions/tasks` lists. + +Other measured facts that shaped the design: Forgejo `actions/runs?limit=N` **ignores limit unless +`page` is present** (a bare `limit=3` returned all 6922 runs); `status=` filters server-side but +`branch=`/`event=` are ignored; PR-triggered runs report `head_branch` as `#`, not the branch, +so branch filtering needs the #12 base/head PR lookup first; run `id` and `index_in_repo` are two +valid id spaces for the same route, so the concept refuses to guess which one it was handed. + +And the extraction evidence: in that GitHub log the first `Error:` line is `[artifact-canvas] Error: +host blew up` — a **passing** test's fixture string at line 1257, while the real failure is at 2471. +`Test Files … passed` appears four times before the failing summary. Every payload line is wrapped in +ANSI SGR codes, so a matcher that does not strip ANSI matches nothing. All three are pinned as tests. + +Plan at `codev/plans/13-ci-forge-concepts.md`. Two open questions for the architect: a Codeberg token +(would upgrade the v16 lane from HTTP-level to full-dispatcher verification), and whether +git.pseudoseed.com is due a Forgejo 16 upgrade. + +Awaiting `plan-approval`. + +## IMPLEMENT phase + +Plan approved 20:59Z. Six commits on `builder/pir-13`. + +Shape that emerged: one shared extraction ladder (`scripts/forge/_ci-extract.sh`) and one shared +envelope/cache/window lib (`_ci-lib.sh`), with thin per-provider scripts. Both providers now fetch +**the same thing** — one job's log from `actions/jobs/{id}/logs` — so they share a cache and an +extractor. `gh run view --log-failed` is not used at all: it returned 2528 lines / 293 KB with +every line tagged UNKNOWN STEP, and the failing step NAME is available from `--json jobs` anyway. + +Measured result on the live GitHub run: **293 KB of log becomes a 1.2 KB response** carrying the +assertion, the test name, the step name and the line range. Cold 2.5s, cached 1.3s. + +### Things found while building that changed the code + +- **`ci_fail` returning 1 aborted its own caller.** Every concept runs under `set -e`, so a + non-zero return killed the script at that line, before the intended `exit 2` for a bad input. + Every input error was arriving as exit 1. `ci_fail` now reports and returns 0; call sites decide + the status. +- **Two subshell-assignment bugs.** `ci_text_json` and `gitea_ci_jobs` set globals that their + callers read — inside `$( )`, so the values were discarded and jq got empty `--argjson`. Both now + return their data (or write files) instead. +- **Forgejo rejects the spelling its own CLI documents.** `tea actions runs list --help` says + `canceled`; the API answers `{"message":"unknown status: canceled"}` and accepts `cancelled` + (2240 runs). Both providers now get `cancelled`. +- **Truncation was under-reported twice.** Stopping at exactly the limit, and hitting the page + ceiling with no client-side filter, both reported `truncated: false`. Fixed and pinned. +- **A CLI that exits 0 with non-JSON** (an auth prompt) reached jq, died under `set -e`, and left + nothing on stdout. Guarded. +- **The Forgejo-15 task scan needed its own, higher page ceiling.** A page of 50 tasks spans ~6 + runs, so 4 pages reached 24 runs back and reported truncation for anything older. Now 20 pages, + with an early stop the moment the walk passes the run — a recent run costs one page. + +### Verification coverage, stated precisely + +- **GitHub — end to end through the real dispatcher** (config → preset → env → script → JSON parse), + all four concepts, live against `pseudoseed/codev`. Timings in the PR body. +- **Forgejo 15.0.2 — end to end through the real dispatcher**, live against `~/dev/entriq` on the + bare gitea preset. entriq was READ ONLY; its working tree carries an unrelated uncommitted config + edit that predates this session (13:34 MDT). +- **Forgejo 16 — HTTP level only.** The v16 routes were verified with unauthenticated curl against + codeberg.org, and the code path is covered by unit tests with a stubbed `tea` serving the REAL + captured codeberg job log. It has NOT been driven through the dispatcher against a live v16 + server; that needs a Codeberg token, which the architect deferred. + +Question for the gate: nothing in codev can invoke a forge concept from the command line, so a +builder reaches these four by script path. A `codev forge ` entry point would fix that; +it is not in the approved plan, so I am asking rather than adding it. diff --git a/packages/codev/scripts/forge/_ci-extract.sh b/packages/codev/scripts/forge/_ci-extract.sh new file mode 100644 index 000000000..7e70b7d9c --- /dev/null +++ b/packages/codev/scripts/forge/_ci-extract.sh @@ -0,0 +1,238 @@ +# Extraction ladder for the CI concepts (#13). SOURCED, not executed. +# +# Turns a raw CI job log into the few lines that say why the job failed, or into +# an honest refusal. Shared by github/ and gitea/ so the two providers cannot +# drift: the runners differ, the failure text does not. +# +# WHY THIS EXISTS AT ALL, ON BOTH PROVIDERS +# +# Issue #13 says `gh run view --log-failed` "already does the extraction for +# you" and tells the implementer not to duplicate it. Measured on run +# 32515040122 of this repository, it does not: it returned 2528 lines / 293 KB +# with every single line tagged "UNKNOWN STEP", i.e. it selected the failing +# JOB and returned all of it. (The architect independently reproduced this on +# run 32448538074: 919 lines, all 919 tagged UNKNOWN STEP.) `gh` attributes log +# lines to steps by matching filenames, and falls back to UNKNOWN STEP when that +# mapping fails — unreliably, not always: on run 32536232930 it attributed all +# 1193 lines correctly. Attributed or not, the output is a whole job or a whole +# step and never the assertion, which is 108 KB at best. So codev extracts on +# GitHub too, and the concepts fetch +# `repos/{owner}/{repo}/actions/jobs/{id}/logs` instead — one job, no invented +# step column, and the same shape Forgejo 16 serves. +# +# THE THREE THINGS THAT MAKE NAIVE EXTRACTION LIE +# +# All three are from the same real log, and all three are pinned by tests: +# +# 1. ANSI. The payload line is not `FAIL src/…`, it is +# `\033[41m\033[1m FAIL \033[22m\033[49m src/…`. A matcher that does not +# strip ANSI first matches NOTHING and reports "no recognized failure" on a +# log that plainly contains one. Cleaning is not cosmetic; it is the +# difference between working and silently giving up. +# 2. "First line matching an error pattern" returns line 1257 of 2528: +# `[artifact-canvas] Error: host blew up` — a fixture string printed by a +# PASSING test. The real failure is at 2471. Hence rung 4 anchors its +# patterns at the START of the line: that fixture's "Error:" is mid-line, so +# anchoring alone kills it, and anchoring holds even in logs with no test +# summary to measure against. +# 3. `Test Files … passed` appears FOUR times before the failing summary. Any +# rule that takes the first match reports a passing suite as the failure. +# +# And the rule that governs the whole ladder: when nothing matches, return +# nothing. A builder handed 50 arbitrary lines treats them as the diagnosis and +# reasons from noise; a builder told extraction failed goes and reads the log, +# which is correct and cheaper. See lessons-critical.md. + +# Clean a raw log on stdin into extractable text on stdout. +# +# Strips, in order: a UTF-8 BOM (GitHub's logs open with one), ANSI CSI/OSC +# escape sequences, carriage returns, and the leading RFC3339 timestamp that +# BOTH providers prefix to every line +# (`2026-08-21T18:47:09.5820646Z Current runner version: …`). +# +# Deliberately does NOT strip gh's `jobstep` prefix: the concepts do +# not use `--log-failed`, so no such prefix exists, and a rule that ate tab- +# separated leading fields would eat real log content on a runner that prints +# tables. +ci_clean_log() { + _esc=$(printf '\033') + LC_ALL=C sed \ + -e '1s/^\xef\xbb\xbf//' \ + -e "s/${_esc}\][^${_esc}]*${_esc}\\\\//g" \ + -e "s/${_esc}\[[0-9;?]*[ -\/]*[@-~]//g" \ + -e "s/${_esc}[()][A-Za-z0-9]//g" \ + -e 's/\r$//' \ + -e 's/^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9][.0-9]*Z //' +} + +# Run the ladder over a CLEANED log file. +# +# ci_extract +# +# On a match, prints a header line `\t\t` followed by +# the extracted lines. On no match, prints NOTHING and returns 1 — which is the +# `extracted: false` path, not an error. +# +# Line numbers are 1-based into the cleaned log, and the cleaner is line-count +# preserving, so they are also line numbers into the raw log — which is what +# makes the `ci-run-log` handoff land where `ci-failures` was looking. +ci_extract() { + awk ' + function is_noise_marker(s) { + # `##[error]Process completed with exit code 1.` is the runner reporting + # that the step failed. True, and not a diagnosis of anything — returning + # it alone would be an arbitrary line dressed as an answer. + return (s ~ /^##\[error\](Process completed with exit code|The (job|operation) was canceled)/) + } + { line[NR] = $0 } + END { + n = NR + if (n == 0) exit 1 + + # ---- pass 1: index the landmarks ------------------------------------- + first_marker = 0 + last_fail_summary = 0 # vitest "Test Files … failed" / jest "Tests: … failed" + last_pass_boundary = 0 # the last "N passed" summary of ANY suite + last_failed_banner = 0 # vitest "⎯⎯ Failed Tests N ⎯⎯" + first_ts_error = 0 + first_go_fail = 0 + first_unhandled = 0 + in_capture = 0 + for (i = 1; i <= n; i++) { + s = line[i] + if (!first_marker && s ~ /^##\[error\]/ && !is_noise_marker(s)) first_marker = i + if (s ~ /Failed Tests/) last_failed_banner = i + if (s ~ /(Test Files|Tests:|Test Suites:)/) { + if (s ~ /fail/) last_fail_summary = i + else if (s ~ /pass/) last_pass_boundary = i + } + if (s ~ /^[0-9]+ (passing|tests? passed)/) last_pass_boundary = i + if (!first_ts_error && s ~ /error TS[0-9]+:/) first_ts_error = i + if (!first_go_fail && s ~ /^ *--- FAIL: /) first_go_fail = i + if (!first_unhandled && s ~ /Unhandled Error/) first_unhandled = i + # Vitest prints a PASSING tests captured output as a block: + # stderr | path.test.ts > suite > name + # + # + # Those lines are a test working as designed, and rung 6 must not anchor + # on one. Marking the region is the only way to tell them apart: the + # captured text is often a perfectly formed "Error: ..." at the start of + # its line, which is exactly what rung 6 looks for. + if (s ~ /^(stdout|stderr) \| /) { in_capture = 1; captured[i] = 1 } + else if (in_capture) { + if (s ~ /^[ \t]*$/) in_capture = 0 + else captured[i] = 1 + } + } + + # ---- rung 1: a recognised test runner -------------------------------- + # Anchored on the summary that says "failed" — NOT the first summary, of + # which there were four saying "passed" in the log this was built from. + if (last_fail_summary) { + start = 0 + if (last_failed_banner && last_failed_banner < last_fail_summary) start = last_failed_banner + if (!start) { + # earliest FAIL after the last passing summary that precedes the + # failing one, so a re-run of the same suite does not drag in the + # passing run above it + floor = 0 + for (i = 1; i < last_fail_summary; i++) + if (line[i] ~ /(Test Files|Tests:|Test Suites:)/ && line[i] ~ /pass/) floor = i + for (i = floor + 1; i < last_fail_summary; i++) + if (line[i] ~ /(^| )FAIL( |$)/ || line[i] ~ /^ *(✕|×) /) { start = i; break } + } + if (start) { emit("vitest", start, last_fail_summary); exit 0 } + } + + # ---- rung 2: go test ------------------------------------------------- + # The Forgejo fixture this was verified against is a Go suite, and it is + # the case that best justifies the whole ladder: the failure sits at line + # 1292 of 1599 and the last 25 lines of that log are git credential + # cleanup, so a tail returns nothing at all. The block runs from the + # --- FAIL: line to the package summary (FAILpkg0.302s). + if (first_go_fail) { + to = first_go_fail + for (i = first_go_fail + 1; i <= n && i <= first_go_fail + 40; i++) { + to = i + if (line[i] ~ /^FAIL\t/ || line[i] ~ /^ok \t/) break + } + emit("go-test", first_go_fail, to) + exit 0 + } + + # ---- rung 3: a compiler ---------------------------------------------- + if (first_ts_error) { + to = first_ts_error + for (i = first_ts_error + 1; i <= n && i <= first_ts_error + 20; i++) { + if (line[i] ~ /error TS[0-9]+:/ || line[i] ~ /^[ \t]/) to = i; else break + } + emit("tsc", first_ts_error, to) + exit 0 + } + + # ---- rung 4: vitest unhandled errors --------------------------------- + # A worker that dies, an unhandled rejection, a native teardown crash. No + # test reports as failed, so the rungs above find nothing, and the run is + # still red. Below the Failed Tests rung because a real assertion beats a + # teardown crash when both are present. + if (first_unhandled) { + to = first_unhandled + 12; if (to > n) to = n + emit("vitest-unhandled", first_unhandled, to) + exit 0 + } + + # ---- rung 5: the runner error marker --------------------------------- + # GitHub puts the message INSIDE the marker (##[error]AssertionError: + # expected null to be unauth), so the marker line is itself the answer and + # what FOLLOWS it is the useful context. What precedes it, on the log this + # was built from, is a vitest duration line and two blank lines. This sits + # below runner recognition, not above it, because rung 1 returns the whole + # Failed Tests block (test name, assertion, expected/received, file:line) + # where this returns one sentence. Issue #13 priority order agrees: + # recognise the runner first. + if (first_marker) { + to = first_marker + 3; if (to > n) to = n + emit("runner-marker", first_marker, to) + exit 0 + } + + # ---- rung 6: the first ANCHORED error, preferring after the last pass - + # Anchoring at the start of the line is what makes this safe: the false + # positive this ladder was built against ("[artifact-canvas] Error: host + # blew up", printed by a passing test) has its "Error:" mid-line and + # cannot match. Searching after the last passing summary is an additional + # preference, not the safety property — logs from a failed install or a + # crashed runner have no summary at all, and refusing to answer for that + # whole class would buy no safety. + for (pass = 1; pass <= 2; pass++) { + start = (pass == 1) ? last_pass_boundary + 1 : 1 + if (pass == 2 && last_pass_boundary == 0) break + for (i = start; i <= n; i++) { + s = line[i] + if (s ~ /^(Error|error|ERROR|Exception|FATAL|fatal( error)?|panic|Traceback \(most recent call last\)|npm ERR!|##\[error\])[: ]/ || + s ~ /^[A-Za-z_][A-Za-z0-9_.]*(Error|Exception): / || + s ~ /^error(\[[A-Z0-9]+\])?: /) { + if (is_noise_marker(s)) continue + # A passing tests captured stdout/stderr is not a diagnosis. Found by + # running ci-failures against this repos own red CI run: it returned + # `Error: Refusing to POST /api/tunnel/disconnect ...` from inside a + # `stderr |` block belonging to a test that PASSED, while the real + # failure sat 350 lines further down. + if (i in captured) continue + from = i - 3; if (from < 1) from = 1 + to = i + 3; if (to > n) to = n + emit("first-error", from, to) + exit 0 + } + } + } + + # ---- rung 7: give up honestly ---------------------------------------- + exit 1 + } + function emit(rung, from, to, i) { + printf "%s\t%d\t%d\n", rung, from, to + for (i = from; i <= to; i++) print line[i] + } + ' "$1" +} diff --git a/packages/codev/scripts/forge/_ci-lib.sh b/packages/codev/scripts/forge/_ci-lib.sh new file mode 100644 index 000000000..38f3d3962 --- /dev/null +++ b/packages/codev/scripts/forge/_ci-lib.sh @@ -0,0 +1,450 @@ +# Shared plumbing for the CI concepts (#13). SOURCED, not executed. +# +# Holds the response envelope, the caps, the log cache, and the window parsing — +# everything the four concepts share that is not the extraction ladder itself +# (that is _ci-extract.sh). POSIX sh; jq is required, as it already is for +# pr-list and every gitea concept. +# +# THE ENVELOPE, AND WHY ERRORS ARE PRINTED ON STDOUT +# +# Every ci-* concept prints ONE JSON object on stdout, on success and on +# failure. A caller therefore always has something structured to read, and never +# has to interpret an empty string. +# +# That includes the failure paths, which is a deliberate departure from the +# other concepts. `executeForgeCommand` collapses every failure mode to `null` +# (forge.ts), so "the API timed out at 60s", "the run does not exist" and "the +# server has no log API" arrive identically — and #12 spent a phase on exactly +# that ambiguity with pr-exists returning null-that-read-as-false. Printing the +# envelope regardless of exit status means the class of failure survives the +# trip. Exit statuses still follow the #12 contract: 0 answered, 1 could not +# answer, 2 missing or unusable input. +# +# The one rule that outranks the rest: a response that carries log text ALWAYS +# carries logLines, returnedLines and truncated, so a trimmed answer can never +# read as a whole one. + +CI_LIMIT_DEFAULT=20 + +# Per-extract and per-response byte caps (issue #13: "a few KB per failing +# step"). A cap that bites is always reported as truncated:true — never +# silently. +CI_MAX_STEP_BYTES=${CODEV_CI_MAX_STEP_BYTES:-2048} +CI_MAX_RESPONSE_BYTES=${CODEV_CI_MAX_BYTES:-8192} +case "$CI_MAX_STEP_BYTES" in ''|*[!0-9]*) CI_MAX_STEP_BYTES=2048 ;; esac +case "$CI_MAX_RESPONSE_BYTES" in ''|*[!0-9]*) CI_MAX_RESPONSE_BYTES=8192 ;; esac + +# How many list pages a client-side filter may walk before it stops and says so. +CI_MAX_PAGES=${CODEV_CI_MAX_PAGES:-4} +case "$CI_MAX_PAGES" in ''|*[!0-9]*|0) CI_MAX_PAGES=4 ;; esac + +# A separate, higher ceiling for the Forgejo-15 task scan, which is a TARGETED +# lookup for one known run rather than an open-ended filter: it stops the moment +# it has walked past that run, so a recent run costs one page and the ceiling +# only bites on old ones. Measured on the reference Forgejo, a page of 50 tasks +# spans about 6 runs (~8 jobs per run), so 4 pages reached only 24 runs back and +# reported truncation for anything older — technically honest and practically +# useless. 20 pages is ~120 runs of history at ~0.25s per page. +CI_TASKS_MAX_PAGES=${CODEV_CI_TASKS_MAX_PAGES:-20} +case "$CI_TASKS_MAX_PAGES" in ''|*[!0-9]*|0) CI_TASKS_MAX_PAGES=20 ;; esac + +# The status vocabulary a caller may use, provider-independent. Taken from what +# Forgejo accepts; GitHub spells one of them differently and is translated in +# ci_status_for. +CI_STATUS_VOCABULARY="success failure pending queued in_progress skipped canceled" + +# Print the error envelope on stdout, a human line on stderr, and return 1. +# +# ci_fail [extra-json-object] +# +# `kind` is a stable machine token — timeout, not-found, unsupported-server, +# forge-error, bad-input — and `detail` is the sentence a person reads. +ci_fail() { + _concept="$1"; _kind="$2"; _detail="$3"; _extra="$4" + [ -n "$_extra" ] || _extra='{}' + jq -cn --arg kind "$_kind" --arg detail "$_detail" --argjson extra "$_extra" \ + '{ok: false, error: $kind, detail: $detail} + $extra' + echo "${_concept}: ${_detail}" >&2 + # Returns 0 DELIBERATELY. Every concept runs under `set -e`, so a non-zero + # return here would abort the script at this line — before the caller's own + # `exit 2` for a bad input could run, turning every input error into a + # generic exit 1. Reporting and deciding the exit status are separate jobs; + # this one only reports. + return 0 +} + +# The timeout envelope. Kept separate from ci_fail so that every concept spells +# a timeout the same way and none of them can quietly turn one into a generic +# failure or an empty result — the specific thing #17, #8 and #12 all ran into. +ci_fail_timeout() { + _concept="$1"; _what="$2"; _seconds="$3" + ci_fail "$_concept" timeout \ + "${_what} did not return within ${_seconds}s" \ + "$(jq -cn --argjson s "$_seconds" '{seconds: $s, remedy: "raise CODEV_FORGE_TIMEOUT"}')" +} + +# Validate a caller-supplied status against the shared vocabulary. +# Unknown values exit 2 naming the accepted set rather than being passed to the +# forge, where GitHub would reject them with its own wording and Forgejo would +# silently return everything. +ci_check_status() { + _concept="$1"; _status="$2" + [ -n "$_status" ] || return 0 + for _s in $CI_STATUS_VOCABULARY; do + [ "$_s" = "$_status" ] && return 0 + done + _msg="CODEV_CI_STATUS=${_status} is not one of: ${CI_STATUS_VOCABULARY}" + jq -cn --arg d "$_msg" '{ok: false, error: "bad-input", detail: $d}' + echo "${_concept}: ${_msg}" >&2 + exit 2 +} + +# Translate the shared vocabulary into what a provider expects on the wire. +# +# The vocabulary spells it `canceled` because that is what `tea actions runs +# list --help` documents. Both forges want `cancelled`: GitHub has always spelt +# it that way, and Forgejo — despite its own CLI help — answers +# `status=canceled` with `{"message":"unknown status: canceled"}` while +# `status=cancelled` returns 2240 runs. Measured, because the two spellings are +# exactly the kind of difference that turns into an empty list nobody questions. +ci_status_for() { + _provider="$1"; _status="$2" + if [ "$_status" = "canceled" ]; then + printf 'cancelled' + else + printf '%s' "$_status" + fi +} + +# Is this job status terminal? Only a terminal job has an immutable log, and +# only an immutable log may be cached. +ci_status_is_terminal() { + case "$1" in + success|failure|skipped|canceled|cancelled|completed|timed_out|neutral|stale|startup_failure|failed) return 0 ;; + *) return 1 ;; + esac +} + +# Require a run/job id to be a plain positive integer. +# +# ci_require_id "" +# +# Both ids are interpolated into a URL path on the gitea side and into a jq +# `--argjson` on both sides, so a non-numeric value either builds a URL nobody +# meant or kills jq under `set -e` with nothing on stdout. Neither is a useful +# answer, and "CODEV_CI_RUN_ID must be numeric" is. +ci_require_id() { + _concept="$1"; _name="$2"; _value="$3" + case "$_value" in + ''|*[!0-9]*) + _msg="${_name} must be a numeric id, got '${_value}' — pass the \`id\` field from ci-runs, not the run \`number\` or a URL" + jq -cn --arg d "$_msg" '{ok: false, error: "bad-input", detail: $d}' + echo "${_concept}: ${_msg}" >&2 + exit 2 + ;; + esac +} + +# Fail loudly and early if TMPDIR is unusable. +# +# Every CI concept writes logs and windows to a temp dir, and the watchdog in +# _timeout.sh needs one for every single call. macOS mktemp quietly falls back +# to the system temp dir when TMPDIR points nowhere; GNU mktemp does not, so an +# unusable TMPDIR fails on Linux and passes on a Mac. Checked ONCE up front, by +# name, because the alternative is what it did before: the first forge call +# failed for a reason nobody could see, and the concept reported "run could +# not be read" — a temp-dir problem wearing the face of a missing run. +ci_require_tmpdir() { + _concept="$1" + # The explicit directory test comes FIRST, and is the reason this is uniform: + # relying on mktemp alone would keep the platform split, since macOS would + # quietly succeed against a TMPDIR that does not exist while Linux failed. A + # TMPDIR naming somewhere that is not a directory is an error on both, and the + # log cache — which reads ${TMPDIR:-/tmp} directly — silently does nothing on + # macOS in that state anyway. + if [ -n "$TMPDIR" ] && [ ! -d "$TMPDIR" ]; then + _msg="TMPDIR=${TMPDIR} is not a directory; no forge call was made" + jq -cn --arg d "$_msg" '{ok: false, error: "forge-error", detail: $d}' 2>/dev/null \ + || printf '{"ok":false,"error":"forge-error","detail":"%s"}\n' "$_msg" + echo "${_concept}: ${_msg}" >&2 + exit 1 + fi + _probe=$(mktemp -d 2>/dev/null) || { + _msg="TMPDIR=${TMPDIR:-/tmp} is not usable (mktemp -d failed); no forge call was made" + jq -cn --arg d "$_msg" '{ok: false, error: "forge-error", detail: $d}' 2>/dev/null \ + || printf '{"ok":false,"error":"forge-error","detail":"%s"}\n' "$_msg" + echo "${_concept}: ${_msg}" >&2 + exit 1 + } + rmdir "$_probe" 2>/dev/null || : +} + +# Assert that a captured payload really is JSON, and emit an envelope if not. +# +# ci_require_json "" "" +# +# A forge CLI that exits 0 and prints something other than JSON — an auth +# prompt, an empty body, an HTML error page — would otherwise reach `jq` and +# kill the script under `set -e`, leaving jq's own diagnostic on stderr and +# NOTHING on stdout. That is the one shape these concepts promised never to +# produce: a caller with no structured answer at all. +ci_require_json() { + _concept="$1"; _payload="$2"; _what="$3" + if printf '%s' "$_payload" | jq -e . >/dev/null 2>&1; then + return 0 + fi + ci_fail "$_concept" forge-error \ + "${_what} did not return JSON: $(printf '%s' "$_payload" | head -c 200 | tr '\n' ' ')" + exit 1 +} + +# --------------------------------------------------------------------------- +# Log cache +# --------------------------------------------------------------------------- +# +# A completed run log is immutable, so the realistic sequence — ci-failures, +# then ci-run-log with a tail, then ci-run-log with a grep — should cost exactly +# one download instead of three. Cached under TMPDIR rather than in the repo: +# these are large and disposable, and TMPDIR inherits the OS cleanup. +# +# In-progress jobs are NEVER cached and never read from cache. Caching a running +# job would hand back a log that stops mid-failure and looks complete, which is +# the failure mode this whole issue is about. + +CI_CACHE_MAX_MB=${CODEV_CI_CACHE_MAX_MB:-32} +case "$CI_CACHE_MAX_MB" in ''|*[!0-9]*) CI_CACHE_MAX_MB=32 ;; esac + +ci_cache_path() { + _provider="$1"; _slug="$2"; _job="$3" + _slug=$(printf '%s' "$_slug" | tr '/ ' '__') + printf '%s/codev-ci-logs/%s/%s/%s.log' "${TMPDIR:-/tmp}" "$_provider" "$_slug" "$_job" +} + +# Copy a cached log to stdout if it exists and caching is enabled. +ci_cache_read() { + [ "$CODEV_CI_NO_CACHE" = "1" ] && return 1 + [ -s "$1" ] || return 1 + cat "$1" +} + +# Store a log, but only for a terminal job and only under the size cap. +ci_cache_write() { + _path="$1"; _src="$2"; _status="$3" + [ "$CODEV_CI_NO_CACHE" = "1" ] && return 0 + ci_status_is_terminal "$_status" || return 0 + _bytes=$(wc -c < "$_src" | tr -d ' ') + [ "$_bytes" -le $((CI_CACHE_MAX_MB * 1024 * 1024)) ] || return 0 + mkdir -p "$(dirname "$_path")" 2>/dev/null || return 0 + cp "$_src" "${_path}.tmp$$" 2>/dev/null && mv "${_path}.tmp$$" "$_path" 2>/dev/null + return 0 +} + +# --------------------------------------------------------------------------- +# Windows for ci-run-log +# --------------------------------------------------------------------------- +# +# Exactly one of head/tail/grep, and no default. A defaulted window is how this +# concept turns into "tail by habit", which is the thing the issue asks for it +# to be separate in order to prevent. Zero windows and two windows are both +# exit 2 with a named message. +ci_window_parse() { + _concept="$1" + CI_WIN_KIND="" + CI_WIN_ARG="" + _count=0 + if [ -n "$CODEV_CI_LOG_TAIL" ]; then CI_WIN_KIND=tail; CI_WIN_ARG="$CODEV_CI_LOG_TAIL"; _count=$((_count + 1)); fi + if [ -n "$CODEV_CI_LOG_HEAD" ]; then CI_WIN_KIND=head; CI_WIN_ARG="$CODEV_CI_LOG_HEAD"; _count=$((_count + 1)); fi + if [ -n "$CODEV_CI_LOG_GREP" ]; then CI_WIN_KIND=grep; CI_WIN_ARG="$CODEV_CI_LOG_GREP"; _count=$((_count + 1)); fi + + if [ "$_count" -eq 0 ]; then + _msg="exactly one window is required: CODEV_CI_LOG_TAIL=N, CODEV_CI_LOG_HEAD=N, or CODEV_CI_LOG_GREP=" + elif [ "$_count" -gt 1 ]; then + _msg="exactly one window may be set; got ${_count} of CODEV_CI_LOG_TAIL / CODEV_CI_LOG_HEAD / CODEV_CI_LOG_GREP" + else + case "$CI_WIN_KIND" in + head|tail) + case "$CI_WIN_ARG" in + ''|*[!0-9]*|0) + _upper=$(printf '%s' "$CI_WIN_KIND" | tr 'a-z' 'A-Z') + _msg="CODEV_CI_LOG_${_upper} must be a positive number of lines, got '${CI_WIN_ARG}'" ;; + *) return 0 ;; + esac + ;; + *) return 0 ;; + esac + fi + + jq -cn --arg d "$_msg" '{ok: false, error: "bad-input", detail: $d}' + echo "${_concept}: ${_msg}" >&2 + exit 2 +} + +CI_GREP_CONTEXT=${CODEV_CI_LOG_CONTEXT:-3} +case "$CI_GREP_CONTEXT" in ''|*[!0-9]*) CI_GREP_CONTEXT=3 ;; esac + +# --------------------------------------------------------------------------- +# Turning text into a capped JSON payload +# --------------------------------------------------------------------------- + +# Copy a file to , capped at bytes, whole lines only, from the head +# — for an extract the assertion is at the top and half a line is worse than a +# missing one. Echoes " ". +# +# The count and the truncation flag are RETURNED, not assigned to globals: every +# caller runs this inside a command substitution, where a subshell assignment is +# discarded and the caller then hands jq an empty --argjson. That mistake was +# made here once and cost a debugging round; the shape now makes it impossible. +ci_cap_file() { + _src="$1"; _cap="$2"; _dest="$3" + _bytes=$(wc -c < "$_src" | tr -d ' ') + if [ "$_bytes" -le "$_cap" ]; then + cp "$_src" "$_dest" + _trunc=false + else + awk -v cap="$_cap" '{ t += length($0) + 1; if (t > cap) exit; print }' "$_src" > "$_dest" + _trunc=true + fi + printf '%s %s' "$(awk 'END {print NR}' "$_dest")" "$_trunc" +} + +# --------------------------------------------------------------------------- +# Tool invocation +# --------------------------------------------------------------------------- + +. "$(dirname "$0")/../_timeout.sh" + +# Wall-clock ceiling for one forge CLI call. Same knob as the gitea concepts +# use, deliberately: an operator raising CODEV_FORGE_TIMEOUT for a slow forge +# should not have to discover that CI has its own. +CI_TIMEOUT=${CODEV_FORGE_TIMEOUT:-60} +case "$CI_TIMEOUT" in ''|*[!0-9]*) CI_TIMEOUT=60 ;; esac + +# Run a forge CLI under the timeout. Returns the command status, or 124 for a +# timeout — which the caller MUST turn into ci_fail_timeout rather than folding +# into a generic failure. This function does not print the envelope itself +# because every caller captures its stdout. +ci_tool() { + _rc=0 + forge_timeout "$CI_TIMEOUT" "$@" || _rc=$? + return "$_rc" +} + +# owner/repo for cache keys and API paths. Honors CODEV_REPO, else derives it +# from the origin remote. Unlike gitea_repo this never fails the concept — the +# only caller that cannot proceed without it says so itself — so an +# underivable slug degrades to "unknown-repo" for the cache path alone. +ci_repo_slug() { + _repo="${CODEV_REPO:-$(git remote get-url origin 2>/dev/null | sed -E -e 's#\.git$##' -e 's#.*[/:]([^/]+/[^/]+)$#\1#')}" + case "$_repo" in + */*) printf '%s' "$_repo" ;; + *) printf 'unknown-repo' ;; + esac +} + + +# --------------------------------------------------------------------------- +# Windowing a cleaned log into the ci-run-log response +# --------------------------------------------------------------------------- +# +# ci_window_emit +# +# Reads /clean.log, applies the window ci_window_parse selected, and +# prints the response. Shared by both providers so that a tail means the same +# thing on GitHub and on Forgejo. +# +# `from` and `to` are line numbers into the FULL log, always, so a caller knows +# where in the log it is standing and whether more exists on either side. In +# grep mode the selected lines are not contiguous, so `contiguous: false` says +# so and `matchLines` lists exactly which lines matched — otherwise a reader +# would have to infer which of the returned lines were hits and which were +# context, and inference is what these concepts exist to remove. +ci_window_emit() { + _concept="$1"; _tmp="$2"; _provider="$3"; _run="$4"; _job="$5"; _jobname="$6"; _cached="$7" + _log="${_tmp}/clean.log" + _total=$(awk 'END {print NR}' "$_log") + # An empty log is an answer, not a crash. Without this, head/tail build + # `sed -n "1,0p"`, which BSD sed tolerates and GNU sed rejects — so the script + # would abort under `set -e` on Linux with NOTHING on stdout, which is the one + # shape these concepts promised never to produce. Found by the claude review + # lane and reproduced with `sed --posix` before fixing. + if [ "$_total" -eq 0 ]; then + jq -cn --arg provider "$_provider" --arg run "$_run" --argjson job "$_job" --arg jobName "$_jobname" \ + --arg kind "$CI_WIN_KIND" --arg arg "$CI_WIN_ARG" --argjson cached "$_cached" \ + '{ok: true, provider: $provider, runId: ($run | tonumber? // $run), jobId: $job, jobName: $jobName, + window: {kind: $kind, arg: $arg}, + logLines: 0, returnedLines: 0, from: 0, to: 0, + contiguous: true, truncated: false, + matches: (if $kind == "grep" then 0 else null end), + matchLines: (if $kind == "grep" then [] else null end), + cached: $cached, lines: []}' + return 0 + fi + _matchlines=null + _matches=null + _contiguous=true + + case "$CI_WIN_KIND" in + head) + _from=1 + _to=$CI_WIN_ARG + [ "$_to" -gt "$_total" ] && _to=$_total + sed -n "${_from},${_to}p" "$_log" > "${_tmp}/window.txt" + ;; + tail) + _from=$((_total - CI_WIN_ARG + 1)) + [ "$_from" -lt 1 ] && _from=1 + _to=$_total + sed -n "${_from},${_to}p" "$_log" > "${_tmp}/window.txt" + ;; + grep) + if ! awk -v pat="$CI_WIN_ARG" -v ctx="$CI_GREP_CONTEXT" \ + -v out="${_tmp}/window.txt" -v nums="${_tmp}/matches.txt" ' + { line[NR] = $0; if ($0 ~ pat) hit[NR] = 1 } + END { + for (i = 1; i <= NR; i++) + if (i in hit) { + lo = i - ctx; if (lo < 1) lo = 1 + hi = i + ctx; if (hi > NR) hi = NR + for (j = lo; j <= hi; j++) sel[j] = 1 + print i > nums + } + first = 0; last = 0 + for (i = 1; i <= NR; i++) + if (i in sel) { print line[i] > out; if (!first) first = i; last = i } + print first, last + }' "$_log" > "${_tmp}/bounds.txt" 2>"${_tmp}/awk.err"; then + ci_fail "$_concept" bad-input \ + "CODEV_CI_LOG_GREP is not a usable pattern: $(tr -d '\n' < "${_tmp}/awk.err")" + exit 2 + fi + _from=$(cut -d' ' -f1 "${_tmp}/bounds.txt") + _to=$(cut -d' ' -f2 "${_tmp}/bounds.txt") + [ -f "${_tmp}/window.txt" ] || : > "${_tmp}/window.txt" + [ -f "${_tmp}/matches.txt" ] || : > "${_tmp}/matches.txt" + _matches=$(wc -l < "${_tmp}/matches.txt" | tr -d ' ') + _matchlines=$(jq -R -s -c 'split("\n") | map(select(. != "") | tonumber)' < "${_tmp}/matches.txt") + _contiguous=false + # No match is an answer, not a failure: an empty window with matches: 0. + [ "$_from" -eq 0 ] && _from=0 && _to=0 + ;; + esac + + _meta=$(ci_cap_file "${_tmp}/window.txt" "$CI_MAX_RESPONSE_BYTES" "${_tmp}/capped.txt") + _returned=${_meta% *} + _trunc=${_meta#* } + _lines=$(jq -R -s -c 'split("\n") | if (.[-1] == "") then .[0:-1] else . end' < "${_tmp}/capped.txt") + + jq -cn \ + --arg provider "$_provider" --arg run "$_run" --argjson job "$_job" --arg jobName "$_jobname" \ + --arg kind "$CI_WIN_KIND" --arg arg "$CI_WIN_ARG" \ + --argjson total "$_total" --argjson returned "$_returned" --argjson truncated "$_trunc" \ + --argjson from "$_from" --argjson to "$_to" --argjson lines "$_lines" \ + --argjson matchLines "$_matchlines" --argjson matches "$_matches" \ + --argjson contiguous "$_contiguous" --argjson cached "$_cached" \ + '{ok: true, provider: $provider, runId: ($run | tonumber? // $run), jobId: $job, jobName: $jobName, + window: {kind: $kind, arg: $arg}, + logLines: $total, returnedLines: $returned, from: $from, to: $to, + contiguous: $contiguous, truncated: $truncated, + matches: $matches, matchLines: $matchLines, + cached: $cached, lines: $lines}' +} diff --git a/packages/codev/scripts/forge/_timeout.sh b/packages/codev/scripts/forge/_timeout.sh new file mode 100644 index 000000000..4dba75463 --- /dev/null +++ b/packages/codev/scripts/forge/_timeout.sh @@ -0,0 +1,109 @@ +# Shared wall-clock timeout for forge concept scripts. +# +# SOURCED, not executed, so it has no shebang and defines only functions. POSIX +# sh only — the concept scripts are #!/bin/sh and forge runs them via `sh -c`. +# The leading underscore keeps it out of the concept namespace (forge.ts builds +# presets from an explicit KNOWN_CONCEPTS allowlist, so this file is never +# registered as a concept). +# +# This started life inside gitea/_lib.sh (#12), where it fixed a `tea api` call +# that stalled a whole porch phase. The CI concepts (#13) need the identical +# guarantee around `gh`, and a second copy of a hand-rolled process watchdog is +# the last thing this repo needs — so it lives here and both providers source +# it. `gitea_timeout` remains a one-line alias in gitea/_lib.sh so #12's scripts +# and the comments that reference it by name still read true. + +# Run a command under a wall-clock limit. Returns the command's own exit status, +# or 124 (the exit status `timeout(1)` uses) if the limit was reached. +# +# Deliberately does NOT use timeout(1)/gtimeout even when present. macOS ships +# neither by default, so the fallback would be the path that actually runs for +# most adopters while the tested-in-CI path would be the one that doesn't — +# exactly the arrangement where the untested path rots. One implementation, +# same behaviour everywhere. +# +# TWO THINGS HERE ARE LOAD-BEARING, and both exist because killing the command +# is not the same as unblocking the caller: +# +# 1. The command's stdout goes to a TEMP FILE, not to the caller's pipe. Every +# caller runs this inside `$(...)`. A killed command can leave a grandchild +# holding the write end of that pipe, and the command substitution then +# blocks forever on a process nobody is waiting for — the timeout fires, the +# message prints, and the script still hangs. Measured, not theorised: with +# the command writing straight to the pipe, a 3s timeout against a wrapper +# that spawns `sleep 300` printed its timeout message at 3s and was still +# blocked two minutes later. +# 2. The watchdog subshell's own stdout goes to /dev/null, for the same reason. +# +# Grandchildren are swept with `pkill -P` where it exists. That is best-effort +# and not the guarantee — (1) is the guarantee, and it holds even where `pkill` +# does not exist. +forge_timeout() { + _limit="$1" + shift + # Both files live in a private mktemp DIRECTORY. The marker's path used to be + # derived from the output file's ("$_tf.fired"), which mktemp does not reserve + # — a predictable name in a world-writable tmpdir that anyone could pre-create + # to make every call report a timeout. + # 125, not 1: a temp dir we could not create is NOT the wrapped command + # failing — the command never ran. Returning 1 made an unusable TMPDIR + # indistinguishable from "gh says no such run", and on Linux (where mktemp + # honours TMPDIR strictly, unlike macOS which falls back) that turned into a + # confident "run 32515040122 could not be read". Reproduced in an ubuntu:24.04 + # container before fixing. + _dir=$(mktemp -d 2>/dev/null) || { + echo "forge: could not create a temporary directory under TMPDIR=${TMPDIR:-/tmp}; the command was never run" >&2 + return 125 + } + _tf="${_dir}/out" + _fired="${_dir}/fired" + "$@" >"$_tf" & + _cmd_pid=$! + ( sleep "$_limit" + # Claim the timeout only if there is still something to kill. Writing the + # marker unconditionally misreports a command that finished in the same + # instant the deadline passed — it succeeded, and saying otherwise discards + # a good answer. `kill -0` narrows that window to the gap between this test + # and the signal; it cannot be closed entirely without a lock, and a + # false timeout is a retryable error rather than a wrong answer. + if kill -0 "$_cmd_pid" 2>/dev/null; then + : > "$_fired" + pkill -TERM -P "$_cmd_pid" 2>/dev/null + kill -TERM "$_cmd_pid" 2>/dev/null + sleep 2 + pkill -KILL -P "$_cmd_pid" 2>/dev/null + kill -KILL "$_cmd_pid" 2>/dev/null + fi + ) >/dev/null 2>&1 & + _wd_pid=$! + # `|| _rc=$?` rather than `wait; _rc=$?`: the concept scripts run under + # `set -e`, which would abort here the moment the wrapped command exited + # non-zero — before the timeout could be classified and named. + _rc=0 + wait "$_cmd_pid" || _rc=$? + # Kill the watchdog and REAP it. Without the wait, the shell prints its own + # "Terminated: 15" notice about the killed background job onto the caller's + # stderr, on every single call — noise that a concept returning structured + # JSON should not be emitting, and that trains a reader to ignore the stream + # where the real diagnostics also arrive. + kill "$_wd_pid" 2>/dev/null + wait "$_wd_pid" 2>/dev/null || : + + # Whether the watchdog fired is recorded by the watchdog, not INFERRED from + # the exit status. Inferring it (status 143/137 = "we killed it") looked + # equivalent and is not: a killed process can still exit 0. A wrapper whose + # own `wait` takes no operand does exactly that — POSIX defines operand-less + # `wait` as always returning zero — so its death by SIGTERM was reported as a + # successful call returning an empty body, and the caller then diagnosed an + # unreadable repository instead of a timeout. Found by the test that pins this + # function; the marker file cannot be wrong the same way. + if [ -f "$_fired" ]; then + rm -rf "$_dir" + return 124 + fi + # A half-written response is worse than no response, so output is emitted only + # on the non-timeout path. + cat "$_tf" + rm -rf "$_dir" + return "$_rc" +} diff --git a/packages/codev/scripts/forge/gitea/_ci.sh b/packages/codev/scripts/forge/gitea/_ci.sh new file mode 100755 index 000000000..1dee791ed --- /dev/null +++ b/packages/codev/scripts/forge/gitea/_ci.sh @@ -0,0 +1,229 @@ +# Shared helpers for the Gitea/Forgejo CI concept scripts (#13). SOURCED. +# +# Sources gitea/_lib.sh for gitea_repo / gitea_api / gitea_api_error (#12) and +# the shared _ci-lib.sh / _ci-extract.sh. POSIX sh, leading underscore, never a +# concept. +# +# WHAT FORGEJO ACTUALLY OFFERS, MEASURED 2026-08-21 +# +# The issue proposed driving this from `tea actions runs list|view|logs`. Those +# subcommands exist on tea 0.14.2, and two of the three do not work against +# Forgejo 15.0.2 (git.pseudoseed.com): +# +# tea actions runs view 11130 → 404 on /api/v1/repos/…/actions/runs/11130/jobs +# tea actions runs logs 6881 --job 40084 +# → 404 on /api/v1/repos/…/actions/jobs/40084/logs +# +# Both routes were added in **Forgejo 16.0** (released 2026-07-16). And +# `tea actions runs list --output json` is lossy even where it works: workflow, +# branch, started and duration all come back as empty strings. So these scripts +# go through `tea api`, as #12 established for the PR concepts. +# +# What 15.0.2 does have: +# +# GET actions/runs?page=1&limit=N[&status=…] runs; 0.3s; ~17.8 KB PER RUN +# (each embeds a full repo object) +# GET actions/runs/{id} one run +# GET actions/tasks?page=1&limit=N[&status=…] JOBS, GitHub-shaped, 482 B each +# +# Three query-parameter facts, all measured, all footguns: +# +# * `limit` is IGNORED unless `page` is also present. `actions/runs?limit=3` +# returned all 6922 runs. Every call here passes page=1. +# * `status=` filters server-side and works. +# * `branch=` and `event=` are silently IGNORED. Branch filtering is +# client-side, and it is not a string compare — see below. +# +# THE BRANCH IS NOT THE BRANCH +# +# For `pull_request` runs Forgejo reports head_branch / prettyref as `#3847` — +# the PR number. Only push and schedule runs carry a real branch name. On the +# reference repo the first 100 tasks were: #3869 ×32, #3865 ×10, main ×7, +# v1.0.230 ×1. So filtering by `builder/pir-13` matches NOTHING on a repo that +# runs CI on pull requests, unless the branch is first resolved to its PR +# number — which is one base/head lookup, the same primitive #12 built. +# +# TWO ID SPACES +# +# Run `id` 11130 has `index_in_repo` 6881, and `actions/runs/6881` resolves to a +# DIFFERENT, real run. The web URL shows the index. So CODEV_CI_RUN_ID is always +# the `id` field from ci-runs, never the number, and these scripts never guess +# which one they were handed. + +. "$(dirname "$0")/_lib.sh" +. "$(dirname "$0")/../_ci-lib.sh" +. "$(dirname "$0")/../_ci-extract.sh" + +# The Forgejo release that first exposed Actions jobs and logs over the API. +GITEA_CI_LOG_MIN_VERSION="16.0" + +# The server version string, for error messages. Best effort: a server that will +# not answer `version` still gets a usable message, just a vaguer one. +gitea_server_version() { + _v=$(gitea_api "version" 2>/dev/null) || _v= + printf '%s' "$_v" | jq -r '.version // "unknown"' 2>/dev/null || printf 'unknown' +} + +# The unsupported-server envelope. +# +# This exists so that an old Forgejo can never be mistaken for a run with no +# failures. Those two are the same observation to a caller that only sees an +# empty array, and they are opposite facts: one means "your CI is fine", the +# other means "I cannot see your CI at all". +gitea_ci_unsupported() { + _concept="$1"; _route="$2"; _extra="$3" + [ -n "$_extra" ] || _extra='{}' + _ver=$(gitea_server_version) + ci_fail "$_concept" unsupported-server \ + "this Forgejo has no Actions ${_route} API (added in Forgejo ${GITEA_CI_LOG_MIN_VERSION}); server reports ${_ver}. ci-runs and ci-run-view still work here." \ + "$(printf '%s' "$_extra" | jq -c --arg v "$_ver" --arg n "$GITEA_CI_LOG_MIN_VERSION" '. + {serverVersion: $v, needs: (">=" + $n)}')" + return 0 +} + +# Fetch a JSON response INTO A FILE and classify it. +# +# gitea_ci_fetch +# 0 = a JSON object or array is in +# 44 = the endpoint said 404 (on Forgejo 15 that is how a missing Actions +# API announces itself) +# 45 = some other error body, left in for the caller to quote +# 124 = timed out (gitea_api already said so on stderr) +# +# A file rather than a shell variable because a single page of `actions/runs` is +# ~892 KB — Forgejo embeds the entire repository object in every run — and the +# jq reduction should read it from disk rather than after it has been copied +# through two shell strings. +gitea_ci_fetch() { + _url="$1"; _dest="$2" + _rc=0 + gitea_api "$_url" > "$_dest" || _rc=$? + [ "$_rc" -eq 0 ] || return "$_rc" + # Classify from the FILE, not from a prefix of it. Reading `head -c 400` and + # handing that to gitea_api_error looked equivalent and is not: a truncated + # JSON prefix does not parse, so every large healthy response classified as an + # error. Found on the first live call against Forgejo. + if jq -e 'type == "array"' "$_dest" >/dev/null 2>&1; then + return 0 + fi + if jq -e 'type == "object"' "$_dest" >/dev/null 2>&1; then + _msg=$(jq -r '.message // empty' "$_dest" 2>/dev/null) || _msg= + case "$_msg" in + '') return 0 ;; + *"couldn't be found"*|*'could not be found'*|*'Not found'*|*'not found'*|*'does not exist'*) return 44 ;; + *) return 45 ;; + esac + fi + # Not JSON at all: Forgejo answers an unknown ROUTE with the bare text + # "404 page not found", which is how a missing Actions API announces itself. + case "$(head -c 40 "$_dest")" in + 404*) return 44 ;; + esac + return 45 +} + +# Resolve a branch name to the PR ref Forgejo will have recorded on its runs. +# Echoes `#` when the branch has a PR, nothing when it does not. Returns 1 +# only when the REPO could not be read — a branch with no PR is an answer. +gitea_ci_pr_ref() { + _repo="$1"; _branch="$2" + _base=${CODEV_PR_BASE:-$(gitea_default_branch "$_repo")} || return 1 + _resp=$(gitea_api "repos/${_repo}/pulls/${_base}/${_branch}") || return 1 + case "$(gitea_api_error "$_resp")" in + ok) ;; + *) return 0 ;; + esac + _n=$(printf '%s' "$_resp" | jq -r 'if type == "object" and (.number | type) == "number" then "#\(.number)" else empty end' 2>/dev/null) || _n= + printf '%s' "$_n" +} + +# The jobs of a run, normalised, plus GITEA_JOB_SOURCE describing where they +# came from. Echoes a JSON array; returns 1 on a hard failure. +# +# Forgejo 16 has `actions/runs/{id}/jobs`, which carries the job `id` the log +# API takes. Forgejo 15 has neither, but `actions/tasks` lists the same jobs +# with a `run_number`, so the run object gives us `index_in_repo` and the tasks +# can be filtered to it. That fallback is why `ci-run-view` still answers on +# 15.0.2 instead of going dark with the log concepts. +# +# On the fallback path the identifier is a TASK id, not a job id, and there is +# no way to obtain a job id on a server that does not expose jobs. So `id` is +# null and `taskId` carries what we have. Emitting the task id as `id` would +# hand callers a number that looks usable with ci-run-log and is not. +gitea_ci_jobs() { + _concept="$1"; _repo="$2"; _runid="$3"; _runindex="$4"; _out="$5" + printf 'false' > "${_out}/jobs.truncated" + + _rc=0 + gitea_ci_fetch "repos/${_repo}/actions/runs/${_runid}/jobs" "${_out}/jobs.raw" || _rc=$? + if [ "$_rc" -eq 0 ]; then + printf 'runs-jobs' > "${_out}/jobs.source" + jq -c '[ .[]? | { + id: .id, taskId: .task_id, name: .name, + status: .status, + conclusion: (if (.status == "success" or .status == "failure" or .status == "skipped" or .status == "canceled" or .status == "cancelled") then .status else null end), + startedAt: null, completedAt: null, failedSteps: [] + } ]' "${_out}/jobs.raw" > "${_out}/jobs.json" + return 0 + fi + [ "$_rc" -eq 44 ] || return 1 + + # Forgejo 15: no jobs route. Recover the jobs from `actions/tasks`, which + # carries run_number. Tasks come back newest-first, so a run occupies one + # contiguous block and the walk can stop the moment it has gone PAST that + # block — otherwise every lookup of an older run would cost the full page + # ceiling and report truncation it did not need to. + printf 'tasks-scan' > "${_out}/jobs.source" + _acc='[]' + _page=1 + _found=0 + _passed=0 + while [ "$_page" -le "$CI_TASKS_MAX_PAGES" ]; do + _rc=0 + gitea_ci_fetch "repos/${_repo}/actions/tasks?page=${_page}&limit=${GITEA_PAGE_LIMIT}" "${_out}/tasks.json" || _rc=$? + [ "$_rc" -eq 0 ] || return 1 + _raw=$(jq '.workflow_runs | length' "${_out}/tasks.json") + [ "$_raw" -eq 0 ] && break + _hits=$(jq -c --argjson r "$_runindex" '[ .workflow_runs[]? | select(.run_number == $r) | { + id: null, taskId: .id, name: .name, status: .status, + conclusion: (if (.status == "success" or .status == "failure" or .status == "skipped" or .status == "canceled" or .status == "cancelled") then .status else null end), + startedAt: .run_started_at, completedAt: .updated_at, failedSteps: [] + } ]' "${_out}/tasks.json") + if [ "$(printf '%s' "$_hits" | jq 'length')" -gt 0 ]; then + _acc=$(printf '%s\n%s' "$_acc" "$_hits" | jq -s -c 'add') + _found=1 + fi + _min=$(jq --argjson r "$_runindex" '[.workflow_runs[]?.run_number] | min // ($r + 1)' "${_out}/tasks.json") + if [ "$_min" -lt "$_runindex" ]; then _passed=1; break; fi + [ "$_raw" -lt "$GITEA_PAGE_LIMIT" ] && break + _page=$((_page + 1)) + done + printf '%s' "$_acc" > "${_out}/jobs.json" + # Truncated only when the walk ran out of allowance BEFORE reaching the run. + # Having walked past it and found nothing is a complete answer. + if [ "$_found" -eq 0 ] && [ "$_passed" -eq 0 ] && [ "$_page" -gt "$CI_TASKS_MAX_PAGES" ]; then + printf 'true' > "${_out}/jobs.truncated" + echo "${_concept}: this Forgejo has no jobs API, and run ${_runid} was not reached within ${CI_TASKS_MAX_PAGES} pages of the task list; raise CODEV_CI_TASKS_MAX_PAGES" >&2 + fi + return 0 +} + +# Fetch one job log (Forgejo 16 only), via the cache when the job is terminal. +gitea_ci_job_log() { + _repo="$1"; _job="$2"; _status="$3"; _dest="$4" + _cache=$(ci_cache_path gitea "$_repo" "$_job") + if ci_cache_read "$_cache" > "$_dest" 2>/dev/null && [ -s "$_dest" ]; then + CI_LOG_FROM_CACHE=true + return 0 + fi + CI_LOG_FROM_CACHE=false + _rc=0 + gitea_api "repos/${_repo}/actions/jobs/${_job}/logs" > "$_dest" || _rc=$? + [ "$_rc" -eq 0 ] || return "$_rc" + # `tea api` exits 0 on HTTP errors and prints the body, so a 404 arrives here + # looking like a log. Classify it before anyone treats an error page as one. + if [ ! -s "$_dest" ] || gitea_api_error "$(head -c 200 "$_dest")" | grep -q 'notfound'; then + return 44 + fi + ci_cache_write "$_cache" "$_dest" "$_status" + return 0 +} diff --git a/packages/codev/scripts/forge/gitea/_lib.sh b/packages/codev/scripts/forge/gitea/_lib.sh index 726c23e17..dad3af36f 100755 --- a/packages/codev/scripts/forge/gitea/_lib.sh +++ b/packages/codev/scripts/forge/gitea/_lib.sh @@ -135,84 +135,17 @@ case "$GITEA_PAGED_DEADLINE" in ''|*[!0-9]*) GITEA_PAGED_DEADLINE=120 ;; esac -# Run a command under a wall-clock limit. Returns the command's own exit status, -# or 124 (the exit status `timeout(1)` uses) if the limit was reached. -# -# Deliberately does NOT use timeout(1)/gtimeout even when present. macOS ships -# neither by default, so the fallback would be the path that actually runs for -# most adopters while the tested-in-CI path would be the one that doesn't — -# exactly the arrangement where the untested path rots. One implementation, -# same behaviour everywhere. -# -# TWO THINGS HERE ARE LOAD-BEARING, and both exist because killing the command -# is not the same as unblocking the caller: -# -# 1. The command's stdout goes to a TEMP FILE, not to the caller's pipe. Every -# caller runs this inside `$(...)`. A killed command can leave a grandchild -# holding the write end of that pipe, and the command substitution then -# blocks forever on a process nobody is waiting for — the timeout fires, the -# message prints, and the script still hangs. Measured, not theorised: with -# the command writing straight to the pipe, a 3s timeout against a wrapper -# that spawns `sleep 300` printed its timeout message at 3s and was still -# blocked two minutes later. -# 2. The watchdog subshell's own stdout goes to /dev/null, for the same reason. -# -# Grandchildren are swept with `pkill -P` where it exists. That is best-effort -# and not the guarantee — (1) is the guarantee, and it holds even where `pkill` -# does not exist. -gitea_timeout() { - _limit="$1" - shift - # Both files live in a private mktemp DIRECTORY. The marker's path used to be - # derived from the output file's ("$_tf.fired"), which mktemp does not reserve - # — a predictable name in a world-writable tmpdir that anyone could pre-create - # to make every call report a timeout. - _dir=$(mktemp -d) || return 1 - _tf="${_dir}/out" - _fired="${_dir}/fired" - "$@" >"$_tf" & - _cmd_pid=$! - ( sleep "$_limit" - # Claim the timeout only if there is still something to kill. Writing the - # marker unconditionally misreports a command that finished in the same - # instant the deadline passed — it succeeded, and saying otherwise discards - # a good answer. `kill -0` narrows that window to the gap between this test - # and the signal; it cannot be closed entirely without a lock, and a - # false timeout is a retryable error rather than a wrong answer. - if kill -0 "$_cmd_pid" 2>/dev/null; then - : > "$_fired" - pkill -TERM -P "$_cmd_pid" 2>/dev/null - kill -TERM "$_cmd_pid" 2>/dev/null - sleep 2 - pkill -KILL -P "$_cmd_pid" 2>/dev/null - kill -KILL "$_cmd_pid" 2>/dev/null - fi - ) >/dev/null 2>&1 & - _wd_pid=$! - # `|| _rc=$?` rather than `wait; _rc=$?`: the concept scripts run under - # `set -e`, which would abort here the moment the wrapped command exited - # non-zero — before the timeout could be classified and named. - _rc=0 - wait "$_cmd_pid" || _rc=$? - kill "$_wd_pid" 2>/dev/null +# The wall-clock watchdog now lives in scripts/forge/_timeout.sh, because the +# CI concepts (#13) need the same guarantee around `gh` and a second hand-rolled +# process watchdog is the last thing this repo needs. `gitea_timeout` stays as +# the name every gitea script and comment already uses; the behaviour, and the +# two load-bearing details documented in _timeout.sh (the command's stdout goes +# to a temp file, and the watchdog records the timeout in a marker file rather +# than having it inferred from an exit status), are unchanged. +. "$(dirname "$0")/../_timeout.sh" - # Whether the watchdog fired is recorded by the watchdog, not INFERRED from - # the exit status. Inferring it (status 143/137 = "we killed it") looked - # equivalent and is not: a killed process can still exit 0. A wrapper whose - # own `wait` takes no operand does exactly that — POSIX defines operand-less - # `wait` as always returning zero — so its death by SIGTERM was reported as a - # successful call returning an empty body, and the caller then diagnosed an - # unreadable repository instead of a timeout. Found by the test that pins this - # function; the marker file cannot be wrong the same way. - if [ -f "$_fired" ]; then - rm -rf "$_dir" - return 124 - fi - # A half-written response is worse than no response, so output is emitted only - # on the non-timeout path. - cat "$_tf" - rm -rf "$_dir" - return "$_rc" +gitea_timeout() { + forge_timeout "$@" } # `tea api` under GITEA_TIMEOUT, with a named error on the timeout path. diff --git a/packages/codev/scripts/forge/gitea/ci-failures.sh b/packages/codev/scripts/forge/gitea/ci-failures.sh new file mode 100755 index 000000000..8e4ea089c --- /dev/null +++ b/packages/codev/scripts/forge/gitea/ci-failures.sh @@ -0,0 +1,171 @@ +#!/bin/sh +# Forge concept: ci-failures (Gitea/Forgejo via tea CLI) +# forge-executable: tea +# Input: CODEV_CI_RUN_ID (required), CODEV_CI_JOB_ID (optional) +# Output: the shared ci-failures envelope; see github/ci-failures.sh +# +# REQUIRES FORGEJO 16.0 OR LATER. The Actions job-log API +# (`actions/jobs/{id}/logs`) landed in Forgejo 16.0, released 2026-07-16; on +# 15.0.2 there is no token-reachable log anywhere — not via `tea actions runs +# logs` (which calls that same route), not via any other API route, and not via +# the web UI route, which is session-only and rejects both an API token and +# basic auth. Verified against a live 15.0.2 instance. +# +# On such a server this returns the `unsupported-server` envelope, naming the +# version it found and the version it needs, and CARRYING THE FAILING JOB NAMES +# it was still able to determine. What it must never do is return an empty +# `failures` array: "your CI is fine" and "I cannot see your CI at all" are +# opposite facts, and they are the same observation to a caller that only sees +# an empty list. +set -e +. "$(dirname "$0")/_ci.sh" + +CONCEPT=ci-failures + +ci_require_tmpdir "$CONCEPT" + +if [ -z "$CODEV_CI_RUN_ID" ]; then + jq -cn '{ok: false, error: "bad-input", detail: "CODEV_CI_RUN_ID is required"}' + echo "${CONCEPT}: CODEV_CI_RUN_ID is required" >&2 + exit 2 +fi +ci_require_id "$CONCEPT" CODEV_CI_RUN_ID "$CODEV_CI_RUN_ID" +[ -z "$CODEV_CI_JOB_ID" ] || ci_require_id "$CONCEPT" CODEV_CI_JOB_ID "$CODEV_CI_JOB_ID" + +REPO="$(gitea_repo)" || exit 1 +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +rc=0 +gitea_ci_fetch "repos/${REPO}/actions/runs/${CODEV_CI_RUN_ID}" "$TMP/run.json" || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "GET repos/${REPO}/actions/runs/${CODEV_CI_RUN_ID}" "$GITEA_TIMEOUT" + exit 1 +fi +if [ "$rc" -eq 44 ]; then + ci_fail "$CONCEPT" not-found "run ${CODEV_CI_RUN_ID} does not exist in ${REPO}; pass the \`id\` from ci-runs, not the run \`number\`" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" '{runId: $r}')" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "Forgejo could not read run ${CODEV_CI_RUN_ID}: $(head -c 200 "$TMP/run.json")" + exit 1 +fi + +RUN_INDEX=$(jq -r '.index_in_repo' "$TMP/run.json") +RUN_STATUS=$(jq -r '.status // "unknown"' "$TMP/run.json") + +rc=0 +gitea_ci_jobs "$CONCEPT" "$REPO" "$CODEV_CI_RUN_ID" "$RUN_INDEX" "$TMP" || rc=$? +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "could not list the jobs of run ${CODEV_CI_RUN_ID}" + exit 1 +fi +JOB_SOURCE=$(cat "$TMP/jobs.source") +FAILED=$(jq -c '[ .[] | select(.status == "failure" or .status == "timed_out" or .conclusion == "failure") ]' "$TMP/jobs.json") +JOBS_FAILED=$(printf '%s' "$FAILED" | jq 'length') + +# Forgejo 15: the jobs came from the task scan, so there is no job id and no log +# API. Say which server this is, and hand back what IS known. +if [ "$JOB_SOURCE" = "tasks-scan" ]; then + gitea_ci_unsupported "$CONCEPT" "job-log" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --arg rs "$RUN_STATUS" --argjson f "$FAILED" --argjson n "$JOBS_FAILED" \ + '{runId: ($r | tonumber? // $r), runStatus: $rs, jobsFailed: $n, + failingJobs: [ $f[] | {taskId: .taskId, jobName: .name} ], + hint: "ci-run-view still works on this server and lists per-job status"}')" + exit 1 +fi + +if [ -n "$CODEV_CI_JOB_ID" ]; then + TARGET=$(jq -c --argjson j "$CODEV_CI_JOB_ID" 'first(.[] | select(.id == $j)) // empty' "$TMP/jobs.json") + if [ -z "$TARGET" ]; then + ci_fail "$CONCEPT" not-found "job ${CODEV_CI_JOB_ID} is not part of run ${CODEV_CI_RUN_ID}" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --arg j "$CODEV_CI_JOB_ID" '{runId: $r, jobId: $j}')" + exit 1 + fi +else + TARGET=$(printf '%s' "$FAILED" | jq -c '.[0] // empty') +fi + +if [ -z "$TARGET" ]; then + jq -cn --arg r "$CODEV_CI_RUN_ID" --arg rs "$RUN_STATUS" '{ + ok: true, provider: "gitea", runId: ($r | tonumber? // $r), + runStatus: $rs, runConclusion: null, jobsFailed: 0, extracted: false, + reason: (if $rs == "success" then "no job in this run failed" else "no failing job found for this run" end), + failures: []}' + exit 0 +fi + +JOB_ID=$(printf '%s' "$TARGET" | jq -r '.id') +JOB_NAME=$(printf '%s' "$TARGET" | jq -r '.name') +JOB_STATE=$(printf '%s' "$TARGET" | jq -r '.status') + +rc=0 +gitea_ci_job_log "$REPO" "$JOB_ID" "$JOB_STATE" "$TMP/raw.log" || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "GET repos/${REPO}/actions/jobs/${JOB_ID}/logs" "$GITEA_TIMEOUT" + exit 1 +fi +if [ "$rc" -eq 44 ]; then + gitea_ci_unsupported "$CONCEPT" "job-log" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --argjson j "$JOB_ID" --arg n "$JOB_NAME" --argjson c "$JOBS_FAILED" \ + '{runId: ($r | tonumber? // $r), jobsFailed: $c, failingJobs: [{jobId: $j, jobName: $n}]}')" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "could not read the log for job ${JOB_ID} (${JOB_NAME})" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --argjson j "$JOB_ID" --arg n "$JOB_NAME" '{runId: $r, jobId: $j, jobName: $n}')" + exit 1 +fi + +ci_clean_log < "$TMP/raw.log" > "$TMP/clean.log" +# awk NR, not `wc -l`: a log with no trailing newline makes wc undercount by +# one, and logLines would then disagree with the from/to the extractor reports. +LOG_LINES=$(awk 'END {print NR}' "$TMP/clean.log") +OTHERS=$(printf '%s' "$FAILED" | jq -c --argjson j "$JOB_ID" '[.[] | select(.id != $j) | {id: .id, name: .name}]') + +if ci_extract "$TMP/clean.log" > "$TMP/extract.txt" 2>/dev/null && [ -s "$TMP/extract.txt" ]; then + MATCHED=$(head -1 "$TMP/extract.txt" | cut -f1) + FROM=$(head -1 "$TMP/extract.txt" | cut -f2) + TO=$(head -1 "$TMP/extract.txt" | cut -f3) + tail -n +2 "$TMP/extract.txt" > "$TMP/text.txt" + CAP=$CI_MAX_STEP_BYTES + [ "$CAP" -gt "$CI_MAX_RESPONSE_BYTES" ] && CAP=$CI_MAX_RESPONSE_BYTES + META=$(ci_cap_file "$TMP/text.txt" "$CAP" "$TMP/capped.txt") + RETURNED_LINES=${META% *} + TRUNCATED=${META#* } + TEXT=$(jq -R -s -c . < "$TMP/capped.txt") + jq -cn \ + --arg run "$CODEV_CI_RUN_ID" --arg rs "$RUN_STATUS" \ + --argjson job "$JOB_ID" --arg jobName "$JOB_NAME" \ + --arg matched "$MATCHED" --argjson text "$TEXT" \ + --argjson from "$FROM" --argjson to "$TO" \ + --argjson logLines "$LOG_LINES" --argjson returned "$RETURNED_LINES" \ + --argjson truncated "$TRUNCATED" --argjson jobsFailed "$JOBS_FAILED" \ + --argjson others "$OTHERS" --argjson cached "$CI_LOG_FROM_CACHE" \ + '{ok: true, provider: "gitea", runId: ($run | tonumber? // $run), + runStatus: $rs, runConclusion: null, + jobsFailed: $jobsFailed, extracted: true, + failures: [{ + jobId: $job, jobName: $jobName, + stepName: null, stepNumber: null, + matchedBy: $matched, text: $text, + from: $from, to: $to, + logLines: $logLines, returnedLines: $returned, truncated: $truncated + }], + otherFailingJobs: $others, cached: $cached}' + exit 0 +fi + +jq -cn \ + --arg run "$CODEV_CI_RUN_ID" --arg rs "$RUN_STATUS" \ + --argjson job "$JOB_ID" --arg jobName "$JOB_NAME" \ + --argjson logLines "$LOG_LINES" --argjson jobsFailed "$JOBS_FAILED" \ + --argjson others "$OTHERS" --argjson cached "$CI_LOG_FROM_CACHE" \ + '{ok: true, provider: "gitea", runId: ($run | tonumber? // $run), + runStatus: $rs, runConclusion: null, + jobsFailed: $jobsFailed, extracted: false, + reason: "no recognized failure pattern", + failures: [{jobId: $job, jobName: $jobName, logLines: $logLines}], + otherFailingJobs: $others, cached: $cached, + next: ("ci-run-log CODEV_CI_RUN_ID=" + $run + " CODEV_CI_JOB_ID=" + ($job|tostring) + " CODEV_CI_LOG_TAIL=80")}' diff --git a/packages/codev/scripts/forge/gitea/ci-run-log.sh b/packages/codev/scripts/forge/gitea/ci-run-log.sh new file mode 100755 index 000000000..a8c8c6dc8 --- /dev/null +++ b/packages/codev/scripts/forge/gitea/ci-run-log.sh @@ -0,0 +1,109 @@ +#!/bin/sh +# Forge concept: ci-run-log (Gitea/Forgejo via tea CLI) +# forge-executable: tea +# Input: CODEV_CI_RUN_ID (required), CODEV_CI_JOB_ID (optional), +# exactly ONE of CODEV_CI_LOG_TAIL / CODEV_CI_LOG_HEAD / CODEV_CI_LOG_GREP +# (+ CODEV_CI_LOG_CONTEXT, default 3) +# Output: the shared ci-run-log envelope; see github/ci-run-log.sh +# +# REQUIRES FORGEJO 16.0 OR LATER, for the same reason as ci-failures: the job +# log API did not exist before it. On an older server this returns the +# unsupported-server envelope rather than an empty window. +# +# Forgejo 16 serves the log as text/plain with `accept-ranges: bytes`, so a tail +# could in principle be fetched as a byte range. It is not, because `tea api` +# sends no Range header and the log cache makes the second and third window over +# the same job free anyway — measured 142 KB / 1.0s for a 1599-line job log. +set -e +. "$(dirname "$0")/_ci.sh" + +CONCEPT=ci-run-log + +ci_require_tmpdir "$CONCEPT" + +if [ -z "$CODEV_CI_RUN_ID" ]; then + jq -cn '{ok: false, error: "bad-input", detail: "CODEV_CI_RUN_ID is required"}' + echo "${CONCEPT}: CODEV_CI_RUN_ID is required" >&2 + exit 2 +fi +ci_require_id "$CONCEPT" CODEV_CI_RUN_ID "$CODEV_CI_RUN_ID" +[ -z "$CODEV_CI_JOB_ID" ] || ci_require_id "$CONCEPT" CODEV_CI_JOB_ID "$CODEV_CI_JOB_ID" + +# Window first: a malformed request should not cost an API call. +ci_window_parse "$CONCEPT" + +REPO="$(gitea_repo)" || exit 1 +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +rc=0 +gitea_ci_fetch "repos/${REPO}/actions/runs/${CODEV_CI_RUN_ID}" "$TMP/run.json" || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "GET repos/${REPO}/actions/runs/${CODEV_CI_RUN_ID}" "$GITEA_TIMEOUT" + exit 1 +fi +if [ "$rc" -eq 44 ]; then + ci_fail "$CONCEPT" not-found "run ${CODEV_CI_RUN_ID} does not exist in ${REPO}; pass the \`id\` from ci-runs, not the run \`number\`" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" '{runId: $r}')" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "Forgejo could not read run ${CODEV_CI_RUN_ID}: $(head -c 200 "$TMP/run.json")" + exit 1 +fi + +RUN_INDEX=$(jq -r '.index_in_repo' "$TMP/run.json") + +rc=0 +gitea_ci_jobs "$CONCEPT" "$REPO" "$CODEV_CI_RUN_ID" "$RUN_INDEX" "$TMP" || rc=$? +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "could not list the jobs of run ${CODEV_CI_RUN_ID}" + exit 1 +fi + +if [ "$(cat "$TMP/jobs.source")" = "tasks-scan" ]; then + gitea_ci_unsupported "$CONCEPT" "job-log" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" '{runId: ($r | tonumber? // $r), hint: "ci-run-view still works on this server and lists per-job status"}')" + exit 1 +fi + +if [ -n "$CODEV_CI_JOB_ID" ]; then + TARGET=$(jq -c --argjson j "$CODEV_CI_JOB_ID" 'first(.[] | select(.id == $j)) // empty' "$TMP/jobs.json") + if [ -z "$TARGET" ]; then + ci_fail "$CONCEPT" not-found "job ${CODEV_CI_JOB_ID} is not part of run ${CODEV_CI_RUN_ID}" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --arg j "$CODEV_CI_JOB_ID" '{runId: $r, jobId: $j}')" + exit 1 + fi +else + TARGET=$(jq -c 'first(.[] | select(.status == "failure" or .status == "timed_out")) // empty' "$TMP/jobs.json") + [ -n "$TARGET" ] || TARGET=$(jq -c 'if length == 1 then .[0] else empty end' "$TMP/jobs.json") + if [ -z "$TARGET" ]; then + ci_fail "$CONCEPT" bad-input "run ${CODEV_CI_RUN_ID} has no failing job and more than one job; set CODEV_CI_JOB_ID" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --slurpfile j "$TMP/jobs.json" '{runId: $r, jobs: [ $j[0][] | {id: .id, name: .name} ]}')" + exit 2 + fi +fi + +JOB_ID=$(printf '%s' "$TARGET" | jq -r '.id') +JOB_NAME=$(printf '%s' "$TARGET" | jq -r '.name') +JOB_STATE=$(printf '%s' "$TARGET" | jq -r '.status') + +rc=0 +gitea_ci_job_log "$REPO" "$JOB_ID" "$JOB_STATE" "$TMP/raw.log" || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "GET repos/${REPO}/actions/jobs/${JOB_ID}/logs" "$GITEA_TIMEOUT" + exit 1 +fi +if [ "$rc" -eq 44 ]; then + gitea_ci_unsupported "$CONCEPT" "job-log" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --argjson j "$JOB_ID" --arg n "$JOB_NAME" '{runId: ($r | tonumber? // $r), jobId: $j, jobName: $n}')" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "could not read the log for job ${JOB_ID} (${JOB_NAME})" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --argjson j "$JOB_ID" --arg n "$JOB_NAME" '{runId: $r, jobId: $j, jobName: $n}')" + exit 1 +fi + +ci_clean_log < "$TMP/raw.log" > "$TMP/clean.log" +ci_window_emit "$CONCEPT" "$TMP" gitea "$CODEV_CI_RUN_ID" "$JOB_ID" "$JOB_NAME" "$CI_LOG_FROM_CACHE" diff --git a/packages/codev/scripts/forge/gitea/ci-run-view.sh b/packages/codev/scripts/forge/gitea/ci-run-view.sh new file mode 100755 index 000000000..c78febbc5 --- /dev/null +++ b/packages/codev/scripts/forge/gitea/ci-run-view.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# Forge concept: ci-run-view (Gitea/Forgejo via tea CLI) +# forge-executable: tea +# Input: CODEV_CI_RUN_ID (required — the `id` from ci-runs, NOT the `number`) +# Output: {ok, provider, run: {...}, jobs: [...], jobSource, truncated} +# +# Still no log bytes. +# +# `jobSource` says which route answered, because the two are not equivalent: +# +# runs-jobs Forgejo 16 `actions/runs/{id}/jobs`. Carries the job `id` that +# ci-failures and ci-run-log need. +# tasks-scan Forgejo 15 has no jobs route, so the jobs are recovered by +# filtering `actions/tasks` on run_number. That yields TASK ids, +# not job ids, so `id` is null and `taskId` carries what exists. +# A task id looks like a job id and is not accepted by the log +# API, so handing it back as `id` would be a trap. +# +# Forgejo exposes no per-step data on either version, so `failedSteps` is always +# [] here. It is present rather than omitted so the field means the same thing +# on both providers. +set -e +. "$(dirname "$0")/_ci.sh" + +CONCEPT=ci-run-view + +ci_require_tmpdir "$CONCEPT" + +if [ -z "$CODEV_CI_RUN_ID" ]; then + jq -cn '{ok: false, error: "bad-input", detail: "CODEV_CI_RUN_ID is required"}' + echo "${CONCEPT}: CODEV_CI_RUN_ID is required" >&2 + exit 2 +fi +ci_require_id "$CONCEPT" CODEV_CI_RUN_ID "$CODEV_CI_RUN_ID" +[ -z "$CODEV_CI_JOB_ID" ] || ci_require_id "$CONCEPT" CODEV_CI_JOB_ID "$CODEV_CI_JOB_ID" + +REPO="$(gitea_repo)" || exit 1 +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +rc=0 +gitea_ci_fetch "repos/${REPO}/actions/runs/${CODEV_CI_RUN_ID}" "$TMP/run.json" || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "GET repos/${REPO}/actions/runs/${CODEV_CI_RUN_ID}" "$GITEA_TIMEOUT" + exit 1 +fi +if [ "$rc" -eq 44 ]; then + ci_fail "$CONCEPT" not-found "run ${CODEV_CI_RUN_ID} does not exist in ${REPO}; pass the \`id\` from ci-runs, not the run \`number\` — on Forgejo both are valid ids for different runs" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" '{runId: $r}')" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "Forgejo could not read run ${CODEV_CI_RUN_ID}: $(head -c 200 "$TMP/run.json")" + exit 1 +fi + +RUN_INDEX=$(jq -r '.index_in_repo' "$TMP/run.json") + +rc=0 +gitea_ci_jobs "$CONCEPT" "$REPO" "$CODEV_CI_RUN_ID" "$RUN_INDEX" "$TMP" || rc=$? +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "could not list the jobs of run ${CODEV_CI_RUN_ID}" + exit 1 +fi + +# Read back from files rather than from a command substitution: gitea_ci_jobs +# has to report three things (the jobs, which route answered, whether the walk +# was cut short) and a subshell would discard two of them. +jq -c --slurpfile jobs "$TMP/jobs.json" --arg source "$(cat "$TMP/jobs.source")" \ + --argjson truncated "$(cat "$TMP/jobs.truncated")" '{ + ok: true, provider: "gitea", jobSource: $source, + run: { + id: .id, number: .index_in_repo, title: .title, + workflow: .workflow_id, status: .status, conclusion: null, + branch: .prettyref, sha: .commit_sha, event: .event, + url: .html_url, createdAt: .created + }, + jobs: $jobs[0], + truncated: $truncated +}' "$TMP/run.json" diff --git a/packages/codev/scripts/forge/gitea/ci-runs.sh b/packages/codev/scripts/forge/gitea/ci-runs.sh new file mode 100755 index 000000000..ddbdff885 --- /dev/null +++ b/packages/codev/scripts/forge/gitea/ci-runs.sh @@ -0,0 +1,116 @@ +#!/bin/sh +# Forge concept: ci-runs (Gitea/Forgejo via tea CLI) +# forge-executable: tea +# Input: CODEV_BRANCH_NAME (optional), CODEV_CI_STATUS (optional), +# CODEV_CI_WORKFLOW (optional — workflow file, e.g. ci.yml), +# CODEV_CI_LIMIT (optional, default 20), CODEV_PR_BASE (optional) +# Output: {ok, provider, runs: [...], truncated, note?} +# +# The cheap question. No log bytes on any path. +# +# `conclusion` is always null here and that is not an oversight: Forgejo has no +# separate conclusion field, `status` carries success/failure/skipped/canceled +# directly. Emitting a fabricated conclusion would make the two providers look +# identical where they are not. +# +# Branch filtering is client-side and PR-aware — see the header of _ci.sh for +# why `branch=` cannot be sent to the server and why `builder/x` has to become +# `#123` before it will match anything. +set -e +. "$(dirname "$0")/_ci.sh" + +CONCEPT=ci-runs + +ci_require_tmpdir "$CONCEPT" + +ci_check_status "$CONCEPT" "$CODEV_CI_STATUS" + +LIMIT=${CODEV_CI_LIMIT:-$CI_LIMIT_DEFAULT} +case "$LIMIT" in ''|*[!0-9]*|0) LIMIT=$CI_LIMIT_DEFAULT ;; esac + +REPO="$(gitea_repo)" || exit 1 + +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +QUERY="" +[ -n "$CODEV_CI_STATUS" ] && QUERY="status=$(ci_status_for gitea "$CODEV_CI_STATUS")" + +NOTE=null +PR_REF="" +if [ -n "$CODEV_BRANCH_NAME" ]; then + PR_REF=$(gitea_ci_pr_ref "$REPO" "$CODEV_BRANCH_NAME") || { + ci_fail "$CONCEPT" forge-error "could not read repository '${REPO}' while resolving branch '${CODEV_BRANCH_NAME}'" + exit 1 + } + if [ -z "$PR_REF" ]; then + NOTE='"branch has no pull request, so only push/schedule runs on that branch can match; Forgejo labels pull_request runs with the PR number, not the branch"' + fi +fi + +ACC='[]' +PAGE=1 +TRUNCATED=false +while [ "$PAGE" -le "$CI_MAX_PAGES" ]; do + rc=0 + gitea_ci_fetch "repos/${REPO}/actions/runs?page=${PAGE}&limit=${GITEA_PAGE_LIMIT}${QUERY:+&$QUERY}" "$TMP/page.json" || rc=$? + if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "GET repos/${REPO}/actions/runs" "$GITEA_TIMEOUT" + exit 1 + fi + if [ "$rc" -eq 44 ]; then + gitea_ci_unsupported "$CONCEPT" "workflow-runs" "$(jq -cn --arg r "$REPO" '{repo: $r}')" + exit 1 + fi + if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "Forgejo could not list workflow runs for ${REPO}: $(head -c 200 "$TMP/page.json")" + exit 1 + fi + + RAW=$(jq '.workflow_runs | length' "$TMP/page.json") + HITS=$(jq -c --arg branch "$CODEV_BRANCH_NAME" --arg prref "$PR_REF" --arg wf "$CODEV_CI_WORKFLOW" ' + [ .workflow_runs[]? + | select($wf == "" or .workflow_id == $wf) + | select($branch == "" or .prettyref == $branch or ($prref != "" and .prettyref == $prref)) + | { + id: .id, + number: .index_in_repo, + name: .title, + workflow: .workflow_id, + status: .status, + conclusion: null, + branch: .prettyref, + sha: .commit_sha, + event: .event, + url: .html_url, + createdAt: .created + } ]' "$TMP/page.json") + ACC=$(printf '%s\n%s' "$ACC" "$HITS" | jq -s -c 'add') + + # -gt, not -ge: stopping at exactly LIMIT would leave `truncated` (computed + # below as length > limit) reading false when there is more to see. One extra + # item is what makes "there are more runs than this" a fact rather than a + # guess — the same reason the github script asks gh for LIMIT + 1. + [ "$(printf '%s' "$ACC" | jq 'length')" -gt "$LIMIT" ] && break + if [ "$RAW" -lt "$GITEA_PAGE_LIMIT" ]; then break; fi + PAGE=$((PAGE + 1)) +done + +# The loop condition — not a break — is what ends a walk that ran out of +# allowance, so PAGE past the ceiling means there was more to see. Checking it +# here rather than inside the loop also covers the UNFILTERED case: asking for +# 200 runs from a 6922-run repo collects exactly 200 and used to report +# truncated=false, because the old check only fired when a client-side filter +# was active. A capped answer that says it is complete is the failure this +# whole issue is about. +if [ "$PAGE" -gt "$CI_MAX_PAGES" ]; then + TRUNCATED=true + echo "${CONCEPT}: stopped after ${CI_MAX_PAGES} pages of runs; raise CODEV_CI_MAX_PAGES for a deeper search" >&2 +fi + +printf '%s' "$ACC" | jq -c --argjson limit "$LIMIT" --argjson truncated "$TRUNCATED" --argjson note "$NOTE" '{ + ok: true, provider: "gitea", + runs: .[0:$limit], + truncated: ($truncated or (length > $limit)), + note: $note +}' diff --git a/packages/codev/scripts/forge/github/_lib.sh b/packages/codev/scripts/forge/github/_lib.sh new file mode 100755 index 000000000..f62d99b08 --- /dev/null +++ b/packages/codev/scripts/forge/github/_lib.sh @@ -0,0 +1,57 @@ +# Shared helpers for the GitHub CI concept scripts (#13). SOURCED, not executed. +# +# POSIX sh, no shebang, leading underscore so forge never registers it as a +# concept. The older github concepts are one-line `exec gh …` scripts and do not +# use this; the CI ones need a run lookup, a job-log fetch and a cache, and none +# of that should be written three times. + +. "$(dirname "$0")/../_ci-lib.sh" +. "$(dirname "$0")/../_ci-extract.sh" + +# The gh `run view` projection every CI concept needs. One call, ~1.3s measured, +# and it already carries per-step conclusions — so the failing STEP name comes +# from structured JSON rather than from parsing a log. +GH_RUN_FIELDS='databaseId,number,displayTitle,workflowName,name,status,conclusion,headBranch,headSha,event,url,createdAt,jobs' + +# Fetch a run as JSON. Echoes the JSON; returns 124 on timeout, 1 otherwise. +gh_run_json() { + ci_tool gh run view "$1" --json "$GH_RUN_FIELDS" +} + +# The jobs of a run that actually failed, as a JSON array, in run order. +# +# `cancelled` is NOT in this list. A cancelled job has no failure to diagnose, +# and treating it as one produces an extract of whatever the runner happened to +# be printing when the cancel landed — an arbitrary slice of log presented as a +# cause, which is the thing this concept exists to avoid. +GH_FAILED_JOBS_JQ='[.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "startup_failure")]' + +# Fetch one job log, via the cache when the job is terminal. +# +# gh_job_log +# +# Uses `gh api repos/{owner}/{repo}/actions/jobs/{id}/logs` — NOT +# `gh run view --log-failed`. The reason is measured, and it contradicts the +# issue that asked for this work: --log-failed returned 2528 lines / 293 KB for +# one failing job on run 32515040122, every line tagged "UNKNOWN STEP". Its step +# attribution is unreliable rather than always absent — on run 32536232930 the +# same command attributed all 1193 lines correctly — but attributed or not, what +# comes back is a whole job or a whole step, never the assertion. The +# per-job endpoint returns the same bytes without the invented step column, for +# exactly the job asked for, and mirrors the shape Forgejo 16 serves at +# `actions/jobs/{id}/logs` — so both providers share one cache and one +# extractor. +gh_job_log() { + _job="$1"; _status="$2"; _dest="$3" + _cache=$(ci_cache_path github "$(ci_repo_slug)" "$_job") + if ci_cache_read "$_cache" > "$_dest" 2>/dev/null && [ -s "$_dest" ]; then + CI_LOG_FROM_CACHE=true + return 0 + fi + CI_LOG_FROM_CACHE=false + _rc=0 + ci_tool gh api "repos/{owner}/{repo}/actions/jobs/${_job}/logs" > "$_dest" || _rc=$? + [ "$_rc" -eq 0 ] || return "$_rc" + ci_cache_write "$_cache" "$_dest" "$_status" + return 0 +} diff --git a/packages/codev/scripts/forge/github/ci-failures.sh b/packages/codev/scripts/forge/github/ci-failures.sh new file mode 100755 index 000000000..e52244472 --- /dev/null +++ b/packages/codev/scripts/forge/github/ci-failures.sh @@ -0,0 +1,164 @@ +#!/bin/sh +# Forge concept: ci-failures (GitHub via gh CLI) +# forge-executable: gh +# Input: CODEV_CI_RUN_ID (required — the `id` from ci-runs) +# CODEV_CI_JOB_ID (optional — pin one job instead of the first failing) +# Output: {ok, provider, runId, runStatus, runConclusion, jobsFailed, extracted, +# failures: [{jobId, jobName, stepName, stepNumber, matchedBy, text, +# from, to, logLines, returnedLines, truncated}], +# otherFailingJobs, cached} +# +# The concept that matters. Two calls: the run (structured, gives the failing +# job and the failing STEP name), then that ONE job's log, which is extracted +# down to the assertion and capped. The other failing jobs are listed by name +# and id and deliberately not fetched — later failures are usually downstream of +# the first, and fetching them is how a bounded answer becomes a log dump again. +# +# WHEN EXTRACTION FAILS IT SAYS SO AND HANDS OVER +# +# {"extracted": false, "reason": "no recognized failure pattern", +# "failures": [{"jobId": …, "jobName": …, "logLines": 1247}], +# "next": "ci-run-log CODEV_CI_RUN_ID=… CODEV_CI_JOB_ID=… CODEV_CI_LOG_TAIL=80"} +# +# It never falls back to "here are the last 50 lines". A builder handed 50 +# arbitrary lines treats them as the diagnosis and reasons from noise; a builder +# told extraction failed reads the log with a targeted ci-run-log call, which is +# both correct and cheaper. The response carries the job id it would need, so +# the refusal is a handoff rather than a dead end. +set -e +. "$(dirname "$0")/_lib.sh" + +CONCEPT=ci-failures + +ci_require_tmpdir "$CONCEPT" + +if [ -z "$CODEV_CI_RUN_ID" ]; then + jq -cn '{ok: false, error: "bad-input", detail: "CODEV_CI_RUN_ID is required"}' + echo "${CONCEPT}: CODEV_CI_RUN_ID is required" >&2 + exit 2 +fi +ci_require_id "$CONCEPT" CODEV_CI_RUN_ID "$CODEV_CI_RUN_ID" +[ -z "$CODEV_CI_JOB_ID" ] || ci_require_id "$CONCEPT" CODEV_CI_JOB_ID "$CODEV_CI_JOB_ID" + +rc=0 +RUN=$(gh_run_json "$CODEV_CI_RUN_ID") || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "gh run view ${CODEV_CI_RUN_ID}" "$CI_TIMEOUT" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" not-found "run ${CODEV_CI_RUN_ID} could not be read (gh exit ${rc}); pass the \`id\` from ci-runs, not the run \`number\`" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" '{runId: $r}')" + exit 1 +fi + +ci_require_json "$CONCEPT" "$RUN" "gh run view ${CODEV_CI_RUN_ID}" + +FAILED=$(printf '%s' "$RUN" | jq -c "$GH_FAILED_JOBS_JQ") +RUN_STATUS=$(printf '%s' "$RUN" | jq -r '.status // "unknown"') +RUN_CONCLUSION=$(printf '%s' "$RUN" | jq -r '.conclusion // "null"') + +# Pin a job if asked. A job id that is not in this run is an error, not an empty +# answer — silently returning "nothing failed" for a mistyped id is the exact +# shape of wrong answer this concept exists to remove. +if [ -n "$CODEV_CI_JOB_ID" ]; then + TARGET=$(printf '%s' "$RUN" | jq -c --argjson j "$CODEV_CI_JOB_ID" 'first(.jobs[] | select(.databaseId == $j)) // empty') + if [ -z "$TARGET" ]; then + ci_fail "$CONCEPT" not-found "job ${CODEV_CI_JOB_ID} is not part of run ${CODEV_CI_RUN_ID}" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --arg j "$CODEV_CI_JOB_ID" '{runId: $r, jobId: $j}')" + exit 1 + fi +else + TARGET=$(printf '%s' "$FAILED" | jq -c '.[0] // empty') +fi + +JOBS_FAILED=$(printf '%s' "$FAILED" | jq 'length') + +# No failing job. Say which of the two reasons it is — a green run and a run +# still going are not the same answer, and neither is "the server could not tell +# me", which is an error envelope rather than this branch. +if [ -z "$TARGET" ]; then + printf '%s' "$RUN" | jq -c --arg concept "$CONCEPT" '{ + ok: true, provider: "github", runId: .databaseId, + runStatus: .status, runConclusion: .conclusion, + jobsFailed: 0, extracted: false, + reason: (if .status != "completed" then "run has not finished" else "no job in this run failed" end), + failures: [] + }' + exit 0 +fi + +JOB_ID=$(printf '%s' "$TARGET" | jq -r '.databaseId') +JOB_NAME=$(printf '%s' "$TARGET" | jq -r '.name') +JOB_STATE=$(printf '%s' "$TARGET" | jq -r '.conclusion // .status') +STEP=$(printf '%s' "$TARGET" | jq -c 'first(.steps[]? | select(.conclusion == "failure" or .conclusion == "timed_out")) // null') + +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +rc=0 +gh_job_log "$JOB_ID" "$JOB_STATE" "$TMP/raw.log" || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "gh api repos/{owner}/{repo}/actions/jobs/${JOB_ID}/logs" "$CI_TIMEOUT" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "could not read the log for job ${JOB_ID} (${JOB_NAME}); gh exit ${rc}" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --argjson j "$JOB_ID" --arg n "$JOB_NAME" '{runId: $r, jobId: $j, jobName: $n}')" + exit 1 +fi + +ci_clean_log < "$TMP/raw.log" > "$TMP/clean.log" +# awk NR, not `wc -l`: a log with no trailing newline makes wc undercount by +# one, and logLines would then disagree with the from/to the extractor reports. +LOG_LINES=$(awk 'END {print NR}' "$TMP/clean.log") + +OTHERS=$(printf '%s' "$FAILED" | jq -c --argjson j "$JOB_ID" '[.[] | select(.databaseId != $j) | {id: .databaseId, name: .name}]') + +if ci_extract "$TMP/clean.log" > "$TMP/extract.txt" 2>/dev/null && [ -s "$TMP/extract.txt" ]; then + MATCHED=$(head -1 "$TMP/extract.txt" | cut -f1) + FROM=$(head -1 "$TMP/extract.txt" | cut -f2) + TO=$(head -1 "$TMP/extract.txt" | cut -f3) + tail -n +2 "$TMP/extract.txt" > "$TMP/text.txt" + CAP=$CI_MAX_STEP_BYTES + [ "$CAP" -gt "$CI_MAX_RESPONSE_BYTES" ] && CAP=$CI_MAX_RESPONSE_BYTES + META=$(ci_cap_file "$TMP/text.txt" "$CAP" "$TMP/capped.txt") + RETURNED_LINES=${META% *} + TRUNCATED=${META#* } + TEXT=$(jq -R -s -c . < "$TMP/capped.txt") + jq -cn \ + --arg run "$CODEV_CI_RUN_ID" --arg rs "$RUN_STATUS" --arg rc "$RUN_CONCLUSION" \ + --argjson job "$JOB_ID" --arg jobName "$JOB_NAME" --argjson step "$STEP" \ + --arg matched "$MATCHED" --argjson text "$TEXT" \ + --argjson from "$FROM" --argjson to "$TO" \ + --argjson logLines "$LOG_LINES" --argjson returned "$RETURNED_LINES" \ + --argjson truncated "$TRUNCATED" --argjson jobsFailed "$JOBS_FAILED" \ + --argjson others "$OTHERS" --argjson cached "$CI_LOG_FROM_CACHE" \ + '{ok: true, provider: "github", runId: ($run | tonumber? // $run), + runStatus: $rs, runConclusion: (if $rc == "null" then null else $rc end), + jobsFailed: $jobsFailed, extracted: true, + failures: [{ + jobId: $job, jobName: $jobName, + stepName: ($step.name // null), stepNumber: ($step.number // null), + matchedBy: $matched, text: $text, + from: $from, to: $to, + logLines: $logLines, returnedLines: $returned, truncated: $truncated + }], + otherFailingJobs: $others, cached: $cached}' + exit 0 +fi + +# Rung 6: nothing recognisable. Hand over the parameters for the follow-up +# rather than inventing a diagnosis. +jq -cn \ + --arg run "$CODEV_CI_RUN_ID" --arg rs "$RUN_STATUS" --arg rc "$RUN_CONCLUSION" \ + --argjson job "$JOB_ID" --arg jobName "$JOB_NAME" \ + --argjson logLines "$LOG_LINES" --argjson jobsFailed "$JOBS_FAILED" \ + --argjson others "$OTHERS" --argjson cached "$CI_LOG_FROM_CACHE" \ + '{ok: true, provider: "github", runId: ($run | tonumber? // $run), + runStatus: $rs, runConclusion: (if $rc == "null" then null else $rc end), + jobsFailed: $jobsFailed, extracted: false, + reason: "no recognized failure pattern", + failures: [{jobId: $job, jobName: $jobName, logLines: $logLines}], + otherFailingJobs: $others, cached: $cached, + next: ("ci-run-log CODEV_CI_RUN_ID=" + $run + " CODEV_CI_JOB_ID=" + ($job|tostring) + " CODEV_CI_LOG_TAIL=80")}' diff --git a/packages/codev/scripts/forge/github/ci-run-log.sh b/packages/codev/scripts/forge/github/ci-run-log.sh new file mode 100755 index 000000000..2d5592bd3 --- /dev/null +++ b/packages/codev/scripts/forge/github/ci-run-log.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# Forge concept: ci-run-log (GitHub via gh CLI) +# forge-executable: gh +# Input: CODEV_CI_RUN_ID (required) +# CODEV_CI_JOB_ID (optional — defaults to the first failing job, or the +# only job if the run has exactly one) +# exactly ONE window: +# CODEV_CI_LOG_TAIL=N last N lines +# CODEV_CI_LOG_HEAD=N first N lines (setup, install, config) +# CODEV_CI_LOG_GREP= matching lines, with CODEV_CI_LOG_CONTEXT +# (default 3) lines either side +# Output: {ok, provider, runId, jobId, jobName, window, logLines, returnedLines, +# from, to, truncated, lines: [...], matchLines?: [...]} +# +# The escape hatch, and deliberately a SEPARATE concept rather than a flag on +# ci-failures: a window parameter on the main call gets passed by habit, and +# then every status question drags a log again — which is the thing this issue +# exists to prevent. Reaching for this should be a deliberate act. +# +# There is no default window. Zero windows and two windows are both exit 2 with +# a named message, because a default is how "deliberate" decays into "always". +# +# This is remote head/tail/grep -C over a log, and nothing more. The value codev +# adds is doing it against two forges with one contract and a bounded response, +# not inventing log analysis. +set -e +. "$(dirname "$0")/_lib.sh" + +CONCEPT=ci-run-log + +ci_require_tmpdir "$CONCEPT" + +if [ -z "$CODEV_CI_RUN_ID" ]; then + jq -cn '{ok: false, error: "bad-input", detail: "CODEV_CI_RUN_ID is required"}' + echo "${CONCEPT}: CODEV_CI_RUN_ID is required" >&2 + exit 2 +fi +ci_require_id "$CONCEPT" CODEV_CI_RUN_ID "$CODEV_CI_RUN_ID" +[ -z "$CODEV_CI_JOB_ID" ] || ci_require_id "$CONCEPT" CODEV_CI_JOB_ID "$CODEV_CI_JOB_ID" + +# Window first: a malformed request should not cost an API call. +ci_window_parse "$CONCEPT" + +rc=0 +RUN=$(gh_run_json "$CODEV_CI_RUN_ID") || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "gh run view ${CODEV_CI_RUN_ID}" "$CI_TIMEOUT" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" not-found "run ${CODEV_CI_RUN_ID} could not be read (gh exit ${rc}); pass the \`id\` from ci-runs, not the run \`number\`" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" '{runId: $r}')" + exit 1 +fi + +ci_require_json "$CONCEPT" "$RUN" "gh run view ${CODEV_CI_RUN_ID}" + +if [ -n "$CODEV_CI_JOB_ID" ]; then + TARGET=$(printf '%s' "$RUN" | jq -c --argjson j "$CODEV_CI_JOB_ID" 'first(.jobs[] | select(.databaseId == $j)) // empty') + if [ -z "$TARGET" ]; then + ci_fail "$CONCEPT" not-found "job ${CODEV_CI_JOB_ID} is not part of run ${CODEV_CI_RUN_ID}" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --arg j "$CODEV_CI_JOB_ID" '{runId: $r, jobId: $j}')" + exit 1 + fi +else + TARGET=$(printf '%s' "$RUN" | jq -c "${GH_FAILED_JOBS_JQ} | .[0] // empty") + # No failing job: fall back to the only job, if there is exactly one. With + # several passing jobs there is no defensible default, and picking one would + # hand back a log the caller did not ask for. + if [ -z "$TARGET" ]; then + TARGET=$(printf '%s' "$RUN" | jq -c 'if (.jobs | length) == 1 then .jobs[0] else empty end') + fi + if [ -z "$TARGET" ]; then + NAMES=$(printf '%s' "$RUN" | jq -c '[.jobs[] | {id: .databaseId, name: .name}]') + ci_fail "$CONCEPT" bad-input "run ${CODEV_CI_RUN_ID} has no failing job and more than one job; set CODEV_CI_JOB_ID" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --argjson jobs "$NAMES" '{runId: $r, jobs: $jobs}')" + exit 2 + fi +fi + +JOB_ID=$(printf '%s' "$TARGET" | jq -r '.databaseId') +JOB_NAME=$(printf '%s' "$TARGET" | jq -r '.name') +JOB_STATE=$(printf '%s' "$TARGET" | jq -r '.conclusion // .status') + +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +rc=0 +gh_job_log "$JOB_ID" "$JOB_STATE" "$TMP/raw.log" || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "gh api repos/{owner}/{repo}/actions/jobs/${JOB_ID}/logs" "$CI_TIMEOUT" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "could not read the log for job ${JOB_ID} (${JOB_NAME}); gh exit ${rc}" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" --argjson j "$JOB_ID" --arg n "$JOB_NAME" '{runId: $r, jobId: $j, jobName: $n}')" + exit 1 +fi + +ci_clean_log < "$TMP/raw.log" > "$TMP/clean.log" +ci_window_emit "$CONCEPT" "$TMP" github "$CODEV_CI_RUN_ID" "$JOB_ID" "$JOB_NAME" "$CI_LOG_FROM_CACHE" diff --git a/packages/codev/scripts/forge/github/ci-run-view.sh b/packages/codev/scripts/forge/github/ci-run-view.sh new file mode 100755 index 000000000..d778c4138 --- /dev/null +++ b/packages/codev/scripts/forge/github/ci-run-view.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# Forge concept: ci-run-view (GitHub via gh CLI) +# forge-executable: gh +# Input: CODEV_CI_RUN_ID (required — the `id` field from ci-runs, not `number`) +# Output: {ok, provider, run: {...}, jobs: [{id, name, status, conclusion, +# startedAt, completedAt, failedSteps: [{name, number, conclusion}]}], +# jobSource} +# +# Still no log bytes. This answers "is it still running, and which job is +# pending" and "which step failed" — the second of which is normally enough to +# know whether a failure is yours, and costs one ~1.3s call. +# +# `failedSteps` comes from gh's structured per-step conclusions, NOT from +# parsing a log. That matters: `gh run view --log-failed` labels its lines +# "UNKNOWN STEP" whenever its filename-to-step mapping misses, which on the run +# this was built against was every line of all 2528. +set -e +. "$(dirname "$0")/_lib.sh" + +CONCEPT=ci-run-view + +ci_require_tmpdir "$CONCEPT" + +if [ -z "$CODEV_CI_RUN_ID" ]; then + jq -cn '{ok: false, error: "bad-input", detail: "CODEV_CI_RUN_ID is required"}' + echo "${CONCEPT}: CODEV_CI_RUN_ID is required" >&2 + exit 2 +fi +ci_require_id "$CONCEPT" CODEV_CI_RUN_ID "$CODEV_CI_RUN_ID" +[ -z "$CODEV_CI_JOB_ID" ] || ci_require_id "$CONCEPT" CODEV_CI_JOB_ID "$CODEV_CI_JOB_ID" + +rc=0 +OUT=$(gh_run_json "$CODEV_CI_RUN_ID") || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "gh run view ${CODEV_CI_RUN_ID}" "$CI_TIMEOUT" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" not-found "run ${CODEV_CI_RUN_ID} could not be read (gh exit ${rc}); pass the \`id\` from ci-runs, not the run \`number\`" \ + "$(jq -cn --arg r "$CODEV_CI_RUN_ID" '{runId: $r}')" + exit 1 +fi + +ci_require_json "$CONCEPT" "$OUT" "gh run view ${CODEV_CI_RUN_ID}" + +printf '%s' "$OUT" | jq -c '{ + ok: true, + provider: "github", + jobSource: "run-view", + run: { + id: .databaseId, + number: .number, + title: .displayTitle, + workflow: .workflowName, + status: .status, + conclusion: .conclusion, + branch: .headBranch, + sha: .headSha, + event: .event, + url: .url, + createdAt: .createdAt + }, + jobs: [ .jobs[] | { + id: .databaseId, + name: .name, + status: .status, + conclusion: .conclusion, + startedAt: .startedAt, + completedAt: .completedAt, + failedSteps: [ .steps[]? | select(.conclusion == "failure" or .conclusion == "timed_out") + | {name: .name, number: .number, conclusion: .conclusion} ] + } ] +}' diff --git a/packages/codev/scripts/forge/github/ci-runs.sh b/packages/codev/scripts/forge/github/ci-runs.sh new file mode 100755 index 000000000..457172aaa --- /dev/null +++ b/packages/codev/scripts/forge/github/ci-runs.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# Forge concept: ci-runs (GitHub via gh CLI) +# forge-executable: gh +# Input: CODEV_BRANCH_NAME (optional) — filter to one branch +# CODEV_CI_STATUS (optional) — success|failure|pending|queued|in_progress|skipped|canceled +# CODEV_CI_WORKFLOW (optional) — workflow file name or display name +# CODEV_CI_LIMIT (optional, default 20) +# Output: {ok, provider, runs: [{id, number, name, workflow, status, conclusion, +# branch, sha, event, url, createdAt}], truncated} +# +# The cheap question, and the whole point of tiering these concepts: "did my +# push pass" must never drag a log through a context window. One `gh run list` +# call, ~0.7s measured, no log bytes at any point. +# +# It also answers "is this mine or is it flaky", which is why CODEV_CI_WORKFLOW +# exists: the same workflow across other commits is a run-history question, and +# on 2026-08-21 two wrong conclusions were drawn about CI state that were both +# answerable from run history and neither checked, because asking was awkward. +# +# `id` is what ci-run-view / ci-failures / ci-run-log take. `number` is the +# human run number. On GitHub they differ; on Forgejo they differ AND both are +# valid inputs to the same route, so the concepts never guess which one they +# were handed. Pass `id`. +set -e +. "$(dirname "$0")/_lib.sh" + +CONCEPT=ci-runs + +ci_require_tmpdir "$CONCEPT" + +ci_check_status "$CONCEPT" "$CODEV_CI_STATUS" + +LIMIT=${CODEV_CI_LIMIT:-$CI_LIMIT_DEFAULT} +case "$LIMIT" in ''|*[!0-9]*|0) LIMIT=$CI_LIMIT_DEFAULT ;; esac + +# One more than asked for, so that "there are more runs than this" can be +# reported instead of guessed. gh has no hasMore of its own, and a list that was +# cut by a limit is indistinguishable from a complete one once printed. +set -- run list --limit "$((LIMIT + 1))" --json databaseId,number,name,workflowName,status,conclusion,headBranch,headSha,event,url,createdAt +[ -n "$CODEV_BRANCH_NAME" ] && set -- "$@" --branch "$CODEV_BRANCH_NAME" +[ -n "$CODEV_CI_WORKFLOW" ] && set -- "$@" --workflow "$CODEV_CI_WORKFLOW" +if [ -n "$CODEV_CI_STATUS" ]; then + set -- "$@" --status "$(ci_status_for github "$CODEV_CI_STATUS")" +fi + +rc=0 +OUT=$(ci_tool gh "$@") || rc=$? +if [ "$rc" -eq 124 ]; then + ci_fail_timeout "$CONCEPT" "gh run list" "$CI_TIMEOUT" + exit 1 +fi +if [ "$rc" -ne 0 ]; then + ci_fail "$CONCEPT" forge-error "gh run list failed (exit ${rc}); see stderr above" + exit 1 +fi + +ci_require_json "$CONCEPT" "$OUT" "gh run list" + +printf '%s' "$OUT" | jq -c --argjson limit "$LIMIT" '{ + ok: true, + provider: "github", + runs: [ .[] | { + id: .databaseId, + number: .number, + name: .name, + workflow: .workflowName, + status: .status, + conclusion: .conclusion, + branch: .headBranch, + sha: .headSha, + event: .event, + url: .url, + createdAt: .createdAt + } ] | .[0:$limit], + truncated: (length > $limit), + note: null +}' diff --git a/packages/codev/src/__tests__/fixtures/pir-13/forgejo-go-failure.log.gz b/packages/codev/src/__tests__/fixtures/pir-13/forgejo-go-failure.log.gz new file mode 100644 index 000000000..47e49a62f Binary files /dev/null and b/packages/codev/src/__tests__/fixtures/pir-13/forgejo-go-failure.log.gz differ diff --git a/packages/codev/src/__tests__/fixtures/pir-13/github-vitest-failure.log.gz b/packages/codev/src/__tests__/fixtures/pir-13/github-vitest-failure.log.gz new file mode 100644 index 000000000..8b62df7ab Binary files /dev/null and b/packages/codev/src/__tests__/fixtures/pir-13/github-vitest-failure.log.gz differ diff --git a/packages/codev/src/__tests__/fixtures/pir-13/github-vitest-worker-crash.log.gz b/packages/codev/src/__tests__/fixtures/pir-13/github-vitest-worker-crash.log.gz new file mode 100644 index 000000000..bee9a6801 Binary files /dev/null and b/packages/codev/src/__tests__/fixtures/pir-13/github-vitest-worker-crash.log.gz differ diff --git a/packages/codev/src/__tests__/forge.test.ts b/packages/codev/src/__tests__/forge.test.ts index 8ee0f2ca8..1250fe596 100644 --- a/packages/codev/src/__tests__/forge.test.ts +++ b/packages/codev/src/__tests__/forge.test.ts @@ -320,7 +320,7 @@ describe('executeForgeCommandSync', () => { // ============================================================================= describe('getKnownConcepts', () => { - it('returns all 18 known concept names', () => { + it('returns all 22 known concept names', () => { const concepts = getKnownConcepts(); expect(concepts).toContain('issue-view'); expect(concepts).toContain('pr-list'); @@ -340,7 +340,11 @@ describe('getKnownConcepts', () => { expect(concepts).toContain('pr-diff'); expect(concepts).toContain('auth-status'); expect(concepts).toContain('repo-archive'); - expect(concepts.length).toBe(18); + expect(concepts).toContain('ci-runs'); + expect(concepts).toContain('ci-run-view'); + expect(concepts).toContain('ci-failures'); + expect(concepts).toContain('ci-run-log'); + expect(concepts.length).toBe(22); }); }); @@ -574,9 +578,9 @@ describe('graceful degradation when command not found', () => { // ============================================================================= describe('resolveAllConcepts', () => { - it('returns all 18 concepts with default source when no config', () => { + it('returns all 22 concepts with default source when no config', () => { const resolutions = resolveAllConcepts(); - expect(resolutions).toHaveLength(18); + expect(resolutions).toHaveLength(22); expect(resolutions.every(r => r.source === 'default')).toBe(true); expect(resolutions.every(r => r.executable !== null)).toBe(true); }); diff --git a/packages/codev/src/__tests__/pir-13-ci-concepts.test.ts b/packages/codev/src/__tests__/pir-13-ci-concepts.test.ts new file mode 100644 index 000000000..be69dbae8 --- /dev/null +++ b/packages/codev/src/__tests__/pir-13-ci-concepts.test.ts @@ -0,0 +1,1122 @@ +/** + * Issue #13 — CI concepts for the forge layer. + * + * What is pinned here, and why each one is worth a test: + * + * 1. **The extraction ladder, against two REAL logs.** Both fixtures are + * verbatim captures, not hand-written samples: a 2528-line GitHub vitest + * failure (run 32515040122 of this repository) and a 1599-line Forgejo Go + * failure (codeberg.org/forgejo/forgejo job 11952749). They are stored + * gzipped because the exact bytes are the point — the ANSI escapes, the + * misleading lines, and the position of the real failure inside the file. + * + * The three traps they carry are why this ladder is not three greps: + * - Every payload line is ANSI-wrapped. The FAIL token in the raw bytes is + * `ESC[41m ESC[1m FAIL ESC[22m ESC[49m src/…`, which no /^ FAIL/ rule + * matches. A matcher that skips the cleaning step reports "no recognized + * failure" on a log that plainly contains one. + * - The first line containing "Error:" is `[artifact-canvas] Error: host + * blew up`, printed by a PASSING test 1214 lines above the real failure. + * - "Test Files … passed" appears FOUR times before the summary that says + * failed. + * + * 2. **Refusal is a first-class outcome.** When nothing matches, the response + * carries `extracted: false`, the job identity, `logLines` and a + * ready-to-run `next`, and contains NO log lines at all. Returning "the last + * 50 lines" instead is the failure mode this issue exists to remove: a + * builder treats them as the diagnosis and reasons from noise. + * + * 3. **A timeout reports as a timeout.** Through the script (envelope on + * stdout) and through the dispatcher (`executeForgeCommandDetailed(). + * timedOut`). `executeForgeCommand` flattens every failure to `null`, which + * is how #12 shipped a `pr-exists` whose null read as "no PR exists". + * + * 4. **An old Forgejo is never mistaken for a green run.** Forgejo gained the + * Actions job-log API in 16.0; on 15.x these concepts return + * `unsupported-server` naming both versions, NOT an empty `failures` array. + * + * 5. **The measured Forgejo footguns.** `limit` is ignored unless `page` is + * also sent, and a `pull_request` run records `#` where a branch + * name would be. Neither is visible in a test that only checks output shape, + * so the requests themselves are asserted. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { execFileSync, spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import * as zlib from 'node:zlib'; +import { getForgeCommand, resolveAllConcepts, executeForgeCommandDetailed } from '../lib/forge.js'; +import { runForgeConcept } from '../commands/forge.js'; + +const codevPkgRoot = path.resolve(import.meta.dirname, '..', '..'); +const forgeScripts = path.join(codevPkgRoot, 'scripts', 'forge'); +const githubDir = path.join(forgeScripts, 'github'); +const giteaDir = path.join(forgeScripts, 'gitea'); +const fixtures = path.join(import.meta.dirname, 'fixtures', 'pir-13'); + +const CI_CONCEPTS = ['ci-runs', 'ci-run-view', 'ci-failures', 'ci-run-log'] as const; + +const ESC = ''; + +function hasJq(): boolean { + try { + execFileSync('sh', ['-c', 'command -v jq'], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +function fixtureLog(name: string): string { + return zlib.gunzipSync(fs.readFileSync(path.join(fixtures, `${name}.log.gz`))).toString('utf-8'); +} + +let tmp: string; + +beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pir13-')); + // The scripts run with TMPDIR pointed here so the log cache is per-test. It + // has to EXIST: macOS mktemp falls back to the system temp dir when TMPDIR + // points nowhere, GNU mktemp does not — so a missing directory passed locally + // and failed 31 tests on the Linux runner, with the concept reporting + // "run could not be read" for what was really an unusable TMPDIR. + fs.mkdirSync(path.join(tmp, 'cache'), { recursive: true }); +}); + +afterEach(() => { + fs.rmSync(tmp, { recursive: true, force: true }); +}); + +interface RunResult { + status: number | null; + stdout: string; + stderr: string; + json: Record | null; +} + +/** Run a provider script with a stubbed CLI on PATH. */ +function run(dir: string, script: string, env: Record = {}): RunResult { + const result = spawnSync('sh', [path.join(dir, script)], { + cwd: tmp, + encoding: 'utf-8', + env: { + ...process.env, + PATH: `${tmp}:${process.env.PATH}`, + TMPDIR: path.join(tmp, 'cache'), + CODEV_REPO: 'o/r', + GH_LOG: path.join(tmp, 'gh.log'), + TEA_LOG: path.join(tmp, 'tea.log'), + // Every CODEV_CI_* the scripts read, cleared, so a variable set in the + // developer's own shell cannot change what a test asserts. + CODEV_CI_NO_CACHE: '1', + CODEV_BRANCH_NAME: '', CODEV_CI_STATUS: '', CODEV_CI_WORKFLOW: '', CODEV_CI_LIMIT: '', + CODEV_CI_RUN_ID: '', CODEV_CI_JOB_ID: '', CODEV_PR_BASE: '', + CODEV_CI_LOG_TAIL: '', CODEV_CI_LOG_HEAD: '', CODEV_CI_LOG_GREP: '', CODEV_CI_LOG_CONTEXT: '', + CODEV_FORGE_TIMEOUT: '', CODEV_CI_MAX_PAGES: '', CODEV_CI_TASKS_MAX_PAGES: '', + CODEV_CI_MAX_STEP_BYTES: '', CODEV_CI_MAX_BYTES: '', + ...env, + }, + }); + let json: Record | null = null; + try { + json = JSON.parse(result.stdout); + } catch { /* not every assertion needs JSON */ } + return { status: result.status, stdout: result.stdout, stderr: result.stderr, json }; +} + +/** Write an executable stub onto the test PATH. */ +function stub(name: string, body: string): void { + const p = path.join(tmp, name); + fs.writeFileSync(p, `#!/bin/sh\n${body}\n`, { mode: 0o755 }); + fs.chmodSync(p, 0o755); +} + +function fileLines(p: string): string[] { + if (!fs.existsSync(p)) return []; + return fs.readFileSync(p, 'utf-8').split('\n').filter(Boolean); +} + +// --------------------------------------------------------------------------- +// Registration +// --------------------------------------------------------------------------- + +describe('#13 — the CI concepts are registered, and disabled where unimplemented', () => { + it.each(CI_CONCEPTS)('routes %s to the github script by default', (concept) => { + const command = getForgeCommand(concept, null); + expect(command).toBe(path.join(githubDir, `${concept}.sh`)); + expect(fs.existsSync(command!)).toBe(true); + expect(fs.statSync(command!).mode & 0o111, 'script is not executable').not.toBe(0); + }); + + it.each(CI_CONCEPTS)('routes %s to the gitea script for provider gitea', (concept) => { + const command = getForgeCommand(concept, { provider: 'gitea' }); + expect(command).toBe(path.join(giteaDir, `${concept}.sh`)); + expect(fs.statSync(command!).mode & 0o111, 'script is not executable').not.toBe(0); + }); + + // The important half. A concept with no script does NOT resolve to nothing — + // it falls through to the github default, so an unimplemented ci-runs on a + // GitLab repo would silently run `gh run list`. Issue #13 asks for gitlab to + // degrade loudly; `disabled` is what loud looks like here. + it.each(['gitlab', 'linear'])( + 'DISABLES every CI concept for %s rather than letting it fall through to gh', + (provider) => { + for (const concept of CI_CONCEPTS) { + expect(getForgeCommand(concept, { provider }), `${provider}/${concept} resolved to a command`).toBeNull(); + const resolution = resolveAllConcepts({ provider }).find((r) => r.concept === concept); + expect(resolution?.source).toBe('disabled'); + } + }, + ); + + it('doctor resolves every CI script to its real CLI, not to a shell builtin', () => { + for (const provider of ['github', 'gitea'] as const) { + const expected = provider === 'github' ? 'gh' : 'tea'; + for (const concept of CI_CONCEPTS) { + const resolution = resolveAllConcepts({ provider }).find((r) => r.concept === concept); + expect(resolution?.executable, `${provider}/${concept}`).toBe(expected); + } + } + }); +}); + +// --------------------------------------------------------------------------- +// The extraction ladder, against the real logs +// --------------------------------------------------------------------------- + +describe.skipIf(!hasJq())('#13 — extraction returns the assertion, not the log and not the wrong line', () => { + /** Run the ladder the way the concepts do: clean, then extract. */ + function extract(raw: string): { rung: string; from: number; to: number; text: string } | null { + const rawPath = path.join(tmp, 'raw.log'); + const cleanPath = path.join(tmp, 'clean.log'); + fs.writeFileSync(rawPath, raw); + const script = [ + `. ${JSON.stringify(path.join(forgeScripts, '_ci-extract.sh'))}`, + `ci_clean_log < ${JSON.stringify(rawPath)} > ${JSON.stringify(cleanPath)}`, + `ci_extract ${JSON.stringify(cleanPath)}`, + ].join('\n'); + const r = spawnSync('sh', ['-c', script], { encoding: 'utf-8' }); + if (!r.stdout.trim()) return null; + const [header, ...rest] = r.stdout.split('\n'); + const [rung, from, to] = header.split('\t'); + return { rung, from: Number(from), to: Number(to), text: rest.join('\n') }; + } + + it('finds the vitest assertion in a 2528-line GitHub log', () => { + const raw = fixtureLog('github-vitest-failure'); + expect(raw.split('\n').length).toBeGreaterThan(2500); + + const got = extract(raw); + expect(got, 'extraction returned nothing on a log containing a plain vitest failure').not.toBeNull(); + expect(got!.rung).toBe('vitest'); + expect(got!.text).toContain('AssertionError: expected null to be'); + expect(got!.text).toContain('agy-auth-cache.test.ts'); + expect(got!.text).toContain('Test Files 1 failed'); + // 23 lines out of 2528. The whole point. + expect(got!.to - got!.from).toBeLessThan(60); + }); + + it('does NOT return the passing test fixture string that a first-error rule picks', () => { + // `[artifact-canvas] Error: host blew up` is printed by a test that PASSES, + // 1214 lines above the real failure. It is the single most plausible wrong + // answer this ladder can give. + const raw = fixtureLog('github-vitest-failure'); + expect(raw, 'the fixture no longer carries the decoy this test exists for').toContain('host blew up'); + expect(extract(raw)!.text).not.toContain('host blew up'); + }); + + it('does NOT anchor on any of the earlier "Test Files … passed" lines', () => { + const raw = fixtureLog('github-vitest-failure'); + const lines = raw.split('\n'); + const failing = lines.findIndex((l) => l.includes('Test Files') && l.includes('failed') && !l.includes('grep')); + const earlier = lines.slice(0, failing).filter((l) => l.includes('Test Files')); + // Four of them: three passing suite summaries, plus a shell line echoing + // `grep -q "Test Files.*passed"`, which is its own flavour of the same trap. + expect(earlier.length, 'the fixture no longer carries the trap this test exists for').toBe(4); + + const got = extract(raw)!; + expect(got.text).toContain('1 failed'); + expect(got.from).toBeGreaterThan(failing - 40); + }); + + it('matches through ANSI escapes — the raw fixture has them and cleaning is load-bearing', () => { + const raw = fixtureLog('github-vitest-failure'); + expect(raw, 'the fixture was normalised and no longer proves anything').toContain(`${ESC}[`); + // The FAIL token in the raw bytes is wrapped, so no plain /FAIL / anchor + // matches it until ci_clean_log has run. + expect(raw).toMatch(new RegExp(`${ESC}\\[[0-9;]*m[^\\n]{0,20}FAIL`)); + const got = extract(raw)!; + expect(got.text).toContain('FAIL'); + expect(got.text, 'escapes survived into the answer').not.toContain(ESC); + }); + + it('finds the Go failure in a 1599-line Forgejo log whose last 25 lines are git cleanup', () => { + const raw = fixtureLog('forgejo-go-failure'); + const lines = raw.split('\n'); + // The case for extraction over tailing, stated as an assertion. + expect(lines.slice(-25).join('\n')).not.toContain('--- FAIL'); + + const got = extract(raw)!; + expect(got.rung).toBe('go-test'); + expect(got.text).toContain('--- FAIL: TestReadPointerFromBuffer'); + expect(got.text).toContain('Should be false'); + expect(got.text).toContain('FAIL\tforgejo.org/modules/lfs'); + }); + + it('does NOT anchor on a PASSING test whose captured stderr contains a real Error line', () => { + // Found by running ci-failures against this repository's own red CI run + // (32536232930) while this PR was open. The extractor returned + // `Error: Refusing to POST /api/tunnel/disconnect ...` from inside a + // `stderr |` block belonging to a test that PASSED — that test asserts the + // error is thrown, so the text is the suite working as designed — while the + // real failure sat 355 lines further down. Same decoy class as the + // artifact-canvas one, arriving through a different door: this decoy IS + // anchored at the start of its line, so anchoring alone does not stop it. + const raw = fixtureLog('github-vitest-worker-crash'); + expect(raw, 'the fixture no longer carries the decoy').toContain('Refusing to POST /api/tunnel/disconnect'); + + const got = extract(raw)!; + expect(got.text).not.toContain('Refusing to POST'); + expect(got.rung).toBe('vitest-unhandled'); + expect(got.text).toContain('Unhandled Error'); + expect(got.text).toContain('Worker forks emitted error'); + }); + + it('reads a captured block as capture until the blank line that ends it', () => { + const log = [ + 'stderr | some.test.ts > a suite > a passing case', + 'Error: this is expected output from a passing test', + ' at somewhere', + '', + ' ✓ some.test.ts (3 tests) 4ms', + ' ✓ other.test.ts (1 test) 2ms', + ' ✓ third.test.ts (2 tests) 3ms', + ' ✓ fourth.test.ts (2 tests) 3ms', + 'Error: this one is real', + 'more', + ].join('\n'); + const got = extract(log)!; + expect(got.rung).toBe('first-error'); + expect(got.text).toContain('this one is real'); + expect(got.text).not.toContain('expected output from a passing test'); + }); + + it('returns NOTHING when nothing is recognisable, rather than an arbitrary slice', () => { + const noise = Array.from( + { length: 400 }, + (_, i) => `2026-08-21T18:47:0${i % 10}.0000000Z step ${i} did a thing`, + ).join('\n'); + expect(extract(noise)).toBeNull(); + }); + + it('refuses a bare "Process completed with exit code 1" as a diagnosis', () => { + const log = ['Running the thing', 'more output', '##[error]Process completed with exit code 1.'].join('\n'); + expect(extract(log)).toBeNull(); + }); + + it('accepts a line-anchored error where a mid-line one is ignored', () => { + const decoy = [ + '[pkg] Error: this one is printed by a passing test', + 'ok', + 'Error: the real one', + 'trailing', + ].join('\n'); + const got = extract(decoy)!; + expect(got.rung).toBe('first-error'); + expect(got.text).toContain('Error: the real one'); + // The decoy may appear as leading CONTEXT — three lines either side is the + // rule — but it must not be the line the rung anchored on. `to` is the + // anchor plus context, so the anchor is the real one at line 3. + expect(got.to).toBe(4); + }); + + it('strips the RFC3339 timestamp both providers prefix to every line', () => { + const got = extract(['2026-08-21T18:47:09.5820646Z Error: boom', '2026-08-21T18:47:10.0000000Z after'].join('\n'))!; + expect(got.text).toContain('Error: boom'); + expect(got.text).not.toContain('2026-08-21T18:47:09'); + }); +}); + +// --------------------------------------------------------------------------- +// ci-failures, end to end against a stubbed gh +// --------------------------------------------------------------------------- + +/** A gh stub serving `run view --json` and `api …/logs` from files. */ +function stubGh(runJson: unknown, logFile?: string): void { + fs.writeFileSync(path.join(tmp, 'run.json'), JSON.stringify(runJson)); + stub('gh', [ + 'printf "%s\\n" "$*" >> "$GH_LOG"', + 'if [ "$1" = "run" ] && [ "$2" = "view" ] && [ "$4" = "--json" ]; then cat "$(dirname "$0")/run.json"; exit 0; fi', + 'if [ "$1" = "run" ] && [ "$2" = "list" ]; then cat "$(dirname "$0")/run.json"; exit 0; fi', + logFile + ? `if [ "$1" = "api" ]; then cat ${JSON.stringify(logFile)}; exit 0; fi` + : 'if [ "$1" = "api" ]; then echo "no log" >&2; exit 1; fi', + 'echo "unexpected gh invocation: $*" >&2', + 'exit 9', + ].join('\n')); +} + +const ONE_FAILING_JOB = { + databaseId: 32515040122, + number: 42, + displayTitle: 'a commit', + workflowName: 'Tests', + name: 'Tests', + status: 'completed', + conclusion: 'failure', + headBranch: 'builder/pir-13', + headSha: 'abc123', + event: 'push', + url: 'https://github.com/o/r/actions/runs/32515040122', + createdAt: '2026-08-21T18:46:21Z', + jobs: [ + { databaseId: 1, name: 'Lint', status: 'completed', conclusion: 'success', startedAt: null, completedAt: null, steps: [] }, + { + databaseId: 96874679182, name: 'Unit Tests', status: 'completed', conclusion: 'failure', + startedAt: '2026-08-21T18:47:08Z', completedAt: '2026-08-21T18:49:07Z', + steps: [ + { name: 'Checkout', number: 1, status: 'completed', conclusion: 'success' }, + { name: 'Run unit tests with coverage', number: 15, status: 'completed', conclusion: 'failure' }, + ], + }, + ], +}; + +describe.skipIf(!hasJq())('#13 — ci-failures returns a bounded extract with its provenance', () => { + function writeFixtureTo(name: string): string { + const p = path.join(tmp, `${name}.log`); + fs.writeFileSync(p, fixtureLog(name)); + return p; + } + + it('turns a 293 KB job log into a response of about a kilobyte', () => { + const logPath = writeFixtureTo('github-vitest-failure'); + stubGh(ONE_FAILING_JOB, logPath); + + const r = run(githubDir, 'ci-failures.sh', { CODEV_CI_RUN_ID: '32515040122' }); + expect(r.status).toBe(0); + expect(r.json!.ok).toBe(true); + expect(r.json!.extracted).toBe(true); + expect(r.json!.jobsFailed).toBe(1); + + const f = r.json!.failures[0]; + expect(f.jobId).toBe(96874679182); + expect(f.jobName).toBe('Unit Tests'); + // The failing STEP name comes from structured JSON, not from parsing a log: + // `gh run view --log-failed` labels its lines UNKNOWN STEP whenever its + // filename-to-step mapping misses, which on the run this was captured from + // was every line of all 2528. + expect(f.stepName).toBe('Run unit tests with coverage'); + expect(f.stepNumber).toBe(15); + expect(f.matchedBy).toBe('vitest'); + expect(f.text).toContain('AssertionError: expected null to be'); + expect(f.logLines).toBe(2528); + expect(f.returnedLines).toBeLessThan(60); + + expect(fs.statSync(logPath).size).toBeGreaterThan(200_000); + expect(r.stdout.length, 'the response is no longer bounded').toBeLessThan(8_192); + }); + + it('always reports logLines, returnedLines and truncated together', () => { + stubGh(ONE_FAILING_JOB, writeFixtureTo('github-vitest-failure')); + const f = run(githubDir, 'ci-failures.sh', { CODEV_CI_RUN_ID: '32515040122' }).json!.failures[0]; + for (const key of ['logLines', 'returnedLines', 'truncated']) { + expect(f, `a trimmed answer without ${key} reads as a whole one`).toHaveProperty(key); + } + }); + + it('marks truncated when the cap bites, and keeps whole lines', () => { + stubGh(ONE_FAILING_JOB, writeFixtureTo('github-vitest-failure')); + const r = run(githubDir, 'ci-failures.sh', { + CODEV_CI_RUN_ID: '32515040122', + CODEV_CI_MAX_STEP_BYTES: '200', + }); + const f = r.json!.failures[0]; + expect(f.truncated).toBe(true); + expect(f.returnedLines).toBeLessThan(23); + expect(f.text.length).toBeLessThanOrEqual(260); + expect(f.logLines).toBe(2528); + }); + + it('hands over instead of guessing when nothing is recognisable', () => { + const noise = path.join(tmp, 'noise.log'); + fs.writeFileSync(noise, `${Array.from({ length: 900 }, (_, i) => `2026-08-21T18:47:00.000Z doing thing ${i}`).join('\n')}\n`); + stubGh(ONE_FAILING_JOB, noise); + + const r = run(githubDir, 'ci-failures.sh', { CODEV_CI_RUN_ID: '32515040122' }); + expect(r.json!.extracted).toBe(false); + expect(r.json!.reason).toBe('no recognized failure pattern'); + expect(r.json!.failures[0]).toEqual({ jobId: 96874679182, jobName: 'Unit Tests', logLines: 900 }); + expect(r.json!.next).toContain('CODEV_CI_JOB_ID=96874679182'); + // The rule this whole issue turns on: no arbitrary lines, ever. + expect(r.stdout).not.toContain('doing thing'); + expect(r.json!.failures[0]).not.toHaveProperty('text'); + }); + + it('says a green run is green, and distinguishes it from a run still going', () => { + stubGh({ ...ONE_FAILING_JOB, conclusion: 'success', jobs: [ONE_FAILING_JOB.jobs[0]] }); + const green = run(githubDir, 'ci-failures.sh', { CODEV_CI_RUN_ID: '32515040122' }); + expect(green.status).toBe(0); + expect(green.json!.jobsFailed).toBe(0); + expect(green.json!.reason).toBe('no job in this run failed'); + + stubGh({ ...ONE_FAILING_JOB, status: 'in_progress', conclusion: null, jobs: [ONE_FAILING_JOB.jobs[0]] }); + const running = run(githubDir, 'ci-failures.sh', { CODEV_CI_RUN_ID: '32515040122' }); + expect(running.json!.reason).toBe('run has not finished'); + }); + + it('rejects a job id that is not in the run instead of answering "nothing failed"', () => { + stubGh(ONE_FAILING_JOB, writeFixtureTo('github-vitest-failure')); + const r = run(githubDir, 'ci-failures.sh', { CODEV_CI_RUN_ID: '32515040122', CODEV_CI_JOB_ID: '999' }); + expect(r.status).not.toBe(0); + expect(r.json!.error).toBe('not-found'); + expect(r.json!.ok).toBe(false); + }); + + it.each([ + ['ci-failures.sh', { CODEV_CI_RUN_ID: '../../etc/passwd' }], + ['ci-run-log.sh', { CODEV_CI_RUN_ID: '32515040122', CODEV_CI_JOB_ID: 'x; echo pwned', CODEV_CI_LOG_TAIL: '5' }], + ['ci-run-view.sh', { CODEV_CI_RUN_ID: 'https://github.com/o/r/actions/runs/123' }], + ])('%s rejects a non-numeric id before it reaches a URL or jq', (script, env) => { + stubGh(ONE_FAILING_JOB); + const r = run(githubDir, script, env); + expect(r.status).toBe(2); + expect(r.json!.error).toBe('bad-input'); + expect(r.json!.detail).toContain('must be a numeric id'); + expect(fileLines(path.join(tmp, 'gh.log'))).toEqual([]); + }); + + it('names an unusable TMPDIR instead of blaming the run', () => { + // The failure this cost a red CI run to find. mktemp -d fails on Linux when + // TMPDIR points nowhere; the watchdog needs one for EVERY call, so the + // first gh invocation died for an invisible reason and the concept answered + // "run 32515040122 could not be read" — a temp-dir problem wearing the face + // of a missing run. macOS hides it by falling back to the system temp dir. + stubGh(ONE_FAILING_JOB); + const r = run(githubDir, 'ci-failures.sh', { + CODEV_CI_RUN_ID: '32515040122', + TMPDIR: path.join(tmp, 'no-such-directory'), + }); + expect(r.status).not.toBe(0); + expect(r.json!.error, 'an unusable TMPDIR was reported as a missing run').toBe('forge-error'); + expect(r.json!.detail).toContain('TMPDIR'); + expect(fileLines(path.join(tmp, 'gh.log')), 'the forge was called despite a broken TMPDIR').toEqual([]); + }); + + it('emits an envelope when the forge CLI exits 0 with something that is not JSON', () => { + // An auth prompt, an empty body, an HTML error page. Without a guard this + // reaches jq, dies under `set -e`, and leaves jq's diagnostic on stderr + // with NOTHING on stdout — the one shape these concepts promised never to + // produce. + stub('gh', 'echo "gh: not logged into any hosts"'); + const r = run(githubDir, 'ci-run-view.sh', { CODEV_CI_RUN_ID: '32515040122' }); + expect(r.status).not.toBe(0); + expect(r.json!.error).toBe('forge-error'); + expect(r.json!.detail).toContain('did not return JSON'); + }); + + it('translates the shared canceled into gh cancelled', () => { + stubGh([]); + run(githubDir, 'ci-runs.sh', { CODEV_CI_STATUS: 'canceled' }); + const calls = fileLines(path.join(tmp, 'gh.log')); + expect(calls.some((c) => c.includes('--status cancelled'))).toBe(true); + }); + + it('fetches the per-job log endpoint, never --log-failed', () => { + stubGh(ONE_FAILING_JOB, writeFixtureTo('github-vitest-failure')); + run(githubDir, 'ci-failures.sh', { CODEV_CI_RUN_ID: '32515040122' }); + const calls = fileLines(path.join(tmp, 'gh.log')); + expect(calls.some((c) => c.includes('actions/jobs/96874679182/logs'))).toBe(true); + expect(calls.some((c) => c.includes('--log-failed')), '--log-failed returns the whole job, UNKNOWN STEP-tagged').toBe(false); + }); + + it('caches a terminal job log so the second question costs no download', () => { + const logPath = writeFixtureTo('github-vitest-failure'); + stubGh(ONE_FAILING_JOB, logPath); + const env = { CODEV_CI_RUN_ID: '32515040122', CODEV_CI_NO_CACHE: '' }; + + const first = run(githubDir, 'ci-failures.sh', env); + expect(first.json!.cached).toBe(false); + const second = run(githubDir, 'ci-failures.sh', env); + expect(second.json!.cached).toBe(true); + expect(second.json!.failures[0].text).toBe(first.json!.failures[0].text); + }); + + it('never caches a job that is still running', () => { + const running = { + ...ONE_FAILING_JOB, + status: 'in_progress', + jobs: [{ ...ONE_FAILING_JOB.jobs[1], status: 'in_progress', conclusion: null }], + }; + stubGh(running, writeFixtureTo('github-vitest-failure')); + const env = { CODEV_CI_RUN_ID: '32515040122', CODEV_CI_JOB_ID: '96874679182', CODEV_CI_NO_CACHE: '' }; + run(githubDir, 'ci-failures.sh', env); + const second = run(githubDir, 'ci-failures.sh', env); + expect(second.json!.cached, 'a half-written log was cached and will read as complete forever').toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// ci-run-log +// --------------------------------------------------------------------------- + +describe.skipIf(!hasJq())('#13 — ci-run-log takes exactly one window, and says where it looked', () => { + function stubWithFixture(): void { + const p = path.join(tmp, 'gh-fixture.log'); + fs.writeFileSync(p, fixtureLog('github-vitest-failure')); + stubGh(ONE_FAILING_JOB, p); + } + + it('refuses with no window, and refuses BEFORE calling the forge', () => { + stubWithFixture(); + const r = run(githubDir, 'ci-run-log.sh', { CODEV_CI_RUN_ID: '32515040122' }); + expect(r.status).toBe(2); + expect(r.json!.error).toBe('bad-input'); + expect(r.json!.detail).toContain('exactly one window'); + expect(fileLines(path.join(tmp, 'gh.log')), 'a malformed request cost an API call').toEqual([]); + }); + + it('refuses two windows — a default is how deliberate decays into always', () => { + stubWithFixture(); + const r = run(githubDir, 'ci-run-log.sh', { + CODEV_CI_RUN_ID: '32515040122', CODEV_CI_LOG_TAIL: '5', CODEV_CI_LOG_HEAD: '5', + }); + expect(r.status).toBe(2); + expect(r.json!.detail).toContain('exactly one window may be set'); + }); + + it('refuses a non-numeric line count, naming the variable in the caller spelling', () => { + stubWithFixture(); + const r = run(githubDir, 'ci-run-log.sh', { CODEV_CI_RUN_ID: '32515040122', CODEV_CI_LOG_TAIL: 'lots' }); + expect(r.status).toBe(2); + expect(r.json!.detail).toContain('CODEV_CI_LOG_TAIL'); + }); + + it('tail returns the last N lines with absolute line numbers', () => { + stubWithFixture(); + const r = run(githubDir, 'ci-run-log.sh', { CODEV_CI_RUN_ID: '32515040122', CODEV_CI_LOG_TAIL: '5' }); + expect(r.status).toBe(0); + expect(r.json!.logLines).toBe(2528); + expect(r.json!.returnedLines).toBe(5); + expect(r.json!.from).toBe(2524); + expect(r.json!.to).toBe(2528); + expect(r.json!.contiguous).toBe(true); + expect(r.json!.lines).toHaveLength(5); + }); + + it('head returns the first N lines', () => { + stubWithFixture(); + const r = run(githubDir, 'ci-run-log.sh', { CODEV_CI_RUN_ID: '32515040122', CODEV_CI_LOG_HEAD: '3' }); + expect(r.json!.from).toBe(1); + expect(r.json!.to).toBe(3); + expect(r.json!.lines[0]).toContain('Current runner version'); + }); + + it('grep returns context, marks itself non-contiguous, and lists which lines matched', () => { + stubWithFixture(); + const r = run(githubDir, 'ci-run-log.sh', { + CODEV_CI_RUN_ID: '32515040122', CODEV_CI_LOG_GREP: 'AssertionError', CODEV_CI_LOG_CONTEXT: '1', + }); + expect(r.json!.matches).toBe(2); + expect(r.json!.matchLines).toEqual([2472, 2497]); + expect(r.json!.contiguous).toBe(false); + expect(r.json!.returnedLines).toBe(6); + }); + + it('honours the context width', () => { + stubWithFixture(); + const narrow = run(githubDir, 'ci-run-log.sh', { + CODEV_CI_RUN_ID: '32515040122', CODEV_CI_LOG_GREP: 'AssertionError', CODEV_CI_LOG_CONTEXT: '0', + }); + expect(narrow.json!.returnedLines).toBe(2); + }); + + it.each(['tail', 'head', 'grep'])('survives an EMPTY log in %s mode', (kind) => { + // Without a guard, head/tail build `sed -n "1,0p"` — tolerated by BSD sed, + // REJECTED by GNU sed, so on Linux the script aborted under `set -e` with + // nothing on stdout. Found by the claude review lane on a macOS box, where + // the defect is invisible. + fs.writeFileSync(path.join(tmp, 'empty.log'), ''); + stubGh(ONE_FAILING_JOB, path.join(tmp, 'empty.log')); + const env: Record = { CODEV_CI_RUN_ID: '32515040122' }; + if (kind === 'tail') env.CODEV_CI_LOG_TAIL = '10'; + if (kind === 'head') env.CODEV_CI_LOG_HEAD = '10'; + if (kind === 'grep') env.CODEV_CI_LOG_GREP = 'anything'; + + const r = run(githubDir, 'ci-run-log.sh', env); + expect(r.status, r.stderr).toBe(0); + expect(r.json!.ok).toBe(true); + expect(r.json!.logLines).toBe(0); + expect(r.json!.returnedLines).toBe(0); + expect(r.json!.lines).toEqual([]); + expect(r.json!.truncated).toBe(false); + }); + + it('builds no reversed sed range for an empty log', () => { + // The portable-behaviour assertion, since the box this runs on may be the + // one that tolerates it: reject the range itself, not just its effect. + const lib = fs.readFileSync(path.join(forgeScripts, '_ci-lib.sh'), 'utf-8'); + expect(lib).toMatch(/if \[ "\$_total" -eq 0 \]; then/); + }); + + it('reports no match as an empty window rather than as a failure', () => { + stubWithFixture(); + const r = run(githubDir, 'ci-run-log.sh', { + CODEV_CI_RUN_ID: '32515040122', CODEV_CI_LOG_GREP: 'zzz-definitely-not-present', + }); + expect(r.status).toBe(0); + expect(r.json!.ok).toBe(true); + expect(r.json!.matches).toBe(0); + expect(r.json!.lines).toEqual([]); + expect(r.json!.logLines).toBe(2528); + }); + + it('refuses to pick a job when several passed and none failed', () => { + const green = { + ...ONE_FAILING_JOB, + conclusion: 'success', + jobs: [ + { databaseId: 1, name: 'A', status: 'completed', conclusion: 'success', steps: [] }, + { databaseId: 2, name: 'B', status: 'completed', conclusion: 'success', steps: [] }, + ], + }; + stubGh(green, path.join(tmp, 'nothing.log')); + const r = run(githubDir, 'ci-run-log.sh', { CODEV_CI_RUN_ID: '32515040122', CODEV_CI_LOG_TAIL: '5' }); + expect(r.status).toBe(2); + expect(r.json!.detail).toContain('set CODEV_CI_JOB_ID'); + expect(r.json!.jobs).toHaveLength(2); + }); +}); + +// --------------------------------------------------------------------------- +// Timeouts +// --------------------------------------------------------------------------- + +describe.skipIf(!hasJq())('#13 — a timeout reports as a timeout, not as an empty result', () => { + it('names the timeout on stdout and on stderr, and exits non-zero', () => { + // A stub that spawns a child and waits: killing the wrapper alone leaves the + // child holding the stdout pipe, which is the shape that made an earlier + // watchdog print its message and hang anyway (#12). + stub('gh', 'sleep 60 &\nwait'); + const started = Date.now(); + const r = run(githubDir, 'ci-runs.sh', { CODEV_FORGE_TIMEOUT: '2' }); + const elapsed = Date.now() - started; + + expect(r.status).not.toBe(0); + expect(r.json!.ok).toBe(false); + expect(r.json!.error).toBe('timeout'); + expect(r.json!.seconds).toBe(2); + expect(r.json!.detail).toContain('gh run list'); + expect(r.json!.remedy).toContain('CODEV_FORGE_TIMEOUT'); + expect(r.stderr).toContain('did not return within 2s'); + expect(elapsed, 'the timeout did not actually unblock the caller').toBeLessThan(30_000); + }, 40_000); + + it('does not leave the shell watchdog notice on a concept stderr', () => { + stubGh(ONE_FAILING_JOB); + const r = run(githubDir, 'ci-run-view.sh', { CODEV_CI_RUN_ID: '32515040122' }); + expect(r.status).toBe(0); + expect(r.stderr, 'the watchdog is reporting its own death onto a clean path').toBe(''); + }); + + it('codev forge puts its own ceiling ABOVE the script watchdog, so the named envelope wins', async () => { + // The inversion the claude lane caught: executeForgeCommandDetailed defaults + // to 30s and the scripts default to a 60s CODEV_FORGE_TIMEOUT, so at the + // DEFAULTS Node killed the command first and the script's named timeout + // envelope never printed. The earlier timeout test forced a 2s watchdog, + // which hid it. This one pins the ordering itself. + const watchdog = 4; + // The stub lives one directory DOWN from a copy of _timeout.sh, because + // _ci-lib.sh resolves its sibling as `$(dirname "$0")/../_timeout.sh` — + // the same layout every real provider directory has. + const providerDir = path.join(tmp, 'fake-provider'); + fs.mkdirSync(providerDir, { recursive: true }); + fs.copyFileSync(path.join(forgeScripts, '_timeout.sh'), path.join(tmp, '_timeout.sh')); + const slow = path.join(providerDir, 'slow-forge.sh'); + fs.writeFileSync( + slow, + [ + '#!/bin/sh', + `. ${JSON.stringify(path.join(forgeScripts, '_ci-lib.sh'))}`, + 'rc=0', + 'ci_tool sleep 30 || rc=$?', + '[ "$rc" -eq 124 ] && ci_fail_timeout ci-runs "the slow forge" "$CI_TIMEOUT"', + 'exit 1', + ].join('\n'), + { mode: 0o755 }, + ); + fs.chmodSync(slow, 0o755); + + const root = path.join(tmp, 'ws-timeout'); + fs.mkdirSync(path.join(root, '.codev'), { recursive: true }); + fs.writeFileSync(path.join(root, '.codev', 'config.json'), JSON.stringify({ forge: { 'ci-runs': slow } })); + + const previous = process.env.CODEV_FORGE_TIMEOUT; + process.env.CODEV_FORGE_TIMEOUT = String(watchdog); + try { + const out: string[] = []; + const err: string[] = []; + const code = await runForgeConcept('ci-runs', { + cwd: root, + stdout: (t) => out.push(t), + stderr: (t) => err.push(t), + }); + // The SCRIPT reported, not Node: a named envelope on stdout, and an exit + // status from the script rather than a signal kill. + expect(JSON.parse(out.join('')), 'the outer ceiling fired first and ate the named envelope').toMatchObject({ + ok: false, + error: 'timeout', + seconds: watchdog, + }); + expect(code).toBe(1); + expect(code, 'Node killed it — 124 is the outer backstop, not the inner watchdog').not.toBe(124); + } finally { + if (previous === undefined) delete process.env.CODEV_FORGE_TIMEOUT; + else process.env.CODEV_FORGE_TIMEOUT = previous; + } + }, 60_000); + + it('executeForgeCommandDetailed distinguishes a timeout from a failure and from silence', async () => { + const slow = path.join(tmp, 'slow.sh'); + fs.writeFileSync(slow, '#!/bin/sh\nsleep 30\n', { mode: 0o755 }); + fs.chmodSync(slow, 0o755); + + const timedOut = await executeForgeCommandDetailed('ci-runs', {}, { + forgeConfig: { 'ci-runs': slow }, + timeoutMs: 500, + }); + expect(timedOut.ok).toBe(false); + expect(timedOut.timedOut).toBe(true); + expect(timedOut.unavailable).toBe(false); + + const failing = path.join(tmp, 'fail.sh'); + fs.writeFileSync(failing, '#!/bin/sh\necho \'{"ok":false,"error":"forge-error"}\'\nexit 1\n', { mode: 0o755 }); + fs.chmodSync(failing, 0o755); + + const failed = await executeForgeCommandDetailed('ci-runs', {}, { forgeConfig: { 'ci-runs': failing } }); + expect(failed.ok).toBe(false); + expect(failed.timedOut, 'a plain failure was reported as a timeout').toBe(false); + // stdout survives a non-zero exit — the whole reason the envelope is printed + // there rather than only on stderr. + expect((failed.data as Record).error).toBe('forge-error'); + + const disabled = await executeForgeCommandDetailed('ci-runs', {}, { forgeConfig: { 'ci-runs': null } }); + expect(disabled.unavailable).toBe(true); + expect(disabled.timedOut).toBe(false); + }, 20_000); +}); + +// --------------------------------------------------------------------------- +// Forgejo: the measured footguns, and the version gate +// --------------------------------------------------------------------------- + +/** + * A fake `tea` serving the `tea api` slice these scripts use. + * + * TEA_ROUTES is a `\t` table, first match wins, body emitted + * verbatim from a FILE — a log contains newlines, which a line-oriented table + * would silently truncate. Every requested endpoint is appended to TEA_LOG, so + * a test can assert what was called AND what was not. + */ +const TEA_STUB = [ + 'if [ "$1" != "api" ]; then echo "unexpected tea invocation: $*" >&2; exit 9; fi', + 'for a in "$@"; do ep=$a; done', + 'printf "%s\\n" "$ep" >> "$TEA_LOG"', + 'while IFS="\t" read -r pattern bodyfile; do', + ' [ -n "$pattern" ] || continue', + ' case "$ep" in', + ' $pattern) cat "$bodyfile"; exit 0 ;;', + ' esac', + 'done < "$TEA_ROUTES"', + 'printf "404 page not found\\n"', + 'exit 0', +].join('\n'); + +function stubTea(routes: Array<[string, unknown | { raw: string }]>): void { + const table: string[] = []; + routes.forEach(([pattern, body], i) => { + const file = path.join(tmp, `route-${i}.body`); + const content = typeof body === 'object' && body !== null && 'raw' in (body as any) + ? (body as { raw: string }).raw + : JSON.stringify(body); + fs.writeFileSync(file, content); + table.push(`${pattern}\t${file}`); + }); + fs.writeFileSync(path.join(tmp, 'routes.tsv'), `${table.join('\n')}\n`); + stub('tea', TEA_STUB); +} + +const FORGEJO_15_VERSION = { version: '15.0.2+gitea-1.22.0' }; + +/** One page of `actions/runs`, reduced to the fields the scripts read. */ +function forgejoRuns(runs: Array>): Record { + return { total_count: runs.length, workflow_runs: runs }; +} + +function forgejoRun(id: number, index: number, extra: Record = {}): Record { + return { + id, index_in_repo: index, title: `run ${index}`, workflow_id: 'ci.yml', + status: 'failure', prettyref: 'main', commit_sha: 'deadbeef', event: 'push', + html_url: `https://forge.example.com/o/r/actions/runs/${index}`, created: '2026-08-18T21:47:24Z', + ...extra, + }; +} + +describe.skipIf(!hasJq())('#13 — Forgejo, as it actually behaves', () => { + function teaEnv(extra: Record = {}): Record { + return { TEA_ROUTES: path.join(tmp, 'routes.tsv'), ...extra }; + } + + it('always sends page= with limit=, because Forgejo ignores limit without it', () => { + // Measured: `actions/runs?limit=3` returned all 6922 runs on the reference + // Forgejo. This is the kind of default that turns into a #12. + stubTea([['repos/o/r/actions/runs?*', forgejoRuns([forgejoRun(11130, 6881)])]]); + run(giteaDir, 'ci-runs.sh', teaEnv({ CODEV_CI_LIMIT: '3' })); + const calls = fileLines(path.join(tmp, 'tea.log')); + expect(calls.length).toBeGreaterThan(0); + for (const call of calls.filter((c) => c.includes('limit='))) { + expect(call, `a list request without page=: ${call}`).toContain('page='); + } + }); + + it('matches a branch by its PULL REQUEST ref, which is what Forgejo records', () => { + // A pull_request run records `#3855`, not `builder/air-364`. Filtering on + // the branch name alone matches nothing on a repo that runs CI on PRs. + stubTea([ + ['repos/o/r', { default_branch: 'main' }], + ['repos/o/r/pulls/main/builder/air-364', { number: 3855, state: 'open', head: { label: 'builder/air-364' } }], + ['repos/o/r/actions/runs?*', forgejoRuns([ + forgejoRun(11142, 6886, { prettyref: '#3855', event: 'pull_request', status: 'success' }), + forgejoRun(11100, 6870, { prettyref: '#9999', event: 'pull_request' }), + forgejoRun(11000, 6800, { prettyref: 'main' }), + ])], + ]); + + const r = run(giteaDir, 'ci-runs.sh', teaEnv({ CODEV_BRANCH_NAME: 'builder/air-364' })); + expect(r.status).toBe(0); + expect(r.json!.runs).toHaveLength(1); + expect(r.json!.runs[0].id).toBe(11142); + expect(r.json!.runs[0].branch).toBe('#3855'); + }); + + it('says so when a branch has no PR, instead of returning a bare empty list', () => { + stubTea([ + ['repos/o/r', { default_branch: 'main' }], + ['repos/o/r/actions/runs?*', forgejoRuns([forgejoRun(11000, 6800, { prettyref: 'main' })])], + ]); + const r = run(giteaDir, 'ci-runs.sh', teaEnv({ CODEV_BRANCH_NAME: 'builder/no-pr' })); + expect(r.json!.runs).toEqual([]); + expect(r.json!.note, 'an empty list with no explanation reads as "CI never ran"').toContain('no pull request'); + }); + + it('emits conclusion: null rather than inventing one Forgejo does not have', () => { + stubTea([['repos/o/r/actions/runs?*', forgejoRuns([forgejoRun(11130, 6881)])]]); + const r = run(giteaDir, 'ci-runs.sh', teaEnv()); + expect(r.json!.runs[0].conclusion).toBeNull(); + expect(r.json!.runs[0].status).toBe('failure'); + }); + + it('ci-run-view recovers jobs from the task list when the jobs API is absent', () => { + stubTea([ + ['repos/o/r/actions/runs/11130', forgejoRun(11130, 6881)], + // no repos/o/r/actions/runs/11130/jobs route -> the stub 404s, as 15.0.2 does + ['repos/o/r/actions/tasks?*', { + total_count: 2, + workflow_runs: [ + { id: 40084, name: 'E2E coverage-drift guard', run_number: 6881, status: 'failure', run_started_at: 'x', updated_at: 'y' }, + { id: 40082, name: 'Lint', run_number: 6881, status: 'success', run_started_at: 'x', updated_at: 'y' }, + ], + }], + ]); + + const r = run(giteaDir, 'ci-run-view.sh', teaEnv({ CODEV_CI_RUN_ID: '11130' })); + expect(r.status).toBe(0); + expect(r.json!.jobSource).toBe('tasks-scan'); + expect(r.json!.jobs).toHaveLength(2); + // A task id is not a job id and the log API does not accept one, so it is + // never reported as `id`. + expect(r.json!.jobs[0].id).toBeNull(); + expect(r.json!.jobs[0].taskId).toBe(40084); + expect(r.json!.jobs[0].status).toBe('failure'); + }); + + it('ci-failures on Forgejo 15 says unsupported-server, names both versions, and still names the failing job', () => { + stubTea([ + ['repos/o/r/actions/runs/11130', forgejoRun(11130, 6881)], + ['repos/o/r/actions/tasks?*', { + total_count: 1, + workflow_runs: [{ id: 40084, name: 'E2E coverage-drift guard', run_number: 6881, status: 'failure', run_started_at: 'x', updated_at: 'y' }], + }], + ['version', FORGEJO_15_VERSION], + ]); + + const r = run(giteaDir, 'ci-failures.sh', teaEnv({ CODEV_CI_RUN_ID: '11130' })); + expect(r.status).not.toBe(0); + expect(r.json!.ok).toBe(false); + expect(r.json!.error).toBe('unsupported-server'); + expect(r.json!.serverVersion).toBe('15.0.2+gitea-1.22.0'); + expect(r.json!.needs).toBe('>=16.0'); + expect(r.json!.detail).toContain('Forgejo 16.0'); + // The rule the architect held this to: an unsupported server must never + // look like a run with no failures. + expect(r.json!, 'an old server produced an empty failures array').not.toHaveProperty('failures'); + expect(r.json!.jobsFailed).toBe(1); + expect(r.json!.failingJobs[0].jobName).toBe('E2E coverage-drift guard'); + }); + + it('ci-run-log on Forgejo 15 says unsupported-server rather than an empty window', () => { + stubTea([ + ['repos/o/r/actions/runs/11130', forgejoRun(11130, 6881)], + ['repos/o/r/actions/tasks?*', { total_count: 0, workflow_runs: [] }], + ['version', FORGEJO_15_VERSION], + ]); + const r = run(giteaDir, 'ci-run-log.sh', teaEnv({ CODEV_CI_RUN_ID: '11130', CODEV_CI_LOG_TAIL: '20' })); + expect(r.json!.error).toBe('unsupported-server'); + expect(r.json!).not.toHaveProperty('lines'); + }); + + it('ci-failures on Forgejo 16 extracts from the job-log endpoint', () => { + stubTea([ + ['repos/o/r/actions/runs/6554924', forgejoRun(6554924, 189242)], + ['repos/o/r/actions/runs/6554924/jobs', [ + { id: 11952743, task_id: 8848592, name: 'backend-checks', status: 'success' }, + { id: 11952749, task_id: 8848703, name: 'test-unit', status: 'failure' }, + ]], + ['repos/o/r/actions/jobs/11952749/logs', { raw: fixtureLog('forgejo-go-failure') }], + ]); + + const r = run(giteaDir, 'ci-failures.sh', teaEnv({ CODEV_CI_RUN_ID: '6554924' })); + expect(r.status).toBe(0); + expect(r.json!.extracted).toBe(true); + const f = r.json!.failures[0]; + // The log route takes the JOB id (11952749), not the task id (8848703). + expect(f.jobId).toBe(11952749); + expect(f.jobName).toBe('test-unit'); + expect(f.matchedBy).toBe('go-test'); + expect(f.text).toContain('--- FAIL: TestReadPointerFromBuffer'); + expect(f.logLines).toBe(1599); + expect(fileLines(path.join(tmp, 'tea.log')).some((c) => c.includes('actions/jobs/11952749/logs'))).toBe(true); + }); + + it('ci-run-log on Forgejo 16 windows the same log', () => { + stubTea([ + ['repos/o/r/actions/runs/6554924', forgejoRun(6554924, 189242)], + ['repos/o/r/actions/runs/6554924/jobs', [{ id: 11952749, task_id: 8848703, name: 'test-unit', status: 'failure' }]], + ['repos/o/r/actions/jobs/11952749/logs', { raw: fixtureLog('forgejo-go-failure') }], + ]); + const r = run(giteaDir, 'ci-run-log.sh', teaEnv({ CODEV_CI_RUN_ID: '6554924', CODEV_CI_LOG_GREP: '--- FAIL', CODEV_CI_LOG_CONTEXT: '0' })); + expect(r.json!.matches).toBe(1); + expect(r.json!.lines[0]).toContain('--- FAIL: TestReadPointerFromBuffer'); + }); + + it('sends status=cancelled, because Forgejo rejects the spelling its own CLI documents', () => { + // `tea actions runs list --help` says `canceled`; the API answers that with + // {"message":"unknown status: canceled"} and answers `cancelled` with 2240 + // runs. Both measured. This is the kind of difference that becomes an empty + // list nobody questions. + stubTea([['repos/o/r/actions/runs?*', forgejoRuns([])]]); + run(giteaDir, 'ci-runs.sh', teaEnv({ CODEV_CI_STATUS: 'canceled' })); + const calls = fileLines(path.join(tmp, 'tea.log')); + expect(calls.some((c) => c.includes('status=cancelled'))).toBe(true); + expect(calls.some((c) => c.includes('status=canceled'))).toBe(false); + }); + + it('rejects a status outside the shared vocabulary instead of passing it to the forge', () => { + stubTea([['repos/o/r/actions/runs?*', forgejoRuns([])]]); + const r = run(giteaDir, 'ci-runs.sh', teaEnv({ CODEV_CI_STATUS: 'broken' })); + expect(r.status).toBe(2); + expect(r.json!.error).toBe('bad-input'); + expect(r.json!.detail).toContain('in_progress'); + expect(fileLines(path.join(tmp, 'tea.log'))).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// codev forge +// --------------------------------------------------------------------------- + +describe.skipIf(!hasJq())('#13 — codev forge runs a concept through the real resolver', () => { + /** A workspace with a .codev/config.json, so resolution has something to resolve. */ + function workspace(forge: Record): string { + const root = path.join(tmp, 'ws'); + fs.mkdirSync(path.join(root, '.codev'), { recursive: true }); + fs.writeFileSync(path.join(root, '.codev', 'config.json'), JSON.stringify({ forge })); + return root; + } + + function capture() { + const out: string[] = []; + const err: string[] = []; + return { out, err, stdout: (t: string) => out.push(t), stderr: (t: string) => err.push(t) }; + } + + it('HONOURS a per-repo override — the reason this command exists', () => { + // Naming the script by path bypasses the config lookup, the provider preset + // and any override. A project that overrides ci-failures would have its + // override silently ignored, and would get GitHub's script against a + // Forgejo repo. The reference Forgejo repo carried three such overrides + // until #12 shipped, so this is not hypothetical. + const custom = path.join(tmp, 'my-ci-failures.sh'); + fs.writeFileSync(custom, '#!/bin/sh\necho \'{"ok":true,"mine":true}\'\n', { mode: 0o755 }); + fs.chmodSync(custom, 0o755); + + const io = capture(); + return runForgeConcept('ci-failures', { cwd: workspace({ 'ci-failures': custom }), ...io }).then((code) => { + expect(code).toBe(0); + expect(JSON.parse(io.out.join(''))).toEqual({ ok: true, mine: true }); + }); + }); + + it('names a concept disabled for the provider and exits non-zero', async () => { + const io = capture(); + const code = await runForgeConcept('team-activity', { cwd: workspace({ provider: 'gitea' }), ...io }); + expect(code).not.toBe(0); + expect(io.out.join('')).toBe(''); + expect(io.err.join('')).toContain('team-activity'); + expect(io.err.join(''), 'a disabled concept must name WHY it is unavailable').toContain('gitea'); + }); + + it('lists the valid concepts when given an unknown name', async () => { + const io = capture(); + const code = await runForgeConcept('ci-failure', { cwd: workspace({}), ...io }); + expect(code).toBe(2); + expect(io.err.join('')).toContain('not a known forge concept'); + expect(io.err.join('')).toContain('ci-failures'); + expect(io.err.join('')).toContain('pr-exists'); + }); + + it('prints stdout verbatim and propagates the exit code, envelope included', async () => { + // A concept that fails still has something to say — the ci-* scripts print + // their error envelope on stdout precisely so the class of failure survives + // a non-zero exit. Swallowing stdout on failure would throw it away at the + // last step. + const failing = path.join(tmp, 'failing.sh'); + fs.writeFileSync(failing, '#!/bin/sh\necho \'{"ok":false,"error":"timeout","seconds":60}\'\nexit 1\n', { mode: 0o755 }); + fs.chmodSync(failing, 0o755); + + const io = capture(); + const code = await runForgeConcept('ci-runs', { cwd: workspace({ 'ci-runs': failing }), ...io }); + expect(code).toBe(1); + expect(JSON.parse(io.out.join('')).error).toBe('timeout'); + }); + + it('passes the ambient CODEV_* environment through to the script', async () => { + const echoer = path.join(tmp, 'echo-env.sh'); + fs.writeFileSync(echoer, '#!/bin/sh\nprintf \'{"runId":"%s"}\' "$CODEV_CI_RUN_ID"\n', { mode: 0o755 }); + fs.chmodSync(echoer, 0o755); + + const previous = process.env.CODEV_CI_RUN_ID; + process.env.CODEV_CI_RUN_ID = '32515040122'; + try { + const io = capture(); + const code = await runForgeConcept('ci-failures', { cwd: workspace({ 'ci-failures': echoer }), ...io }); + expect(code).toBe(0); + expect(JSON.parse(io.out.join('')).runId).toBe('32515040122'); + } finally { + if (previous === undefined) delete process.env.CODEV_CI_RUN_ID; + else process.env.CODEV_CI_RUN_ID = previous; + } + }); +}); diff --git a/packages/codev/src/__tests__/spec-1280-measurement-instrument.test.ts b/packages/codev/src/__tests__/spec-1280-measurement-instrument.test.ts index eefd3c10f..efdb2b57f 100644 --- a/packages/codev/src/__tests__/spec-1280-measurement-instrument.test.ts +++ b/packages/codev/src/__tests__/spec-1280-measurement-instrument.test.ts @@ -22,6 +22,19 @@ import * as path from 'node:path'; const repoRoot = path.resolve(import.meta.dirname, '../../../..'); const script = path.join(repoRoot, 'scripts/measure-prompt-surface.sh'); +/** + * Per-test ceiling for anything that shells out to the instrument. + * + * Measured on this repo: one `measure-prompt-surface.sh` invocation costs ~25-30s + * on a quiet machine, and several of these tests invoke it two or three times + * (two locales, two runs for determinism, a fixture plus the live repo). The old + * 60s ceiling therefore sat *below* the honest cost of the slowest cases, so + * under full-suite load they were killed mid-run — and a different one lost the + * race each time, which reads as flakiness rather than as a ceiling set too low. + * The whole file passes in isolation once given room. + */ +const INSTRUMENT_TIMEOUT_MS = 240_000; + function run(root: string = repoRoot, env: Record = {}): string { return execFileSync('bash', [script, root], { encoding: 'utf-8', @@ -130,7 +143,7 @@ describe('T1b — per-file four-tier resolution, not directory-level selection', const out = run(dir); // Directory-level selection would have missed the override entirely. expect(out).toMatch(/\| spir \| \d+ \| 8 \| \d+ \|/); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); it('prefers .codev/ over codev/ over codev-skeleton/', () => { const src = fs.readFileSync(script, 'utf-8'); @@ -154,7 +167,7 @@ describe('T2 — phantom-savings proof: includes are expanded', () => { fs.writeFileSync(path.join(dir, 'codev-skeleton/protocols/spir/templates/frag.md'), 'cc dd ee\n'); expect(num(run(dir), 'ALWAYS_ON_WORDS')).toBe(before); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); it('expands non-markdown includes too — protocol.json delivery depends on it (P6)', () => { const dir = makeFixture(); @@ -169,7 +182,7 @@ describe('T2 — phantom-savings proof: includes are expanded', () => { ); // The JSON's words must appear in the served count, not vanish. expect(num(run(dir), 'ALWAYS_ON_WORDS')).toBeGreaterThan(before); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); }); describe('T3 — per-surface reporting completeness (not a ceiling)', () => { @@ -189,11 +202,11 @@ describe('T3 — per-surface reporting completeness (not a ceiling)', () => { new RegExp(`^\\| ${name} \\|`, 'm'), ); } - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); it('includes codev-only protocols with no skeleton twin (release)', () => { expect(run()).toMatch(/^\| release \|/m); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); }); describe('T11 — buckets vs audience loads are reported on different bases', () => { @@ -202,7 +215,7 @@ describe('T11 — buckets vs audience loads are reported on different bases', () expect(out).toContain('ALWAYS_ON(builder,p,I) = SHARED + BUILDER_SPAWN[p]'); expect(out).toMatch(/OVERLAP by design/); expect(out).toMatch(/these SUM/); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); it('one bucket growing while another shrinks shows BOTH movements, not a netted zero', () => { const dir = makeFixture(); @@ -219,7 +232,7 @@ describe('T11 — buckets vs audience loads are reported on different bases', () expect(sharedAfter).toBeLessThan(sharedBefore); expect(archAfter).toBeGreaterThan(archBefore); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); }); describe('T15 — relocation is visible, never reported as deletion (M0c)', () => { @@ -236,7 +249,7 @@ describe('T15 — relocation is visible, never reported as deletion (M0c)', () = const after = run(dir); expect(num(after, 'ALWAYS_ON_WORDS')).toBeLessThan(num(before, 'ALWAYS_ON_WORDS')); expect(num(after, 'TOTAL_AUTHORED_WORDS')).toBe(num(before, 'TOTAL_AUTHORED_WORDS')); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); it('counts all four skill trees — one-tree counting would report relocation as deletion', () => { const src = fs.readFileSync(script, 'utf-8'); @@ -271,13 +284,13 @@ describe('portability — the count must not depend on the host', () => { const utf8 = num(run(repoRoot, { LC_ALL: 'en_US.UTF-8' }), 'ALWAYS_ON_WORDS'); const c = num(run(repoRoot, { LC_ALL: 'C' }), 'ALWAYS_ON_WORDS'); expect(c).toBe(utf8); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); }); describe('T12 — determinism', () => { it('emits byte-identical output twice at the same commit', () => { expect(run()).toBe(run()); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); }); describe('instrument correctness — asserted without pinning the live surface', () => { @@ -304,7 +317,7 @@ describe('instrument correctness — asserted without pinning the live surface', // human reads them as findings rather than maintaining them as expectations. let out: string; - beforeAll(() => { out = run(); }, 60_000); + beforeAll(() => { out = run(); }, INSTRUMENT_TIMEOUT_MS); describe('layer 1 — invariants over the live repo (hold at any surface size)', () => { it('ALWAYS_ON = SHARED + BUILDER_SPAWN[spir] + I x (HOT + PHASE mean[spir])', () => { @@ -330,7 +343,7 @@ describe('instrument correctness — asserted without pinning the live surface', const spir = out.match(/^\| spir \| \d+ \| (\d+) \| \d+ \|$/m)!; const hot = Number(out.match(/lessons-critical\(\d+\) = (\d+)/)![1]); expect(two - one).toBe(hot + Number(spir[1])); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); }); describe('layer 2 — absolute values on a fixture whose arithmetic a human can check', () => { @@ -343,13 +356,13 @@ describe('instrument correctness — asserted without pinning the live surface', it('reports the hand-computed total for a known surface', () => { const out2 = run(makeFixture()); expect(num(out2, 'ALWAYS_ON_WORDS')).toBe(107); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); it('reports the hand-computed architect and consultant loads', () => { const out2 = run(makeFixture()); expect(out2).toMatch(/\| Architect \(per session\) \| 10 \|/); expect(out2).toMatch(/\| Consultant \(per review, spir\) \| 4 \|/); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); it('a component silently omitted would fail here even though invariants still hold', () => { // The blind spot invariants cannot see: drop the hot tier from SHARED and every @@ -357,7 +370,7 @@ describe('instrument correctness — asserted without pinning the live surface', // Only an externally-known expected value catches it. const out2 = run(makeFixture()); expect(Number(out2.match(/\| SHARED [^|]*\| (\d+) \|/)![1])).toBe(7); - }, 60_000); + }, INSTRUMENT_TIMEOUT_MS); }); describe('layer 3 — the frozen baseline artifacts are historical records', () => { diff --git a/packages/codev/src/cli.ts b/packages/codev/src/cli.ts index 8d065a2b5..d22dbe7b3 100644 --- a/packages/codev/src/cli.ts +++ b/packages/codev/src/cli.ts @@ -8,6 +8,7 @@ import { existsSync } from 'node:fs'; import { join } from 'node:path'; import { Command } from 'commander'; import { doctor } from './commands/doctor.js'; +import { runForgeConcept } from './commands/forge.js'; import { init } from './commands/init.js'; import { adopt } from './commands/adopt.js'; import { update } from './commands/update.js'; @@ -61,6 +62,24 @@ program } }); +// Forge command +// +// Deliberately thin. Its whole reason for existing is that naming a concept +// script by path bypasses resolution — the config lookup, the provider preset +// and any per-repo override — so a project that overrides a concept would have +// its override silently ignored. See commands/forge.ts. +program + .command('forge ') + .description('Run one forge concept through the configured provider (CODEV_* env is passed through)') + .action(async (concept: string) => { + try { + process.exit(await runForgeConcept(concept)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } + }); + // Init command program .command('init [project-name]') diff --git a/packages/codev/src/commands/forge.ts b/packages/codev/src/commands/forge.ts new file mode 100644 index 000000000..ee6ca1b6d --- /dev/null +++ b/packages/codev/src/commands/forge.ts @@ -0,0 +1,112 @@ +/** + * `codev forge ` — run one forge concept through the real resolver. + * + * WHY THIS EXISTS (issue #13) + * + * Before this, the only way to invoke a concept from a shell was to name its + * script by path — `packages/codev/scripts/forge/github/ci-failures.sh`. That + * path **bypasses resolution**: it skips the `.codev/config.json` lookup, the + * provider preset, and any per-repo override. A project that overrides + * `ci-failures` would have its override silently ignored by anyone following + * those instructions, and would get GitHub's script against a Forgejo repo. + * That is not hypothetical — the reference Forgejo repo carried three concept + * overrides until #12 shipped. + * + * So this is deliberately a THIN dispatcher and nothing else: resolve the + * concept exactly as `executeForgeCommand` does, pass the ambient `CODEV_*` + * environment through, print stdout verbatim, and exit with the script's own + * exit code. No parsing, no reformatting, no second code path — it delegates to + * `executeForgeCommandDetailed` so there is exactly one place where a forge + * command is actually run. + * + * The two things it adds are the two ways a caller can be wrong: + * + * - a concept **disabled for this provider** says so by name and exits + * non-zero, rather than printing nothing and letting silence read as an + * empty answer; + * - an **unknown concept name** lists the valid ones. + */ + +import { + executeForgeCommandDetailed, + getKnownConcepts, + getForgeCommand, + loadForgeConfig, + describeUnavailableConcept, +} from '../lib/forge.js'; + +export interface ForgeCommandCliOptions { + /** Workspace root to resolve `.codev/config.json` from. Defaults to cwd. */ + cwd?: string; + /** Sinks, injectable for tests. Default to the real streams. */ + stdout?: (text: string) => void; + stderr?: (text: string) => void; +} + +/** + * Run one concept. Returns the process exit code; never calls process.exit, so + * the caller (cli.ts) decides and tests can assert. + */ +export async function runForgeConcept( + concept: string | undefined, + options: ForgeCommandCliOptions = {}, +): Promise { + const cwd = options.cwd ?? process.cwd(); + const out = options.stdout ?? ((t: string) => process.stdout.write(t)); + const err = options.stderr ?? ((t: string) => process.stderr.write(t)); + + const known = getKnownConcepts(); + + if (!concept) { + err(`codev forge: a concept name is required.\nKnown concepts: ${known.join(', ')}\n`); + return 2; + } + + if (!known.includes(concept)) { + err(`codev forge: '${concept}' is not a known forge concept.\nKnown concepts: ${known.join(', ')}\n`); + return 2; + } + + const forgeConfig = loadForgeConfig(cwd); + + // Disabled is not an error in the command; it is an answer the caller must be + // able to see. `describeUnavailableConcept` names the provider or the config, + // which is the difference between "why is this empty" and "right, Forgejo has + // no GraphQL". + if (getForgeCommand(concept, forgeConfig) === null) { + err(`codev forge: ${describeUnavailableConcept(concept, forgeConfig)}\n`); + return 3; + } + + // The outer ceiling MUST sit above the script watchdog, or the wrong one wins. + // executeForgeCommandDetailed defaults to 30s while the scripts default to a + // 60s CODEV_FORGE_TIMEOUT, so with the defaults a stalled forge was killed by + // Node first and the script's named timeout envelope — the whole point of the + // inner watchdog — never printed. Found by the claude review lane, which also + // noted that the test hid it by forcing a 2s watchdog. Give the script its + // full allowance plus a margin, and keep this as the backstop it is meant to + // be: it fires only if the watchdog itself fails. + const watchdogSeconds = Number.parseInt(process.env.CODEV_FORGE_TIMEOUT ?? '', 10); + const innerMs = (Number.isFinite(watchdogSeconds) && watchdogSeconds > 0 ? watchdogSeconds : 60) * 1000; + + const result = await executeForgeCommandDetailed(concept, undefined, { + cwd, + forgeConfig, + timeoutMs: innerMs + 30_000, + }); + + // stdout verbatim, including on the failure path: the ci-* concepts print + // their structured error envelope there precisely so the class of failure + // survives a non-zero exit. + if (result.stdout) out(result.stdout.endsWith('\n') ? result.stdout : `${result.stdout}\n`); + if (result.stderr) err(result.stderr.endsWith('\n') ? result.stderr : `${result.stderr}\n`); + + if (result.timedOut) { + err(`codev forge: '${concept}' was killed for exceeding the command timeout\n`); + // 124 is what timeout(1) uses, and what scripts/forge/_timeout.sh reports + // internally — so a caller sees the same number wherever the clock ran out. + return 124; + } + + return result.exitCode ?? (result.ok ? 0 : 1); +} diff --git a/packages/codev/src/lib/forge-contracts.ts b/packages/codev/src/lib/forge-contracts.ts index 8a8e3e444..6efc9140b 100644 --- a/packages/codev/src/lib/forge-contracts.ts +++ b/packages/codev/src/lib/forge-contracts.ts @@ -201,3 +201,184 @@ export interface PrCreateResult { /** Output of the `auth-status` concept command: exit code only. */ // Exit code 0 = authenticated, non-zero = not authenticated. + +// ============================================================================= +// CI concepts (#13) +// ============================================================================= +// +// Four concepts, tiered so that the cheap question stays cheap: `ci-runs` and +// `ci-run-view` never read a log, `ci-failures` reads exactly one job's and +// returns a bounded extract, and `ci-run-log` is the deliberate raw window. +// +// Two contract-wide rules, both of which exist because breaking them produced +// real wrong answers: +// +// 1. **Every response that carries log text also carries `logLines`, +// `returnedLines` and `truncated`.** A trimmed answer must never read as a +// whole one. +// 2. **Failure is a value, not an absence.** These concepts print `CiError` on +// stdout even when they exit non-zero, so a timeout, a missing run and a +// server too old to answer stay distinguishable after +// `executeForgeCommand` has flattened everything else to `null`. Read them +// with `executeForgeCommandDetailed` when the distinction matters. + +/** Error envelope printed on stdout by any ci-* concept that cannot answer. */ +export interface CiError { + ok: false; + /** + * Stable machine token: + * - `timeout` — the forge did not answer inside the watchdog; carries `seconds` + * - `not-found` — no such run or job + * - `unsupported-server` — the forge is too old to have the API; carries + * `serverVersion` and `needs`. Distinct from "nothing failed", deliberately: + * Forgejo gained the Actions job-log API only in 16.0, and an empty + * `failures` array on an older server would say "your CI is fine" when the + * truth is "I cannot see your CI at all". + * - `forge-error` — the forge answered with an error + * - `bad-input` — missing or unusable input (exit 2) + */ + error: 'timeout' | 'not-found' | 'unsupported-server' | 'forge-error' | 'bad-input'; + detail: string; + seconds?: number; + remedy?: string; + serverVersion?: string; + needs?: string; + runId?: number | string; + jobId?: number | string; + jobName?: string; + /** Failing jobs the concept could still name on an unsupported server. */ + failingJobs?: Array<{ jobId?: number; taskId?: number; jobName: string }>; +} + +/** One run in `ci-runs` output. */ +export interface CiRunItem { + /** The API id. This is what ci-run-view / ci-failures / ci-run-log take. */ + id: number; + /** + * The human run number (`index_in_repo` on Forgejo, `number` on GitHub). + * Never pass this as CODEV_CI_RUN_ID: on Forgejo both id spaces resolve on + * the same route, so a number is silently a *different, real* run. + */ + number: number; + name: string; + workflow: string; + status: string; + /** GitHub only. Forgejo has no conclusion field — `status` carries it. */ + conclusion: string | null; + /** + * The branch as the forge recorded it. On Forgejo a `pull_request` run + * records `#` rather than a branch name; ci-runs resolves + * CODEV_BRANCH_NAME to that form before filtering. + */ + branch: string | null; + sha: string | null; + event: string; + url: string; + createdAt: string; +} + +/** Output of the `ci-runs` concept. */ +export interface CiRunsResult { + ok: true; + provider: string; + runs: CiRunItem[]; + /** True when more runs exist than were returned, for any reason. */ + truncated: boolean; + note?: string | null; +} + +/** One job in `ci-run-view` output. */ +export interface CiJobItem { + /** + * The job id the log concepts take. **Null on Forgejo < 16**, which exposes + * no jobs API: those jobs are recovered from `actions/tasks` and only a + * `taskId` exists. A task id looks like a job id and is not one, so it is + * never reported as `id`. + */ + id: number | null; + taskId?: number; + name: string; + status: string; + conclusion: string | null; + startedAt: string | null; + completedAt: string | null; + /** GitHub only; Forgejo exposes no per-step data, so this is `[]` there. */ + failedSteps: Array<{ name: string; number: number; conclusion: string }>; +} + +/** Output of the `ci-run-view` concept. */ +export interface CiRunViewResult { + ok: true; + provider: string; + /** Which route produced `jobs`: `run-view` | `runs-jobs` | `tasks-scan`. */ + jobSource: string; + run: Omit & { title?: string; name?: string }; + jobs: CiJobItem[]; + truncated?: boolean; +} + +/** One extracted failure in `ci-failures` output. */ +export interface CiFailureItem { + jobId: number; + jobName: string; + stepName?: string | null; + stepNumber?: number | null; + /** + * Which rung of the extraction ladder fired: `vitest`, `go-test`, `tsc`, + * `runner-marker`, `first-error`. Present so a reader can weigh the answer — + * a `first-error` match is a weaker claim than a `vitest` one. + */ + matchedBy?: string; + text?: string; + from?: number; + to?: number; + logLines: number; + returnedLines?: number; + truncated?: boolean; +} + +/** Output of the `ci-failures` concept. */ +export interface CiFailuresResult { + ok: true; + provider: string; + runId: number | string; + runStatus: string; + runConclusion: string | null; + jobsFailed: number; + /** + * False means extraction found nothing it recognised — NOT that the job + * passed. The response then carries `reason`, the job identity, `logLines`, + * and a ready-to-run `next` command, so the refusal is a handoff to + * ci-run-log rather than a dead end. It never falls back to returning + * arbitrary lines, which a reader would treat as a diagnosis. + */ + extracted: boolean; + reason?: string; + failures: CiFailureItem[]; + otherFailingJobs?: Array<{ id: number; name: string }>; + cached?: boolean; + next?: string; +} + +/** Output of the `ci-run-log` concept. */ +export interface CiRunLogResult { + ok: true; + provider: string; + runId: number | string; + jobId: number; + jobName: string; + window: { kind: 'head' | 'tail' | 'grep'; arg: string }; + logLines: number; + returnedLines: number; + /** First and last line numbers returned, into the FULL log. 0/0 for no match. */ + from: number; + to: number; + /** False in grep mode: the returned lines have gaps. See `matchLines`. */ + contiguous: boolean; + truncated: boolean; + /** grep mode only: how many lines matched, and which. */ + matches: number | null; + matchLines: number[] | null; + cached?: boolean; + lines: string[]; +} diff --git a/packages/codev/src/lib/forge.ts b/packages/codev/src/lib/forge.ts index 611df59fa..527bd0abb 100644 --- a/packages/codev/src/lib/forge.ts +++ b/packages/codev/src/lib/forge.ts @@ -36,6 +36,26 @@ function resolveScriptPath(provider: string, concept: string): string { /** Default maxBuffer for forge commands (10MB). Prevents truncation for large diffs. */ const DEFAULT_MAX_BUFFER = 10 * 1024 * 1024; +/** + * Default wall-clock ceiling for a forge command (30s), unchanged since before + * the scripts had watchdogs of their own. + * + * **It is SHORTER than the script watchdog, not longer** (CODEV_FORGE_TIMEOUT, + * default 60s in scripts/forge/_timeout.sh), and the ordering matters: whichever + * ceiling fires first decides what the caller learns. This one can only report + * that something died; the inner one reports WHICH endpoint stopped answering + * and how long it was given. With the defaults as they stand, a stalled forge + * hits this 30s kill first and the named message never gets to fire. + * + * That inversion predates the CI concepts — #12 gave the gitea scripts a 60s + * watchdog under this same 30s ceiling — and correcting it globally would change + * the timeout behaviour of every concept and every caller, so it is not done + * here. What IS done: any caller that needs the named envelope passes + * `timeoutMs` above the script watchdog. `codev forge` does exactly that (see + * commands/forge.ts), which is why its timeout path reports a timeout by name. + */ +const DEFAULT_TIMEOUT_MS = 30_000; + // ============================================================================= // Types // ============================================================================= @@ -55,6 +75,8 @@ export interface ForgeCommandOptions { raw?: boolean; /** Maximum stdout buffer size in bytes. Defaults to 10MB. */ maxBuffer?: number; + /** Wall-clock ceiling in ms. Defaults to 30s. */ + timeoutMs?: number; } // ============================================================================= @@ -66,6 +88,12 @@ const KNOWN_CONCEPTS = [ 'recently-closed', 'recently-merged', 'user-identity', 'team-activity', 'on-it-timestamps', 'pr-create', 'pr-merge', 'pr-search', 'pr-view', 'pr-diff', 'auth-status', 'repo-archive', + // CI concepts (#13), tiered so the cheap question stays cheap: ci-runs and + // ci-run-view answer "did it pass" and "which job" without touching a log, + // ci-failures fetches exactly one job's log and returns a bounded extract, + // and ci-run-log is the deliberate raw-window escape hatch. Only the last two + // ever read log bytes. + 'ci-runs', 'ci-run-view', 'ci-failures', 'ci-run-log', ] as const; // ============================================================================= @@ -125,7 +153,16 @@ function getProviderPresets(): Record> { if (_providerPresets) return _providerPresets; _providerPresets = { github: getDefaultCommands(), - gitlab: buildPresetFromScripts('gitlab', ['team-activity', 'on-it-timestamps']), + // The ci-* concepts are DISABLED for gitlab, not merely unimplemented. A + // concept with no script falls through to the github default, so leaving + // them out would make a GitLab repo silently run `gh run list` against + // whatever GitHub remote gh happens to resolve — the silent-fallthrough + // class #1455 closed. Issue #13 asks for gitlab to "degrade loudly, not + // silently"; this is what makes it loud. + gitlab: buildPresetFromScripts('gitlab', [ + 'team-activity', 'on-it-timestamps', + 'ci-runs', 'ci-run-view', 'ci-failures', 'ci-run-log', + ]), // pr-search and pr-diff were disabled here until #12 shipped gitea scripts // for them. team-activity and on-it-timestamps stay disabled and are not // coming: both are `gh api graphql` pass-throughs and Forgejo has no @@ -136,7 +173,11 @@ function getProviderPresets(): Record> { // concept of its own, and without this it silently falls through to the // github default (`gh pr create`) instead of failing loudly. That's the // exact silent-fallthrough bug class #1455 closes. - linear: buildPresetFromScripts('linear', ['team-activity', 'on-it-timestamps', 'pr-create']), + linear: buildPresetFromScripts('linear', [ + 'team-activity', 'on-it-timestamps', 'pr-create', + // Linear has no CI of its own, and the same silent fallthrough applies. + 'ci-runs', 'ci-run-view', 'ci-failures', 'ci-run-log', + ]), }; return _providerPresets; } @@ -392,7 +433,7 @@ export async function executeForgeCommand( const { stdout } = await execAsync(command, { cwd: options?.cwd, env: { ...process.env, ...forgeEnv, ...env }, - timeout: 30_000, + timeout: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS, maxBuffer: options?.maxBuffer ?? DEFAULT_MAX_BUFFER, }); @@ -403,6 +444,97 @@ export async function executeForgeCommand( } } +/** Outcome of executeForgeCommandDetailed — every failure mode kept distinct. */ +export interface ForgeCommandResult { + /** True when the command exited 0. */ + ok: boolean; + /** Parsed stdout (JSON, or the raw string when `raw` is set). Null if unparseable or empty. */ + data: unknown | null; + /** Raw stdout, kept even on failure — the ci-* concepts print their error envelope there. */ + stdout: string; + stderr: string; + /** Process exit code, or null when the process was killed by a signal. */ + exitCode: number | null; + /** True when the command was killed for exceeding the timeout. */ + timedOut: boolean; + /** True when the concept has no command at all (disabled, or no provider script). */ + unavailable: boolean; + durationMs: number; +} + +/** + * Execute a forge concept command and report HOW it went, not merely whether. + * + * `executeForgeCommand` returns `null` for every failure mode: a timeout, a + * non-zero exit, unparseable output and a disabled concept are one value. That + * ambiguity has now cost this project several rounds — #12 shipped a fix for + * `pr-exists` returning a null that porch read as "no PR exists", and #17 and + * #8 both turned a stalled call into a generic failure with nothing naming the + * stall. A caller that needs to tell those apart uses this instead. + * + * Note in particular that `stdout` is returned even when `ok` is false. The CI + * concepts print a structured error envelope on stdout precisely so that the + * class of failure survives; discarding stdout on a non-zero exit would throw + * it away at the last step. + * + * Additive: `executeForgeCommand` is unchanged and no existing caller moves. + */ +export async function executeForgeCommandDetailed( + concept: string, + env?: Record, + options?: ForgeCommandOptions, +): Promise { + const started = Date.now(); + const forgeConfig = resolveForgeConfig(options); + const command = getForgeCommand(concept, forgeConfig); + + if (command === null) { + return { + ok: false, data: null, stdout: '', stderr: '', + exitCode: null, timedOut: false, unavailable: true, + durationMs: Date.now() - started, + }; + } + + const timeout = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const forgeEnv = buildForgeEnv(forgeConfig); + + try { + const { stdout, stderr } = await execAsync(command, { + cwd: options?.cwd, + env: { ...process.env, ...forgeEnv, ...env }, + timeout, + maxBuffer: options?.maxBuffer ?? DEFAULT_MAX_BUFFER, + }); + return { + ok: true, data: parseOutput(stdout, options?.raw), + stdout: String(stdout), stderr: String(stderr), + exitCode: 0, timedOut: false, unavailable: false, + durationMs: Date.now() - started, + }; + } catch (err: unknown) { + logDebug(concept, err); + const e = err as { code?: number | string; killed?: boolean; signal?: string; stdout?: string; stderr?: string }; + // `killed` plus a signal is how Node reports the timeout it enforced — + // verified during #12 against a command whose grandchild held the stdout + // pipe. An exit code alone cannot be read as a timeout: a killed process + // can still exit with a status, and a script that times out INTERNALLY (the + // shell watchdog in scripts/forge/_timeout.sh) exits non-zero with its own + // timeout envelope on stdout, which is why stdout is preserved below. + const timedOut = e.killed === true && typeof e.signal === 'string'; + return { + ok: false, + data: e.stdout ? parseOutput(e.stdout, options?.raw) : null, + stdout: e.stdout ?? '', + stderr: e.stderr ?? (err instanceof Error ? err.message : String(err)), + exitCode: typeof e.code === 'number' ? e.code : null, + timedOut, + unavailable: false, + durationMs: Date.now() - started, + }; + } +} + /** * Execute a forge concept command synchronously. * @@ -428,7 +560,7 @@ export function executeForgeCommandSync( cwd: options?.cwd, env: { ...process.env, ...forgeEnv, ...env }, encoding: 'utf-8', - timeout: 30_000, + timeout: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS, maxBuffer: options?.maxBuffer ?? DEFAULT_MAX_BUFFER, stdio: ['pipe', 'pipe', 'pipe'], });