From 05af5efe895b15e331b6ffd4afafb53ba95e9403 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 23:38:59 +0000 Subject: [PATCH 01/20] Add prerelease-test skill for release QA campaigns Captures the workflow used to validate the 0.9.9 release train as a repeatable skill, so the same analysis can be kicked off before any release. SKILL.md lays out the phases: discover what shipped from the pre-release branch changelogs and confirm every package published to PyPI, de-risk with a smoke test, fan out feature clusters that build real apps and drive them in a browser, upgrade-test reflex-examples apps from the previous stable, regression-test reflex-enterprise demos, audit wheel/sdist packaging, then triage into fix-before-release vs file-as-issue. Testing installs only published packages, baselines against the previous stable to tell regressions from pre-existing bugs, and verifies each claimed issue with an independent reproducer before reporting it. references/ holds the brief handed to each test agent (hard rules, environment traps, known-benign console noise), cluster decomposition guidance, the Workflow explore/verify pipeline with schemas and a port map, and report templates. scripts/ bundles three helpers, each validated against the released 0.9.9 train: - check_release_versions.py: changelog versions -> PyPI publication check (16 packages, all published) - audit_pyi.py: stubs present in both wheel and sdist, byte-identical, no foreign stubs, counts vs pyi_hashes.json (121 stubs, clean) - drive_app.py: Chromium driver with console/network capture, a small JSON action language and benign-noise filtering Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/SKILL.md | 170 +++++++++++ .../prerelease-test/references/agent-brief.md | 109 +++++++ .../prerelease-test/references/clusters.md | 52 ++++ .../references/orchestration.md | 140 +++++++++ .../prerelease-test/references/reporting.md | 104 +++++++ .../prerelease-test/scripts/audit_pyi.py | 266 ++++++++++++++++++ .../scripts/check_release_versions.py | 199 +++++++++++++ .../prerelease-test/scripts/drive_app.py | 252 +++++++++++++++++ pyproject.toml | 2 + 9 files changed, 1294 insertions(+) create mode 100644 .claude/skills/prerelease-test/SKILL.md create mode 100644 .claude/skills/prerelease-test/references/agent-brief.md create mode 100644 .claude/skills/prerelease-test/references/clusters.md create mode 100644 .claude/skills/prerelease-test/references/orchestration.md create mode 100644 .claude/skills/prerelease-test/references/reporting.md create mode 100755 .claude/skills/prerelease-test/scripts/audit_pyi.py create mode 100755 .claude/skills/prerelease-test/scripts/check_release_versions.py create mode 100755 .claude/skills/prerelease-test/scripts/drive_app.py diff --git a/.claude/skills/prerelease-test/SKILL.md b/.claude/skills/prerelease-test/SKILL.md new file mode 100644 index 00000000000..2a3d6548689 --- /dev/null +++ b/.claude/skills/prerelease-test/SKILL.md @@ -0,0 +1,170 @@ +--- +name: prerelease-test +description: Run the independent pre-release QA campaign for a Reflex release train. Discovers what shipped by reading every CHANGELOG.md on the pre-release branch, checks each package actually published to PyPI, then exercises the new features end-to-end in real apps driven by a real browser using ONLY published packages, upgrade-tests reflex-examples apps from the previous stable, regression-tests reflex-enterprise demos, audits wheel/sdist packaging, and produces a triaged FINDINGS.md plus a fix-before-release plan. Use this whenever a Reflex release is being prepared or checked — the user mentions a pre-release, an alpha/beta/rc, an `r/pre-*` branch, "about to release", release QA, validating a release candidate, or re-verifying that release-blocking fixes landed in a newer alpha. Also use it for the narrower slices on their own: smoke-testing published packages, upgrade/regression testing example apps against a new version, or auditing that packages ship their `.pyi` stubs in both wheel and sdist. +--- + +# Reflex pre-release testing + +This skill runs the QA campaign that stands between a pre-release and a release: independent, +adversarial, end-to-end exercise of what actually got published, from the perspective of a user +who runs `pip install reflex` and opens a browser. + +Its value comes from three habits, not from breadth of assertions: + +1. **Only published packages.** Everything installs from PyPI into throwaway venvs. The checkout + has unreleased code, different metadata, and a different dependency graph — testing it tells + you nothing about what users will get, and a package that failed to publish is itself a + release blocker you would otherwise miss. +2. **Baseline against the previous stable.** A defect that also reproduces on the last release is + a bug; one that only reproduces on the new version is a regression and usually a blocker. + Without the baseline run you cannot tell them apart, and the triage that follows is guesswork. +3. **Adversarially verify before reporting.** Have a second agent reproduce each claimed issue + from the written repro alone and actively try to refute it. In practice this refutes or + reclassifies a meaningful share of findings — and it is what makes the report trustworthy + enough for maintainers to act on directly. + +## Deliverables + +Produce these under `prerelease-testing/-/` in the repo and commit them to a work +branch as you go (never to `main`): + +- `FINDINGS.md` — executive summary, numbered findings with repro + evidence + regression status, + refuted claims, and per-cluster summaries. Templates: `references/reporting.md`. +- `RELEASE_PLAN.md` — triage into fix-before-release vs file-as-issue. Rubric below. +- One directory per test cluster containing the sample app sources (no `.web/`, `node_modules/`, + venvs), the driver scripts, `NOTES.md` with exact rerun commands, logs and screenshots. +- `README.md` — what each cluster covers and how to reuse it for the next release. + +Findings must be reproducible by a stranger from `NOTES.md` alone. That is the bar: a fix agent +gets handed the finding and nothing else. + +## Phases + +Work these in order. Phases 2–5 can overlap; 0 and 1 gate everything. + +### Phase 0 — Scope discovery + +Find the pre-release branch (`git ls-remote --heads origin 'r/pre-*'`, or the user names it), then +read **every** `CHANGELOG.md` on it: the root one plus `packages/*/CHANGELOG.md`. The top entry of +each is what this train ships. + +Run `scripts/check_release_versions.py ` to extract every package's version and confirm +each is published on PyPI. An unpublished version is a finding in its own right — report it +immediately rather than working around it. + +Read the linked PRs for anything whose intent is not obvious from the changelog line; the GitHub +MCP tools (`mcp__github__pull_request_read`) do this well. Understanding what a change was *for* +is what lets you test it as a user rather than as a checklist. + +### Phase 1 — De-risk with a smoke test + +Before any fan-out: one venv, `reflex init --template blank`, `reflex run`, drive it in Chromium +with `scripts/drive_app.py`. This shakes out environment problems (proxy, bun installs, browser +path) once, in a context where you can debug them, instead of inside ten parallel agents. + +Also confirm the dependency graph resolves the way the changelog says it should (e.g. optional +dependencies genuinely absent, sub-packages pinned as intended). + +### Phase 2 — Feature exploration fan-out + +Decompose the changelog into clusters of related changes and give each to its own agent, with an +adversarial verification stage behind it. See `references/clusters.md` for how to cut the clusters +and a standing coverage list; `references/orchestration.md` for the Workflow script, schemas and +port map. + +The instruction that makes this productive: **real-world exploration, not coverage filling.** +Combine each new feature with the things users actually combine it with — State vars, +`rx._x.client_state`, `rx.ComponentState`, `@rx.memo` wrapping, `rx.foreach`/`rx.cond`, event +chains, background tasks, multiple pages and navigation, dev *and* prod mode. Bugs live in the +interactions, and the original PR's tests already cover the happy path. + +Every run inspects all four channels: server log, browser console (errors *and* warnings), network +tab (failed requests, 4xx/5xx), and the rendered page. Findings frequently show up in a channel +nobody asserted on. + +### Phase 3 — Upgrade regression + +Users upgrade in place; that path has its own failure modes (lockfile migration, pruned packages, +stale `.web/`). Clone `reflex-dev/reflex-examples`, pick apps that span the feature surface — +including ones using third-party packages (`reflex-local-auth`, `reflex-global-hotkey`) and +`reflex[db]`/API apps — then for each: + +1. Install the **previous stable** (no `--prerelease` flag, so requirements resolve as a user's + would), run it, and drive its real user flows in the browser. This is your baseline. +2. Upgrade the *same* venv and *same* app directory in place to the new version, preserving + `.web/` and `reflex.lock/`. Watch the first run's log closely — that is where migration + happens. Re-drive the identical flows and compare. +3. Do one cold run (`rm -rf .web`) to check the fresh-install path converges to the same state. + +Diff `.web/package.json` before and after; unexpected dependency changes are findings. + +### Phase 4 — Enterprise regression + +Downstream breakage is the most common release blocker, because removing a public name is +invisible until something imports it. Install the **published** `reflex-enterprise` (never the +checkout) against the new reflex and run its demos — ag-grid, map, dnd, mantine, flow, the MCP +plugin, OIDC. Baseline against the previous stable whenever something fails, so you can say +whether the new release broke it. + +Before running anything, grep the published enterprise wheel for names the changelog says moved or +were removed; that finds the breakage in seconds instead of hours. + +### Phase 5 — Packaging audit + +Run `scripts/audit_pyi.py` over every package in the train. It verifies each ships its own +generated `.pyi` stubs in **both** wheel and sdist, that they are byte-identical between the two, +that no package leaks another package's stubs, and that counts line up with `pyi_hashes.json`. + +Stub content hashes legitimately differ from the committed manifest because the build hook +regenerates them at release time — compare *presence and counts*, and wheel against sdist. + +### Phase 6 — Triage and report + +Write `FINDINGS.md` first (everything confirmed), then `RELEASE_PLAN.md` splitting findings into: + +**Fix before release** — anything meeting one of: +- a confirmed regression against the previous stable, +- security-relevant (path traversal, unauthenticated input handling, injection), +- significant user impact, or trivially small to fix (a one-line guard, a missing default). + +**File as issues, fix after** — everything else, including pre-existing defects the campaign +happened to surface. Downstream (enterprise) issues go to that project's tracker, not this repo's. + +Also flag decisions only a maintainer can make — an undocumented behavior change is either a bug +or a missing changelog entry, and which one it is is their call. Ask rather than assume. + +### Phase 7 — Re-verify the next build + +When a new alpha ships with the fixes, re-run the *original failing repro* for each finding +(not just the unit tests) against the new published packages, plus a general smoke. Append the +pass/fail table to `FINDINGS.md`. When the final ships, do one last stock-install smoke: +`pip install reflex==` with no prerelease flag, init, run, browser, and confirm the +resolved graph is all-final. + +## Working rules + +- **Isolation.** Every agent gets its own venv and its own reserved port range; never install into + a shared venv. See the port map in `references/orchestration.md`. +- **The traps are real.** `references/agent-brief.md` is the brief to hand every agent — it carries + the environment gotchas (checkout shadowing, proxy variables breaking installs) and the list of + known-benign console noise. Agents that skip it rediscover the same problems and report noise as + findings. +- **Concurrency.** Dev servers are heavy (bun + vite + granian). On a 4-CPU box run about two + agents at a time and one dev server per agent; more just makes everything slow and flaky. +- **Commit as you go.** Each cluster's artifacts land on the work branch when that cluster + finishes, so a long campaign is never one unrecoverable batch. +- **Never fix framework code during the campaign.** Testing and fixing are separate engagements + with separate review standards; mixing them costs you the independence that makes the findings + credible. Record the repro and move on. (If the user then asks for fixes, that is a new task — + one PR per finding, regression test first, `references/reporting.md` has the PR conventions.) + +## Scaling the campaign + +Full campaign is roughly 30 agents over several hours. Scale down by dropping whole phases rather +than by testing each phase more shallowly — a shallow pass produces false confidence: + +- **Quick check** (~30 min): phases 0, 1, 5, plus grep-based downstream checks from phase 4. +- **Standard** (~2 h): add phase 2 over the three or four highest-risk clusters. +- **Full**: everything, with adversarial verification on every claimed issue. + +Ask the user which they want if the request is ambiguous and the difference matters. diff --git a/.claude/skills/prerelease-test/references/agent-brief.md b/.claude/skills/prerelease-test/references/agent-brief.md new file mode 100644 index 00000000000..7754c1a3584 --- /dev/null +++ b/.claude/skills/prerelease-test/references/agent-brief.md @@ -0,0 +1,109 @@ +# The agent brief + +Copy this into the campaign scratchpad (e.g. `$SB/AGENT_BRIEF.md`), fill the `<...>` placeholders, +and tell every test agent to read it first. It carries the rules that keep results trustworthy and +the environment traps that otherwise get rediscovered — expensively — by each agent in turn. + +Keep it in a file rather than pasting it into each prompt: agents re-read it when they get +confused, and updating one file updates the whole fleet. + +--- + +## BRIEF TEMPLATE (copy from here) + +You are one of several agents independently exercising the reflex `` pre-release +(published ``) as a real-world user of the framework. Your job: build small sample apps and +repro scripts for your assigned feature cluster, run them END-TO-END (real server, real Chromium), +and hunt for anomalies. You REPORT issues; you never fix framework code. + +### HARD RULES + +1. **NEVER install reflex (or any workspace package) from the local checkout at ``.** No + `uv sync`, no `uv run`, no `pip install -e`, no `uv pip install .` anywhere under it. + Everything installs from PyPI. We are testing what users receive, not what the tree contains. +2. **Never run python with the checkout as your working directory.** `/reflex/` shadows the + installed package, so `import reflex` silently picks up unreleased source and your results + become fiction. Run scripts from a neutral directory and start each repro with an assertion: + ```python + import reflex; assert "" in reflex.__file__, reflex.__file__ + ``` +3. Reading the checkout is fine and encouraged — release source is on branch ``. + For PR context load the GitHub MCP tools via ToolSearch (`select:mcp__github__pull_request_read`). +4. Do NOT run any `git` write commands (add/commit/checkout/...) in the checkout. The orchestrator + commits artifacts. +5. Do NOT fix bugs you find — record precise repro steps instead. +6. Kill every server and browser you start before you finish (track PIDs; verify with `ps`). Other + agents share this machine — run at most ONE dev server at a time unless your cluster needs two. + +### Environment + +- Scratchpad root: `SB=` +- **Prebuilt shared venv (READ-ONLY — never install into it):** `$SB/envs/` has + `reflex==` and the alpha sub-packages. Use `$SB/envs//bin/reflex` directly. +- Need other deps or versions? Make your OWN venv: + ``` + uv venv $SB/envs/ --python 3.11 + uv pip install --python $SB/envs//bin/python --prerelease=allow 'reflex==' + ``` + Always pass `--prerelease=allow` for alphas; PyPI is the default index — never point it at the + checkout. Python 3.12/3.13/3.14 are available via `uv venv --python 3.14` etc. +- **Playwright driver venv:** `$SB/envs/driver/bin/python` (playwright, httpx, websockets). + Chromium: `/opt/pw-browsers/chromium` — launch with + `p.chromium.launch(executable_path="/opt/pw-browsers/chromium")`. + A ready-made driver with console/network capture is at `/scripts/drive_app.py`. +- **Local HTTP needs proxy bypass on the CLIENT side only:** prefix curl/Playwright commands with + `NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1` (curl: `--noproxy '*'`). + Do NOT export those variables into the reflex server's environment — it breaks bun's package + installs through the proxy, which looks like a framework bug and is not. +- Set `REFLEX_TELEMETRY_ENABLED=false` for every reflex command. +- Node and bun are preinstalled. Enterprise apps need `CI=true` to bypass the dev login gate. +- App working dirs: `$SB/apps//...`. Run servers on YOUR ASSIGNED PORTS: + `reflex run --frontend-port --backend-port `. The first run does a bun install (1–2 min); + poll the frontend URL until it returns 200 for up to ~6 minutes before concluding failure. +- `--loglevel debug` gives verbose server logs; redirect to a file and actually read it. + +### What "testing" means here + +- Real-world exploration, not coverage filling. Combine the feature with State vars, + `rx._x.client_state`, `@rx.memo` wrapping, `rx.ComponentState`, `rx.foreach`/`rx.cond`, event + chains (`yield Other.handler()`), background tasks, multiple pages and navigation — whatever + plausibly interacts. The original PR already tested the happy path. +- Drive the app in Chromium as a user would: click, type, navigate, upload, drag, use the keyboard. +- On EVERY run capture and inspect: (a) the server log file, (b) browser console messages (errors + AND warnings), (c) failed requests / 4xx-5xx responses, (d) screenshots at key moments. +- Baseline comparisons are what make findings actionable: if behavior looks wrong, check the + previous stable (`uv pip install 'reflex=='` in its own venv). Only-on-new is a + regression and high severity; both-versions is context worth noting, not a blocker. +- Prod matters too: `reflex run --env prod` compiles and serves the built frontend. Test both modes + when your feature could differ (hydration, memoization, routing, prerender). + +### Known-benign noise — do not report these as findings + +- Browser console: the React Router "💿 Hey developer" HydrateFallback log, vite + `connecting.../connected` debug lines, the React DevTools download info line. +- `reflex init` logging a few "Failed to connect to https://registry.npmmirror.com" lines before + falling back (this environment blocks that mirror). Do report it if the fallback itself fails. +- Transient bun "incorrect peer dependency" warnings during a one-time upgrade migration, provided + the final lockfile is consistent. + +If you see something surprising that is not on this list, investigate it — several real findings +have surfaced first as an unexplained warning. + +### Deliverables (mandatory) + +1. Copy reusable artifacts into the repo (plain `cp`/`rsync`, no git): + `DEST=//` + - the app source dirs, EXCLUDING `.web/`, `node_modules/`, `.states/`, `assets/external/`, + `*.db`, venvs + - your Playwright/repro scripts + - `NOTES.md`: what you tested, how to rerun it (exact commands), what you observed, including + benign quirks +2. Return structured findings: every discrete check as a test entry (pass/fail/anomaly/skipped) + with enough repro detail that another agent can reproduce a failure from `NOTES.md` and your + scripts alone, without your conversation. An "anomaly" is anything surprising (console error, + warning, traceback, visual glitch, perf cliff) even when functionality works. + +### Timeboxing + +Be thorough but keep moving: if one sub-test resists debugging for ~10 minutes, record it as an +anomaly with logs attached and continue. Finishing the whole cluster beats perfecting one test. diff --git a/.claude/skills/prerelease-test/references/clusters.md b/.claude/skills/prerelease-test/references/clusters.md new file mode 100644 index 00000000000..2ae20b7144d --- /dev/null +++ b/.claude/skills/prerelease-test/references/clusters.md @@ -0,0 +1,52 @@ +# Cutting the changelog into clusters + +A cluster is one agent's assignment: a group of changelog entries that share a subsystem, so the +agent can build one or two apps that exercise all of them together and notice how they interact. + +## How to cut them + +Group by **where the change lives at runtime**, not by which PR shipped it. Entries that touch the +same subsystem belong together even when they came from unrelated PRs, because the interesting +bugs are in their interaction — and one app can cover them all. + +Aim for 8–12 clusters of 2–6 changelog entries each. A cluster that needs more than two apps is +too big; one with a single trivial entry should be folded into a neighbour. + +Each cluster brief should carry: +- the changelog lines verbatim (agents should not have to re-derive what shipped), +- the PR numbers, so the agent can read intent, +- concrete "combine it with" suggestions — this is what turns a checklist into exploration, +- anything the entry implies but does not state (a perf claim to measure, a behavior that should + differ between dev and prod). + +## Standing coverage checklist + +Most Reflex releases touch some subset of these. Use it to check you have not left a shipped change +unassigned, and as a source of interaction ideas. + +| Area | What to build | Interactions worth forcing | +|---|---|---| +| Routing / navigation | Multi-page app with dynamic (`[id]`), catchall (`[[...splat]]`) and static-sibling routes; slow `on_load` | Navigate away mid-`on_load`; two tabs on different dynamic values; browser back/forward; direct load vs client-side nav; prod mode | +| State & event loop | Counter hammered by clicks while a background task writes | Background task + foreground handler racing; `yield`-streaming handlers; `async with self`; abrupt disconnect mid-stream | +| Components / props | One page per changed component | Prop bound to a State var, to a `client_state` var, and inside `@rx.memo`; explicit `style=` merged with forwarded props | +| Memoization | `@rx.memo` component with state-bound props and an event-handler prop, plus render counters | Memo in `rx.foreach`; memo wrapping `ComponentState`; two memos sharing a name in different modules; dev vs prod | +| Typing / annotations | State vars using every annotation shape the entry mentions | Assign at runtime (not just compile); pass the handler *uncalled* to a trigger; run on the oldest and newest supported Python | +| Uploads | `rx.upload` with buffered and streamed handlers | Hostile filenames (traversal, unicode, all-dots, spaces); raw multipart via httpx alongside browser uploads | +| Logging / CLI | Run every CLI verb with and without the logging flags | Strict JSON-lines parsing of stdout; a user app that configures `logging` itself; missing optional sub-packages | +| Config / plugins | App with the config knob set, unset, and set via env var | Deprecated form alongside the new form; plugin explicitly enabled vs implicit | +| Build / prod / export | Multi-page app through `run --env prod`, `export`, `run --env preview` | Serve exported frontend statically with a backend-only server; inspect generated `.web/package.json`; custom component with a changed library | +| Registration / multi-app | Two apps in one process; `AppHarness` twice | Sequential and simultaneous; check for registration leakage between them | +| Optional dependencies | Bare install, each extra, and the upgrade path | Import the optional package's feature without the extra and judge the error message quality | +| DevTools / perf | App with nested substates, custom components, client-only wrappers | Walk React fibers for `displayName`; record navigation main-thread cost; verify perf claims rather than trusting them | + +## Downstream and packaging clusters + +Two clusters that are not feature-shaped but catch the highest-severity problems: + +- **Breaking-change surface.** For every entry under "Breaking Changes" (and every removal you spot + in the diff), ask what a downstream caller sees. Grep the published `reflex-enterprise` wheel and + the popular third-party packages (`reflex-local-auth`, `reflex-global-hotkey`) for the removed + names. A bare `AttributeError`/`ImportError` with no pointer to the replacement is a finding even + when the removal itself was intended. +- **Packaging.** `scripts/audit_pyi.py`, plus a check that each package's declared dependency pins + match what the changelog says and that installing from the sdist works. diff --git a/.claude/skills/prerelease-test/references/orchestration.md b/.claude/skills/prerelease-test/references/orchestration.md new file mode 100644 index 00000000000..915c75f2baf --- /dev/null +++ b/.claude/skills/prerelease-test/references/orchestration.md @@ -0,0 +1,140 @@ +# Orchestrating the fleet + +## Choosing the mechanism + +Invoking this skill is itself the opt-in for the Workflow tool, so a full campaign should use it — +the explore→verify pipeline is deterministic control flow over many agents, exactly what Workflow +is for. For a quick check or a single cluster, plain `Agent` calls are simpler and cheaper. + +Use one workflow per phase rather than one giant script. Phase results change what you want to ask +next (a cluster that finds a breaking change reshapes the enterprise phase), and you stay in the +loop between phases. + +## The explore → verify pipeline + +The shape that matters: each cluster's findings go straight into verification without waiting for +the other clusters. Use `pipeline`, not a barrier — a slow cluster should not hold up verification +of a fast one. + +```js +export const meta = { + name: 'prerelease-explore', + description: 'Exercise feature clusters end-to-end, then adversarially verify findings', + phases: [{ title: 'Explore' }, { title: 'Verify' }], +} + +const SB = '' +const DEST = '' + +const FINDINGS_SCHEMA = { + type: 'object', + required: ['cluster', 'summary', 'tests', 'issues', 'artifacts_dir'], + properties: { + cluster: { type: 'string' }, + summary: { type: 'string' }, + tests: { + type: 'array', + items: { + type: 'object', + required: ['name', 'status', 'details'], + properties: { + name: { type: 'string' }, + status: { type: 'string', enum: ['pass', 'fail', 'anomaly', 'skipped'] }, + details: { type: 'string' }, + repro: { type: 'string' }, + }, + }, + }, + issues: { + type: 'array', + items: { + type: 'object', + required: ['title', 'severity', 'repro'], + properties: { + title: { type: 'string' }, + severity: { type: 'string', enum: ['critical', 'high', 'medium', 'low'] }, + repro: { type: 'string' }, + evidence: { type: 'string' }, + regression: { type: 'boolean', description: 'true if the previous stable behaves correctly' }, + }, + }, + }, + artifacts_dir: { type: 'string' }, + }, +} + +const VERDICT_SCHEMA = { + type: 'object', + required: ['confirmed', 'notes'], + properties: { + confirmed: { type: 'boolean' }, + notes: { type: 'string' }, + root_cause_guess: { type: 'string' }, + }, +} + +phase('Explore') +const results = await pipeline( + CLUSTERS, + (c, _item, i) => agent(brief(c, i), { label: `explore:${c.key}`, phase: 'Explore', schema: FINDINGS_SCHEMA }), + async (res, c, i) => { + if (!res) return { cluster: c.key, agent_died: true } + const issues = (res.issues || []).slice(0, 4) // cap verification fan-out per cluster + if (!issues.length) return res + const verdicts = await parallel( + issues.map((iss, j) => () => + agent(verifyBrief(c, iss, j), { label: `verify:${c.key}:${j}`, phase: 'Verify', schema: VERDICT_SCHEMA }) + .then((v) => ({ issue: iss, verdict: v })), + ), + ) + return { ...res, verified_issues: verdicts.filter(Boolean) } + }, +) +return results.filter(Boolean) +``` + +The verifier's prompt is what makes this work. It must: +- reproduce **from the written repro alone**, in its own working directory, not by reading the + reporter's conversation — that is simultaneously a test of the repro's quality, +- be told to actively **refute**: is this an environment quirk, an API misuse, pre-existing on the + previous stable, or a genuine defect? +- set `confirmed: true` only for a genuine defect a fix agent should act on, +- append a `## VERIFICATION` section to the cluster's `NOTES.md` so the record travels with the + artifacts. + +## Port map + +Give every agent a disjoint range and tell it to always pass `--frontend-port/--backend-port` +explicitly. Never let anything use the defaults (3000/8000) — that is how two agents collide and +produce confusing, unreproducible results. + +| Stage | Frontend | Backend | +|---|---|---| +| Explore agent *i* | `3100 + 40i` … +19 | `8100 + 40i` … +19 | +| Verify agent *i.j* | `3600 + 40i + 4j` … +3 | `8600 + 40i + 4j` … +3 | +| Orchestrator smoke | 3050–3059 | 8050–8059 | + +## Prompt scaffolding for each agent + +Every cluster prompt should open with the same preamble, then the cluster-specific brief: + +``` +FIRST read /AGENT_BRIEF.md and follow every rule in it (isolated PyPI-only installs — never +install from the checkout; end-to-end browser testing; artifact + NOTES.md deliverables; kill your +processes). +Your cluster key: "". Working dir: /apps//. Artifact destination: //. +Your RESERVED ports: frontend -, backend - — always pass them explicitly. + +Also in scope: anything adjacent you notice (log warnings, console anomalies, network errors, +visual glitches). Record benign-but-surprising observations in NOTES.md and as 'anomaly' entries. +``` + +## Running it + +- Two agents at a time is right for a 4-CPU box; the workflow's own concurrency cap handles the + rest of the queue. +- Between phases, read the results before launching the next one. +- Schedule a check-in (`send_later`, ~45 min) during long runs so a wedged agent or a leaked dev + server gets noticed. Check `ps` for stray `reflex run` processes and commit finished clusters. +- When the workflow completes, its full structured result is in the task output file; parse it for + the report rather than re-reading transcripts. diff --git a/.claude/skills/prerelease-test/references/reporting.md b/.claude/skills/prerelease-test/references/reporting.md new file mode 100644 index 00000000000..242eae436f7 --- /dev/null +++ b/.claude/skills/prerelease-test/references/reporting.md @@ -0,0 +1,104 @@ +# Reporting + +## FINDINGS.md + +The audience is a maintainer deciding what blocks the release, and fix agents who will get handed +one finding and nothing else. Write for both: a summary that supports a go/no-go call, and findings +self-contained enough to act on in isolation. + +```markdown +# Findings — reflex pre-release testing () + + + +## Versions under test + + + +## Executive summary + + + + +Index: +- FINDING-001: (SEVERITY) +- ... + +## FINDING-00N: <title> (SEVERITY) + +- Cluster: `<cluster>` | Regression vs <prev stable>: yes/no | Verifier: CONFIRMED +- Repro: <exact commands/steps, self-contained> +- Evidence: <log excerpt, console error, screenshot path> +- Root cause (verifier analysis): <file:line and mechanism, when known> +- Verification notes: <what the independent reproducer found> + +## Refuted / reclassified claims +- **<title>** (`<cluster>`): <why it is not an actionable defect for this release> + +## Cluster summaries +### `<cluster>` (pass:N, anomaly:N, fail:N) +<2-4 sentences: what was built, what was verified, what was found.> +``` + +Two things carry disproportionate weight and are easy to skip: + +- **The refuted list.** It shows the findings were filtered, which is why a maintainer can trust the + ones that remain. Never quietly drop a refuted claim. +- **Regression status on every finding.** "Broken in both versions" and "newly broken" lead to + opposite decisions, and the reader cannot recover that from the description. + +## RELEASE_PLAN.md + +```markdown +# Release plan — what blocks <version> vs what gets filed + +## Already in flight +| PR | Covers | Gap check | +<For each open PR, which findings it addresses and — importantly — what it does NOT cover.> + +## Fix before release +### Security +### Confirmed regressions +### High impact and/or trivially small +<Each item: finding ref, one-line rationale tied to the rubric, and the shape of the fix.> + +## File as issues, fix after release +### <this repo> +### <downstream repo> + +## Decisions needed from a maintainer +<Behavior changes that are either bugs or missing changelog entries — their call, not yours.> + +## Suggested sequencing +<Order of PRs, noting which can land in parallel and which touch the same files.> +``` + +The rubric (confirmed regression / security / significant-impact-or-trivial) is in `SKILL.md`. +Apply it visibly: each entry should say which arm of the rubric put it there, so the maintainer can +disagree with a specific judgment rather than the whole list. + +## Filing issues + +For post-release items: search for duplicates first (2–3 keyword variants, open *and* closed), then +file with a self-contained repro, observed vs expected, environment, a note that it came from +`<version>` pre-release testing, and a pointer to the artifacts branch and cluster directory. +Mirror `.github/ISSUE_TEMPLATE/bug_report.md` where it fits. + +## If the user asks for the fixes afterwards + +That is a separate engagement. What worked well: + +- **One PR per finding**, branched from current `main`, in its own git worktree so parallel fix + agents never collide. +- **Regression test first**: write it, show it failing against unfixed `main`, then fix, then show + it passing. Report both states in the PR body — a reviewer should not have to take it on faith. +- **News fragment** under the right package's `news/` directory. The changelog CI gate checks per + package: a fix touching `packages/reflex-base/` needs a fragment in + `packages/reflex-base/news/`, and a root-only fragment will fail the gate. +- **Adversarial review of each PR** before handing it over: an independent agent re-runs the + regression test against unfixed source to confirm it actually pins the bug, and checks repo + conventions. This catches PRs whose test passes either way. +- PR body: cite the FINDING number and that it came from pre-release testing, describe the defect + mechanism, the fix, and the test plan including the failing-before evidence. +- Watch CI after opening (`subscribe_pr_activity`) and drive each PR to green. diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py new file mode 100755 index 00000000000..85b28e4fb86 --- /dev/null +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Audit that published packages ship their generated .pyi stubs correctly. + +For each package it downloads the wheel and the sdist from PyPI and checks: + + * the package ships stubs for its own modules in BOTH artifacts, + * those stubs are byte-identical between wheel and sdist, + * no artifact carries stubs belonging to a different package, + * per-package stub counts match the repo's pyi_hashes.json manifest (optional). + +Stub *content* legitimately differs from the committed manifest hashes because the build +hook regenerates stubs at release time, so this compares presence and counts rather than +hashes. A package with no manifest entries is expected to ship no stubs. + +Usage: + audit_pyi.py reflex==0.9.9 reflex-base==0.9.9 + check_release_versions.py --ref origin/main --specs | xargs audit_pyi.py --manifest-ref origin/main +""" + +from __future__ import annotations + +import argparse +import collections +import json +import subprocess +import sys +import tarfile +import tempfile +import urllib.request +import zipfile +from pathlib import Path + + +def download_artifacts(name: str, version: str, dest: Path) -> dict[str, Path]: + """Download the wheel and sdist for a package version. + + Args: + name: The PyPI distribution name. + version: The version to download. + dest: Directory to download into. + + Returns: + A mapping of ``"wheel"``/``"sdist"`` to the downloaded file paths. + """ + with urllib.request.urlopen( + f"https://pypi.org/pypi/{name}/{version}/json", timeout=60 + ) as r: + data = json.load(r) + out: dict[str, Path] = {} + for entry in data.get("urls", []): + kind = entry["packagetype"] + if kind not in ("bdist_wheel", "sdist"): + continue + path = dest / entry["filename"] + if not path.exists(): + urllib.request.urlretrieve(entry["url"], path) + out["wheel" if kind == "bdist_wheel" else "sdist"] = path + return out + + +def wheel_stubs(path: Path) -> dict[str, bytes]: + """Read every stub in a wheel. + + Args: + path: Path to the wheel file. + + Returns: + A mapping of archive-relative stub path to its contents. + """ + with zipfile.ZipFile(path) as z: + return {n: z.read(n) for n in z.namelist() if n.endswith(".pyi")} + + +def sdist_stubs(path: Path) -> dict[str, bytes]: + """Read every stub in an sdist, normalized to the wheel's layout. + + The ``<name>-<version>/`` prefix and any ``src/`` layout directory are stripped so the + keys line up with the wheel's, making the two directly comparable. + + Args: + path: Path to the sdist tarball. + + Returns: + A mapping of normalized stub path to its contents. + """ + out: dict[str, bytes] = {} + with tarfile.open(path) as t: + for member in t.getmembers(): + if not member.name.endswith(".pyi"): + continue + parts = member.name.split("/")[1:] # strip the <name>-<version>/ prefix + if parts and parts[0] == "src": # src-layout packages + parts = parts[1:] + handle = t.extractfile(member) + if handle is not None: + out["/".join(parts)] = handle.read() + return out + + +def import_root(dist_name: str) -> str: + """Guess the top-level import package for a distribution name. + + Args: + dist_name: The PyPI distribution name. + + Returns: + The import package name, which differs only by underscores by convention. + """ + return dist_name.replace("-", "_") + + +def manifest_counts(ref: str, repo: str) -> dict[str, int]: + """Count the stubs each distribution is expected to ship. + + Args: + ref: The git ref to read ``pyi_hashes.json`` from. + repo: Path to the reflex checkout. + + Returns: + A mapping of distribution name to expected stub count, empty when the manifest is + unavailable at that ref. + """ + result = subprocess.run( + ["git", "-C", repo, "show", f"{ref}:pyi_hashes.json"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return {} + counts: collections.Counter[str] = collections.Counter() + for key in json.loads(result.stdout): + # "packages/<dist>/src/<pkg>/x.pyi" or a root-package path like "reflex/x.pyi" + counts[key.split("/")[1] if key.startswith("packages/") else "reflex"] += 1 + return dict(counts) + + +def audit(spec: str, workdir: Path, expected: dict[str, int]) -> dict: + """Audit the stubs shipped by one package version. + + Args: + spec: A ``name==version`` spec. + workdir: Directory to download artifacts into. + expected: Expected stub counts by distribution name, from the manifest. + + Returns: + A record with the package, version, stub count and any problems found. + """ + name, _, version = spec.partition("==") + root = import_root(name) + problems: list[str] = [] + + artifacts = download_artifacts(name, version, workdir) + if "wheel" not in artifacts or "sdist" not in artifacts: + problems.append(f"missing artifact kinds: has {sorted(artifacts)}") + return {"package": name, "version": version, "problems": problems, "own": 0} + + wheel = wheel_stubs(artifacts["wheel"]) + sdist = sdist_stubs(artifacts["sdist"]) + + def split(stubs: dict[str, bytes]) -> tuple[dict[str, bytes], dict[str, bytes]]: + own = {k: v for k, v in stubs.items() if k.split("/")[0] == root} + foreign = { + k: v + for k, v in stubs.items() + if k.split("/")[0] != root and not k.split("/")[0].endswith(".dist-info") + } + return own, foreign + + wheel_own, wheel_foreign = split(wheel) + sdist_own, sdist_foreign = split(sdist) + + if wheel_foreign: + problems.append( + f"wheel carries {len(wheel_foreign)} foreign stub(s): {sorted(wheel_foreign)[:3]}" + ) + if sdist_foreign: + problems.append( + f"sdist carries {len(sdist_foreign)} foreign stub(s): {sorted(sdist_foreign)[:3]}" + ) + + only_wheel = sorted(set(wheel_own) - set(sdist_own)) + only_sdist = sorted(set(sdist_own) - set(wheel_own)) + if only_wheel: + problems.append( + f"{len(only_wheel)} stub(s) in wheel but not sdist: {only_wheel[:3]}" + ) + if only_sdist: + problems.append( + f"{len(only_sdist)} stub(s) in sdist but not wheel: {only_sdist[:3]}" + ) + + differing = sorted( + k for k in set(wheel_own) & set(sdist_own) if wheel_own[k] != sdist_own[k] + ) + if differing: + problems.append( + f"{len(differing)} stub(s) differ between wheel and sdist: {differing[:3]}" + ) + + want = expected.get(name) + if want is not None and want != len(wheel_own): + problems.append( + f"expected {want} stub(s) from manifest, wheel ships {len(wheel_own)}" + ) + + return { + "package": name, + "version": version, + "own": len(wheel_own), + "expected": want, + "problems": problems, + } + + +def main() -> int: + """Audit every requested package and print a summary. + + Returns: + ``1`` when any package has packaging problems, otherwise ``0``. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("specs", nargs="+", help="package==version specs") + parser.add_argument( + "--manifest-ref", help="git ref to read pyi_hashes.json from for count checks" + ) + parser.add_argument( + "--repo", default=".", help="path to the reflex checkout (default: cwd)" + ) + parser.add_argument("--keep", help="directory to keep downloaded artifacts in") + args = parser.parse_args() + + expected = ( + manifest_counts(args.manifest_ref, args.repo) if args.manifest_ref else {} + ) + + with tempfile.TemporaryDirectory() as tmp: + workdir = Path(args.keep) if args.keep else Path(tmp) + workdir.mkdir(parents=True, exist_ok=True) + results = [audit(spec, workdir, expected) for spec in args.specs] + + width = max(len(r["package"]) for r in results) + print(f"pyi packaging audit — {len(results)} package(s)\n") + for r in results: + mark = "OK " if not r["problems"] else "!! " + want = "" if r["expected"] is None else f" (manifest: {r['expected']})" + print( + f" {mark}{r['package']:<{width}} {r['version']:<12} stubs={r['own']}{want}" + ) + for problem in r["problems"]: + print(f" - {problem}") + + failed = [r for r in results if r["problems"]] + print() + if failed: + print(f"FAIL: {len(failed)} package(s) with packaging problems.") + else: + total = sum(r["own"] for r in results) + print( + f"PASS: {total} stubs ship correctly in both wheel and sdist; no foreign stubs." + ) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/skills/prerelease-test/scripts/check_release_versions.py b/.claude/skills/prerelease-test/scripts/check_release_versions.py new file mode 100755 index 00000000000..79615de530b --- /dev/null +++ b/.claude/skills/prerelease-test/scripts/check_release_versions.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Discover what a release train ships and confirm every package published to PyPI. + +Reads the root CHANGELOG.md plus every packages/*/CHANGELOG.md at a git ref, takes the +top version entry from each, maps it to the PyPI distribution name declared in that +package's pyproject.toml, and checks the version exists on PyPI. + +An unpublished version is a release blocker in its own right, so a missing package makes +this exit non-zero. + +Usage: + check_release_versions.py --ref origin/r/pre-2026-08-27-1234 + check_release_versions.py --ref origin/main --json + check_release_versions.py --ref origin/main --specs # pkg==ver lines for audit_pyi.py +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import urllib.error +import urllib.request + +VERSION_HEADING = re.compile(r"^##\s+v?([0-9][^\s]*)", re.MULTILINE) +PROJECT_NAME = re.compile(r"^name\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) + + +def git_show(ref: str, path: str, repo: str) -> str | None: + """Read a file's contents at a git ref. + + Args: + ref: The git ref to read from. + path: Repo-relative path of the file. + repo: Path to the reflex checkout. + + Returns: + The file contents, or ``None`` when the path does not exist at that ref. + """ + result = subprocess.run( + ["git", "-C", repo, "show", f"{ref}:{path}"], + capture_output=True, + text=True, + check=False, + ) + return result.stdout if result.returncode == 0 else None + + +def changelog_paths(ref: str, repo: str) -> list[str]: + """Find the changelogs that describe a release train. + + Args: + ref: The git ref to list files from. + repo: Path to the reflex checkout. + + Returns: + Paths of the root ``CHANGELOG.md`` and every ``packages/*/CHANGELOG.md``. + """ + listing = subprocess.run( + ["git", "-C", repo, "ls-tree", "-r", "--name-only", ref], + capture_output=True, + text=True, + check=True, + ).stdout.splitlines() + return [ + p + for p in listing + if p == "CHANGELOG.md" or re.fullmatch(r"packages/[^/]+/CHANGELOG.md", p) + ] + + +def dist_name(changelog_path: str, ref: str, repo: str) -> str: + """Resolve the PyPI distribution name for the package owning a changelog. + + Args: + changelog_path: Repo-relative path of the package's ``CHANGELOG.md``. + ref: The git ref to read ``pyproject.toml`` from. + repo: Path to the reflex checkout. + + Returns: + The declared project name, falling back to the package directory name. + """ + pyproject_path = changelog_path.replace("CHANGELOG.md", "pyproject.toml") + content = git_show(ref, pyproject_path, repo) or "" + match = PROJECT_NAME.search(content) + if match: + return match.group(1) + # Fall back to the directory name, which matches the dist name by convention. + parts = changelog_path.split("/") + return parts[1] if len(parts) > 2 else "reflex" + + +def pypi_status(name: str, version: str) -> tuple[bool, str]: + """Check whether a distribution version is installable from PyPI. + + Args: + name: The PyPI distribution name. + version: The version to look for. + + Returns: + A ``(published, detail)`` tuple, where detail names the artifact kinds found or + why the check failed. + """ + url = f"https://pypi.org/pypi/{name}/{version}/json" + try: + with urllib.request.urlopen(url, timeout=30) as response: + data = json.load(response) + kinds = sorted({u["packagetype"] for u in data.get("urls", [])}) + if not kinds: + return False, "published but no files" + return True, "+".join( + k.replace("bdist_wheel", "wheel").replace("sdist", "sdist") for k in kinds + ) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return False, "NOT ON PYPI" + return False, f"HTTP {exc.code}" + except ( + Exception + ) as exc: # network/proxy trouble should not look like a missing package + return False, f"check failed: {type(exc).__name__}" + + +def main() -> int: + """Run the discovery and print the results. + + Returns: + ``1`` when any package is missing from PyPI, otherwise ``0``. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--ref", required=True, help="git ref carrying the release changelogs" + ) + parser.add_argument( + "--repo", default=".", help="path to the reflex checkout (default: cwd)" + ) + parser.add_argument( + "--json", action="store_true", help="emit JSON instead of a table" + ) + parser.add_argument( + "--specs", + action="store_true", + help="emit 'name==version' lines only (feed to audit_pyi.py)", + ) + args = parser.parse_args() + + rows = [] + for path in sorted(changelog_paths(args.ref, args.repo)): + content = git_show(args.ref, path, args.repo) or "" + match = VERSION_HEADING.search(content) + if not match: + continue + version = match.group(1) + name = dist_name(path, args.ref, args.repo) + published, detail = pypi_status(name, version) + rows.append({ + "package": name, + "version": version, + "published": published, + "detail": detail, + "changelog": path, + "prerelease": bool(re.search(r"[abc]|rc|dev", version.split(".")[-1])), + }) + + missing = [r for r in rows if not r["published"]] + + if args.specs: + for row in rows: + print(f"{row['package']}=={row['version']}") + elif args.json: + print( + json.dumps( + {"ref": args.ref, "packages": rows, "missing": len(missing)}, indent=2 + ) + ) + else: + width = max((len(r["package"]) for r in rows), default=10) + print(f"Release train at {args.ref}: {len(rows)} packages\n") + for row in rows: + mark = "OK " if row["published"] else "!! " + tag = " (prerelease)" if row["prerelease"] else "" + print( + f" {mark}{row['package']:<{width}} {row['version']:<12} {row['detail']}{tag}" + ) + print() + if missing: + print(f"BLOCKER: {len(missing)} package(s) not installable from PyPI:") + for row in missing: + print(f" - {row['package']}=={row['version']} ({row['detail']})") + else: + print("All packages are published and installable.") + + return 1 if missing else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py new file mode 100755 index 00000000000..928e07fa118 --- /dev/null +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +r"""Drive a running Reflex app in Chromium and report anomalies. + +Loads a page, optionally runs a small action script against it, and captures the four +channels worth watching on every run: console messages, page errors, failed requests, and +4xx/5xx responses. Known-benign dev-server noise is filtered out by default so that a +clean run really means clean. + +Run it with the driver venv (playwright installed), not the app's venv: + + NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 \\ + $SB/envs/driver/bin/python drive_app.py http://localhost:3100/ \\ + --screenshot shots/index.png --report logs/index.json + +Actions are JSON, either inline or in a file, applied in order after load: + + [{"click": "text=Increment"}, + {"expect_text": "Count: 1"}, + {"fill": ["input[name=todo]", "buy milk"]}, + {"press": ["input[name=todo]", "Enter"]}, + {"goto": "http://localhost:3100/about"}, + {"wait": 500}, + {"screenshot": "shots/after.png"}, + {"expect_missing": "text=Error"}] + +Exit status is 0 when every action succeeded and nothing anomalous was captured, 1 +otherwise — so it works directly as a check in a loop. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +from playwright.sync_api import Page, sync_playwright + +CHROMIUM = "/opt/pw-browsers/chromium" + +# Dev-server chatter that appears on healthy runs; see references/agent-brief.md. +BENIGN_CONSOLE = [ + re.compile( + r"Hey developer.*HydrateFallback|reactrouter\.com/start/framework/route-module" + ), + re.compile(r"\[vite\] (connecting|connected)"), + re.compile(r"Download the React DevTools"), +] + + +def is_benign(text: str) -> bool: + """Report whether a console message is known dev-server noise. + + Args: + text: The console message text. + + Returns: + ``True`` when the message matches a known-benign pattern. + """ + return any(pattern.search(text) for pattern in BENIGN_CONSOLE) + + +def load_actions(raw: str | None) -> list[dict]: + """Parse the action script. + + Args: + raw: Inline JSON, a path to a JSON file, or ``None``. + + Returns: + The list of actions to apply, empty when none were given. + """ + if not raw: + return [] + candidate = Path(raw) + return json.loads(candidate.read_text() if candidate.exists() else raw) + + +def run_action(page: Page, action: dict, timeout: int) -> str: + """Apply one action to the page. + + Args: + page: The Playwright page to act on. + action: A single-entry mapping of verb to its argument(s). + timeout: Per-action timeout in milliseconds. + + Returns: + A human-readable description of what was done. + """ + ((verb, value),) = action.items() + if verb == "click": + page.click(value, timeout=timeout) + elif verb == "fill": + page.fill(value[0], value[1], timeout=timeout) + elif verb == "press": + page.press(value[0], value[1], timeout=timeout) + elif verb == "goto": + page.goto(value, wait_until="networkidle", timeout=timeout) + elif verb == "wait": + page.wait_for_timeout(value) + elif verb == "wait_for": + page.wait_for_selector(value, timeout=timeout) + elif verb == "expect_text": + page.wait_for_selector(f"text={value}", timeout=timeout) + elif verb == "expect_missing": + page.wait_for_selector(value, state="detached", timeout=timeout) + elif verb == "screenshot": + Path(value).parent.mkdir(parents=True, exist_ok=True) + page.screenshot(path=value, full_page=True) + elif verb == "eval": + return f"eval -> {page.evaluate(value)!r}" + else: + msg = f"unknown action {verb!r}" + raise ValueError(msg) + return f"{verb}: {value!r}" + + +def try_action(page: Page, action: dict, timeout: int) -> tuple[str | None, str | None]: + """Run one action, turning a failure into a value rather than an exception. + + Reporting the failure this way keeps the record of the actions that already + succeeded, which is usually the most useful part of a failed run. + + Args: + page: The Playwright page to act on. + action: A single-entry mapping of verb to its argument(s). + timeout: Per-action timeout in milliseconds. + + Returns: + A ``(description, error)`` tuple; exactly one side is set. + """ + try: + return run_action(page, action, timeout), None + except Exception as exc: + return None, f"{type(exc).__name__}: {exc}" + + +def main() -> int: + """Drive the app and print a report. + + Returns: + ``0`` when the run was clean, ``1`` when anything anomalous was captured. + """ + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("url", help="page to load, e.g. http://localhost:3100/") + parser.add_argument("--actions", help="JSON list of actions, inline or a file path") + parser.add_argument( + "--screenshot", help="path for a screenshot after the actions run" + ) + parser.add_argument("--report", help="path to write the full JSON report to") + parser.add_argument( + "--timeout", type=int, default=15000, help="per-action timeout in ms" + ) + parser.add_argument( + "--settle", type=int, default=1500, help="ms to wait after load for hydration" + ) + parser.add_argument( + "--headed", action="store_true", help="run with a visible browser" + ) + parser.add_argument( + "--all-console", action="store_true", help="do not filter benign console noise" + ) + args = parser.parse_args() + + console: list[dict] = [] + page_errors: list[str] = [] + failed: list[dict] = [] + bad_status: list[dict] = [] + performed: list[str] = [] + action_error: str | None = None + + with sync_playwright() as p: + browser = p.chromium.launch(executable_path=CHROMIUM, headless=not args.headed) + page = browser.new_context().new_page() + page.on("console", lambda m: console.append({"type": m.type, "text": m.text})) + page.on("pageerror", lambda e: page_errors.append(str(e))) + page.on( + "requestfailed", + lambda r: failed.append({"url": r.url, "failure": str(r.failure)}), + ) + page.on( + "response", + lambda r: ( + bad_status.append({"url": r.url, "status": r.status}) + if r.status >= 400 + else None + ), + ) + + page.goto(args.url, wait_until="networkidle", timeout=60000) + page.wait_for_timeout(args.settle) + + for action in load_actions(args.actions): + done, action_error = try_action(page, action, args.timeout) + if done is not None: + performed.append(done) + if action_error: + break + + title = page.title() + body = page.inner_text("body")[:2000] + if args.screenshot: + Path(args.screenshot).parent.mkdir(parents=True, exist_ok=True) + page.screenshot(path=args.screenshot, full_page=True) + browser.close() + + shown = ( + console + if args.all_console + else [m for m in console if not is_benign(m["text"])] + ) + problems = [m for m in shown if m["type"] in ("error", "warning")] + clean = not (problems or page_errors or failed or bad_status or action_error) + + report = { + "url": args.url, + "title": title, + "clean": clean, + "actions_performed": performed, + "action_error": action_error, + "console": shown, + "page_errors": page_errors, + "failed_requests": failed, + "http_4xx_5xx": bad_status, + "body_text": body, + } + if args.report: + Path(args.report).parent.mkdir(parents=True, exist_ok=True) + Path(args.report).write_text(json.dumps(report, indent=2)) + + print(f"TITLE: {title}") + print(f"BODY: {body[:300].replace(chr(10), ' | ')}") + for item in performed: + print(f" did {item}") + if action_error: + print(f" ACTION FAILED: {action_error}") + for m in shown: + print(f" [{m['type']}] {m['text'][:300]}") + for e in page_errors: + print(f" PAGE ERROR: {e[:300]}") + for f in failed: + print(f" REQUEST FAILED: {f['url']} ({f['failure']})") + for b in bad_status: + print(f" HTTP {b['status']}: {b['url']}") + print("RESULT:", "clean" if clean else "ANOMALIES FOUND") + return 0 if clean else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml index 959df1a5633..e1e6a77cdd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -269,6 +269,8 @@ preview = true "*/blank.py" = ["I001"] "docs/package/scripts/*.py" = ["INP001"] "scripts/check_min_deps.py" = ["T201"] +# Skill helper scripts are CLIs whose whole job is printing a report. +".claude/skills/*/scripts/*.py" = ["T201"] [tool.pytest.ini_options] filterwarnings = "ignore:fields may not start with an underscore:RuntimeWarning" From 74d23c8b0fbe9fc3feafc538db08726c0dc3533d Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 28 Aug 2026 23:48:41 +0000 Subject: [PATCH 02/20] Format the python snippet in the agent brief ruff-format runs on markdown in pre-commit, so embedded python blocks are formatted too; the import/assert one-liner becomes two statements. Reproduced locally with the locked ruff 0.16.3, which now reports the whole skill directory as already formatted. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/references/agent-brief.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.claude/skills/prerelease-test/references/agent-brief.md b/.claude/skills/prerelease-test/references/agent-brief.md index 7754c1a3584..257d99b4210 100644 --- a/.claude/skills/prerelease-test/references/agent-brief.md +++ b/.claude/skills/prerelease-test/references/agent-brief.md @@ -25,7 +25,9 @@ and hunt for anomalies. You REPORT issues; you never fix framework code. installed package, so `import reflex` silently picks up unreleased source and your results become fiction. Run scripts from a neutral directory and start each repro with an assertion: ```python - import reflex; assert "<VENV_PATH>" in reflex.__file__, reflex.__file__ + import reflex + + assert "<VENV_PATH>" in reflex.__file__, reflex.__file__ ``` 3. Reading the checkout is fine and encouraged — release source is on branch `<PRERELEASE_BRANCH>`. For PR context load the GitHub MCP tools via ToolSearch (`select:mcp__github__pull_request_read`). From 0487b40ec19e929fd8ebbe9b2457bc74616fca1a Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 28 Aug 2026 23:56:28 +0000 Subject: [PATCH 03/20] Address cubic review findings All eight were valid. check_release_versions: publication status is now tri-state so a network failure is indeterminate (exit 2) rather than a false 'unpublished' blocker, and never-publish-packages from the release config are excluded from the scan. audit_pyi: the missing-artifact early return omitted 'expected' and crashed main() with KeyError, and a package absent from the manifest is now held to zero stubs instead of going unchecked. drive_app: a server that never came up raised out of page.goto before the report was written; startup failures are captured and exit 1. orchestration: a dead verifier no longer leaves verdict:null in verified_issues. SKILL.md: the documented invocation omitted --ref and bare scripts/... paths collide with the repo's own scripts/ directory. Re-validated after the changes: discovery finds 16 published packages, the documented Phase 5 pipe audits 121 stubs clean, and the driver's clean, action-failure and dead-server paths all behave as documented. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/SKILL.md | 29 ++++-- .../prerelease-test/references/agent-brief.md | 3 +- .../references/orchestration.md | 4 +- .../prerelease-test/scripts/audit_pyi.py | 21 +++- .../scripts/check_release_versions.py | 98 ++++++++++++++----- .../prerelease-test/scripts/drive_app.py | 26 +++-- 6 files changed, 133 insertions(+), 48 deletions(-) diff --git a/.claude/skills/prerelease-test/SKILL.md b/.claude/skills/prerelease-test/SKILL.md index 2a3d6548689..d02478ff06f 100644 --- a/.claude/skills/prerelease-test/SKILL.md +++ b/.claude/skills/prerelease-test/SKILL.md @@ -48,9 +48,14 @@ Find the pre-release branch (`git ls-remote --heads origin 'r/pre-*'`, or the us read **every** `CHANGELOG.md` on it: the root one plus `packages/*/CHANGELOG.md`. The top entry of each is what this train ships. -Run `scripts/check_release_versions.py <git-ref>` to extract every package's version and confirm -each is published on PyPI. An unpublished version is a finding in its own right — report it -immediately rather than working around it. +Run the discovery script to extract every package's version and confirm each is published: + + .claude/skills/prerelease-test/scripts/check_release_versions.py --ref <git-ref> + +(Script paths in this skill are repo-root-relative — the repo has its own unrelated `scripts/` +directory, so the prefix matters.) It exits 1 when a package is confirmed missing and 2 when a +check could not complete, so a proxy hiccup never reads as a missing package. An unpublished +version is a finding in its own right — report it immediately rather than working around it. Read the linked PRs for anything whose intent is not obvious from the changelog line; the GitHub MCP tools (`mcp__github__pull_request_read`) do this well. Understanding what a change was *for* @@ -58,9 +63,10 @@ is what lets you test it as a user rather than as a checklist. ### Phase 1 — De-risk with a smoke test -Before any fan-out: one venv, `reflex init --template blank`, `reflex run`, drive it in Chromium -with `scripts/drive_app.py`. This shakes out environment problems (proxy, bun installs, browser -path) once, in a context where you can debug them, instead of inside ten parallel agents. +Before any fan-out: one venv, `reflex init --template blank`, `reflex run`, then drive it in +Chromium with `.claude/skills/prerelease-test/scripts/drive_app.py`. This shakes out environment +problems (proxy, bun installs, browser path) once, in a context where you can debug them, instead +of inside ten parallel agents. Also confirm the dependency graph resolves the way the changelog says it should (e.g. optional dependencies genuinely absent, sub-packages pinned as intended). @@ -111,9 +117,14 @@ were removed; that finds the breakage in seconds instead of hours. ### Phase 5 — Packaging audit -Run `scripts/audit_pyi.py` over every package in the train. It verifies each ships its own -generated `.pyi` stubs in **both** wheel and sdist, that they are byte-identical between the two, -that no package leaks another package's stubs, and that counts line up with `pyi_hashes.json`. +Audit every package in the train; the discovery script feeds it the whole list: + + .claude/skills/prerelease-test/scripts/check_release_versions.py --ref <ref> --specs \ + | xargs .claude/skills/prerelease-test/scripts/audit_pyi.py --manifest-ref <ref> + +It verifies each package ships its own generated `.pyi` stubs in **both** wheel and sdist, that +they are byte-identical between the two, that no package leaks another package's stubs, and that +counts line up with `pyi_hashes.json` (a package absent from the manifest must ship none). Stub content hashes legitimately differ from the committed manifest because the build hook regenerates them at release time — compare *presence and counts*, and wheel against sdist. diff --git a/.claude/skills/prerelease-test/references/agent-brief.md b/.claude/skills/prerelease-test/references/agent-brief.md index 257d99b4210..4ca743341fa 100644 --- a/.claude/skills/prerelease-test/references/agent-brief.md +++ b/.claude/skills/prerelease-test/references/agent-brief.md @@ -52,7 +52,8 @@ and hunt for anomalies. You REPORT issues; you never fix framework code. - **Playwright driver venv:** `$SB/envs/driver/bin/python` (playwright, httpx, websockets). Chromium: `/opt/pw-browsers/chromium` — launch with `p.chromium.launch(executable_path="/opt/pw-browsers/chromium")`. - A ready-made driver with console/network capture is at `<SKILL>/scripts/drive_app.py`. + A ready-made driver with console/network capture is at + `.claude/skills/prerelease-test/scripts/drive_app.py` in the checkout (read-only use is fine). - **Local HTTP needs proxy bypass on the CLIENT side only:** prefix curl/Playwright commands with `NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1` (curl: `--noproxy '*'`). Do NOT export those variables into the reflex server's environment — it breaks bun's package diff --git a/.claude/skills/prerelease-test/references/orchestration.md b/.claude/skills/prerelease-test/references/orchestration.md index 915c75f2baf..8bfc46b5dca 100644 --- a/.claude/skills/prerelease-test/references/orchestration.md +++ b/.claude/skills/prerelease-test/references/orchestration.md @@ -84,7 +84,9 @@ const results = await pipeline( const verdicts = await parallel( issues.map((iss, j) => () => agent(verifyBrief(c, iss, j), { label: `verify:${c.key}:${j}`, phase: 'Verify', schema: VERDICT_SCHEMA }) - .then((v) => ({ issue: iss, verdict: v })), + // A dead verifier resolves falsy; drop it rather than recording verdict: null, + // which would read downstream as an unverified issue that passed verification. + .then((v) => (v ? { issue: iss, verdict: v } : null)), ), ) return { ...res, verified_issues: verdicts.filter(Boolean) } diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py index 85b28e4fb86..b87bbf0946b 100755 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -135,13 +135,17 @@ def manifest_counts(ref: str, repo: str) -> dict[str, int]: return dict(counts) -def audit(spec: str, workdir: Path, expected: dict[str, int]) -> dict: +def audit( + spec: str, workdir: Path, expected: dict[str, int], have_manifest: bool +) -> dict: """Audit the stubs shipped by one package version. Args: spec: A ``name==version`` spec. workdir: Directory to download artifacts into. expected: Expected stub counts by distribution name, from the manifest. + have_manifest: Whether a manifest was loaded. When it was, a package absent from + it is expected to ship no stubs at all; without one, counts go unchecked. Returns: A record with the package, version, stub count and any problems found. @@ -153,7 +157,13 @@ def audit(spec: str, workdir: Path, expected: dict[str, int]) -> dict: artifacts = download_artifacts(name, version, workdir) if "wheel" not in artifacts or "sdist" not in artifacts: problems.append(f"missing artifact kinds: has {sorted(artifacts)}") - return {"package": name, "version": version, "problems": problems, "own": 0} + return { + "package": name, + "version": version, + "problems": problems, + "own": 0, + "expected": expected.get(name) if have_manifest else None, + } wheel = wheel_stubs(artifacts["wheel"]) sdist = sdist_stubs(artifacts["sdist"]) @@ -198,7 +208,7 @@ def split(stubs: dict[str, bytes]) -> tuple[dict[str, bytes], dict[str, bytes]]: f"{len(differing)} stub(s) differ between wheel and sdist: {differing[:3]}" ) - want = expected.get(name) + want = expected.get(name, 0) if have_manifest else None if want is not None and want != len(wheel_own): problems.append( f"expected {want} stub(s) from manifest, wheel ships {len(wheel_own)}" @@ -233,11 +243,14 @@ def main() -> int: expected = ( manifest_counts(args.manifest_ref, args.repo) if args.manifest_ref else {} ) + # An empty result means the manifest was unreadable at that ref, so counts stay + # unchecked rather than every package being held to a zero-stub expectation. + have_manifest = bool(expected) with tempfile.TemporaryDirectory() as tmp: workdir = Path(args.keep) if args.keep else Path(tmp) workdir.mkdir(parents=True, exist_ok=True) - results = [audit(spec, workdir, expected) for spec in args.specs] + results = [audit(spec, workdir, expected, have_manifest) for spec in args.specs] width = max(len(r["package"]) for r in results) print(f"pyi packaging audit — {len(results)} package(s)\n") diff --git a/.claude/skills/prerelease-test/scripts/check_release_versions.py b/.claude/skills/prerelease-test/scripts/check_release_versions.py index 79615de530b..0f9df8e3dde 100755 --- a/.claude/skills/prerelease-test/scripts/check_release_versions.py +++ b/.claude/skills/prerelease-test/scripts/check_release_versions.py @@ -26,6 +26,8 @@ VERSION_HEADING = re.compile(r"^##\s+v?([0-9][^\s]*)", re.MULTILINE) PROJECT_NAME = re.compile(r"^name\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) +NEVER_PUBLISH = re.compile(r"^never-publish-packages\s*=\s*\[([^\]]*)\]", re.MULTILINE) +QUOTED = re.compile(r"[\"']([^\"']+)[\"']") def git_show(ref: str, path: str, repo: str) -> str | None: @@ -48,6 +50,27 @@ def git_show(ref: str, path: str, repo: str) -> str | None: return result.stdout if result.returncode == 0 else None +def never_published(ref: str, repo: str) -> set[str]: + """Read the packages the release config never publishes. + + Those are built for other purposes (the docs site) and have no PyPI release, so a + changelog in one of them must not be reported as an unpublished blocker. Parsed with a + regex rather than a TOML library to keep this script dependency-free on Python 3.10. + + Args: + ref: The git ref to read the root ``pyproject.toml`` from. + repo: Path to the reflex checkout. + + Returns: + The set of package directory names to skip. + """ + content = git_show(ref, "pyproject.toml", repo) or "" + match = NEVER_PUBLISH.search(content) + if not match: + return set() + return set(QUOTED.findall(match.group(1))) + + def changelog_paths(ref: str, repo: str) -> list[str]: """Find the changelogs that describe a release train. @@ -64,11 +87,16 @@ def changelog_paths(ref: str, repo: str) -> list[str]: text=True, check=True, ).stdout.splitlines() - return [ - p - for p in listing - if p == "CHANGELOG.md" or re.fullmatch(r"packages/[^/]+/CHANGELOG.md", p) - ] + skip = never_published(ref, repo) + paths = [] + for path in listing: + if path == "CHANGELOG.md": + paths.append(path) + continue + match = re.fullmatch(r"packages/([^/]+)/CHANGELOG.md", path) + if match and match.group(1) not in skip: + paths.append(path) + return paths def dist_name(changelog_path: str, ref: str, repo: str) -> str: @@ -92,16 +120,19 @@ def dist_name(changelog_path: str, ref: str, repo: str) -> str: return parts[1] if len(parts) > 2 else "reflex" -def pypi_status(name: str, version: str) -> tuple[bool, str]: +def pypi_status(name: str, version: str) -> tuple[str, str]: """Check whether a distribution version is installable from PyPI. + The result distinguishes "definitely not published" from "could not tell", because a + proxy hiccup reported as a missing package would be a false release blocker. + Args: name: The PyPI distribution name. version: The version to look for. Returns: - A ``(published, detail)`` tuple, where detail names the artifact kinds found or - why the check failed. + A ``(status, detail)`` tuple where status is ``"published"``, ``"missing"`` or + ``"error"``, and detail names the artifact kinds found or why the check failed. """ url = f"https://pypi.org/pypi/{name}/{version}/json" try: @@ -109,25 +140,23 @@ def pypi_status(name: str, version: str) -> tuple[bool, str]: data = json.load(response) kinds = sorted({u["packagetype"] for u in data.get("urls", [])}) if not kinds: - return False, "published but no files" - return True, "+".join( - k.replace("bdist_wheel", "wheel").replace("sdist", "sdist") for k in kinds - ) + return "missing", "published but no files" + return "published", "+".join(k.replace("bdist_wheel", "wheel") for k in kinds) except urllib.error.HTTPError as exc: if exc.code == 404: - return False, "NOT ON PYPI" - return False, f"HTTP {exc.code}" - except ( - Exception - ) as exc: # network/proxy trouble should not look like a missing package - return False, f"check failed: {type(exc).__name__}" + return "missing", "NOT ON PYPI" + return "error", f"HTTP {exc.code}" + except Exception as exc: + # Network/proxy trouble is indeterminate, not evidence the package is missing. + return "error", f"check failed: {type(exc).__name__}" def main() -> int: """Run the discovery and print the results. Returns: - ``1`` when any package is missing from PyPI, otherwise ``0``. + ``1`` when a package is confirmed missing from PyPI (a release blocker), ``2`` + when a check could not be completed and the result is indeterminate, else ``0``. """ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -154,17 +183,19 @@ def main() -> int: continue version = match.group(1) name = dist_name(path, args.ref, args.repo) - published, detail = pypi_status(name, version) + status, detail = pypi_status(name, version) rows.append({ "package": name, "version": version, - "published": published, + "status": status, "detail": detail, "changelog": path, "prerelease": bool(re.search(r"[abc]|rc|dev", version.split(".")[-1])), }) - missing = [r for r in rows if not r["published"]] + missing = [r for r in rows if r["status"] == "missing"] + errors = [r for r in rows if r["status"] == "error"] + marks = {"published": "OK ", "missing": "!! ", "error": "?? "} if args.specs: for row in rows: @@ -172,27 +203,40 @@ def main() -> int: elif args.json: print( json.dumps( - {"ref": args.ref, "packages": rows, "missing": len(missing)}, indent=2 + { + "ref": args.ref, + "packages": rows, + "missing": len(missing), + "errors": len(errors), + }, + indent=2, ) ) else: width = max((len(r["package"]) for r in rows), default=10) print(f"Release train at {args.ref}: {len(rows)} packages\n") for row in rows: - mark = "OK " if row["published"] else "!! " tag = " (prerelease)" if row["prerelease"] else "" print( - f" {mark}{row['package']:<{width}} {row['version']:<12} {row['detail']}{tag}" + f" {marks[row['status']]}{row['package']:<{width}} " + f"{row['version']:<12} {row['detail']}{tag}" ) print() if missing: print(f"BLOCKER: {len(missing)} package(s) not installable from PyPI:") for row in missing: print(f" - {row['package']}=={row['version']} ({row['detail']})") - else: + if errors: + print(f"INDETERMINATE: {len(errors)} package(s) could not be checked:") + for row in errors: + print(f" - {row['package']}=={row['version']} ({row['detail']})") + print(" Re-run before drawing conclusions; this is not evidence of a gap.") + if not missing and not errors: print("All packages are published and installable.") - return 1 if missing else 0 + if missing: + return 1 + return 2 if errors else 0 if __name__ == "__main__": diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py index 928e07fa118..39c1bdeb778 100755 --- a/.claude/skills/prerelease-test/scripts/drive_app.py +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -170,6 +170,7 @@ def main() -> int: bad_status: list[dict] = [] performed: list[str] = [] action_error: str | None = None + load_error: str | None = None with sync_playwright() as p: browser = p.chromium.launch(executable_path=CHROMIUM, headless=not args.headed) @@ -189,18 +190,26 @@ def main() -> int: ), ) - page.goto(args.url, wait_until="networkidle", timeout=60000) - page.wait_for_timeout(args.settle) + try: + page.goto(args.url, wait_until="networkidle", timeout=60000) + page.wait_for_timeout(args.settle) + except Exception as exc: + # A server that never came up is the most common failure here, and it still + # deserves a report and the documented exit status rather than a traceback. + load_error = f"{type(exc).__name__}: {exc}" - for action in load_actions(args.actions): + for action in [] if load_error else load_actions(args.actions): done, action_error = try_action(page, action, args.timeout) if done is not None: performed.append(done) if action_error: break - title = page.title() - body = page.inner_text("body")[:2000] + try: + title = page.title() + body = page.inner_text("body")[:2000] + except Exception as exc: + title, body = "", f"<unreadable: {type(exc).__name__}>" if args.screenshot: Path(args.screenshot).parent.mkdir(parents=True, exist_ok=True) page.screenshot(path=args.screenshot, full_page=True) @@ -212,12 +221,15 @@ def main() -> int: else [m for m in console if not is_benign(m["text"])] ) problems = [m for m in shown if m["type"] in ("error", "warning")] - clean = not (problems or page_errors or failed or bad_status or action_error) + clean = not ( + problems or page_errors or failed or bad_status or action_error or load_error + ) report = { "url": args.url, "title": title, "clean": clean, + "load_error": load_error, "actions_performed": performed, "action_error": action_error, "console": shown, @@ -230,6 +242,8 @@ def main() -> int: Path(args.report).parent.mkdir(parents=True, exist_ok=True) Path(args.report).write_text(json.dumps(report, indent=2)) + if load_error: + print(f" LOAD FAILED: {load_error}") print(f"TITLE: {title}") print(f"BODY: {body[:300].replace(chr(10), ' | ')}") for item in performed: From e59fdbda878274ac42dfd71d7d06ef9528cff0df Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:00:22 +0000 Subject: [PATCH 04/20] Address the second review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four further valid findings, distinct from the first round: - drive_app: inline action JSON longer than the filesystem's filename limit made Path(raw).exists() raise OSError before parsing, so the documented inline form failed for any non-trivial script (verified: a 1109-byte payload raised 'File name too long' before, runs now). - drive_app: the report was read immediately after the last action, so anything it triggered could land after the browser closed. - audit_pyi: the import root came from a dist-name heuristic; it now comes from the stub manifest, which is authoritative and — unlike deriving it from the artifact — keeps the foreign-stub check meaningful. - scripts: dropped bare python3 shebangs for PEP 723 metadata and 'uv run --script', matching scripts/check_min_deps.py and CLAUDE.md. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/SKILL.md | 12 ++- .../prerelease-test/scripts/audit_pyi.py | 81 +++++++++++++++---- .../scripts/check_release_versions.py | 5 +- .../prerelease-test/scripts/drive_app.py | 23 +++++- 4 files changed, 100 insertions(+), 21 deletions(-) mode change 100755 => 100644 .claude/skills/prerelease-test/scripts/audit_pyi.py mode change 100755 => 100644 .claude/skills/prerelease-test/scripts/check_release_versions.py mode change 100755 => 100644 .claude/skills/prerelease-test/scripts/drive_app.py diff --git a/.claude/skills/prerelease-test/SKILL.md b/.claude/skills/prerelease-test/SKILL.md index d02478ff06f..eb2c1d804f1 100644 --- a/.claude/skills/prerelease-test/SKILL.md +++ b/.claude/skills/prerelease-test/SKILL.md @@ -50,7 +50,7 @@ each is what this train ships. Run the discovery script to extract every package's version and confirm each is published: - .claude/skills/prerelease-test/scripts/check_release_versions.py --ref <git-ref> + uv run --script .claude/skills/prerelease-test/scripts/check_release_versions.py --ref <git-ref> (Script paths in this skill are repo-root-relative — the repo has its own unrelated `scripts/` directory, so the prefix matters.) It exits 1 when a package is confirmed missing and 2 when a @@ -68,6 +68,10 @@ Chromium with `.claude/skills/prerelease-test/scripts/drive_app.py`. This shakes problems (proxy, bun installs, browser path) once, in a context where you can debug them, instead of inside ten parallel agents. +The scripts carry PEP 723 metadata, so `uv run --script <path>` runs each one in its own isolated +environment (the driver pulls in playwright that way); running them with an existing driver venv's +interpreter works too. + Also confirm the dependency graph resolves the way the changelog says it should (e.g. optional dependencies genuinely absent, sub-packages pinned as intended). @@ -119,8 +123,10 @@ were removed; that finds the breakage in seconds instead of hours. Audit every package in the train; the discovery script feeds it the whole list: - .claude/skills/prerelease-test/scripts/check_release_versions.py --ref <ref> --specs \ - | xargs .claude/skills/prerelease-test/scripts/audit_pyi.py --manifest-ref <ref> + uv run --script .claude/skills/prerelease-test/scripts/check_release_versions.py \ + --ref <ref> --specs \ + | xargs uv run --script .claude/skills/prerelease-test/scripts/audit_pyi.py \ + --manifest-ref <ref> It verifies each package ships its own generated `.pyi` stubs in **both** wheel and sdist, that they are byte-identical between the two, that no package leaks another package's stubs, and that diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py old mode 100755 new mode 100644 index b87bbf0946b..9ba637367f7 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Audit that published packages ship their generated .pyi stubs correctly. For each package it downloads the wheel and the sdist from PyPI and checks: @@ -17,6 +16,10 @@ check_release_versions.py --ref origin/main --specs | xargs audit_pyi.py --manifest-ref origin/main """ +# /// script +# requires-python = ">=3.10" +# /// + from __future__ import annotations import argparse @@ -97,28 +100,56 @@ def sdist_stubs(path: Path) -> dict[str, bytes]: return out -def import_root(dist_name: str) -> str: - """Guess the top-level import package for a distribution name. +def import_root(dist_name: str, roots: dict[str, str]) -> str: + """Resolve the top-level import package for a distribution. + + The manifest is authoritative when available because it records each stub's real + source path; the underscore convention is only a fallback. Deriving the root from the + artifact itself would be circular — foreign stubs would define themselves as native + and the leak check could never fail. Args: dist_name: The PyPI distribution name. + roots: Import roots by distribution name, from the manifest. Returns: - The import package name, which differs only by underscores by convention. + The import package name. """ - return dist_name.replace("-", "_") + return roots.get(dist_name, dist_name.replace("-", "_")) -def manifest_counts(ref: str, repo: str) -> dict[str, int]: - """Count the stubs each distribution is expected to ship. +def manifest_roots(ref: str, repo: str) -> dict[str, str]: + """Read each distribution's real import root from the stub manifest. Args: ref: The git ref to read ``pyi_hashes.json`` from. repo: Path to the reflex checkout. Returns: - A mapping of distribution name to expected stub count, empty when the manifest is - unavailable at that ref. + A mapping of distribution name to import package name, empty when the manifest is + unavailable. + """ + roots: dict[str, str] = {} + for key in _manifest_keys(ref, repo): + parts = key.split("/") + if key.startswith("packages/"): + # packages/<dist>/<src dir>/<import root>/... + if len(parts) > 3: + roots[parts[1]] = parts[3] + elif parts: + roots["reflex"] = parts[0] + return roots + + +def _manifest_keys(ref: str, repo: str) -> list[str]: + """Read the stub paths recorded in the manifest. + + Args: + ref: The git ref to read ``pyi_hashes.json`` from. + repo: Path to the reflex checkout. + + Returns: + The manifest's stub paths, empty when it is unavailable at that ref. """ result = subprocess.run( ["git", "-C", repo, "show", f"{ref}:pyi_hashes.json"], @@ -127,16 +158,34 @@ def manifest_counts(ref: str, repo: str) -> dict[str, int]: check=False, ) if result.returncode != 0: - return {} + return [] + return list(json.loads(result.stdout)) + + +def manifest_counts(ref: str, repo: str) -> dict[str, int]: + """Count the stubs each distribution is expected to ship. + + Args: + ref: The git ref to read ``pyi_hashes.json`` from. + repo: Path to the reflex checkout. + + Returns: + A mapping of distribution name to expected stub count, empty when the manifest is + unavailable at that ref. + """ counts: collections.Counter[str] = collections.Counter() - for key in json.loads(result.stdout): + for key in _manifest_keys(ref, repo): # "packages/<dist>/src/<pkg>/x.pyi" or a root-package path like "reflex/x.pyi" counts[key.split("/")[1] if key.startswith("packages/") else "reflex"] += 1 return dict(counts) def audit( - spec: str, workdir: Path, expected: dict[str, int], have_manifest: bool + spec: str, + workdir: Path, + expected: dict[str, int], + have_manifest: bool, + roots: dict[str, str], ) -> dict: """Audit the stubs shipped by one package version. @@ -144,6 +193,7 @@ def audit( spec: A ``name==version`` spec. workdir: Directory to download artifacts into. expected: Expected stub counts by distribution name, from the manifest. + roots: Import roots by distribution name, from the manifest. have_manifest: Whether a manifest was loaded. When it was, a package absent from it is expected to ship no stubs at all; without one, counts go unchecked. @@ -151,7 +201,7 @@ def audit( A record with the package, version, stub count and any problems found. """ name, _, version = spec.partition("==") - root = import_root(name) + root = import_root(name, roots) problems: list[str] = [] artifacts = download_artifacts(name, version, workdir) @@ -246,11 +296,14 @@ def main() -> int: # An empty result means the manifest was unreadable at that ref, so counts stay # unchecked rather than every package being held to a zero-stub expectation. have_manifest = bool(expected) + roots = manifest_roots(args.manifest_ref, args.repo) if args.manifest_ref else {} with tempfile.TemporaryDirectory() as tmp: workdir = Path(args.keep) if args.keep else Path(tmp) workdir.mkdir(parents=True, exist_ok=True) - results = [audit(spec, workdir, expected, have_manifest) for spec in args.specs] + results = [ + audit(spec, workdir, expected, have_manifest, roots) for spec in args.specs + ] width = max(len(r["package"]) for r in results) print(f"pyi packaging audit — {len(results)} package(s)\n") diff --git a/.claude/skills/prerelease-test/scripts/check_release_versions.py b/.claude/skills/prerelease-test/scripts/check_release_versions.py old mode 100755 new mode 100644 index 0f9df8e3dde..b796e845b71 --- a/.claude/skills/prerelease-test/scripts/check_release_versions.py +++ b/.claude/skills/prerelease-test/scripts/check_release_versions.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Discover what a release train ships and confirm every package published to PyPI. Reads the root CHANGELOG.md plus every packages/*/CHANGELOG.md at a git ref, takes the @@ -14,6 +13,10 @@ check_release_versions.py --ref origin/main --specs # pkg==ver lines for audit_pyi.py """ +# /// script +# requires-python = ">=3.10" +# /// + from __future__ import annotations import argparse diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py old mode 100755 new mode 100644 index 39c1bdeb778..1f81363d806 --- a/.claude/skills/prerelease-test/scripts/drive_app.py +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 r"""Drive a running Reflex app in Chromium and report anomalies. Loads a page, optionally runs a small action script against it, and captures the four @@ -27,6 +26,11 @@ otherwise — so it works directly as a check in a loop. """ +# /// script +# requires-python = ">=3.10" +# dependencies = ["playwright"] +# /// + from __future__ import annotations import argparse @@ -72,8 +76,15 @@ def load_actions(raw: str | None) -> list[dict]: """ if not raw: return [] - candidate = Path(raw) - return json.loads(candidate.read_text() if candidate.exists() else raw) + try: + candidate = Path(raw) + if candidate.is_file(): + raw = candidate.read_text() + except OSError: + # Inline JSON longer than the filesystem's filename limit makes the stat itself + # raise, so treat any path error as "this was not a path". + pass + return json.loads(raw) def run_action(page: Page, action: dict, timeout: int) -> str: @@ -205,6 +216,12 @@ def main() -> int: if action_error: break + if performed: + # Playwright returns as soon as an action's own wait resolves, but the + # resulting state update, network call and any error it triggers land after + # that; without this the last action's fallout is invisible to the report. + page.wait_for_timeout(args.settle) + try: title = page.title() body = page.inner_text("body")[:2000] From 1f71177ce63d0c9a5d4898a814721b59bdd07873 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:05:27 +0000 Subject: [PATCH 05/20] Keep indeterminate packages out of the packaging audit --specs feeds a downloader, so a package whose PyPI check could not complete was turning a controlled indeterminate result into a traceback one stage later. Emit only confirmed-published rows, and report the skipped ones on stderr so a short list is never mistaken for a complete one. Guard the download in audit() too, so a package that cannot be fetched for any other reason fails on its own row instead of aborting the whole audit. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/scripts/audit_pyi.py | 13 ++++++++++++- .../scripts/check_release_versions.py | 11 ++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py index 9ba637367f7..7cd2a41be3e 100644 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -204,7 +204,18 @@ def audit( root = import_root(name, roots) problems: list[str] = [] - artifacts = download_artifacts(name, version, workdir) + try: + artifacts = download_artifacts(name, version, workdir) + except Exception as exc: + # One unreachable package should not abort the audit of all the others. + problems.append(f"could not fetch artifacts: {type(exc).__name__}") + return { + "package": name, + "version": version, + "problems": problems, + "own": 0, + "expected": expected.get(name) if have_manifest else None, + } if "wheel" not in artifacts or "sdist" not in artifacts: problems.append(f"missing artifact kinds: has {sorted(artifacts)}") return { diff --git a/.claude/skills/prerelease-test/scripts/check_release_versions.py b/.claude/skills/prerelease-test/scripts/check_release_versions.py index b796e845b71..abf179dc973 100644 --- a/.claude/skills/prerelease-test/scripts/check_release_versions.py +++ b/.claude/skills/prerelease-test/scripts/check_release_versions.py @@ -201,8 +201,17 @@ def main() -> int: marks = {"published": "OK ", "missing": "!! ", "error": "?? "} if args.specs: + # Only confirmed-published rows: specs feed a downloader, so emitting a missing or + # indeterminate package just moves the failure downstream. Skips go to stderr so + # a short list is never silently mistaken for a complete one. for row in rows: - print(f"{row['package']}=={row['version']}") + if row["status"] == "published": + print(f"{row['package']}=={row['version']}") + for row in missing + errors: + print( + f"skipped {row['package']}=={row['version']} ({row['detail']})", + file=sys.stderr, + ) elif args.json: print( json.dumps( From 6404a3845c9e2b294cb42cf9fef50ab26137f095 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:08:22 +0000 Subject: [PATCH 06/20] Report unchecked packages instead of dropping them A changelog with no version heading was skipped silently, so the one package whose publication went unverified was also the one the report never mentioned. Record it as indeterminate, which keeps it out of --specs and exits 2. Also treat a fully yanked release as not installable, note partial yanks in the detail column, and give the packaging cluster the same repo-root-relative script path the rest of the skill uses. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .../prerelease-test/references/clusters.md | 2 +- .../scripts/check_release_versions.py | 35 +++++++++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.claude/skills/prerelease-test/references/clusters.md b/.claude/skills/prerelease-test/references/clusters.md index 2ae20b7144d..5c5d86aba5f 100644 --- a/.claude/skills/prerelease-test/references/clusters.md +++ b/.claude/skills/prerelease-test/references/clusters.md @@ -48,5 +48,5 @@ Two clusters that are not feature-shaped but catch the highest-severity problems the popular third-party packages (`reflex-local-auth`, `reflex-global-hotkey`) for the removed names. A bare `AttributeError`/`ImportError` with no pointer to the replacement is a finding even when the removal itself was intended. -- **Packaging.** `scripts/audit_pyi.py`, plus a check that each package's declared dependency pins +- **Packaging.** `.claude/skills/prerelease-test/scripts/audit_pyi.py`, plus a check that each package's declared dependency pins match what the changelog says and that installing from the sdist works. diff --git a/.claude/skills/prerelease-test/scripts/check_release_versions.py b/.claude/skills/prerelease-test/scripts/check_release_versions.py index abf179dc973..ca8cef57670 100644 --- a/.claude/skills/prerelease-test/scripts/check_release_versions.py +++ b/.claude/skills/prerelease-test/scripts/check_release_versions.py @@ -141,10 +141,6 @@ def pypi_status(name: str, version: str) -> tuple[str, str]: try: with urllib.request.urlopen(url, timeout=30) as response: data = json.load(response) - kinds = sorted({u["packagetype"] for u in data.get("urls", [])}) - if not kinds: - return "missing", "published but no files" - return "published", "+".join(k.replace("bdist_wheel", "wheel") for k in kinds) except urllib.error.HTTPError as exc: if exc.code == 404: return "missing", "NOT ON PYPI" @@ -153,6 +149,19 @@ def pypi_status(name: str, version: str) -> tuple[str, str]: # Network/proxy trouble is indeterminate, not evidence the package is missing. return "error", f"check failed: {type(exc).__name__}" + files = data.get("urls", []) + if not files: + return "missing", "published but no files" + live = [u for u in files if not u.get("yanked")] + if not live: + # The version exists but every file is withdrawn: resolvers skip a yanked release + # unless it is pinned exactly, and it was withdrawn for a reason. + return "missing", "published but all files yanked" + kinds = sorted({u["packagetype"] for u in live}) + detail = "+".join(k.replace("bdist_wheel", "wheel") for k in kinds) + yanked = len(files) - len(live) + return "published", f"{detail} ({yanked} yanked)" if yanked else detail + def main() -> int: """Run the discovery and print the results. @@ -181,12 +190,16 @@ def main() -> int: rows = [] for path in sorted(changelog_paths(args.ref, args.repo)): content = git_show(args.ref, path, args.repo) or "" - match = VERSION_HEADING.search(content) - if not match: - continue - version = match.group(1) name = dist_name(path, args.ref, args.repo) - status, detail = pypi_status(name, version) + match = VERSION_HEADING.search(content) + if match: + version = match.group(1) + status, detail = pypi_status(name, version) + else: + # Skipping the package silently would leave it unchecked, which is the exact + # failure this script exists to catch; report it as indeterminate instead. + version = "?" + status, detail = "error", "no version heading in changelog" rows.append({ "package": name, "version": version, @@ -242,7 +255,9 @@ def main() -> int: print(f"INDETERMINATE: {len(errors)} package(s) could not be checked:") for row in errors: print(f" - {row['package']}=={row['version']} ({row['detail']})") - print(" Re-run before drawing conclusions; this is not evidence of a gap.") + print( + " Unchecked is not the same as missing — resolve each before releasing." + ) if not missing and not errors: print("All packages are published and installable.") From a33b4ed9a5b7559d87f386672866470703e11a82 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:12:19 +0000 Subject: [PATCH 07/20] Stop the packaging pipeline from hiding an unchecked package Documenting phase 5 as a shell pipe meant the audit's exit status replaced the discovery script's, so a package dropped for an incomplete PyPI check took the audit's PASS with it. Document it as two commands instead. Parse --actions before launching the browser: doing it after spent a launch and a page load on a typo and left a traceback where the report should be. A list that is not a list now says so too. Say plainly in the orchestration example that dead explorers are kept as markers, since filter(Boolean) does not drop them and should not. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/SKILL.md | 10 ++++++---- .../references/orchestration.md | 3 +++ .../prerelease-test/scripts/drive_app.py | 20 +++++++++++++++++-- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.claude/skills/prerelease-test/SKILL.md b/.claude/skills/prerelease-test/SKILL.md index eb2c1d804f1..8fc011cf948 100644 --- a/.claude/skills/prerelease-test/SKILL.md +++ b/.claude/skills/prerelease-test/SKILL.md @@ -121,12 +121,14 @@ were removed; that finds the breakage in seconds instead of hours. ### Phase 5 — Packaging audit -Audit every package in the train; the discovery script feeds it the whole list: +Audit every package in the train; the discovery script feeds it the whole list. Run it as two +commands, not one pipe — a pipe reports only the audit's exit status, so a package whose PyPI +check never completed would be dropped from the list and the audit would still print PASS: uv run --script .claude/skills/prerelease-test/scripts/check_release_versions.py \ - --ref <ref> --specs \ - | xargs uv run --script .claude/skills/prerelease-test/scripts/audit_pyi.py \ - --manifest-ref <ref> + --ref <ref> --specs > specs.txt # non-zero if anything is unpublished or unchecked + xargs uv run --script .claude/skills/prerelease-test/scripts/audit_pyi.py \ + --manifest-ref <ref> < specs.txt It verifies each package ships its own generated `.pyi` stubs in **both** wheel and sdist, that they are byte-identical between the two, that no package leaks another package's stubs, and that diff --git a/.claude/skills/prerelease-test/references/orchestration.md b/.claude/skills/prerelease-test/references/orchestration.md index 8bfc46b5dca..336a4746cfd 100644 --- a/.claude/skills/prerelease-test/references/orchestration.md +++ b/.claude/skills/prerelease-test/references/orchestration.md @@ -92,6 +92,9 @@ const results = await pipeline( return { ...res, verified_issues: verdicts.filter(Boolean) } }, ) +// Dead explorers stay in as `agent_died` markers rather than being filtered out: a cluster +// that produced nothing is a hole in the campaign, and silently dropping it is how a hole +// gets mistaken for a clean result. Handle them explicitly when you write the report. return results.filter(Boolean) ``` diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py index 1f81363d806..30f368a060b 100644 --- a/.claude/skills/prerelease-test/scripts/drive_app.py +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -73,6 +73,9 @@ def load_actions(raw: str | None) -> list[dict]: Returns: The list of actions to apply, empty when none were given. + + Raises: + ValueError: The JSON is malformed or is not a list of actions. """ if not raw: return [] @@ -84,7 +87,11 @@ def load_actions(raw: str | None) -> list[dict]: # Inline JSON longer than the filesystem's filename limit makes the stat itself # raise, so treat any path error as "this was not a path". pass - return json.loads(raw) + actions = json.loads(raw) + if not isinstance(actions, list): + msg = f"expected a JSON list of actions, got {type(actions).__name__}" + raise ValueError(msg) + return actions def run_action(page: Page, action: dict, timeout: int) -> str: @@ -175,6 +182,15 @@ def main() -> int: ) args = parser.parse_args() + try: + actions = load_actions(args.actions) + except ValueError as exc: + # Parsing after the browser is up spends a launch and a page load before failing, + # and leaves a traceback where the report should be. A malformed --actions is a + # usage error, so say so up front and exit with the documented failure status. + print(f"INVALID --actions: {exc}", file=sys.stderr) + return 1 + console: list[dict] = [] page_errors: list[str] = [] failed: list[dict] = [] @@ -209,7 +225,7 @@ def main() -> int: # deserves a report and the documented exit status rather than a traceback. load_error = f"{type(exc).__name__}: {exc}" - for action in [] if load_error else load_actions(args.actions): + for action in [] if load_error else actions: done, action_error = try_action(page, action, args.timeout) if done is not None: performed.append(done) From 43be3edd5c5c564fe5c3e10a1ab1e802e29e1f22 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:14:09 +0000 Subject: [PATCH 08/20] Bound artifact downloads and survive a corrupt one urlretrieve honours only the global socket timeout, which is unset, so a stalled PyPI transfer would hang the whole audit instead of failing one row. Stream the download through urlopen with a timeout. A corrupt wheel or sdist raised out of audit() and took the rest of the train with it, the same way an unreachable package used to. Handle it per package. Every exit from audit() now goes through one record builder, so the reporting loop can no longer meet a record that is missing a key it reads. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .../prerelease-test/scripts/audit_pyi.py | 73 +++++++++++++------ 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py index 7cd2a41be3e..a3d403f7c35 100644 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -25,6 +25,7 @@ import argparse import collections import json +import shutil import subprocess import sys import tarfile @@ -56,7 +57,13 @@ def download_artifacts(name: str, version: str, dest: Path) -> dict[str, Path]: continue path = dest / entry["filename"] if not path.exists(): - urllib.request.urlretrieve(entry["url"], path) + # urlretrieve honours only the global socket timeout, which is unset, so a + # stalled transfer would hang the whole audit rather than failing this row. + with ( + urllib.request.urlopen(entry["url"], timeout=60) as response, + path.open("wb") as handle, + ): + shutil.copyfileobj(response, handle) out["wheel" if kind == "bdist_wheel" else "sdist"] = path return out @@ -180,6 +187,36 @@ def manifest_counts(ref: str, repo: str) -> dict[str, int]: return dict(counts) +def _record( + name: str, + version: str, + problems: list[str], + expected: dict[str, int], + have_manifest: bool, + own: int = 0, +) -> dict: + """Build one package's audit record. + + Args: + name: The PyPI distribution name. + version: The audited version. + problems: Problems found, empty when the package is clean. + expected: Expected stub counts by distribution name, from the manifest. + have_manifest: Whether a manifest was loaded. + own: Number of the package's own stubs found in the wheel. + + Returns: + The record ``main`` prints, with every key it reads. + """ + return { + "package": name, + "version": version, + "own": own, + "expected": expected.get(name, 0) if have_manifest else None, + "problems": problems, + } + + def audit( spec: str, workdir: Path, @@ -209,25 +246,19 @@ def audit( except Exception as exc: # One unreachable package should not abort the audit of all the others. problems.append(f"could not fetch artifacts: {type(exc).__name__}") - return { - "package": name, - "version": version, - "problems": problems, - "own": 0, - "expected": expected.get(name) if have_manifest else None, - } + return _record(name, version, problems, expected, have_manifest) if "wheel" not in artifacts or "sdist" not in artifacts: problems.append(f"missing artifact kinds: has {sorted(artifacts)}") - return { - "package": name, - "version": version, - "problems": problems, - "own": 0, - "expected": expected.get(name) if have_manifest else None, - } + return _record(name, version, problems, expected, have_manifest) - wheel = wheel_stubs(artifacts["wheel"]) - sdist = sdist_stubs(artifacts["sdist"]) + try: + wheel = wheel_stubs(artifacts["wheel"]) + sdist = sdist_stubs(artifacts["sdist"]) + except (zipfile.BadZipFile, tarfile.TarError, OSError) as exc: + # A corrupt artifact is a finding about that package, not a reason to abandon the + # rest of the train. + problems.append(f"could not read artifacts: {type(exc).__name__}") + return _record(name, version, problems, expected, have_manifest) def split(stubs: dict[str, bytes]) -> tuple[dict[str, bytes], dict[str, bytes]]: own = {k: v for k, v in stubs.items() if k.split("/")[0] == root} @@ -275,13 +306,7 @@ def split(stubs: dict[str, bytes]) -> tuple[dict[str, bytes], dict[str, bytes]]: f"expected {want} stub(s) from manifest, wheel ships {len(wheel_own)}" ) - return { - "package": name, - "version": version, - "own": len(wheel_own), - "expected": want, - "problems": problems, - } + return _record(name, version, problems, expected, have_manifest, len(wheel_own)) def main() -> int: From a152cb1dca6c4b7b6d1968313b8b7324838dff08 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:16:47 +0000 Subject: [PATCH 09/20] Skip the packages whose changelog is not a release version Internal packages are patch-released on every push to main rather than through the changelog, so a changelog heading in one names a version that was never cut and would be checked against PyPI as a false blocker. Exclude them alongside the never-published ones, which the release config already groups the same way. Never audit a yanked artifact when a live one of the same kind exists: it is not what a user installs, so the audit would be reporting on the wrong file. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .../prerelease-test/scripts/audit_pyi.py | 5 +++- .../scripts/check_release_versions.py | 29 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py index a3d403f7c35..3383620ca78 100644 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -53,7 +53,10 @@ def download_artifacts(name: str, version: str, dest: Path) -> dict[str, Path]: out: dict[str, Path] = {} for entry in data.get("urls", []): kind = entry["packagetype"] - if kind not in ("bdist_wheel", "sdist"): + if kind not in ("bdist_wheel", "sdist") or entry.get("yanked"): + # A yanked file is not what a user would install, so auditing it would report + # on the wrong artifact. With every file yanked this leaves nothing, which the + # caller already reports as a missing artifact kind. continue path = dest / entry["filename"] if not path.exists(): diff --git a/.claude/skills/prerelease-test/scripts/check_release_versions.py b/.claude/skills/prerelease-test/scripts/check_release_versions.py index ca8cef57670..e7307c07959 100644 --- a/.claude/skills/prerelease-test/scripts/check_release_versions.py +++ b/.claude/skills/prerelease-test/scripts/check_release_versions.py @@ -29,8 +29,11 @@ VERSION_HEADING = re.compile(r"^##\s+v?([0-9][^\s]*)", re.MULTILINE) PROJECT_NAME = re.compile(r"^name\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) -NEVER_PUBLISH = re.compile(r"^never-publish-packages\s*=\s*\[([^\]]*)\]", re.MULTILINE) QUOTED = re.compile(r"[\"']([^\"']+)[\"']") +# Packages whose CHANGELOG.md does not name a release version, for two different +# reasons: never-published ones have no PyPI release at all, and internal ones are +# patch-released on every push to main rather than through the changelog. +EXCLUDED_KEYS = ("never-publish-packages", "internal-packages") def git_show(ref: str, path: str, repo: str) -> str | None: @@ -53,12 +56,14 @@ def git_show(ref: str, path: str, repo: str) -> str | None: return result.stdout if result.returncode == 0 else None -def never_published(ref: str, repo: str) -> set[str]: - """Read the packages the release config never publishes. +def excluded_packages(ref: str, repo: str) -> set[str]: + """Read the packages whose changelog does not name a release version. - Those are built for other purposes (the docs site) and have no PyPI release, so a - changelog in one of them must not be reported as an unpublished blocker. Parsed with a - regex rather than a TOML library to keep this script dependency-free on Python 3.10. + A changelog in one of these must not be read as the version this train ships: a + never-published package has no PyPI release for it to name, and an internal package + is patch-released outside the changelog flow, so its heading would be checked against + a version that was never cut. Either way the result would be a false blocker. Parsed + with a regex rather than a TOML library to keep this script dependency-free on 3.10. Args: ref: The git ref to read the root ``pyproject.toml`` from. @@ -68,10 +73,12 @@ def never_published(ref: str, repo: str) -> set[str]: The set of package directory names to skip. """ content = git_show(ref, "pyproject.toml", repo) or "" - match = NEVER_PUBLISH.search(content) - if not match: - return set() - return set(QUOTED.findall(match.group(1))) + skip: set[str] = set() + for key in EXCLUDED_KEYS: + match = re.search(rf"^{key}\s*=\s*\[([^\]]*)\]", content, re.MULTILINE) + if match: + skip.update(QUOTED.findall(match.group(1))) + return skip def changelog_paths(ref: str, repo: str) -> list[str]: @@ -90,7 +97,7 @@ def changelog_paths(ref: str, repo: str) -> list[str]: text=True, check=True, ).stdout.splitlines() - skip = never_published(ref, repo) + skip = excluded_packages(ref, repo) paths = [] for path in listing: if path == "CHANGELOG.md": From 7cb79ee40df246bf602b8189116a1b8a54ed7261 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:18:55 +0000 Subject: [PATCH 10/20] Separate a package that could not be audited from a broken one An unreachable PyPI or an unreadable archive was being recorded as a packaging defect, so a healthy package would fail the release on infrastructure trouble. Track those as unchecked instead and exit 2, the same missing-versus-unknown split check_release_versions.py already makes, and exit 1 only for a defect observed in artifacts that were actually read. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .../prerelease-test/scripts/audit_pyi.py | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py index 3383620ca78..b0f71945d6a 100644 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -11,6 +11,9 @@ hook regenerates stubs at release time, so this compares presence and counts rather than hashes. A package with no manifest entries is expected to ship no stubs. +It exits 1 when a package's packaging is wrong and 2 when a package could not be audited +at all, so an unreachable PyPI never reads as a defect in a package nobody looked at. + Usage: audit_pyi.py reflex==0.9.9 reflex-base==0.9.9 check_release_versions.py --ref origin/main --specs | xargs audit_pyi.py --manifest-ref origin/main @@ -197,16 +200,20 @@ def _record( expected: dict[str, int], have_manifest: bool, own: int = 0, + unchecked: str | None = None, ) -> dict: """Build one package's audit record. Args: name: The PyPI distribution name. version: The audited version. - problems: Problems found, empty when the package is clean. + problems: Packaging defects found, empty when the package is clean. expected: Expected stub counts by distribution name, from the manifest. have_manifest: Whether a manifest was loaded. own: Number of the package's own stubs found in the wheel. + unchecked: Why the package could not be audited at all, when it could not be. + Kept apart from ``problems`` so an unreachable PyPI never reads as a defect + in a package nobody managed to look at. Returns: The record ``main`` prints, with every key it reads. @@ -217,6 +224,7 @@ def _record( "own": own, "expected": expected.get(name, 0) if have_manifest else None, "problems": problems, + "unchecked": unchecked, } @@ -248,8 +256,11 @@ def audit( artifacts = download_artifacts(name, version, workdir) except Exception as exc: # One unreachable package should not abort the audit of all the others. - problems.append(f"could not fetch artifacts: {type(exc).__name__}") - return _record(name, version, problems, expected, have_manifest) + # Never reached the package, so nothing is known about its packaging. + reason = f"could not fetch artifacts: {type(exc).__name__}" + return _record( + name, version, problems, expected, have_manifest, unchecked=reason + ) if "wheel" not in artifacts or "sdist" not in artifacts: problems.append(f"missing artifact kinds: has {sorted(artifacts)}") return _record(name, version, problems, expected, have_manifest) @@ -258,10 +269,12 @@ def audit( wheel = wheel_stubs(artifacts["wheel"]) sdist = sdist_stubs(artifacts["sdist"]) except (zipfile.BadZipFile, tarfile.TarError, OSError) as exc: - # A corrupt artifact is a finding about that package, not a reason to abandon the - # rest of the train. - problems.append(f"could not read artifacts: {type(exc).__name__}") - return _record(name, version, problems, expected, have_manifest) + # A truncated download and an artifact PyPI really serves broken look identical + # from here, so this is unchecked rather than a defect: re-run to tell them apart. + reason = f"could not read artifacts: {type(exc).__name__}" + return _record( + name, version, problems, expected, have_manifest, unchecked=reason + ) def split(stubs: dict[str, bytes]) -> tuple[dict[str, bytes], dict[str, bytes]]: own = {k: v for k, v in stubs.items() if k.split("/")[0] == root} @@ -316,7 +329,8 @@ def main() -> int: """Audit every requested package and print a summary. Returns: - ``1`` when any package has packaging problems, otherwise ``0``. + ``1`` when any package has packaging problems, ``2`` when a package could not be + audited and the result is indeterminate, otherwise ``0``. """ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("specs", nargs="+", help="package==version specs") @@ -344,27 +358,38 @@ def main() -> int: audit(spec, workdir, expected, have_manifest, roots) for spec in args.specs ] + failed = [r for r in results if r["problems"]] + unchecked = [r for r in results if r["unchecked"]] + width = max(len(r["package"]) for r in results) print(f"pyi packaging audit — {len(results)} package(s)\n") for r in results: - mark = "OK " if not r["problems"] else "!! " + mark = "!! " if r["problems"] else "?? " if r["unchecked"] else "OK " want = "" if r["expected"] is None else f" (manifest: {r['expected']})" print( f" {mark}{r['package']:<{width}} {r['version']:<12} stubs={r['own']}{want}" ) for problem in r["problems"]: print(f" - {problem}") + if r["unchecked"]: + print(f" - {r['unchecked']}") - failed = [r for r in results if r["problems"]] print() if failed: print(f"FAIL: {len(failed)} package(s) with packaging problems.") - else: + if unchecked: + print(f"INDETERMINATE: {len(unchecked)} package(s) could not be audited:") + for r in unchecked: + print(f" - {r['package']}=={r['version']} ({r['unchecked']})") + print(" Unaudited is not the same as broken — resolve each before releasing.") + if not failed and not unchecked: total = sum(r["own"] for r in results) print( f"PASS: {total} stubs ship correctly in both wheel and sdist; no foreign stubs." ) - return 1 if failed else 0 + if failed: + return 1 + return 2 if unchecked else 0 if __name__ == "__main__": From 906416cb6e6207045bdd89253cc7a8682cfbd680 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:21:59 +0000 Subject: [PATCH 11/20] Account for the work a failed action already started A failing action skipped the settle wait because nothing had been recorded as performed, so a click that reached the server before its assertion failed took the resulting console error and failed request with it. Waiting is exactly what that case needs: the response it triggered is often the whole finding. Give a dead verifier its own marker too. Dropping it left a hole that read the same as an issue the fan-out cap never sent to a verifier, which is the failure the explorer marker two lines below exists to prevent. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .../prerelease-test/references/orchestration.md | 16 ++++++++++------ .../skills/prerelease-test/scripts/drive_app.py | 4 +++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.claude/skills/prerelease-test/references/orchestration.md b/.claude/skills/prerelease-test/references/orchestration.md index 336a4746cfd..5f577840fc5 100644 --- a/.claude/skills/prerelease-test/references/orchestration.md +++ b/.claude/skills/prerelease-test/references/orchestration.md @@ -79,22 +79,26 @@ const results = await pipeline( (c, _item, i) => agent(brief(c, i), { label: `explore:${c.key}`, phase: 'Explore', schema: FINDINGS_SCHEMA }), async (res, c, i) => { if (!res) return { cluster: c.key, agent_died: true } - const issues = (res.issues || []).slice(0, 4) // cap verification fan-out per cluster + // Cap the fan-out per cluster. Issues past the cap are absent from verified_issues but + // still present in res.issues, which is how the report tells them from verified ones. + const issues = (res.issues || []).slice(0, 4) if (!issues.length) return res const verdicts = await parallel( issues.map((iss, j) => () => agent(verifyBrief(c, iss, j), { label: `verify:${c.key}:${j}`, phase: 'Verify', schema: VERDICT_SCHEMA }) - // A dead verifier resolves falsy; drop it rather than recording verdict: null, - // which would read downstream as an unverified issue that passed verification. - .then((v) => (v ? { issue: iss, verdict: v } : null)), + // A dead verifier resolves falsy. Give it its own marker: `verdict: null` would + // read as a verdict that cleared the issue, and dropping the entry would leave a + // hole indistinguishable from an issue the cap never sent to a verifier. + .then((v) => (v ? { issue: iss, verdict: v } : { issue: iss, verifier_died: true })), ), ) - return { ...res, verified_issues: verdicts.filter(Boolean) } + return { ...res, verified_issues: verdicts } }, ) // Dead explorers stay in as `agent_died` markers rather than being filtered out: a cluster // that produced nothing is a hole in the campaign, and silently dropping it is how a hole -// gets mistaken for a clean result. Handle them explicitly when you write the report. +// gets mistaken for a clean result. Same for `verifier_died` above. Every marker is work +// the campaign did not do, so the report must account for each one rather than skip it. return results.filter(Boolean) ``` diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py index 30f368a060b..55a0e861e7d 100644 --- a/.claude/skills/prerelease-test/scripts/drive_app.py +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -232,10 +232,12 @@ def main() -> int: if action_error: break - if performed: + if performed or action_error: # Playwright returns as soon as an action's own wait resolves, but the # resulting state update, network call and any error it triggers land after # that; without this the last action's fallout is invisible to the report. + # A failed action counts: the click that raised on its assertion may still + # have reached the server, and that response is often the whole finding. page.wait_for_timeout(args.settle) try: From 846c786c3814958f30d6e8d93c72b2fbd2b51878 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:23:30 +0000 Subject: [PATCH 12/20] Stop the agent preamble from overpromising ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared preamble handed every agent a 20-port range, but the map gives a verify agent only 4, so a verifier following the bootstrap verbatim would reach into its sibling's range — the collision the map exists to prevent. Take the ends from the table instead of a fixed span. Say which venv the checkout-shadowing guard should name. It is the brief's most important instruction and it was the one placeholder with three plausible fills. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/references/agent-brief.md | 5 ++++- .claude/skills/prerelease-test/references/orchestration.md | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.claude/skills/prerelease-test/references/agent-brief.md b/.claude/skills/prerelease-test/references/agent-brief.md index 4ca743341fa..4e95c07ef9d 100644 --- a/.claude/skills/prerelease-test/references/agent-brief.md +++ b/.claude/skills/prerelease-test/references/agent-brief.md @@ -23,12 +23,15 @@ and hunt for anomalies. You REPORT issues; you never fix framework code. Everything installs from PyPI. We are testing what users receive, not what the tree contains. 2. **Never run python with the checkout as your working directory.** `<REPO>/reflex/` shadows the installed package, so `import reflex` silently picks up unreleased source and your results - become fiction. Run scripts from a neutral directory and start each repro with an assertion: + become fiction. Run scripts from a neutral directory and start each repro with an assertion + naming the venv whose python is running it — the shared one below, or your own: ```python import reflex assert "<VENV_PATH>" in reflex.__file__, reflex.__file__ ``` + A guard that names some other venv fails on every run and gets deleted, which leaves you with + no guard at all. 3. Reading the checkout is fine and encouraged — release source is on branch `<PRERELEASE_BRANCH>`. For PR context load the GitHub MCP tools via ToolSearch (`select:mcp__github__pull_request_read`). 4. Do NOT run any `git` write commands (add/commit/checkout/...) in the checkout. The orchestrator diff --git a/.claude/skills/prerelease-test/references/orchestration.md b/.claude/skills/prerelease-test/references/orchestration.md index 5f577840fc5..da311d6812b 100644 --- a/.claude/skills/prerelease-test/references/orchestration.md +++ b/.claude/skills/prerelease-test/references/orchestration.md @@ -132,12 +132,17 @@ FIRST read <SB>/AGENT_BRIEF.md and follow every rule in it (isolated PyPI-only i install from the checkout; end-to-end browser testing; artifact + NOTES.md deliverables; kill your processes). Your cluster key: "<key>". Working dir: <SB>/apps/<key>/. Artifact destination: <DEST>/<key>/. -Your RESERVED ports: frontend <FP>-<FP+19>, backend <BP>-<BP+19> — always pass them explicitly. +Your RESERVED ports: frontend <FP_START>-<FP_END>, backend <BP_START>-<BP_END> — always pass them +explicitly, and never bind outside them. <cluster brief: changelog lines verbatim, PR numbers, combine-with suggestions> Also in scope: anything adjacent you notice (log warnings, console anomalies, network errors, visual glitches). Record benign-but-surprising observations in NOTES.md and as 'anomaly' entries. ``` +Fill the port ends from the table above, not from a fixed span: an explore agent's range is 20 +ports wide and a verify agent's is 4. A verifier told it owns 20 would reach into the next +verifier's range, which is the collision the map exists to prevent. + ## Running it - Two agents at a time is right for a 4-CPU box; the workflow's own concurrency cap handles the From 0e1c17dd2bf08c5cb0824b3cecc67e07d5826a46 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:27:04 +0000 Subject: [PATCH 13/20] Keep three more failure paths from ending the run A 200 response that is not PyPI's shape was read as a confirmed missing release: only urls present and empty means that, so anything else is now indeterminate rather than a false blocker. A truncated gzip sdist raises EOFError, which is neither OSError nor TarError, so the audit aborted on exactly the interrupted download the surrounding handler was written for. A page an action crashes or closes made the settle and the screenshot raise outside every guard, so the run ended in a traceback with no report at all -- the report being the one thing the caller was waiting for. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .../prerelease-test/scripts/audit_pyi.py | 2 +- .../scripts/check_release_versions.py | 6 ++++- .../prerelease-test/scripts/drive_app.py | 25 ++++++++++++++++--- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py index b0f71945d6a..79f1e735140 100644 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -268,7 +268,7 @@ def audit( try: wheel = wheel_stubs(artifacts["wheel"]) sdist = sdist_stubs(artifacts["sdist"]) - except (zipfile.BadZipFile, tarfile.TarError, OSError) as exc: + except (zipfile.BadZipFile, tarfile.TarError, OSError, EOFError) as exc: # A truncated download and an artifact PyPI really serves broken look identical # from here, so this is unchecked rather than a defect: re-run to tell them apart. reason = f"could not read artifacts: {type(exc).__name__}" diff --git a/.claude/skills/prerelease-test/scripts/check_release_versions.py b/.claude/skills/prerelease-test/scripts/check_release_versions.py index e7307c07959..cb8f6e31d35 100644 --- a/.claude/skills/prerelease-test/scripts/check_release_versions.py +++ b/.claude/skills/prerelease-test/scripts/check_release_versions.py @@ -156,7 +156,11 @@ def pypi_status(name: str, version: str) -> tuple[str, str]: # Network/proxy trouble is indeterminate, not evidence the package is missing. return "error", f"check failed: {type(exc).__name__}" - files = data.get("urls", []) + if not isinstance(data, dict) or not isinstance(data.get("urls"), list): + # A proxy or error page that happens to parse as JSON is not evidence about the + # release. Only PyPI's own shape, with urls present and empty, means "no files". + return "error", "unexpected PyPI response shape" + files = data["urls"] if not files: return "missing", "published but no files" live = [u for u in files if not u.get("yanked")] diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py index 55a0e861e7d..8da58d4795f 100644 --- a/.claude/skills/prerelease-test/scripts/drive_app.py +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -198,6 +198,7 @@ def main() -> int: performed: list[str] = [] action_error: str | None = None load_error: str | None = None + screenshot_error: str | None = None with sync_playwright() as p: browser = p.chromium.launch(executable_path=CHROMIUM, headless=not args.headed) @@ -238,7 +239,12 @@ def main() -> int: # that; without this the last action's fallout is invisible to the report. # A failed action counts: the click that raised on its assertion may still # have reached the server, and that response is often the whole finding. - page.wait_for_timeout(args.settle) + try: + page.wait_for_timeout(args.settle) + except Exception as exc: + # An action can close the page or navigate it away. That is an anomaly to + # record, not a reason to abandon the report the caller is waiting on. + action_error = action_error or f"{type(exc).__name__}: {exc}" try: title = page.title() @@ -247,7 +253,11 @@ def main() -> int: title, body = "", f"<unreadable: {type(exc).__name__}>" if args.screenshot: Path(args.screenshot).parent.mkdir(parents=True, exist_ok=True) - page.screenshot(path=args.screenshot, full_page=True) + try: + page.screenshot(path=args.screenshot, full_page=True) + except Exception as exc: + # Same closed-page case; a missing screenshot must not cost the report. + screenshot_error = f"{type(exc).__name__}: {exc}" browser.close() shown = ( @@ -257,7 +267,13 @@ def main() -> int: ) problems = [m for m in shown if m["type"] in ("error", "warning")] clean = not ( - problems or page_errors or failed or bad_status or action_error or load_error + problems + or page_errors + or failed + or bad_status + or action_error + or load_error + or screenshot_error ) report = { @@ -267,6 +283,7 @@ def main() -> int: "load_error": load_error, "actions_performed": performed, "action_error": action_error, + "screenshot_error": screenshot_error, "console": shown, "page_errors": page_errors, "failed_requests": failed, @@ -285,6 +302,8 @@ def main() -> int: print(f" did {item}") if action_error: print(f" ACTION FAILED: {action_error}") + if screenshot_error: + print(f" SCREENSHOT FAILED: {screenshot_error}") for m in shown: print(f" [{m['type']}] {m['text'][:300]}") for e in page_errors: From 778ac15a3c879327b907ff9aefe67881ab2bfa29 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:29:16 +0000 Subject: [PATCH 14/20] Never leave a partial artifact where the next run will reuse it A download that died partway wrote its bytes straight to the final path, and --keep skips whatever already exists, so every later run reused the truncated file. The EOFError handling added last commit made that worse: it tells you to re-run to tell a bad download from a bad artifact, and re-running could not clear the bad download. Write to .part and rename only once the copy completes. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .../prerelease-test/scripts/audit_pyi.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py index 79f1e735140..1c06362daf0 100644 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -65,11 +65,19 @@ def download_artifacts(name: str, version: str, dest: Path) -> dict[str, Path]: if not path.exists(): # urlretrieve honours only the global socket timeout, which is unset, so a # stalled transfer would hang the whole audit rather than failing this row. - with ( - urllib.request.urlopen(entry["url"], timeout=60) as response, - path.open("wb") as handle, - ): - shutil.copyfileobj(response, handle) + partial = path.with_name(path.name + ".part") + try: + with ( + urllib.request.urlopen(entry["url"], timeout=60) as response, + partial.open("wb") as handle, + ): + shutil.copyfileobj(response, handle) + # Only a whole file earns the real name. A partial one left there would be + # reused by every later run, since --keep skips what already exists, and + # the re-run this failure tells you to do could never clear it. + partial.replace(path) + finally: + partial.unlink(missing_ok=True) out["wheel" if kind == "bdist_wheel" else "sdist"] = path return out From 5bf408aec3ebb67ccb7be7fb13a9bb445c45ab14 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:37:01 +0000 Subject: [PATCH 15/20] Gate the audit on discovery, and stop two more raises escaping Two commands on separate lines never gated anything: the audit ran whatever discovery returned, so the truncated-list PASS the prose claims to prevent still happened. Chain them with && so a non-zero discovery skips the audit. A urls list whose entries are not PyPI's raised out of pypi_status instead of reporting one bad check, and a corrupted deflate stream raises zlib.error, which subclasses only Exception and so missed the archive handler entirely. Name exceptions module-qualified unless they are builtins: zlib.error is called just "error", and "could not read artifacts: error" tells the reader nothing. Report an unknown --ref rather than letting CalledProcessError print a stack about subprocess internals over what is almost always a typo. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/SKILL.md | 13 +++++---- .../prerelease-test/scripts/audit_pyi.py | 29 +++++++++++++++++-- .../scripts/check_release_versions.py | 29 ++++++++++++++++--- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/.claude/skills/prerelease-test/SKILL.md b/.claude/skills/prerelease-test/SKILL.md index 8fc011cf948..ebe2af93e20 100644 --- a/.claude/skills/prerelease-test/SKILL.md +++ b/.claude/skills/prerelease-test/SKILL.md @@ -121,14 +121,15 @@ were removed; that finds the breakage in seconds instead of hours. ### Phase 5 — Packaging audit -Audit every package in the train; the discovery script feeds it the whole list. Run it as two -commands, not one pipe — a pipe reports only the audit's exit status, so a package whose PyPI -check never completed would be dropped from the list and the audit would still print PASS: +Audit every package in the train; the discovery script feeds it the whole list. Chain the two with +`&&`, never a pipe — a pipe reports only the audit's exit status, so a package whose PyPI check +never completed would be dropped from the list and the audit would still print PASS over what was +left. The `&&` is what makes that impossible: the audit runs only when discovery exits 0. uv run --script .claude/skills/prerelease-test/scripts/check_release_versions.py \ - --ref <ref> --specs > specs.txt # non-zero if anything is unpublished or unchecked - xargs uv run --script .claude/skills/prerelease-test/scripts/audit_pyi.py \ - --manifest-ref <ref> < specs.txt + --ref <ref> --specs > specs.txt \ + && xargs uv run --script .claude/skills/prerelease-test/scripts/audit_pyi.py \ + --manifest-ref <ref> < specs.txt It verifies each package ships its own generated `.pyi` stubs in **both** wheel and sdist, that they are byte-identical between the two, that no package leaks another package's stubs, and that diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py index 1c06362daf0..3265049728e 100644 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -35,9 +35,26 @@ import tempfile import urllib.request import zipfile +import zlib from pathlib import Path +def _exc_name(exc: BaseException) -> str: + """Name an exception well enough for the reader to act on it. + + Args: + exc: The exception to name. + + Returns: + The class name, module-qualified unless it is a builtin. ``zlib.error`` is called + just ``error``, which says nothing at all on its own in a report. + """ + cls = type(exc) + if cls.__module__ == "builtins": + return cls.__name__ + return f"{cls.__module__}.{cls.__name__}" + + def download_artifacts(name: str, version: str, dest: Path) -> dict[str, Path]: """Download the wheel and sdist for a package version. @@ -265,7 +282,7 @@ def audit( except Exception as exc: # One unreachable package should not abort the audit of all the others. # Never reached the package, so nothing is known about its packaging. - reason = f"could not fetch artifacts: {type(exc).__name__}" + reason = f"could not fetch artifacts: {_exc_name(exc)}" return _record( name, version, problems, expected, have_manifest, unchecked=reason ) @@ -276,10 +293,16 @@ def audit( try: wheel = wheel_stubs(artifacts["wheel"]) sdist = sdist_stubs(artifacts["sdist"]) - except (zipfile.BadZipFile, tarfile.TarError, OSError, EOFError) as exc: + except ( + zipfile.BadZipFile, + tarfile.TarError, + OSError, + EOFError, + zlib.error, + ) as exc: # A truncated download and an artifact PyPI really serves broken look identical # from here, so this is unchecked rather than a defect: re-run to tell them apart. - reason = f"could not read artifacts: {type(exc).__name__}" + reason = f"could not read artifacts: {_exc_name(exc)}" return _record( name, version, problems, expected, have_manifest, unchecked=reason ) diff --git a/.claude/skills/prerelease-test/scripts/check_release_versions.py b/.claude/skills/prerelease-test/scripts/check_release_versions.py index cb8f6e31d35..c071f1040a2 100644 --- a/.claude/skills/prerelease-test/scripts/check_release_versions.py +++ b/.claude/skills/prerelease-test/scripts/check_release_versions.py @@ -90,13 +90,22 @@ def changelog_paths(ref: str, repo: str) -> list[str]: Returns: Paths of the root ``CHANGELOG.md`` and every ``packages/*/CHANGELOG.md``. + + Raises: + LookupError: The ref does not exist in that checkout. """ - listing = subprocess.run( + result = subprocess.run( ["git", "-C", repo, "ls-tree", "-r", "--name-only", ref], capture_output=True, text=True, - check=True, - ).stdout.splitlines() + check=False, + ) + if result.returncode != 0: + # A mistyped ref is the likeliest way to misuse this, and a CalledProcessError + # traceback buries that under a stack it has no use for. + msg = f"cannot read ref {ref!r} in {repo}: {result.stderr.strip()}" + raise LookupError(msg) + listing = result.stdout.splitlines() skip = excluded_packages(ref, repo) paths = [] for path in listing: @@ -163,6 +172,12 @@ def pypi_status(name: str, version: str) -> tuple[str, str]: files = data["urls"] if not files: return "missing", "published but no files" + if not all( + isinstance(f, dict) and isinstance(f.get("packagetype"), str) for f in files + ): + # The list is there but its entries are not PyPI's, so reading them would raise + # out of this function and end the run rather than reporting one bad check. + return "error", "unexpected PyPI response shape" live = [u for u in files if not u.get("yanked")] if not live: # The version exists but every file is withdrawn: resolvers skip a yanked release @@ -198,8 +213,14 @@ def main() -> int: ) args = parser.parse_args() + try: + paths = sorted(changelog_paths(args.ref, args.repo)) + except LookupError as exc: + print(exc, file=sys.stderr) + return 2 + rows = [] - for path in sorted(changelog_paths(args.ref, args.repo)): + for path in paths: content = git_show(args.ref, path, args.repo) or "" name = dist_name(path, args.ref, args.repo) match = VERSION_HEADING.search(content) From c356b8b63aaf85218a16cdfd1358586294c86256 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:38:55 +0000 Subject: [PATCH 16/20] Guard the destinations too, not just the writes The screenshot's mkdir sat outside the handler added for the screenshot itself, so a destination that cannot be created still ended the run in a traceback -- the same hole the handler was there to close, one line above it. The report write had no guard at all: losing the file should not also lose the findings, so the summary now prints either way. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .../skills/prerelease-test/scripts/drive_app.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py index 8da58d4795f..815a62445fe 100644 --- a/.claude/skills/prerelease-test/scripts/drive_app.py +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -252,11 +252,12 @@ def main() -> int: except Exception as exc: title, body = "", f"<unreadable: {type(exc).__name__}>" if args.screenshot: - Path(args.screenshot).parent.mkdir(parents=True, exist_ok=True) try: + Path(args.screenshot).parent.mkdir(parents=True, exist_ok=True) page.screenshot(path=args.screenshot, full_page=True) except Exception as exc: - # Same closed-page case; a missing screenshot must not cost the report. + # A closed page, or a destination that cannot be created. Either way a + # missing screenshot must not cost the report. screenshot_error = f"{type(exc).__name__}: {exc}" browser.close() @@ -291,8 +292,13 @@ def main() -> int: "body_text": body, } if args.report: - Path(args.report).parent.mkdir(parents=True, exist_ok=True) - Path(args.report).write_text(json.dumps(report, indent=2)) + try: + Path(args.report).parent.mkdir(parents=True, exist_ok=True) + Path(args.report).write_text(json.dumps(report, indent=2)) + except OSError as exc: + # Losing the file must not also lose the findings: the summary below is + # printed either way. + print(f" REPORT NOT WRITTEN: {exc}", file=sys.stderr) if load_error: print(f" LOAD FAILED: {load_error}") From dd3dbac703fec8bc0d50578463a4dfdb27c1b31d Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:42:34 +0000 Subject: [PATCH 17/20] Stop an unreadable page from reporting a clean run Reading the title and body was guarded, but the failure only reached the body text, so a page that died after a successful load and actions produced RESULT: clean and exit 0. A QA driver that says clean when it inspected nothing is the one result it must never produce, so the read failure is now recorded and counts against the run. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/scripts/drive_app.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py index 815a62445fe..e624116bd18 100644 --- a/.claude/skills/prerelease-test/scripts/drive_app.py +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -199,6 +199,7 @@ def main() -> int: action_error: str | None = None load_error: str | None = None screenshot_error: str | None = None + read_error: str | None = None with sync_playwright() as p: browser = p.chromium.launch(executable_path=CHROMIUM, headless=not args.headed) @@ -250,7 +251,11 @@ def main() -> int: title = page.title() body = page.inner_text("body")[:2000] except Exception as exc: + # Never fold this into the body text alone: a page that cannot be read is a + # run that inspected nothing, and reporting that as clean is the one result + # this driver must never produce. title, body = "", f"<unreadable: {type(exc).__name__}>" + read_error = f"{type(exc).__name__}: {exc}" if args.screenshot: try: Path(args.screenshot).parent.mkdir(parents=True, exist_ok=True) @@ -275,6 +280,7 @@ def main() -> int: or action_error or load_error or screenshot_error + or read_error ) report = { @@ -285,6 +291,7 @@ def main() -> int: "actions_performed": performed, "action_error": action_error, "screenshot_error": screenshot_error, + "read_error": read_error, "console": shown, "page_errors": page_errors, "failed_requests": failed, @@ -308,6 +315,8 @@ def main() -> int: print(f" did {item}") if action_error: print(f" ACTION FAILED: {action_error}") + if read_error: + print(f" PAGE UNREADABLE: {read_error}") if screenshot_error: print(f" SCREENSHOT FAILED: {screenshot_error}") for m in shown: From e8602654e5d682f9f8b288acf5f5bacf9e7ac234 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:43:33 +0000 Subject: [PATCH 18/20] Fix the pipe left in the audit script's own usage SKILL.md moved to && so a failed discovery cannot hand its dropped packages to a passing audit, but this script still documented the pipe it warns about, so anyone reading the script instead of the skill would copy the masking form. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/scripts/audit_pyi.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py index 3265049728e..f3dc18679fb 100644 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -16,7 +16,12 @@ Usage: audit_pyi.py reflex==0.9.9 reflex-base==0.9.9 - check_release_versions.py --ref origin/main --specs | xargs audit_pyi.py --manifest-ref origin/main + +Chain it to discovery with && rather than a pipe: a pipe reports only this script's exit +status, so a package dropped for a failed lookup would take the audit's PASS with it. + + check_release_versions.py --ref origin/main --specs > specs.txt \ + && xargs audit_pyi.py --manifest-ref origin/main < specs.txt """ # /// script From e2bbca3c33247ba6658e0423ce75433ab675fb48 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:45:01 +0000 Subject: [PATCH 19/20] Fail the run when a requested report cannot be written The guard added two commits ago printed REPORT NOT WRITTEN and carried on, so a healthy app with an unwritable --report path still printed RESULT: clean and exited 0. A caller that asked for a report and got none has no result to trust, and automation would have read that as a pass. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .claude/skills/prerelease-test/scripts/drive_app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py index e624116bd18..1c2a2a3072d 100644 --- a/.claude/skills/prerelease-test/scripts/drive_app.py +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -304,8 +304,10 @@ def main() -> int: Path(args.report).write_text(json.dumps(report, indent=2)) except OSError as exc: # Losing the file must not also lose the findings: the summary below is - # printed either way. + # printed either way. It must not be reported as a pass either — a caller + # that asked for a report and got none has no result to trust. print(f" REPORT NOT WRITTEN: {exc}", file=sys.stderr) + clean = False if load_error: print(f" LOAD FAILED: {load_error}") From 3b965b9331d02c68e0895839be07552937fd71ab Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 29 Aug 2026 00:48:59 +0000 Subject: [PATCH 20/20] Make the documented commands runnable and reject bad actions early Both usage docstrings showed bare script invocations, but these scripts have no shebang and no exec bit by design -- the PEP 723 metadata is there so uv builds the environment. Copying either documented line failed. Show uv run --script, the same form the skill's canonical command uses. A list whose items are not single-key objects passed validation and failed later inside the action loop, spending a browser launch and a page load on a typo the early exit was added to catch. A yanked flag that is present but not a boolean no longer reads as live: absent still means not yanked, but a value that is not PyPI's cannot be interpreted. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EMjBXPozsNeQNSBZecNH8x --- .../skills/prerelease-test/scripts/audit_pyi.py | 8 ++++---- .../scripts/check_release_versions.py | 15 ++++++++++----- .../skills/prerelease-test/scripts/drive_app.py | 6 ++++++ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py index f3dc18679fb..8daa83076d2 100644 --- a/.claude/skills/prerelease-test/scripts/audit_pyi.py +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -14,14 +14,14 @@ It exits 1 when a package's packaging is wrong and 2 when a package could not be audited at all, so an unreachable PyPI never reads as a defect in a package nobody looked at. -Usage: - audit_pyi.py reflex==0.9.9 reflex-base==0.9.9 +Usage (the script carries PEP 723 metadata and no shebang, so run it through uv): + uv run --script audit_pyi.py reflex==0.9.9 reflex-base==0.9.9 Chain it to discovery with && rather than a pipe: a pipe reports only this script's exit status, so a package dropped for a failed lookup would take the audit's PASS with it. - check_release_versions.py --ref origin/main --specs > specs.txt \ - && xargs audit_pyi.py --manifest-ref origin/main < specs.txt + uv run --script check_release_versions.py --ref origin/main --specs > specs.txt \ + && xargs uv run --script audit_pyi.py --manifest-ref origin/main < specs.txt """ # /// script diff --git a/.claude/skills/prerelease-test/scripts/check_release_versions.py b/.claude/skills/prerelease-test/scripts/check_release_versions.py index c071f1040a2..35580c82325 100644 --- a/.claude/skills/prerelease-test/scripts/check_release_versions.py +++ b/.claude/skills/prerelease-test/scripts/check_release_versions.py @@ -7,10 +7,10 @@ An unpublished version is a release blocker in its own right, so a missing package makes this exit non-zero. -Usage: - check_release_versions.py --ref origin/r/pre-2026-08-27-1234 - check_release_versions.py --ref origin/main --json - check_release_versions.py --ref origin/main --specs # pkg==ver lines for audit_pyi.py +Usage (the script carries PEP 723 metadata and no shebang, so run it through uv): + uv run --script check_release_versions.py --ref origin/r/pre-2026-08-27-1234 + uv run --script check_release_versions.py --ref origin/main --json + uv run --script check_release_versions.py --ref origin/main --specs # pkg==ver lines """ # /// script @@ -173,7 +173,12 @@ def pypi_status(name: str, version: str) -> tuple[str, str]: if not files: return "missing", "published but no files" if not all( - isinstance(f, dict) and isinstance(f.get("packagetype"), str) for f in files + isinstance(f, dict) + and isinstance(f.get("packagetype"), str) + # Absent means not yanked, which is fine; present but not a boolean means the + # entry is not PyPI's and its yank state cannot be read from it. + and isinstance(f.get("yanked", False), bool) + for f in files ): # The list is there but its entries are not PyPI's, so reading them would raise # out of this function and end the run rather than reporting one bad check. diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py index 1c2a2a3072d..61eac672a93 100644 --- a/.claude/skills/prerelease-test/scripts/drive_app.py +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -91,6 +91,12 @@ def load_actions(raw: str | None) -> list[dict]: if not isinstance(actions, list): msg = f"expected a JSON list of actions, got {type(actions).__name__}" raise ValueError(msg) + for action in actions: + # run_action unpacks exactly one (verb, value) pair, so anything else would fail + # mid-run, after a browser launch and a page load that a caller typo did not earn. + if not isinstance(action, dict) or len(action) != 1: + msg = f"each action must be a single-key object, got {action!r}" + raise ValueError(msg) return actions