diff --git a/.claude/skills/prerelease-test/SKILL.md b/.claude/skills/prerelease-test/SKILL.md new file mode 100644 index 00000000000..ebe2af93e20 --- /dev/null +++ b/.claude/skills/prerelease-test/SKILL.md @@ -0,0 +1,190 @@ +--- +name: prerelease-test +description: Run the independent pre-release QA campaign for a Reflex release train. Discovers what shipped by reading every CHANGELOG.md on the pre-release branch, checks each package actually published to PyPI, then exercises the new features end-to-end in real apps driven by a real browser using ONLY published packages, upgrade-tests reflex-examples apps from the previous stable, regression-tests reflex-enterprise demos, audits wheel/sdist packaging, and produces a triaged FINDINGS.md plus a fix-before-release plan. Use this whenever a Reflex release is being prepared or checked — the user mentions a pre-release, an alpha/beta/rc, an `r/pre-*` branch, "about to release", release QA, validating a release candidate, or re-verifying that release-blocking fixes landed in a newer alpha. Also use it for the narrower slices on their own: smoke-testing published packages, upgrade/regression testing example apps against a new version, or auditing that packages ship their `.pyi` stubs in both wheel and sdist. +--- + +# Reflex pre-release testing + +This skill runs the QA campaign that stands between a pre-release and a release: independent, +adversarial, end-to-end exercise of what actually got published, from the perspective of a user +who runs `pip install reflex` and opens a browser. + +Its value comes from three habits, not from breadth of assertions: + +1. **Only published packages.** Everything installs from PyPI into throwaway venvs. The checkout + has unreleased code, different metadata, and a different dependency graph — testing it tells + you nothing about what users will get, and a package that failed to publish is itself a + release blocker you would otherwise miss. +2. **Baseline against the previous stable.** A defect that also reproduces on the last release is + a bug; one that only reproduces on the new version is a regression and usually a blocker. + Without the baseline run you cannot tell them apart, and the triage that follows is guesswork. +3. **Adversarially verify before reporting.** Have a second agent reproduce each claimed issue + from the written repro alone and actively try to refute it. In practice this refutes or + reclassifies a meaningful share of findings — and it is what makes the report trustworthy + enough for maintainers to act on directly. + +## Deliverables + +Produce these under `prerelease-testing/-/` in the repo and commit them to a work +branch as you go (never to `main`): + +- `FINDINGS.md` — executive summary, numbered findings with repro + evidence + regression status, + refuted claims, and per-cluster summaries. Templates: `references/reporting.md`. +- `RELEASE_PLAN.md` — triage into fix-before-release vs file-as-issue. Rubric below. +- One directory per test cluster containing the sample app sources (no `.web/`, `node_modules/`, + venvs), the driver scripts, `NOTES.md` with exact rerun commands, logs and screenshots. +- `README.md` — what each cluster covers and how to reuse it for the next release. + +Findings must be reproducible by a stranger from `NOTES.md` alone. That is the bar: a fix agent +gets handed the finding and nothing else. + +## Phases + +Work these in order. Phases 2–5 can overlap; 0 and 1 gate everything. + +### Phase 0 — Scope discovery + +Find the pre-release branch (`git ls-remote --heads origin 'r/pre-*'`, or the user names it), then +read **every** `CHANGELOG.md` on it: the root one plus `packages/*/CHANGELOG.md`. The top entry of +each is what this train ships. + +Run the discovery script to extract every package's version and confirm each is published: + + uv run --script .claude/skills/prerelease-test/scripts/check_release_versions.py --ref + +(Script paths in this skill are repo-root-relative — the repo has its own unrelated `scripts/` +directory, so the prefix matters.) It exits 1 when a package is confirmed missing and 2 when a +check could not complete, so a proxy hiccup never reads as a missing package. An unpublished +version is a finding in its own right — report it immediately rather than working around it. + +Read the linked PRs for anything whose intent is not obvious from the changelog line; the GitHub +MCP tools (`mcp__github__pull_request_read`) do this well. Understanding what a change was *for* +is what lets you test it as a user rather than as a checklist. + +### Phase 1 — De-risk with a smoke test + +Before any fan-out: one venv, `reflex init --template blank`, `reflex run`, then drive it in +Chromium with `.claude/skills/prerelease-test/scripts/drive_app.py`. This shakes out environment +problems (proxy, bun installs, browser path) once, in a context where you can debug them, instead +of inside ten parallel agents. + +The scripts carry PEP 723 metadata, so `uv run --script ` runs each one in its own isolated +environment (the driver pulls in playwright that way); running them with an existing driver venv's +interpreter works too. + +Also confirm the dependency graph resolves the way the changelog says it should (e.g. optional +dependencies genuinely absent, sub-packages pinned as intended). + +### Phase 2 — Feature exploration fan-out + +Decompose the changelog into clusters of related changes and give each to its own agent, with an +adversarial verification stage behind it. See `references/clusters.md` for how to cut the clusters +and a standing coverage list; `references/orchestration.md` for the Workflow script, schemas and +port map. + +The instruction that makes this productive: **real-world exploration, not coverage filling.** +Combine each new feature with the things users actually combine it with — State vars, +`rx._x.client_state`, `rx.ComponentState`, `@rx.memo` wrapping, `rx.foreach`/`rx.cond`, event +chains, background tasks, multiple pages and navigation, dev *and* prod mode. Bugs live in the +interactions, and the original PR's tests already cover the happy path. + +Every run inspects all four channels: server log, browser console (errors *and* warnings), network +tab (failed requests, 4xx/5xx), and the rendered page. Findings frequently show up in a channel +nobody asserted on. + +### Phase 3 — Upgrade regression + +Users upgrade in place; that path has its own failure modes (lockfile migration, pruned packages, +stale `.web/`). Clone `reflex-dev/reflex-examples`, pick apps that span the feature surface — +including ones using third-party packages (`reflex-local-auth`, `reflex-global-hotkey`) and +`reflex[db]`/API apps — then for each: + +1. Install the **previous stable** (no `--prerelease` flag, so requirements resolve as a user's + would), run it, and drive its real user flows in the browser. This is your baseline. +2. Upgrade the *same* venv and *same* app directory in place to the new version, preserving + `.web/` and `reflex.lock/`. Watch the first run's log closely — that is where migration + happens. Re-drive the identical flows and compare. +3. Do one cold run (`rm -rf .web`) to check the fresh-install path converges to the same state. + +Diff `.web/package.json` before and after; unexpected dependency changes are findings. + +### Phase 4 — Enterprise regression + +Downstream breakage is the most common release blocker, because removing a public name is +invisible until something imports it. Install the **published** `reflex-enterprise` (never the +checkout) against the new reflex and run its demos — ag-grid, map, dnd, mantine, flow, the MCP +plugin, OIDC. Baseline against the previous stable whenever something fails, so you can say +whether the new release broke it. + +Before running anything, grep the published enterprise wheel for names the changelog says moved or +were removed; that finds the breakage in seconds instead of hours. + +### Phase 5 — Packaging audit + +Audit every package in the train; the discovery script feeds it the whole list. Chain the two with +`&&`, never a pipe — a pipe reports only the audit's exit status, so a package whose PyPI check +never completed would be dropped from the list and the audit would still print PASS over what was +left. The `&&` is what makes that impossible: the audit runs only when discovery exits 0. + + uv run --script .claude/skills/prerelease-test/scripts/check_release_versions.py \ + --ref --specs > specs.txt \ + && xargs uv run --script .claude/skills/prerelease-test/scripts/audit_pyi.py \ + --manifest-ref < specs.txt + +It verifies each package ships its own generated `.pyi` stubs in **both** wheel and sdist, that +they are byte-identical between the two, that no package leaks another package's stubs, and that +counts line up with `pyi_hashes.json` (a package absent from the manifest must ship none). + +Stub content hashes legitimately differ from the committed manifest because the build hook +regenerates them at release time — compare *presence and counts*, and wheel against sdist. + +### Phase 6 — Triage and report + +Write `FINDINGS.md` first (everything confirmed), then `RELEASE_PLAN.md` splitting findings into: + +**Fix before release** — anything meeting one of: +- a confirmed regression against the previous stable, +- security-relevant (path traversal, unauthenticated input handling, injection), +- significant user impact, or trivially small to fix (a one-line guard, a missing default). + +**File as issues, fix after** — everything else, including pre-existing defects the campaign +happened to surface. Downstream (enterprise) issues go to that project's tracker, not this repo's. + +Also flag decisions only a maintainer can make — an undocumented behavior change is either a bug +or a missing changelog entry, and which one it is is their call. Ask rather than assume. + +### Phase 7 — Re-verify the next build + +When a new alpha ships with the fixes, re-run the *original failing repro* for each finding +(not just the unit tests) against the new published packages, plus a general smoke. Append the +pass/fail table to `FINDINGS.md`. When the final ships, do one last stock-install smoke: +`pip install reflex==` with no prerelease flag, init, run, browser, and confirm the +resolved graph is all-final. + +## Working rules + +- **Isolation.** Every agent gets its own venv and its own reserved port range; never install into + a shared venv. See the port map in `references/orchestration.md`. +- **The traps are real.** `references/agent-brief.md` is the brief to hand every agent — it carries + the environment gotchas (checkout shadowing, proxy variables breaking installs) and the list of + known-benign console noise. Agents that skip it rediscover the same problems and report noise as + findings. +- **Concurrency.** Dev servers are heavy (bun + vite + granian). On a 4-CPU box run about two + agents at a time and one dev server per agent; more just makes everything slow and flaky. +- **Commit as you go.** Each cluster's artifacts land on the work branch when that cluster + finishes, so a long campaign is never one unrecoverable batch. +- **Never fix framework code during the campaign.** Testing and fixing are separate engagements + with separate review standards; mixing them costs you the independence that makes the findings + credible. Record the repro and move on. (If the user then asks for fixes, that is a new task — + one PR per finding, regression test first, `references/reporting.md` has the PR conventions.) + +## Scaling the campaign + +Full campaign is roughly 30 agents over several hours. Scale down by dropping whole phases rather +than by testing each phase more shallowly — a shallow pass produces false confidence: + +- **Quick check** (~30 min): phases 0, 1, 5, plus grep-based downstream checks from phase 4. +- **Standard** (~2 h): add phase 2 over the three or four highest-risk clusters. +- **Full**: everything, with adversarial verification on every claimed issue. + +Ask the user which they want if the request is ambiguous and the difference matters. diff --git a/.claude/skills/prerelease-test/references/agent-brief.md b/.claude/skills/prerelease-test/references/agent-brief.md new file mode 100644 index 00000000000..4e95c07ef9d --- /dev/null +++ b/.claude/skills/prerelease-test/references/agent-brief.md @@ -0,0 +1,115 @@ +# The agent brief + +Copy this into the campaign scratchpad (e.g. `$SB/AGENT_BRIEF.md`), fill the `<...>` placeholders, +and tell every test agent to read it first. It carries the rules that keep results trustworthy and +the environment traps that otherwise get rediscovered — expensively — by each agent in turn. + +Keep it in a file rather than pasting it into each prompt: agents re-read it when they get +confused, and updating one file updates the whole fleet. + +--- + +## BRIEF TEMPLATE (copy from here) + +You are one of several agents independently exercising the reflex `` pre-release +(published ``) as a real-world user of the framework. Your job: build small sample apps and +repro scripts for your assigned feature cluster, run them END-TO-END (real server, real Chromium), +and hunt for anomalies. You REPORT issues; you never fix framework code. + +### HARD RULES + +1. **NEVER install reflex (or any workspace package) from the local checkout at ``.** No + `uv sync`, no `uv run`, no `pip install -e`, no `uv pip install .` anywhere under it. + Everything installs from PyPI. We are testing what users receive, not what the tree contains. +2. **Never run python with the checkout as your working directory.** `/reflex/` shadows the + installed package, so `import reflex` silently picks up unreleased source and your results + become fiction. Run scripts from a neutral directory and start each repro with an assertion + naming the venv whose python is running it — the shared one below, or your own: + ```python + import reflex + + assert "" in reflex.__file__, reflex.__file__ + ``` + A guard that names some other venv fails on every run and gets deleted, which leaves you with + no guard at all. +3. Reading the checkout is fine and encouraged — release source is on branch ``. + For PR context load the GitHub MCP tools via ToolSearch (`select:mcp__github__pull_request_read`). +4. Do NOT run any `git` write commands (add/commit/checkout/...) in the checkout. The orchestrator + commits artifacts. +5. Do NOT fix bugs you find — record precise repro steps instead. +6. Kill every server and browser you start before you finish (track PIDs; verify with `ps`). Other + agents share this machine — run at most ONE dev server at a time unless your cluster needs two. + +### Environment + +- Scratchpad root: `SB=` +- **Prebuilt shared venv (READ-ONLY — never install into it):** `$SB/envs/` has + `reflex==` and the alpha sub-packages. Use `$SB/envs//bin/reflex` directly. +- Need other deps or versions? Make your OWN venv: + ``` + uv venv $SB/envs/ --python 3.11 + uv pip install --python $SB/envs//bin/python --prerelease=allow 'reflex==' + ``` + Always pass `--prerelease=allow` for alphas; PyPI is the default index — never point it at the + checkout. Python 3.12/3.13/3.14 are available via `uv venv --python 3.14` etc. +- **Playwright driver venv:** `$SB/envs/driver/bin/python` (playwright, httpx, websockets). + Chromium: `/opt/pw-browsers/chromium` — launch with + `p.chromium.launch(executable_path="/opt/pw-browsers/chromium")`. + A ready-made driver with console/network capture is at + `.claude/skills/prerelease-test/scripts/drive_app.py` in the checkout (read-only use is fine). +- **Local HTTP needs proxy bypass on the CLIENT side only:** prefix curl/Playwright commands with + `NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1` (curl: `--noproxy '*'`). + Do NOT export those variables into the reflex server's environment — it breaks bun's package + installs through the proxy, which looks like a framework bug and is not. +- Set `REFLEX_TELEMETRY_ENABLED=false` for every reflex command. +- Node and bun are preinstalled. Enterprise apps need `CI=true` to bypass the dev login gate. +- App working dirs: `$SB/apps//...`. Run servers on YOUR ASSIGNED PORTS: + `reflex run --frontend-port --backend-port `. The first run does a bun install (1–2 min); + poll the frontend URL until it returns 200 for up to ~6 minutes before concluding failure. +- `--loglevel debug` gives verbose server logs; redirect to a file and actually read it. + +### What "testing" means here + +- Real-world exploration, not coverage filling. Combine the feature with State vars, + `rx._x.client_state`, `@rx.memo` wrapping, `rx.ComponentState`, `rx.foreach`/`rx.cond`, event + chains (`yield Other.handler()`), background tasks, multiple pages and navigation — whatever + plausibly interacts. The original PR already tested the happy path. +- Drive the app in Chromium as a user would: click, type, navigate, upload, drag, use the keyboard. +- On EVERY run capture and inspect: (a) the server log file, (b) browser console messages (errors + AND warnings), (c) failed requests / 4xx-5xx responses, (d) screenshots at key moments. +- Baseline comparisons are what make findings actionable: if behavior looks wrong, check the + previous stable (`uv pip install 'reflex=='` in its own venv). Only-on-new is a + regression and high severity; both-versions is context worth noting, not a blocker. +- Prod matters too: `reflex run --env prod` compiles and serves the built frontend. Test both modes + when your feature could differ (hydration, memoization, routing, prerender). + +### Known-benign noise — do not report these as findings + +- Browser console: the React Router "💿 Hey developer" HydrateFallback log, vite + `connecting.../connected` debug lines, the React DevTools download info line. +- `reflex init` logging a few "Failed to connect to https://registry.npmmirror.com" lines before + falling back (this environment blocks that mirror). Do report it if the fallback itself fails. +- Transient bun "incorrect peer dependency" warnings during a one-time upgrade migration, provided + the final lockfile is consistent. + +If you see something surprising that is not on this list, investigate it — several real findings +have surfaced first as an unexplained warning. + +### Deliverables (mandatory) + +1. Copy reusable artifacts into the repo (plain `cp`/`rsync`, no git): + `DEST=//` + - the app source dirs, EXCLUDING `.web/`, `node_modules/`, `.states/`, `assets/external/`, + `*.db`, venvs + - your Playwright/repro scripts + - `NOTES.md`: what you tested, how to rerun it (exact commands), what you observed, including + benign quirks +2. Return structured findings: every discrete check as a test entry (pass/fail/anomaly/skipped) + with enough repro detail that another agent can reproduce a failure from `NOTES.md` and your + scripts alone, without your conversation. An "anomaly" is anything surprising (console error, + warning, traceback, visual glitch, perf cliff) even when functionality works. + +### Timeboxing + +Be thorough but keep moving: if one sub-test resists debugging for ~10 minutes, record it as an +anomaly with logs attached and continue. Finishing the whole cluster beats perfecting one test. diff --git a/.claude/skills/prerelease-test/references/clusters.md b/.claude/skills/prerelease-test/references/clusters.md new file mode 100644 index 00000000000..5c5d86aba5f --- /dev/null +++ b/.claude/skills/prerelease-test/references/clusters.md @@ -0,0 +1,52 @@ +# Cutting the changelog into clusters + +A cluster is one agent's assignment: a group of changelog entries that share a subsystem, so the +agent can build one or two apps that exercise all of them together and notice how they interact. + +## How to cut them + +Group by **where the change lives at runtime**, not by which PR shipped it. Entries that touch the +same subsystem belong together even when they came from unrelated PRs, because the interesting +bugs are in their interaction — and one app can cover them all. + +Aim for 8–12 clusters of 2–6 changelog entries each. A cluster that needs more than two apps is +too big; one with a single trivial entry should be folded into a neighbour. + +Each cluster brief should carry: +- the changelog lines verbatim (agents should not have to re-derive what shipped), +- the PR numbers, so the agent can read intent, +- concrete "combine it with" suggestions — this is what turns a checklist into exploration, +- anything the entry implies but does not state (a perf claim to measure, a behavior that should + differ between dev and prod). + +## Standing coverage checklist + +Most Reflex releases touch some subset of these. Use it to check you have not left a shipped change +unassigned, and as a source of interaction ideas. + +| Area | What to build | Interactions worth forcing | +|---|---|---| +| Routing / navigation | Multi-page app with dynamic (`[id]`), catchall (`[[...splat]]`) and static-sibling routes; slow `on_load` | Navigate away mid-`on_load`; two tabs on different dynamic values; browser back/forward; direct load vs client-side nav; prod mode | +| State & event loop | Counter hammered by clicks while a background task writes | Background task + foreground handler racing; `yield`-streaming handlers; `async with self`; abrupt disconnect mid-stream | +| Components / props | One page per changed component | Prop bound to a State var, to a `client_state` var, and inside `@rx.memo`; explicit `style=` merged with forwarded props | +| Memoization | `@rx.memo` component with state-bound props and an event-handler prop, plus render counters | Memo in `rx.foreach`; memo wrapping `ComponentState`; two memos sharing a name in different modules; dev vs prod | +| Typing / annotations | State vars using every annotation shape the entry mentions | Assign at runtime (not just compile); pass the handler *uncalled* to a trigger; run on the oldest and newest supported Python | +| Uploads | `rx.upload` with buffered and streamed handlers | Hostile filenames (traversal, unicode, all-dots, spaces); raw multipart via httpx alongside browser uploads | +| Logging / CLI | Run every CLI verb with and without the logging flags | Strict JSON-lines parsing of stdout; a user app that configures `logging` itself; missing optional sub-packages | +| Config / plugins | App with the config knob set, unset, and set via env var | Deprecated form alongside the new form; plugin explicitly enabled vs implicit | +| Build / prod / export | Multi-page app through `run --env prod`, `export`, `run --env preview` | Serve exported frontend statically with a backend-only server; inspect generated `.web/package.json`; custom component with a changed library | +| Registration / multi-app | Two apps in one process; `AppHarness` twice | Sequential and simultaneous; check for registration leakage between them | +| Optional dependencies | Bare install, each extra, and the upgrade path | Import the optional package's feature without the extra and judge the error message quality | +| DevTools / perf | App with nested substates, custom components, client-only wrappers | Walk React fibers for `displayName`; record navigation main-thread cost; verify perf claims rather than trusting them | + +## Downstream and packaging clusters + +Two clusters that are not feature-shaped but catch the highest-severity problems: + +- **Breaking-change surface.** For every entry under "Breaking Changes" (and every removal you spot + in the diff), ask what a downstream caller sees. Grep the published `reflex-enterprise` wheel and + the popular third-party packages (`reflex-local-auth`, `reflex-global-hotkey`) for the removed + names. A bare `AttributeError`/`ImportError` with no pointer to the replacement is a finding even + when the removal itself was intended. +- **Packaging.** `.claude/skills/prerelease-test/scripts/audit_pyi.py`, plus a check that each package's declared dependency pins + match what the changelog says and that installing from the sdist works. diff --git a/.claude/skills/prerelease-test/references/orchestration.md b/.claude/skills/prerelease-test/references/orchestration.md new file mode 100644 index 00000000000..da311d6812b --- /dev/null +++ b/.claude/skills/prerelease-test/references/orchestration.md @@ -0,0 +1,154 @@ +# Orchestrating the fleet + +## Choosing the mechanism + +Invoking this skill is itself the opt-in for the Workflow tool, so a full campaign should use it — +the explore→verify pipeline is deterministic control flow over many agents, exactly what Workflow +is for. For a quick check or a single cluster, plain `Agent` calls are simpler and cheaper. + +Use one workflow per phase rather than one giant script. Phase results change what you want to ask +next (a cluster that finds a breaking change reshapes the enterprise phase), and you stay in the +loop between phases. + +## The explore → verify pipeline + +The shape that matters: each cluster's findings go straight into verification without waiting for +the other clusters. Use `pipeline`, not a barrier — a slow cluster should not hold up verification +of a fast one. + +```js +export const meta = { + name: 'prerelease-explore', + description: 'Exercise feature clusters end-to-end, then adversarially verify findings', + phases: [{ title: 'Explore' }, { title: 'Verify' }], +} + +const SB = '' +const DEST = '' + +const FINDINGS_SCHEMA = { + type: 'object', + required: ['cluster', 'summary', 'tests', 'issues', 'artifacts_dir'], + properties: { + cluster: { type: 'string' }, + summary: { type: 'string' }, + tests: { + type: 'array', + items: { + type: 'object', + required: ['name', 'status', 'details'], + properties: { + name: { type: 'string' }, + status: { type: 'string', enum: ['pass', 'fail', 'anomaly', 'skipped'] }, + details: { type: 'string' }, + repro: { type: 'string' }, + }, + }, + }, + issues: { + type: 'array', + items: { + type: 'object', + required: ['title', 'severity', 'repro'], + properties: { + title: { type: 'string' }, + severity: { type: 'string', enum: ['critical', 'high', 'medium', 'low'] }, + repro: { type: 'string' }, + evidence: { type: 'string' }, + regression: { type: 'boolean', description: 'true if the previous stable behaves correctly' }, + }, + }, + }, + artifacts_dir: { type: 'string' }, + }, +} + +const VERDICT_SCHEMA = { + type: 'object', + required: ['confirmed', 'notes'], + properties: { + confirmed: { type: 'boolean' }, + notes: { type: 'string' }, + root_cause_guess: { type: 'string' }, + }, +} + +phase('Explore') +const results = await pipeline( + CLUSTERS, + (c, _item, i) => agent(brief(c, i), { label: `explore:${c.key}`, phase: 'Explore', schema: FINDINGS_SCHEMA }), + async (res, c, i) => { + if (!res) return { cluster: c.key, agent_died: true } + // Cap the fan-out per cluster. Issues past the cap are absent from verified_issues but + // still present in res.issues, which is how the report tells them from verified ones. + const issues = (res.issues || []).slice(0, 4) + if (!issues.length) return res + const verdicts = await parallel( + issues.map((iss, j) => () => + agent(verifyBrief(c, iss, j), { label: `verify:${c.key}:${j}`, phase: 'Verify', schema: VERDICT_SCHEMA }) + // A dead verifier resolves falsy. Give it its own marker: `verdict: null` would + // read as a verdict that cleared the issue, and dropping the entry would leave a + // hole indistinguishable from an issue the cap never sent to a verifier. + .then((v) => (v ? { issue: iss, verdict: v } : { issue: iss, verifier_died: true })), + ), + ) + return { ...res, verified_issues: verdicts } + }, +) +// Dead explorers stay in as `agent_died` markers rather than being filtered out: a cluster +// that produced nothing is a hole in the campaign, and silently dropping it is how a hole +// gets mistaken for a clean result. Same for `verifier_died` above. Every marker is work +// the campaign did not do, so the report must account for each one rather than skip it. +return results.filter(Boolean) +``` + +The verifier's prompt is what makes this work. It must: +- reproduce **from the written repro alone**, in its own working directory, not by reading the + reporter's conversation — that is simultaneously a test of the repro's quality, +- be told to actively **refute**: is this an environment quirk, an API misuse, pre-existing on the + previous stable, or a genuine defect? +- set `confirmed: true` only for a genuine defect a fix agent should act on, +- append a `## VERIFICATION` section to the cluster's `NOTES.md` so the record travels with the + artifacts. + +## Port map + +Give every agent a disjoint range and tell it to always pass `--frontend-port/--backend-port` +explicitly. Never let anything use the defaults (3000/8000) — that is how two agents collide and +produce confusing, unreproducible results. + +| Stage | Frontend | Backend | +|---|---|---| +| Explore agent *i* | `3100 + 40i` … +19 | `8100 + 40i` … +19 | +| Verify agent *i.j* | `3600 + 40i + 4j` … +3 | `8600 + 40i + 4j` … +3 | +| Orchestrator smoke | 3050–3059 | 8050–8059 | + +## Prompt scaffolding for each agent + +Every cluster prompt should open with the same preamble, then the cluster-specific brief: + +``` +FIRST read /AGENT_BRIEF.md and follow every rule in it (isolated PyPI-only installs — never +install from the checkout; end-to-end browser testing; artifact + NOTES.md deliverables; kill your +processes). +Your cluster key: "". Working dir: /apps//. Artifact destination: //. +Your RESERVED ports: frontend -, backend - — always pass them +explicitly, and never bind outside them. + +Also in scope: anything adjacent you notice (log warnings, console anomalies, network errors, +visual glitches). Record benign-but-surprising observations in NOTES.md and as 'anomaly' entries. +``` + +Fill the port ends from the table above, not from a fixed span: an explore agent's range is 20 +ports wide and a verify agent's is 4. A verifier told it owns 20 would reach into the next +verifier's range, which is the collision the map exists to prevent. + +## Running it + +- Two agents at a time is right for a 4-CPU box; the workflow's own concurrency cap handles the + rest of the queue. +- Between phases, read the results before launching the next one. +- Schedule a check-in (`send_later`, ~45 min) during long runs so a wedged agent or a leaked dev + server gets noticed. Check `ps` for stray `reflex run` processes and commit finished clusters. +- When the workflow completes, its full structured result is in the task output file; parse it for + the report rather than re-reading transcripts. diff --git a/.claude/skills/prerelease-test/references/reporting.md b/.claude/skills/prerelease-test/references/reporting.md new file mode 100644 index 00000000000..242eae436f7 --- /dev/null +++ b/.claude/skills/prerelease-test/references/reporting.md @@ -0,0 +1,104 @@ +# Reporting + +## FINDINGS.md + +The audience is a maintainer deciding what blocks the release, and fix agents who will get handed +one finding and nothing else. Write for both: a summary that supports a go/no-go call, and findings +self-contained enough to act on in isolation. + +```markdown +# Findings — reflex pre-release testing () + + + +## Versions under test + + + +## Executive summary + + + + +Index: +- FINDING-001: (SEVERITY) +- ... + +## FINDING-00N: <title> (SEVERITY) + +- Cluster: `<cluster>` | Regression vs <prev stable>: yes/no | Verifier: CONFIRMED +- Repro: <exact commands/steps, self-contained> +- Evidence: <log excerpt, console error, screenshot path> +- Root cause (verifier analysis): <file:line and mechanism, when known> +- Verification notes: <what the independent reproducer found> + +## Refuted / reclassified claims +- **<title>** (`<cluster>`): <why it is not an actionable defect for this release> + +## Cluster summaries +### `<cluster>` (pass:N, anomaly:N, fail:N) +<2-4 sentences: what was built, what was verified, what was found.> +``` + +Two things carry disproportionate weight and are easy to skip: + +- **The refuted list.** It shows the findings were filtered, which is why a maintainer can trust the + ones that remain. Never quietly drop a refuted claim. +- **Regression status on every finding.** "Broken in both versions" and "newly broken" lead to + opposite decisions, and the reader cannot recover that from the description. + +## RELEASE_PLAN.md + +```markdown +# Release plan — what blocks <version> vs what gets filed + +## Already in flight +| PR | Covers | Gap check | +<For each open PR, which findings it addresses and — importantly — what it does NOT cover.> + +## Fix before release +### Security +### Confirmed regressions +### High impact and/or trivially small +<Each item: finding ref, one-line rationale tied to the rubric, and the shape of the fix.> + +## File as issues, fix after release +### <this repo> +### <downstream repo> + +## Decisions needed from a maintainer +<Behavior changes that are either bugs or missing changelog entries — their call, not yours.> + +## Suggested sequencing +<Order of PRs, noting which can land in parallel and which touch the same files.> +``` + +The rubric (confirmed regression / security / significant-impact-or-trivial) is in `SKILL.md`. +Apply it visibly: each entry should say which arm of the rubric put it there, so the maintainer can +disagree with a specific judgment rather than the whole list. + +## Filing issues + +For post-release items: search for duplicates first (2–3 keyword variants, open *and* closed), then +file with a self-contained repro, observed vs expected, environment, a note that it came from +`<version>` pre-release testing, and a pointer to the artifacts branch and cluster directory. +Mirror `.github/ISSUE_TEMPLATE/bug_report.md` where it fits. + +## If the user asks for the fixes afterwards + +That is a separate engagement. What worked well: + +- **One PR per finding**, branched from current `main`, in its own git worktree so parallel fix + agents never collide. +- **Regression test first**: write it, show it failing against unfixed `main`, then fix, then show + it passing. Report both states in the PR body — a reviewer should not have to take it on faith. +- **News fragment** under the right package's `news/` directory. The changelog CI gate checks per + package: a fix touching `packages/reflex-base/` needs a fragment in + `packages/reflex-base/news/`, and a root-only fragment will fail the gate. +- **Adversarial review of each PR** before handing it over: an independent agent re-runs the + regression test against unfixed source to confirm it actually pins the bug, and checks repo + conventions. This catches PRs whose test passes either way. +- PR body: cite the FINDING number and that it came from pre-release testing, describe the defect + mechanism, the fix, and the test plan including the failing-before evidence. +- Watch CI after opening (`subscribe_pr_activity`) and drive each PR to green. diff --git a/.claude/skills/prerelease-test/scripts/audit_pyi.py b/.claude/skills/prerelease-test/scripts/audit_pyi.py new file mode 100644 index 00000000000..8daa83076d2 --- /dev/null +++ b/.claude/skills/prerelease-test/scripts/audit_pyi.py @@ -0,0 +1,432 @@ +"""Audit that published packages ship their generated .pyi stubs correctly. + +For each package it downloads the wheel and the sdist from PyPI and checks: + + * the package ships stubs for its own modules in BOTH artifacts, + * those stubs are byte-identical between wheel and sdist, + * no artifact carries stubs belonging to a different package, + * per-package stub counts match the repo's pyi_hashes.json manifest (optional). + +Stub *content* legitimately differs from the committed manifest hashes because the build +hook regenerates stubs at release time, so this compares presence and counts rather than +hashes. A package with no manifest entries is expected to ship no stubs. + +It exits 1 when a package's packaging is wrong and 2 when a package could not be audited +at all, so an unreachable PyPI never reads as a defect in a package nobody looked at. + +Usage (the script carries PEP 723 metadata and no shebang, so run it through uv): + uv run --script audit_pyi.py reflex==0.9.9 reflex-base==0.9.9 + +Chain it to discovery with && rather than a pipe: a pipe reports only this script's exit +status, so a package dropped for a failed lookup would take the audit's PASS with it. + + uv run --script check_release_versions.py --ref origin/main --specs > specs.txt \ + && xargs uv run --script audit_pyi.py --manifest-ref origin/main < specs.txt +""" + +# /// script +# requires-python = ">=3.10" +# /// + +from __future__ import annotations + +import argparse +import collections +import json +import shutil +import subprocess +import sys +import tarfile +import tempfile +import urllib.request +import zipfile +import zlib +from pathlib import Path + + +def _exc_name(exc: BaseException) -> str: + """Name an exception well enough for the reader to act on it. + + Args: + exc: The exception to name. + + Returns: + The class name, module-qualified unless it is a builtin. ``zlib.error`` is called + just ``error``, which says nothing at all on its own in a report. + """ + cls = type(exc) + if cls.__module__ == "builtins": + return cls.__name__ + return f"{cls.__module__}.{cls.__name__}" + + +def download_artifacts(name: str, version: str, dest: Path) -> dict[str, Path]: + """Download the wheel and sdist for a package version. + + Args: + name: The PyPI distribution name. + version: The version to download. + dest: Directory to download into. + + Returns: + A mapping of ``"wheel"``/``"sdist"`` to the downloaded file paths. + """ + with urllib.request.urlopen( + f"https://pypi.org/pypi/{name}/{version}/json", timeout=60 + ) as r: + data = json.load(r) + out: dict[str, Path] = {} + for entry in data.get("urls", []): + kind = entry["packagetype"] + if kind not in ("bdist_wheel", "sdist") or entry.get("yanked"): + # A yanked file is not what a user would install, so auditing it would report + # on the wrong artifact. With every file yanked this leaves nothing, which the + # caller already reports as a missing artifact kind. + continue + path = dest / entry["filename"] + if not path.exists(): + # urlretrieve honours only the global socket timeout, which is unset, so a + # stalled transfer would hang the whole audit rather than failing this row. + partial = path.with_name(path.name + ".part") + try: + with ( + urllib.request.urlopen(entry["url"], timeout=60) as response, + partial.open("wb") as handle, + ): + shutil.copyfileobj(response, handle) + # Only a whole file earns the real name. A partial one left there would be + # reused by every later run, since --keep skips what already exists, and + # the re-run this failure tells you to do could never clear it. + partial.replace(path) + finally: + partial.unlink(missing_ok=True) + out["wheel" if kind == "bdist_wheel" else "sdist"] = path + return out + + +def wheel_stubs(path: Path) -> dict[str, bytes]: + """Read every stub in a wheel. + + Args: + path: Path to the wheel file. + + Returns: + A mapping of archive-relative stub path to its contents. + """ + with zipfile.ZipFile(path) as z: + return {n: z.read(n) for n in z.namelist() if n.endswith(".pyi")} + + +def sdist_stubs(path: Path) -> dict[str, bytes]: + """Read every stub in an sdist, normalized to the wheel's layout. + + The ``<name>-<version>/`` prefix and any ``src/`` layout directory are stripped so the + keys line up with the wheel's, making the two directly comparable. + + Args: + path: Path to the sdist tarball. + + Returns: + A mapping of normalized stub path to its contents. + """ + out: dict[str, bytes] = {} + with tarfile.open(path) as t: + for member in t.getmembers(): + if not member.name.endswith(".pyi"): + continue + parts = member.name.split("/")[1:] # strip the <name>-<version>/ prefix + if parts and parts[0] == "src": # src-layout packages + parts = parts[1:] + handle = t.extractfile(member) + if handle is not None: + out["/".join(parts)] = handle.read() + return out + + +def import_root(dist_name: str, roots: dict[str, str]) -> str: + """Resolve the top-level import package for a distribution. + + The manifest is authoritative when available because it records each stub's real + source path; the underscore convention is only a fallback. Deriving the root from the + artifact itself would be circular — foreign stubs would define themselves as native + and the leak check could never fail. + + Args: + dist_name: The PyPI distribution name. + roots: Import roots by distribution name, from the manifest. + + Returns: + The import package name. + """ + return roots.get(dist_name, dist_name.replace("-", "_")) + + +def manifest_roots(ref: str, repo: str) -> dict[str, str]: + """Read each distribution's real import root from the stub manifest. + + Args: + ref: The git ref to read ``pyi_hashes.json`` from. + repo: Path to the reflex checkout. + + Returns: + A mapping of distribution name to import package name, empty when the manifest is + unavailable. + """ + roots: dict[str, str] = {} + for key in _manifest_keys(ref, repo): + parts = key.split("/") + if key.startswith("packages/"): + # packages/<dist>/<src dir>/<import root>/... + if len(parts) > 3: + roots[parts[1]] = parts[3] + elif parts: + roots["reflex"] = parts[0] + return roots + + +def _manifest_keys(ref: str, repo: str) -> list[str]: + """Read the stub paths recorded in the manifest. + + Args: + ref: The git ref to read ``pyi_hashes.json`` from. + repo: Path to the reflex checkout. + + Returns: + The manifest's stub paths, empty when it is unavailable at that ref. + """ + result = subprocess.run( + ["git", "-C", repo, "show", f"{ref}:pyi_hashes.json"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return [] + return list(json.loads(result.stdout)) + + +def manifest_counts(ref: str, repo: str) -> dict[str, int]: + """Count the stubs each distribution is expected to ship. + + Args: + ref: The git ref to read ``pyi_hashes.json`` from. + repo: Path to the reflex checkout. + + Returns: + A mapping of distribution name to expected stub count, empty when the manifest is + unavailable at that ref. + """ + counts: collections.Counter[str] = collections.Counter() + for key in _manifest_keys(ref, repo): + # "packages/<dist>/src/<pkg>/x.pyi" or a root-package path like "reflex/x.pyi" + counts[key.split("/")[1] if key.startswith("packages/") else "reflex"] += 1 + return dict(counts) + + +def _record( + name: str, + version: str, + problems: list[str], + expected: dict[str, int], + have_manifest: bool, + own: int = 0, + unchecked: str | None = None, +) -> dict: + """Build one package's audit record. + + Args: + name: The PyPI distribution name. + version: The audited version. + problems: Packaging defects found, empty when the package is clean. + expected: Expected stub counts by distribution name, from the manifest. + have_manifest: Whether a manifest was loaded. + own: Number of the package's own stubs found in the wheel. + unchecked: Why the package could not be audited at all, when it could not be. + Kept apart from ``problems`` so an unreachable PyPI never reads as a defect + in a package nobody managed to look at. + + Returns: + The record ``main`` prints, with every key it reads. + """ + return { + "package": name, + "version": version, + "own": own, + "expected": expected.get(name, 0) if have_manifest else None, + "problems": problems, + "unchecked": unchecked, + } + + +def audit( + spec: str, + workdir: Path, + expected: dict[str, int], + have_manifest: bool, + roots: dict[str, str], +) -> dict: + """Audit the stubs shipped by one package version. + + Args: + spec: A ``name==version`` spec. + workdir: Directory to download artifacts into. + expected: Expected stub counts by distribution name, from the manifest. + roots: Import roots by distribution name, from the manifest. + have_manifest: Whether a manifest was loaded. When it was, a package absent from + it is expected to ship no stubs at all; without one, counts go unchecked. + + Returns: + A record with the package, version, stub count and any problems found. + """ + name, _, version = spec.partition("==") + root = import_root(name, roots) + problems: list[str] = [] + + try: + artifacts = download_artifacts(name, version, workdir) + except Exception as exc: + # One unreachable package should not abort the audit of all the others. + # Never reached the package, so nothing is known about its packaging. + reason = f"could not fetch artifacts: {_exc_name(exc)}" + return _record( + name, version, problems, expected, have_manifest, unchecked=reason + ) + if "wheel" not in artifacts or "sdist" not in artifacts: + problems.append(f"missing artifact kinds: has {sorted(artifacts)}") + return _record(name, version, problems, expected, have_manifest) + + try: + wheel = wheel_stubs(artifacts["wheel"]) + sdist = sdist_stubs(artifacts["sdist"]) + except ( + zipfile.BadZipFile, + tarfile.TarError, + OSError, + EOFError, + zlib.error, + ) as exc: + # A truncated download and an artifact PyPI really serves broken look identical + # from here, so this is unchecked rather than a defect: re-run to tell them apart. + reason = f"could not read artifacts: {_exc_name(exc)}" + return _record( + name, version, problems, expected, have_manifest, unchecked=reason + ) + + def split(stubs: dict[str, bytes]) -> tuple[dict[str, bytes], dict[str, bytes]]: + own = {k: v for k, v in stubs.items() if k.split("/")[0] == root} + foreign = { + k: v + for k, v in stubs.items() + if k.split("/")[0] != root and not k.split("/")[0].endswith(".dist-info") + } + return own, foreign + + wheel_own, wheel_foreign = split(wheel) + sdist_own, sdist_foreign = split(sdist) + + if wheel_foreign: + problems.append( + f"wheel carries {len(wheel_foreign)} foreign stub(s): {sorted(wheel_foreign)[:3]}" + ) + if sdist_foreign: + problems.append( + f"sdist carries {len(sdist_foreign)} foreign stub(s): {sorted(sdist_foreign)[:3]}" + ) + + only_wheel = sorted(set(wheel_own) - set(sdist_own)) + only_sdist = sorted(set(sdist_own) - set(wheel_own)) + if only_wheel: + problems.append( + f"{len(only_wheel)} stub(s) in wheel but not sdist: {only_wheel[:3]}" + ) + if only_sdist: + problems.append( + f"{len(only_sdist)} stub(s) in sdist but not wheel: {only_sdist[:3]}" + ) + + differing = sorted( + k for k in set(wheel_own) & set(sdist_own) if wheel_own[k] != sdist_own[k] + ) + if differing: + problems.append( + f"{len(differing)} stub(s) differ between wheel and sdist: {differing[:3]}" + ) + + want = expected.get(name, 0) if have_manifest else None + if want is not None and want != len(wheel_own): + problems.append( + f"expected {want} stub(s) from manifest, wheel ships {len(wheel_own)}" + ) + + return _record(name, version, problems, expected, have_manifest, len(wheel_own)) + + +def main() -> int: + """Audit every requested package and print a summary. + + Returns: + ``1`` when any package has packaging problems, ``2`` when a package could not be + audited and the result is indeterminate, otherwise ``0``. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("specs", nargs="+", help="package==version specs") + parser.add_argument( + "--manifest-ref", help="git ref to read pyi_hashes.json from for count checks" + ) + parser.add_argument( + "--repo", default=".", help="path to the reflex checkout (default: cwd)" + ) + parser.add_argument("--keep", help="directory to keep downloaded artifacts in") + args = parser.parse_args() + + expected = ( + manifest_counts(args.manifest_ref, args.repo) if args.manifest_ref else {} + ) + # An empty result means the manifest was unreadable at that ref, so counts stay + # unchecked rather than every package being held to a zero-stub expectation. + have_manifest = bool(expected) + roots = manifest_roots(args.manifest_ref, args.repo) if args.manifest_ref else {} + + with tempfile.TemporaryDirectory() as tmp: + workdir = Path(args.keep) if args.keep else Path(tmp) + workdir.mkdir(parents=True, exist_ok=True) + results = [ + audit(spec, workdir, expected, have_manifest, roots) for spec in args.specs + ] + + failed = [r for r in results if r["problems"]] + unchecked = [r for r in results if r["unchecked"]] + + width = max(len(r["package"]) for r in results) + print(f"pyi packaging audit — {len(results)} package(s)\n") + for r in results: + mark = "!! " if r["problems"] else "?? " if r["unchecked"] else "OK " + want = "" if r["expected"] is None else f" (manifest: {r['expected']})" + print( + f" {mark}{r['package']:<{width}} {r['version']:<12} stubs={r['own']}{want}" + ) + for problem in r["problems"]: + print(f" - {problem}") + if r["unchecked"]: + print(f" - {r['unchecked']}") + + print() + if failed: + print(f"FAIL: {len(failed)} package(s) with packaging problems.") + if unchecked: + print(f"INDETERMINATE: {len(unchecked)} package(s) could not be audited:") + for r in unchecked: + print(f" - {r['package']}=={r['version']} ({r['unchecked']})") + print(" Unaudited is not the same as broken — resolve each before releasing.") + if not failed and not unchecked: + total = sum(r["own"] for r in results) + print( + f"PASS: {total} stubs ship correctly in both wheel and sdist; no foreign stubs." + ) + if failed: + return 1 + return 2 if unchecked else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/skills/prerelease-test/scripts/check_release_versions.py b/.claude/skills/prerelease-test/scripts/check_release_versions.py new file mode 100644 index 00000000000..35580c82325 --- /dev/null +++ b/.claude/skills/prerelease-test/scripts/check_release_versions.py @@ -0,0 +1,307 @@ +"""Discover what a release train ships and confirm every package published to PyPI. + +Reads the root CHANGELOG.md plus every packages/*/CHANGELOG.md at a git ref, takes the +top version entry from each, maps it to the PyPI distribution name declared in that +package's pyproject.toml, and checks the version exists on PyPI. + +An unpublished version is a release blocker in its own right, so a missing package makes +this exit non-zero. + +Usage (the script carries PEP 723 metadata and no shebang, so run it through uv): + uv run --script check_release_versions.py --ref origin/r/pre-2026-08-27-1234 + uv run --script check_release_versions.py --ref origin/main --json + uv run --script check_release_versions.py --ref origin/main --specs # pkg==ver lines +""" + +# /// script +# requires-python = ">=3.10" +# /// + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import urllib.error +import urllib.request + +VERSION_HEADING = re.compile(r"^##\s+v?([0-9][^\s]*)", re.MULTILINE) +PROJECT_NAME = re.compile(r"^name\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) +QUOTED = re.compile(r"[\"']([^\"']+)[\"']") +# Packages whose CHANGELOG.md does not name a release version, for two different +# reasons: never-published ones have no PyPI release at all, and internal ones are +# patch-released on every push to main rather than through the changelog. +EXCLUDED_KEYS = ("never-publish-packages", "internal-packages") + + +def git_show(ref: str, path: str, repo: str) -> str | None: + """Read a file's contents at a git ref. + + Args: + ref: The git ref to read from. + path: Repo-relative path of the file. + repo: Path to the reflex checkout. + + Returns: + The file contents, or ``None`` when the path does not exist at that ref. + """ + result = subprocess.run( + ["git", "-C", repo, "show", f"{ref}:{path}"], + capture_output=True, + text=True, + check=False, + ) + return result.stdout if result.returncode == 0 else None + + +def excluded_packages(ref: str, repo: str) -> set[str]: + """Read the packages whose changelog does not name a release version. + + A changelog in one of these must not be read as the version this train ships: a + never-published package has no PyPI release for it to name, and an internal package + is patch-released outside the changelog flow, so its heading would be checked against + a version that was never cut. Either way the result would be a false blocker. Parsed + with a regex rather than a TOML library to keep this script dependency-free on 3.10. + + Args: + ref: The git ref to read the root ``pyproject.toml`` from. + repo: Path to the reflex checkout. + + Returns: + The set of package directory names to skip. + """ + content = git_show(ref, "pyproject.toml", repo) or "" + skip: set[str] = set() + for key in EXCLUDED_KEYS: + match = re.search(rf"^{key}\s*=\s*\[([^\]]*)\]", content, re.MULTILINE) + if match: + skip.update(QUOTED.findall(match.group(1))) + return skip + + +def changelog_paths(ref: str, repo: str) -> list[str]: + """Find the changelogs that describe a release train. + + Args: + ref: The git ref to list files from. + repo: Path to the reflex checkout. + + Returns: + Paths of the root ``CHANGELOG.md`` and every ``packages/*/CHANGELOG.md``. + + Raises: + LookupError: The ref does not exist in that checkout. + """ + result = subprocess.run( + ["git", "-C", repo, "ls-tree", "-r", "--name-only", ref], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + # A mistyped ref is the likeliest way to misuse this, and a CalledProcessError + # traceback buries that under a stack it has no use for. + msg = f"cannot read ref {ref!r} in {repo}: {result.stderr.strip()}" + raise LookupError(msg) + listing = result.stdout.splitlines() + skip = excluded_packages(ref, repo) + paths = [] + for path in listing: + if path == "CHANGELOG.md": + paths.append(path) + continue + match = re.fullmatch(r"packages/([^/]+)/CHANGELOG.md", path) + if match and match.group(1) not in skip: + paths.append(path) + return paths + + +def dist_name(changelog_path: str, ref: str, repo: str) -> str: + """Resolve the PyPI distribution name for the package owning a changelog. + + Args: + changelog_path: Repo-relative path of the package's ``CHANGELOG.md``. + ref: The git ref to read ``pyproject.toml`` from. + repo: Path to the reflex checkout. + + Returns: + The declared project name, falling back to the package directory name. + """ + pyproject_path = changelog_path.replace("CHANGELOG.md", "pyproject.toml") + content = git_show(ref, pyproject_path, repo) or "" + match = PROJECT_NAME.search(content) + if match: + return match.group(1) + # Fall back to the directory name, which matches the dist name by convention. + parts = changelog_path.split("/") + return parts[1] if len(parts) > 2 else "reflex" + + +def pypi_status(name: str, version: str) -> tuple[str, str]: + """Check whether a distribution version is installable from PyPI. + + The result distinguishes "definitely not published" from "could not tell", because a + proxy hiccup reported as a missing package would be a false release blocker. + + Args: + name: The PyPI distribution name. + version: The version to look for. + + Returns: + A ``(status, detail)`` tuple where status is ``"published"``, ``"missing"`` or + ``"error"``, and detail names the artifact kinds found or why the check failed. + """ + url = f"https://pypi.org/pypi/{name}/{version}/json" + try: + with urllib.request.urlopen(url, timeout=30) as response: + data = json.load(response) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return "missing", "NOT ON PYPI" + return "error", f"HTTP {exc.code}" + except Exception as exc: + # Network/proxy trouble is indeterminate, not evidence the package is missing. + return "error", f"check failed: {type(exc).__name__}" + + if not isinstance(data, dict) or not isinstance(data.get("urls"), list): + # A proxy or error page that happens to parse as JSON is not evidence about the + # release. Only PyPI's own shape, with urls present and empty, means "no files". + return "error", "unexpected PyPI response shape" + files = data["urls"] + if not files: + return "missing", "published but no files" + if not all( + isinstance(f, dict) + and isinstance(f.get("packagetype"), str) + # Absent means not yanked, which is fine; present but not a boolean means the + # entry is not PyPI's and its yank state cannot be read from it. + and isinstance(f.get("yanked", False), bool) + for f in files + ): + # The list is there but its entries are not PyPI's, so reading them would raise + # out of this function and end the run rather than reporting one bad check. + return "error", "unexpected PyPI response shape" + live = [u for u in files if not u.get("yanked")] + if not live: + # The version exists but every file is withdrawn: resolvers skip a yanked release + # unless it is pinned exactly, and it was withdrawn for a reason. + return "missing", "published but all files yanked" + kinds = sorted({u["packagetype"] for u in live}) + detail = "+".join(k.replace("bdist_wheel", "wheel") for k in kinds) + yanked = len(files) - len(live) + return "published", f"{detail} ({yanked} yanked)" if yanked else detail + + +def main() -> int: + """Run the discovery and print the results. + + Returns: + ``1`` when a package is confirmed missing from PyPI (a release blocker), ``2`` + when a check could not be completed and the result is indeterminate, else ``0``. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--ref", required=True, help="git ref carrying the release changelogs" + ) + parser.add_argument( + "--repo", default=".", help="path to the reflex checkout (default: cwd)" + ) + parser.add_argument( + "--json", action="store_true", help="emit JSON instead of a table" + ) + parser.add_argument( + "--specs", + action="store_true", + help="emit 'name==version' lines only (feed to audit_pyi.py)", + ) + args = parser.parse_args() + + try: + paths = sorted(changelog_paths(args.ref, args.repo)) + except LookupError as exc: + print(exc, file=sys.stderr) + return 2 + + rows = [] + for path in paths: + content = git_show(args.ref, path, args.repo) or "" + name = dist_name(path, args.ref, args.repo) + match = VERSION_HEADING.search(content) + if match: + version = match.group(1) + status, detail = pypi_status(name, version) + else: + # Skipping the package silently would leave it unchecked, which is the exact + # failure this script exists to catch; report it as indeterminate instead. + version = "?" + status, detail = "error", "no version heading in changelog" + rows.append({ + "package": name, + "version": version, + "status": status, + "detail": detail, + "changelog": path, + "prerelease": bool(re.search(r"[abc]|rc|dev", version.split(".")[-1])), + }) + + missing = [r for r in rows if r["status"] == "missing"] + errors = [r for r in rows if r["status"] == "error"] + marks = {"published": "OK ", "missing": "!! ", "error": "?? "} + + if args.specs: + # Only confirmed-published rows: specs feed a downloader, so emitting a missing or + # indeterminate package just moves the failure downstream. Skips go to stderr so + # a short list is never silently mistaken for a complete one. + for row in rows: + if row["status"] == "published": + print(f"{row['package']}=={row['version']}") + for row in missing + errors: + print( + f"skipped {row['package']}=={row['version']} ({row['detail']})", + file=sys.stderr, + ) + elif args.json: + print( + json.dumps( + { + "ref": args.ref, + "packages": rows, + "missing": len(missing), + "errors": len(errors), + }, + indent=2, + ) + ) + else: + width = max((len(r["package"]) for r in rows), default=10) + print(f"Release train at {args.ref}: {len(rows)} packages\n") + for row in rows: + tag = " (prerelease)" if row["prerelease"] else "" + print( + f" {marks[row['status']]}{row['package']:<{width}} " + f"{row['version']:<12} {row['detail']}{tag}" + ) + print() + if missing: + print(f"BLOCKER: {len(missing)} package(s) not installable from PyPI:") + for row in missing: + print(f" - {row['package']}=={row['version']} ({row['detail']})") + if errors: + print(f"INDETERMINATE: {len(errors)} package(s) could not be checked:") + for row in errors: + print(f" - {row['package']}=={row['version']} ({row['detail']})") + print( + " Unchecked is not the same as missing — resolve each before releasing." + ) + if not missing and not errors: + print("All packages are published and installable.") + + if missing: + return 1 + return 2 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/skills/prerelease-test/scripts/drive_app.py b/.claude/skills/prerelease-test/scripts/drive_app.py new file mode 100644 index 00000000000..61eac672a93 --- /dev/null +++ b/.claude/skills/prerelease-test/scripts/drive_app.py @@ -0,0 +1,343 @@ +r"""Drive a running Reflex app in Chromium and report anomalies. + +Loads a page, optionally runs a small action script against it, and captures the four +channels worth watching on every run: console messages, page errors, failed requests, and +4xx/5xx responses. Known-benign dev-server noise is filtered out by default so that a +clean run really means clean. + +Run it with the driver venv (playwright installed), not the app's venv: + + NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 \\ + $SB/envs/driver/bin/python drive_app.py http://localhost:3100/ \\ + --screenshot shots/index.png --report logs/index.json + +Actions are JSON, either inline or in a file, applied in order after load: + + [{"click": "text=Increment"}, + {"expect_text": "Count: 1"}, + {"fill": ["input[name=todo]", "buy milk"]}, + {"press": ["input[name=todo]", "Enter"]}, + {"goto": "http://localhost:3100/about"}, + {"wait": 500}, + {"screenshot": "shots/after.png"}, + {"expect_missing": "text=Error"}] + +Exit status is 0 when every action succeeded and nothing anomalous was captured, 1 +otherwise — so it works directly as a check in a loop. +""" + +# /// script +# requires-python = ">=3.10" +# dependencies = ["playwright"] +# /// + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +from playwright.sync_api import Page, sync_playwright + +CHROMIUM = "/opt/pw-browsers/chromium" + +# Dev-server chatter that appears on healthy runs; see references/agent-brief.md. +BENIGN_CONSOLE = [ + re.compile( + r"Hey developer.*HydrateFallback|reactrouter\.com/start/framework/route-module" + ), + re.compile(r"\[vite\] (connecting|connected)"), + re.compile(r"Download the React DevTools"), +] + + +def is_benign(text: str) -> bool: + """Report whether a console message is known dev-server noise. + + Args: + text: The console message text. + + Returns: + ``True`` when the message matches a known-benign pattern. + """ + return any(pattern.search(text) for pattern in BENIGN_CONSOLE) + + +def load_actions(raw: str | None) -> list[dict]: + """Parse the action script. + + Args: + raw: Inline JSON, a path to a JSON file, or ``None``. + + Returns: + The list of actions to apply, empty when none were given. + + Raises: + ValueError: The JSON is malformed or is not a list of actions. + """ + if not raw: + return [] + try: + candidate = Path(raw) + if candidate.is_file(): + raw = candidate.read_text() + except OSError: + # Inline JSON longer than the filesystem's filename limit makes the stat itself + # raise, so treat any path error as "this was not a path". + pass + actions = json.loads(raw) + if not isinstance(actions, list): + msg = f"expected a JSON list of actions, got {type(actions).__name__}" + raise ValueError(msg) + for action in actions: + # run_action unpacks exactly one (verb, value) pair, so anything else would fail + # mid-run, after a browser launch and a page load that a caller typo did not earn. + if not isinstance(action, dict) or len(action) != 1: + msg = f"each action must be a single-key object, got {action!r}" + raise ValueError(msg) + return actions + + +def run_action(page: Page, action: dict, timeout: int) -> str: + """Apply one action to the page. + + Args: + page: The Playwright page to act on. + action: A single-entry mapping of verb to its argument(s). + timeout: Per-action timeout in milliseconds. + + Returns: + A human-readable description of what was done. + """ + ((verb, value),) = action.items() + if verb == "click": + page.click(value, timeout=timeout) + elif verb == "fill": + page.fill(value[0], value[1], timeout=timeout) + elif verb == "press": + page.press(value[0], value[1], timeout=timeout) + elif verb == "goto": + page.goto(value, wait_until="networkidle", timeout=timeout) + elif verb == "wait": + page.wait_for_timeout(value) + elif verb == "wait_for": + page.wait_for_selector(value, timeout=timeout) + elif verb == "expect_text": + page.wait_for_selector(f"text={value}", timeout=timeout) + elif verb == "expect_missing": + page.wait_for_selector(value, state="detached", timeout=timeout) + elif verb == "screenshot": + Path(value).parent.mkdir(parents=True, exist_ok=True) + page.screenshot(path=value, full_page=True) + elif verb == "eval": + return f"eval -> {page.evaluate(value)!r}" + else: + msg = f"unknown action {verb!r}" + raise ValueError(msg) + return f"{verb}: {value!r}" + + +def try_action(page: Page, action: dict, timeout: int) -> tuple[str | None, str | None]: + """Run one action, turning a failure into a value rather than an exception. + + Reporting the failure this way keeps the record of the actions that already + succeeded, which is usually the most useful part of a failed run. + + Args: + page: The Playwright page to act on. + action: A single-entry mapping of verb to its argument(s). + timeout: Per-action timeout in milliseconds. + + Returns: + A ``(description, error)`` tuple; exactly one side is set. + """ + try: + return run_action(page, action, timeout), None + except Exception as exc: + return None, f"{type(exc).__name__}: {exc}" + + +def main() -> int: + """Drive the app and print a report. + + Returns: + ``0`` when the run was clean, ``1`` when anything anomalous was captured. + """ + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("url", help="page to load, e.g. http://localhost:3100/") + parser.add_argument("--actions", help="JSON list of actions, inline or a file path") + parser.add_argument( + "--screenshot", help="path for a screenshot after the actions run" + ) + parser.add_argument("--report", help="path to write the full JSON report to") + parser.add_argument( + "--timeout", type=int, default=15000, help="per-action timeout in ms" + ) + parser.add_argument( + "--settle", type=int, default=1500, help="ms to wait after load for hydration" + ) + parser.add_argument( + "--headed", action="store_true", help="run with a visible browser" + ) + parser.add_argument( + "--all-console", action="store_true", help="do not filter benign console noise" + ) + args = parser.parse_args() + + try: + actions = load_actions(args.actions) + except ValueError as exc: + # Parsing after the browser is up spends a launch and a page load before failing, + # and leaves a traceback where the report should be. A malformed --actions is a + # usage error, so say so up front and exit with the documented failure status. + print(f"INVALID --actions: {exc}", file=sys.stderr) + return 1 + + console: list[dict] = [] + page_errors: list[str] = [] + failed: list[dict] = [] + bad_status: list[dict] = [] + performed: list[str] = [] + action_error: str | None = None + load_error: str | None = None + screenshot_error: str | None = None + read_error: str | None = None + + with sync_playwright() as p: + browser = p.chromium.launch(executable_path=CHROMIUM, headless=not args.headed) + page = browser.new_context().new_page() + page.on("console", lambda m: console.append({"type": m.type, "text": m.text})) + page.on("pageerror", lambda e: page_errors.append(str(e))) + page.on( + "requestfailed", + lambda r: failed.append({"url": r.url, "failure": str(r.failure)}), + ) + page.on( + "response", + lambda r: ( + bad_status.append({"url": r.url, "status": r.status}) + if r.status >= 400 + else None + ), + ) + + try: + page.goto(args.url, wait_until="networkidle", timeout=60000) + page.wait_for_timeout(args.settle) + except Exception as exc: + # A server that never came up is the most common failure here, and it still + # deserves a report and the documented exit status rather than a traceback. + load_error = f"{type(exc).__name__}: {exc}" + + for action in [] if load_error else actions: + done, action_error = try_action(page, action, args.timeout) + if done is not None: + performed.append(done) + if action_error: + break + + if performed or action_error: + # Playwright returns as soon as an action's own wait resolves, but the + # resulting state update, network call and any error it triggers land after + # that; without this the last action's fallout is invisible to the report. + # A failed action counts: the click that raised on its assertion may still + # have reached the server, and that response is often the whole finding. + try: + page.wait_for_timeout(args.settle) + except Exception as exc: + # An action can close the page or navigate it away. That is an anomaly to + # record, not a reason to abandon the report the caller is waiting on. + action_error = action_error or f"{type(exc).__name__}: {exc}" + + try: + title = page.title() + body = page.inner_text("body")[:2000] + except Exception as exc: + # Never fold this into the body text alone: a page that cannot be read is a + # run that inspected nothing, and reporting that as clean is the one result + # this driver must never produce. + title, body = "", f"<unreadable: {type(exc).__name__}>" + read_error = f"{type(exc).__name__}: {exc}" + if args.screenshot: + try: + Path(args.screenshot).parent.mkdir(parents=True, exist_ok=True) + page.screenshot(path=args.screenshot, full_page=True) + except Exception as exc: + # A closed page, or a destination that cannot be created. Either way a + # missing screenshot must not cost the report. + screenshot_error = f"{type(exc).__name__}: {exc}" + browser.close() + + shown = ( + console + if args.all_console + else [m for m in console if not is_benign(m["text"])] + ) + problems = [m for m in shown if m["type"] in ("error", "warning")] + clean = not ( + problems + or page_errors + or failed + or bad_status + or action_error + or load_error + or screenshot_error + or read_error + ) + + report = { + "url": args.url, + "title": title, + "clean": clean, + "load_error": load_error, + "actions_performed": performed, + "action_error": action_error, + "screenshot_error": screenshot_error, + "read_error": read_error, + "console": shown, + "page_errors": page_errors, + "failed_requests": failed, + "http_4xx_5xx": bad_status, + "body_text": body, + } + if args.report: + try: + Path(args.report).parent.mkdir(parents=True, exist_ok=True) + Path(args.report).write_text(json.dumps(report, indent=2)) + except OSError as exc: + # Losing the file must not also lose the findings: the summary below is + # printed either way. It must not be reported as a pass either — a caller + # that asked for a report and got none has no result to trust. + print(f" REPORT NOT WRITTEN: {exc}", file=sys.stderr) + clean = False + + if load_error: + print(f" LOAD FAILED: {load_error}") + print(f"TITLE: {title}") + print(f"BODY: {body[:300].replace(chr(10), ' | ')}") + for item in performed: + print(f" did {item}") + if action_error: + print(f" ACTION FAILED: {action_error}") + if read_error: + print(f" PAGE UNREADABLE: {read_error}") + if screenshot_error: + print(f" SCREENSHOT FAILED: {screenshot_error}") + for m in shown: + print(f" [{m['type']}] {m['text'][:300]}") + for e in page_errors: + print(f" PAGE ERROR: {e[:300]}") + for f in failed: + print(f" REQUEST FAILED: {f['url']} ({f['failure']})") + for b in bad_status: + print(f" HTTP {b['status']}: {b['url']}") + print("RESULT:", "clean" if clean else "ANOMALIES FOUND") + return 0 if clean else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml index 959df1a5633..e1e6a77cdd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -269,6 +269,8 @@ preview = true "*/blank.py" = ["I001"] "docs/package/scripts/*.py" = ["INP001"] "scripts/check_min_deps.py" = ["T201"] +# Skill helper scripts are CLIs whose whole job is printing a report. +".claude/skills/*/scripts/*.py" = ["T201"] [tool.pytest.ini_options] filterwarnings = "ignore:fields may not start with an underscore:RuntimeWarning"