From ad80eacdf0bc30aeb8cea453c775d2c53ccb39e9 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 2 Sep 2026 12:35:48 -0400 Subject: [PATCH 1/3] feat(reward): add identity-swap family to the MockMed ExtraDup proof Scripted policy writes a correct-looking note onto the wrong patient_id. visual_only pays; certified_sor reads the named identity_keys and pays 0. Seeds 101,202,303. Synthetic scope. No GPU, not a trained policy, not a Seal. docs/reward/proof_2026-09-01.* is unchanged. --- docs/reward/README.md | 55 +- docs/reward/proof_2026-09-02.json | 1517 +++++++++++++++++++++++++++++ docs/reward/proof_2026-09-02.md | 47 + openadapt_evals/reward/proof.py | 248 ++++- tests/test_reward_proof.py | 99 +- 5 files changed, 1885 insertions(+), 81 deletions(-) create mode 100644 docs/reward/proof_2026-09-02.json create mode 100644 docs/reward/proof_2026-09-02.md diff --git a/docs/reward/README.md b/docs/reward/README.md index a93a96a..731f836 100644 --- a/docs/reward/README.md +++ b/docs/reward/README.md @@ -164,24 +164,33 @@ python -m openadapt_evals.extradup kill-scan It runs the frozen MockMed corpus, prints silent-incorrect-success on gold-FAIL mutants vs honest-write on the control, keeps `execute_seal: false` / `production_seal: false`, and does not mix those mutants into a training -reward. Same numbers as below. `--verdicts path.json` scores someone else's -checker on the same cells. +reward. Same numbers as the ExtraDup-only snapshot. `--verdicts path.json` +scores someone else's checker on the same cells. `python -m openadapt_evals.reward.proof` is the underlying generator: scripted policies against the in-memory MockMed store from `openadapt_evals.extradup`, two rewards through the same `assess_receipt` path the adapters use. No model, -no GPU. Three trials per condition on the seed schedule `[101, 202, 303]`; -the seed picks the banner wording and the receipt identities. The committed -output is [`proof_2026-09-01.md`](proof_2026-09-01.md) and -[`proof_2026-09-01.json`](proof_2026-09-01.json); a test regenerates it and -fails if the committed file drifts from the code. +no GPU, not a trained policy, not a Production Seal. Three trials per +condition on the seed schedule `[101, 202, 303]`; the seed picks the banner +wording and the receipt identities. + +The ExtraDup-only snapshot is +[`proof_2026-09-01.md`](proof_2026-09-01.md) and +[`proof_2026-09-01.json`](proof_2026-09-01.json). A test regenerates it from +the frozen condition set and fails if that file drifts. The current committed +output, which adds the `identity_swap` family, is +[`proof_2026-09-02.md`](proof_2026-09-02.md) and +[`proof_2026-09-02.json`](proof_2026-09-02.json). `visual_only` is a tier-0 reward that believes the saved banner, the filled -form, or the policy's claim. `certified_sor` is a tier-2 reward that runs -ExtraDup's `sor_check` (record count, no extra field, every spec field) behind -a self-signed synthetic-scope certificate. `oracle_outage` is not an ExtraDup -family: the write is correct but the store cannot be read, so the certified -reward answers `failed_platform`. +form, or the policy's claim. `certified_sor` is a tier-2 reward that reads the +named `oracle.identity_keys` and runs ExtraDup's `sor_check` (record count, +no extra field, every spec field) on that record, behind a self-signed +synthetic-scope certificate. `identity_swap` writes a correct-looking note +onto the wrong `patient_id`; the banner still says saved, and the named +record is unchanged. `oracle_outage` is not an ExtraDup family: the write is +correct but the store cannot be read, so the certified reward answers +`failed_platform`. | condition | gold | reward | tier | trials | paid | silent incorrect success | over-refusal | unscored | certified | scope | |---|---|---|---|---|---|---|---|---|---|---| @@ -197,17 +206,21 @@ reward answers `failed_platform`. | unsubmit | FAIL | certified_sor | 2 | 3 | 0 | 0.00 | n/a | 0 | yes | synthetic | | claim | FAIL | visual_only | 0 | 3 | 3 | 1.00 | n/a | 0 | no | none | | claim | FAIL | certified_sor | 2 | 3 | 0 | 0.00 | n/a | 0 | yes | synthetic | +| identity_swap | FAIL | visual_only | 0 | 3 | 3 | 1.00 | n/a | 0 | no | none | +| identity_swap | FAIL | certified_sor | 2 | 3 | 0 | 0.00 | n/a | 0 | yes | synthetic | | oracle_outage | PASS | visual_only | 0 | 3 | 3 | n/a | 0.00 | 0 | no | none | | oracle_outage | PASS | certified_sor | 2 | 3 | 0 | n/a | 0.00 | 3 | yes | synthetic | -The visual reward paid all 15 gold-FAIL rollouts. The certified reward paid -none of them and paid the 3 controls. Its certificate's epsilon, 0.181036, is -the exact one-sided 95% Clopper-Pearson upper bound from those 15 trials; it -was computed from the run, not chosen. The `certified` column reads `yes` -for the tier-2 reward because the certificate is current at update 7, names -its calibration corpus by digest, and states its scope. That scope is -`synthetic`, and a self-signed certificate can state no other; the contract -(openadapt-types 0.17.0) rejects one that tries. +The visual reward paid all 18 gold-FAIL rollouts, including identity_swap. +The certified reward paid none of them and paid the 3 controls. Its +certificate's epsilon, 0.153318, is the exact one-sided 95% Clopper-Pearson +upper bound from those 18 trials; it was computed from the run, not chosen. +The 09-01 snapshot's epsilon, 0.181036, is the same bound over the 15 +ExtraDup-only trials. The `certified` column reads `yes` for the tier-2 reward because the +certificate is current at update 7, names its calibration corpus by digest, +and states its scope. That scope is `synthetic`, and a self-signed +certificate can state no other; the contract (openadapt-types 0.17.0) +rejects one that tries. The expiry check re-assesses the certified control receipts at policy update 100, the first update at which the certificate has expired: none stays @@ -217,7 +230,7 @@ certified and the adapter logs three expiry warnings. It does not train anything, so it says nothing about how the two rewards change a policy. It does not touch a real system of record. The bound is for -15 synthetic trials at one seed schedule and would move with either. The +18 synthetic trials at one seed schedule and would move with either. The reward worker that issues production receipts is a separate piece of work in `openadapt-flow`; until it ships, the only receipts this package has seen were signed by its own development key. diff --git a/docs/reward/proof_2026-09-02.json b/docs/reward/proof_2026-09-02.json new file mode 100644 index 0000000..289a657 --- /dev/null +++ b/docs/reward/proof_2026-09-02.json @@ -0,0 +1,1517 @@ +{ + "calibration": { + "calibration_corpus_digest": "sha256:e62395f26f383c1c82544a08868b44c15bde31cd48af3a26683e954f6da500a7", + "clopper_pearson_upper_95": { + "certified_sor": 0.153318, + "visual_only": 1.0 + }, + "false_accepts": { + "certified_sor": 0, + "visual_only": 18 + }, + "gold_fail_trials": 18 + }, + "certificate": { + "calibration_scope": "synthetic", + "certificate_id": "reward.certificate.proof-synthetic", + "delta": 0.05, + "digest": "sha256:a999c9e8f983731d9dee1d6c4cfed47070a61d61954e3c320015f9cf31854d0c", + "epsilon": 0.153318, + "expiry_policy_updates": 100, + "issued_at_policy_update": 0, + "issuer": "self_signed", + "issuer_key_id": "dev.key.synthetic-proof", + "issuer_public_key_ed25519_b64": "fPWkNmYi+TRIHZ8wF4qfk1prCUKorlEqTzuWcTL9EnM=", + "threshold": 1.0 + }, + "conditions": [ + "control", + "dup", + "extra", + "omit", + "unsubmit", + "claim", + "identity_swap", + "oracle_outage" + ], + "contract": { + "certificate_policy": { + "calibration_corpus_digest": "sha256:e62395f26f383c1c82544a08868b44c15bde31cd48af3a26683e954f6da500a7", + "delta": 0.05, + "epsilon": 0.153318, + "expiry_policy_updates": 100, + "threshold": 1.0 + }, + "digest": "sha256:587c45fae205d6dc10dabdd04ff8ed9755ccd7ef3a9aa84f2cb6e5f65e876ef2" + }, + "expiry_check": { + "certified_after_expiry": 0, + "expiry_warnings_logged": 3, + "policy_update": 100, + "receipts_checked": 3 + }, + "policy_update": 7, + "proof": "reward-proof-mockmed-extradup", + "rollouts": [ + { + "banner_text": "Saved to chart", + "condition": "control", + "episode_id": "proof.control.t0.s101", + "evidence_digest": "sha256:48e80adb41d7ee2d09c32c2acd25f2b8fb99eb652f897f24ba36d1655381e5a9", + "gold": "PASS", + "new_count": 1, + "operator": "control", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.control.t0.s101.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.control.t0.s101", + "reward_oracle_tier": 2, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.control.t0.s101.certified_sor", + "reward_unscored": false, + "scalar": 1.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.control.t0.s101.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.control.t0.s101", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.control.t0.s101.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 101, + "sor_reasons": [], + "spec_count": 1, + "trial": 0 + }, + { + "banner_text": "Saved", + "condition": "control", + "episode_id": "proof.control.t1.s202", + "evidence_digest": "sha256:d86b12be8fa98776558575e5491a7d8718a3947b184d521b6ba74ca94b0a3180", + "gold": "PASS", + "new_count": 1, + "operator": "control", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.control.t1.s202.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.control.t1.s202", + "reward_oracle_tier": 2, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.control.t1.s202.certified_sor", + "reward_unscored": false, + "scalar": 1.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.control.t1.s202.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.control.t1.s202", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.control.t1.s202.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 202, + "sor_reasons": [], + "spec_count": 1, + "trial": 1 + }, + { + "banner_text": "Record saved successfully", + "condition": "control", + "episode_id": "proof.control.t2.s303", + "evidence_digest": "sha256:a72870b17e169a4cf9c6191e88eea5c515ac958d204de54e625bb820401bf533", + "gold": "PASS", + "new_count": 1, + "operator": "control", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.control.t2.s303.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.control.t2.s303", + "reward_oracle_tier": 2, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.control.t2.s303.certified_sor", + "reward_unscored": false, + "scalar": 1.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.control.t2.s303.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.control.t2.s303", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.control.t2.s303.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 303, + "sor_reasons": [], + "spec_count": 1, + "trial": 2 + }, + { + "banner_text": "Saved", + "condition": "dup", + "episode_id": "proof.dup.t0.s101", + "evidence_digest": "sha256:d36d73490511b75e197c5612b70bf9c653c024f13bb85ec2c1dd0bedaf90fd31", + "gold": "FAIL", + "new_count": 2, + "operator": "dup", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.dup.t0.s101.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.dup.t0.s101", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.dup.t0.s101.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.dup.t0.s101.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.dup.t0.s101", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.dup.t0.s101.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 101, + "sor_reasons": [ + "|new(M)|=2 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 0 + }, + { + "banner_text": "Saved to chart", + "condition": "dup", + "episode_id": "proof.dup.t1.s202", + "evidence_digest": "sha256:a6fb465b7ebf4b2663b4ee8253b39d41bcc15950bde2db5c9b5efd02a0298781", + "gold": "FAIL", + "new_count": 2, + "operator": "dup", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.dup.t1.s202.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.dup.t1.s202", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.dup.t1.s202.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.dup.t1.s202.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.dup.t1.s202", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.dup.t1.s202.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 202, + "sor_reasons": [ + "|new(M)|=2 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 1 + }, + { + "banner_text": "Record saved successfully", + "condition": "dup", + "episode_id": "proof.dup.t2.s303", + "evidence_digest": "sha256:1da8b6e9ca4505ed6c34705da4175610906d81042db537d8ed0062b1d1503c14", + "gold": "FAIL", + "new_count": 2, + "operator": "dup", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.dup.t2.s303.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.dup.t2.s303", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.dup.t2.s303.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.dup.t2.s303.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.dup.t2.s303", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.dup.t2.s303.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 303, + "sor_reasons": [ + "|new(M)|=2 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 2 + }, + { + "banner_text": "Saved", + "condition": "extra", + "episode_id": "proof.extra.t0.s101", + "evidence_digest": "sha256:6e93e550e3e9975747979ebb0d42ce60ff1ab3128b2d38f973c89e4f568a8736", + "gold": "FAIL", + "new_count": 1, + "operator": "extra", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.extra.t0.s101.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.extra.t0.s101", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.extra.t0.s101.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.extra.t0.s101.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.extra.t0.s101", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.extra.t0.s101.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 101, + "sor_reasons": [ + "extra field(s): priority" + ], + "spec_count": 1, + "trial": 0 + }, + { + "banner_text": "Record saved successfully", + "condition": "extra", + "episode_id": "proof.extra.t1.s202", + "evidence_digest": "sha256:116432eacd2a5b9fb63086c5868fc5cca211a993e526e5bed27597d81c8df033", + "gold": "FAIL", + "new_count": 1, + "operator": "extra", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.extra.t1.s202.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.extra.t1.s202", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.extra.t1.s202.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.extra.t1.s202.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.extra.t1.s202", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.extra.t1.s202.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 202, + "sor_reasons": [ + "extra field(s): priority" + ], + "spec_count": 1, + "trial": 1 + }, + { + "banner_text": "Saved to chart", + "condition": "extra", + "episode_id": "proof.extra.t2.s303", + "evidence_digest": "sha256:ac392a0d0546718a3637997161a881abd8a2ec7e3f9b6486fb8734e879b55355", + "gold": "FAIL", + "new_count": 1, + "operator": "extra", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.extra.t2.s303.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.extra.t2.s303", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.extra.t2.s303.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.extra.t2.s303.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.extra.t2.s303", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.extra.t2.s303.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 303, + "sor_reasons": [ + "extra field(s): priority" + ], + "spec_count": 1, + "trial": 2 + }, + { + "banner_text": "Saved to chart", + "condition": "omit", + "episode_id": "proof.omit.t0.s101", + "evidence_digest": "sha256:0644298e70c27943433db84c08da46a5267696dc82e1f34e5503395670edb317", + "gold": "FAIL", + "new_count": 1, + "operator": "omit", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.omit.t0.s101.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.omit.t0.s101", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.omit.t0.s101.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.omit.t0.s101.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.omit.t0.s101", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.omit.t0.s101.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 101, + "sor_reasons": [ + "missing or wrong note='Follow-up in 2 weeks; BP recheck.'" + ], + "spec_count": 1, + "trial": 0 + }, + { + "banner_text": "Saved", + "condition": "omit", + "episode_id": "proof.omit.t1.s202", + "evidence_digest": "sha256:d7f5c629be1f4cdb820fa12111145d52509214f91600242bd51174d081733446", + "gold": "FAIL", + "new_count": 1, + "operator": "omit", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.omit.t1.s202.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.omit.t1.s202", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.omit.t1.s202.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.omit.t1.s202.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.omit.t1.s202", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.omit.t1.s202.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 202, + "sor_reasons": [ + "missing or wrong note='Follow-up in 2 weeks; BP recheck.'" + ], + "spec_count": 1, + "trial": 1 + }, + { + "banner_text": "Encounter saved", + "condition": "omit", + "episode_id": "proof.omit.t2.s303", + "evidence_digest": "sha256:7048426983045e691734367db1adcb0b568f216e987b49bcbc24fef89e13d953", + "gold": "FAIL", + "new_count": 1, + "operator": "omit", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.omit.t2.s303.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.omit.t2.s303", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.omit.t2.s303.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.omit.t2.s303.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.omit.t2.s303", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.omit.t2.s303.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 303, + "sor_reasons": [ + "missing or wrong note='Follow-up in 2 weeks; BP recheck.'" + ], + "spec_count": 1, + "trial": 2 + }, + { + "banner_text": "", + "condition": "unsubmit", + "episode_id": "proof.unsubmit.t0.s101", + "evidence_digest": "sha256:e9ad0c874acda9045c5678105d76fc4bbc76f06cb8adf8abbe9f890cff0856b7", + "gold": "FAIL", + "new_count": 0, + "operator": "unsubmit", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.unsubmit.t0.s101.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.unsubmit.t0.s101", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.unsubmit.t0.s101.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.unsubmit.t0.s101.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.unsubmit.t0.s101", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.unsubmit.t0.s101.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": false, + "claimed_success": false, + "form_looks_complete": true + }, + "seed": 101, + "sor_reasons": [ + "|new(M)|=0 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 0 + }, + { + "banner_text": "", + "condition": "unsubmit", + "episode_id": "proof.unsubmit.t1.s202", + "evidence_digest": "sha256:e9ad0c874acda9045c5678105d76fc4bbc76f06cb8adf8abbe9f890cff0856b7", + "gold": "FAIL", + "new_count": 0, + "operator": "unsubmit", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.unsubmit.t1.s202.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.unsubmit.t1.s202", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.unsubmit.t1.s202.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.unsubmit.t1.s202.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.unsubmit.t1.s202", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.unsubmit.t1.s202.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": false, + "claimed_success": false, + "form_looks_complete": true + }, + "seed": 202, + "sor_reasons": [ + "|new(M)|=0 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 1 + }, + { + "banner_text": "", + "condition": "unsubmit", + "episode_id": "proof.unsubmit.t2.s303", + "evidence_digest": "sha256:e9ad0c874acda9045c5678105d76fc4bbc76f06cb8adf8abbe9f890cff0856b7", + "gold": "FAIL", + "new_count": 0, + "operator": "unsubmit", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.unsubmit.t2.s303.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.unsubmit.t2.s303", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.unsubmit.t2.s303.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.unsubmit.t2.s303.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.unsubmit.t2.s303", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.unsubmit.t2.s303.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": false, + "claimed_success": false, + "form_looks_complete": true + }, + "seed": 303, + "sor_reasons": [ + "|new(M)|=0 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 2 + }, + { + "banner_text": "Record saved successfully", + "condition": "claim", + "episode_id": "proof.claim.t0.s101", + "evidence_digest": "sha256:78b816c71d65b7e053e15509547ac9078514a883fd12fc491df3a211f7d70fd5", + "gold": "FAIL", + "new_count": 0, + "operator": "claim", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.claim.t0.s101.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.claim.t0.s101", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.claim.t0.s101.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.claim.t0.s101.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.claim.t0.s101", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.claim.t0.s101.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": true, + "form_looks_complete": false + }, + "seed": 101, + "sor_reasons": [ + "|new(M)|=0 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 0 + }, + { + "banner_text": "Encounter saved", + "condition": "claim", + "episode_id": "proof.claim.t1.s202", + "evidence_digest": "sha256:e67c7bea9139f7eaa06633fd16652932cf664afc7a142f4dda13c5be84b28806", + "gold": "FAIL", + "new_count": 0, + "operator": "claim", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.claim.t1.s202.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.claim.t1.s202", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.claim.t1.s202.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.claim.t1.s202.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.claim.t1.s202", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.claim.t1.s202.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": true, + "form_looks_complete": false + }, + "seed": 202, + "sor_reasons": [ + "|new(M)|=0 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 1 + }, + { + "banner_text": "Saved", + "condition": "claim", + "episode_id": "proof.claim.t2.s303", + "evidence_digest": "sha256:c14bcd106fe8b358e68e39ef1aa88b56ec2e9abe0f7c1f28d5905f969c6bb684", + "gold": "FAIL", + "new_count": 0, + "operator": "claim", + "oracle_available": true, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.claim.t2.s303.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.claim.t2.s303", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.claim.t2.s303.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.claim.t2.s303.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.claim.t2.s303", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.claim.t2.s303.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": true, + "form_looks_complete": false + }, + "seed": 303, + "sor_reasons": [ + "|new(M)|=0 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 2 + }, + { + "banner_text": "Encounter saved", + "condition": "identity_swap", + "episode_id": "proof.identity_swap.t0.s101", + "evidence_digest": "sha256:4d7d6bc4d0e5cc2e84e69d25d9ce865d9073710308d6c31ee1b11c00fb8fcb0c", + "gold": "FAIL", + "named_new_count": 0, + "new_count": 1, + "operator": "identity_swap", + "oracle_available": true, + "oracle_identity": { + "patient_id": "p1", + "type": "Triage" + }, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.identity_swap.t0.s101.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.identity_swap.t0.s101", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.identity_swap.t0.s101.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.identity_swap.t0.s101.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.identity_swap.t0.s101", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.identity_swap.t0.s101.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 101, + "sor_reasons": [ + "|new(M)|=0 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 0, + "written_patient_id": "p2" + }, + { + "banner_text": "Saved to chart", + "condition": "identity_swap", + "episode_id": "proof.identity_swap.t1.s202", + "evidence_digest": "sha256:1c627b60ef85d507c508d5a0a8872c964bf8d7fae110e5f5775b46c59c2623a8", + "gold": "FAIL", + "named_new_count": 0, + "new_count": 1, + "operator": "identity_swap", + "oracle_available": true, + "oracle_identity": { + "patient_id": "p1", + "type": "Triage" + }, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.identity_swap.t1.s202.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.identity_swap.t1.s202", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.identity_swap.t1.s202.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.identity_swap.t1.s202.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.identity_swap.t1.s202", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.identity_swap.t1.s202.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 202, + "sor_reasons": [ + "|new(M)|=0 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 1, + "written_patient_id": "p2" + }, + { + "banner_text": "Saved to chart", + "condition": "identity_swap", + "episode_id": "proof.identity_swap.t2.s303", + "evidence_digest": "sha256:1c627b60ef85d507c508d5a0a8872c964bf8d7fae110e5f5775b46c59c2623a8", + "gold": "FAIL", + "named_new_count": 0, + "new_count": 1, + "operator": "identity_swap", + "oracle_available": true, + "oracle_identity": { + "patient_id": "p1", + "type": "Triage" + }, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.identity_swap.t2.s303.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.identity_swap.t2.s303", + "reward_oracle_tier": 2, + "reward_outcome": "wrong_effect", + "reward_receipt_id": "receipt.proof.identity_swap.t2.s303.certified_sor", + "reward_unscored": false, + "scalar": 0.0 + }, + "visual_only": { + "receipt_id": "receipt.proof.identity_swap.t2.s303.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.identity_swap.t2.s303", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.identity_swap.t2.s303.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 303, + "sor_reasons": [ + "|new(M)|=0 != |spec(M)|=1" + ], + "spec_count": 1, + "trial": 2, + "written_patient_id": "p2" + }, + { + "banner_text": "Saved", + "condition": "oracle_outage", + "episode_id": "proof.oracle_outage.t0.s101", + "evidence_digest": "sha256:d86b12be8fa98776558575e5491a7d8718a3947b184d521b6ba74ca94b0a3180", + "gold": "PASS", + "new_count": 1, + "operator": "control", + "oracle_available": false, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.oracle_outage.t0.s101.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.oracle_outage.t0.s101", + "reward_oracle_tier": 2, + "reward_outcome": "failed_platform", + "reward_receipt_id": "receipt.proof.oracle_outage.t0.s101.certified_sor", + "reward_unscored": true, + "scalar": null + }, + "visual_only": { + "receipt_id": "receipt.proof.oracle_outage.t0.s101.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.oracle_outage.t0.s101", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.oracle_outage.t0.s101.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 101, + "sor_reasons": [], + "spec_count": 1, + "trial": 0 + }, + { + "banner_text": "Record saved successfully", + "condition": "oracle_outage", + "episode_id": "proof.oracle_outage.t1.s202", + "evidence_digest": "sha256:a72870b17e169a4cf9c6191e88eea5c515ac958d204de54e625bb820401bf533", + "gold": "PASS", + "new_count": 1, + "operator": "control", + "oracle_available": false, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.oracle_outage.t1.s202.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.oracle_outage.t1.s202", + "reward_oracle_tier": 2, + "reward_outcome": "failed_platform", + "reward_receipt_id": "receipt.proof.oracle_outage.t1.s202.certified_sor", + "reward_unscored": true, + "scalar": null + }, + "visual_only": { + "receipt_id": "receipt.proof.oracle_outage.t1.s202.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.oracle_outage.t1.s202", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.oracle_outage.t1.s202.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 202, + "sor_reasons": [], + "spec_count": 1, + "trial": 1 + }, + { + "banner_text": "Record saved successfully", + "condition": "oracle_outage", + "episode_id": "proof.oracle_outage.t2.s303", + "evidence_digest": "sha256:a72870b17e169a4cf9c6191e88eea5c515ac958d204de54e625bb820401bf533", + "gold": "PASS", + "new_count": 1, + "operator": "control", + "oracle_available": false, + "rewards": { + "certified_sor": { + "receipt_id": "receipt.proof.oracle_outage.t2.s303.certified_sor", + "reward_calibration_scope": "synthetic", + "reward_certificate_id": "reward.certificate.proof-synthetic", + "reward_certificate_state": "current", + "reward_certified": true, + "reward_development_only": false, + "reward_episode_id": "proof.oracle_outage.t2.s303", + "reward_oracle_tier": 2, + "reward_outcome": "failed_platform", + "reward_receipt_id": "receipt.proof.oracle_outage.t2.s303.certified_sor", + "reward_unscored": true, + "scalar": null + }, + "visual_only": { + "receipt_id": "receipt.proof.oracle_outage.t2.s303.visual_only", + "reward_calibration_scope": null, + "reward_certificate_id": null, + "reward_certificate_state": "absent", + "reward_certified": false, + "reward_development_only": true, + "reward_episode_id": "proof.oracle_outage.t2.s303", + "reward_oracle_tier": 0, + "reward_outcome": "verified", + "reward_receipt_id": "receipt.proof.oracle_outage.t2.s303.visual_only", + "reward_unscored": false, + "scalar": 1.0 + } + }, + "screen": { + "banner_saved": true, + "claimed_success": false, + "form_looks_complete": false + }, + "seed": 303, + "sor_reasons": [], + "spec_count": 1, + "trial": 2 + } + ], + "schema": 1, + "scoring": { + "halted_before_effect_reward": 0.0, + "platform_failures": "unscored", + "refused_reward": 0.0, + "rejected_policy_reward": 0.0, + "uncertain_episodes": "unscored", + "verified_reward": 1.0, + "wrong_effect_reward": 0.0 + }, + "seed_schedule": [ + 101, + 202, + 303 + ], + "table": [ + { + "calibration_scope": null, + "certified": false, + "condition": "control", + "gold": "PASS", + "oracle_tier": 0, + "over_refusal_rate": 0.0, + "paid": 3, + "reward": "visual_only", + "silent_incorrect_success_rate": null, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": "synthetic", + "certified": true, + "condition": "control", + "gold": "PASS", + "oracle_tier": 2, + "over_refusal_rate": 0.0, + "paid": 3, + "reward": "certified_sor", + "silent_incorrect_success_rate": null, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": null, + "certified": false, + "condition": "dup", + "gold": "FAIL", + "oracle_tier": 0, + "over_refusal_rate": null, + "paid": 3, + "reward": "visual_only", + "silent_incorrect_success_rate": 1.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": "synthetic", + "certified": true, + "condition": "dup", + "gold": "FAIL", + "oracle_tier": 2, + "over_refusal_rate": null, + "paid": 0, + "reward": "certified_sor", + "silent_incorrect_success_rate": 0.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": null, + "certified": false, + "condition": "extra", + "gold": "FAIL", + "oracle_tier": 0, + "over_refusal_rate": null, + "paid": 3, + "reward": "visual_only", + "silent_incorrect_success_rate": 1.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": "synthetic", + "certified": true, + "condition": "extra", + "gold": "FAIL", + "oracle_tier": 2, + "over_refusal_rate": null, + "paid": 0, + "reward": "certified_sor", + "silent_incorrect_success_rate": 0.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": null, + "certified": false, + "condition": "omit", + "gold": "FAIL", + "oracle_tier": 0, + "over_refusal_rate": null, + "paid": 3, + "reward": "visual_only", + "silent_incorrect_success_rate": 1.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": "synthetic", + "certified": true, + "condition": "omit", + "gold": "FAIL", + "oracle_tier": 2, + "over_refusal_rate": null, + "paid": 0, + "reward": "certified_sor", + "silent_incorrect_success_rate": 0.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": null, + "certified": false, + "condition": "unsubmit", + "gold": "FAIL", + "oracle_tier": 0, + "over_refusal_rate": null, + "paid": 3, + "reward": "visual_only", + "silent_incorrect_success_rate": 1.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": "synthetic", + "certified": true, + "condition": "unsubmit", + "gold": "FAIL", + "oracle_tier": 2, + "over_refusal_rate": null, + "paid": 0, + "reward": "certified_sor", + "silent_incorrect_success_rate": 0.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": null, + "certified": false, + "condition": "claim", + "gold": "FAIL", + "oracle_tier": 0, + "over_refusal_rate": null, + "paid": 3, + "reward": "visual_only", + "silent_incorrect_success_rate": 1.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": "synthetic", + "certified": true, + "condition": "claim", + "gold": "FAIL", + "oracle_tier": 2, + "over_refusal_rate": null, + "paid": 0, + "reward": "certified_sor", + "silent_incorrect_success_rate": 0.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": null, + "certified": false, + "condition": "identity_swap", + "gold": "FAIL", + "oracle_tier": 0, + "over_refusal_rate": null, + "paid": 3, + "reward": "visual_only", + "silent_incorrect_success_rate": 1.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": "synthetic", + "certified": true, + "condition": "identity_swap", + "gold": "FAIL", + "oracle_tier": 2, + "over_refusal_rate": null, + "paid": 0, + "reward": "certified_sor", + "silent_incorrect_success_rate": 0.0, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": null, + "certified": false, + "condition": "oracle_outage", + "gold": "PASS", + "oracle_tier": 0, + "over_refusal_rate": 0.0, + "paid": 3, + "reward": "visual_only", + "silent_incorrect_success_rate": null, + "trials": 3, + "unscored": 0 + }, + { + "calibration_scope": "synthetic", + "certified": true, + "condition": "oracle_outage", + "gold": "PASS", + "oracle_tier": 2, + "over_refusal_rate": 0.0, + "paid": 0, + "reward": "certified_sor", + "silent_incorrect_success_rate": null, + "trials": 3, + "unscored": 3 + } + ], + "trials_per_condition": 3, + "versions": { + "openadapt-evals": "0.94.1", + "openadapt-types": "0.17.0" + } +} diff --git a/docs/reward/proof_2026-09-02.md b/docs/reward/proof_2026-09-02.md new file mode 100644 index 0000000..c35b479 --- /dev/null +++ b/docs/reward/proof_2026-09-02.md @@ -0,0 +1,47 @@ +# Reward proof on MockMed ExtraDup, 2026-09-02 + +Generated by `python -m openadapt_evals.reward.proof` with seed schedule [101, 202, 303] and 3 trials per condition, scored at policy update 7. openadapt-evals 0.94.1, openadapt-types 0.17.0. No model was trained and no GPU was used. The store is the in-memory MockMed encounter table from `openadapt_evals.extradup`; every record is synthetic. + +Contract digest `sha256:587c45fae205d6dc10dabdd04ff8ed9755ccd7ef3a9aa84f2cb6e5f65e876ef2`. Certificate `reward.certificate.proof-synthetic`, scope **synthetic**, self-signed by the harness key `dev.key.synthetic-proof` (ed25519 public key `fPWkNmYi+TRIHZ8wF4qfk1prCUKorlEqTzuWcTL9EnM=`), epsilon 0.153318, delta 0.05, current for 100 policy updates from update 0. + +## Table + +`paid` counts trials with scalar > 0. `silent incorrect success` is paid over trials on a gold-FAIL condition. `over-refusal` is scalar <= 0 over trials on a gold-PASS condition. `unscored` trials carry no scalar and are dropped by the adapters, never scored 0. `certified` is the adapter's recomputed flag, not the receipt's. + +| condition | gold | reward | tier | trials | paid | silent incorrect success | over-refusal | unscored | certified | scope | +|---|---|---|---|---|---|---|---|---|---|---| +| control | PASS | visual_only | 0 | 3 | 3 | n/a | 0.00 | 0 | no | none | +| control | PASS | certified_sor | 2 | 3 | 3 | n/a | 0.00 | 0 | yes | synthetic | +| dup | FAIL | visual_only | 0 | 3 | 3 | 1.00 | n/a | 0 | no | none | +| dup | FAIL | certified_sor | 2 | 3 | 0 | 0.00 | n/a | 0 | yes | synthetic | +| extra | FAIL | visual_only | 0 | 3 | 3 | 1.00 | n/a | 0 | no | none | +| extra | FAIL | certified_sor | 2 | 3 | 0 | 0.00 | n/a | 0 | yes | synthetic | +| omit | FAIL | visual_only | 0 | 3 | 3 | 1.00 | n/a | 0 | no | none | +| omit | FAIL | certified_sor | 2 | 3 | 0 | 0.00 | n/a | 0 | yes | synthetic | +| unsubmit | FAIL | visual_only | 0 | 3 | 3 | 1.00 | n/a | 0 | no | none | +| unsubmit | FAIL | certified_sor | 2 | 3 | 0 | 0.00 | n/a | 0 | yes | synthetic | +| claim | FAIL | visual_only | 0 | 3 | 3 | 1.00 | n/a | 0 | no | none | +| claim | FAIL | certified_sor | 2 | 3 | 0 | 0.00 | n/a | 0 | yes | synthetic | +| identity_swap | FAIL | visual_only | 0 | 3 | 3 | 1.00 | n/a | 0 | no | none | +| identity_swap | FAIL | certified_sor | 2 | 3 | 0 | 0.00 | n/a | 0 | yes | synthetic | +| oracle_outage | PASS | visual_only | 0 | 3 | 3 | n/a | 0.00 | 0 | no | none | +| oracle_outage | PASS | certified_sor | 2 | 3 | 0 | n/a | 0.00 | 3 | yes | synthetic | + +## Calibration + +Over the 18 gold-FAIL rollouts (five ExtraDup families plus identity_swap, 3 trials each): + +- `visual_only` paid 18 of 18. One-sided 95% Clopper-Pearson upper bound on its false-accept rate: 1.0. +- `certified_sor` paid 0 of 18. One-sided 95% Clopper-Pearson upper bound: 0.153318. This number is the certificate's epsilon. + +The bound covers this synthetic run only. It is not a production epsilon and it says nothing about any real system of record. + +## Expiry + +The 3 certified control receipts were re-assessed at policy update 100, the first update at which the certificate is expired. 0 stayed certified; the adapter logged 3 expiry warnings. + +## Reading it + +The visual reward pays every mutant: the banner says saved after a duplicate CREATE, an extra field, a dropped field, or a correct-looking note written onto the wrong patient_id (p2), the filled form looks complete when nothing was posted, and the claim condition is the policy's own report. It also pays the outage condition, where the write was correct but nobody could check. The certified reward pays the control and refuses the ExtraDup families on the record count and field set; on identity_swap it reads the named identity_keys ['patient_id', 'type'] ({'patient_id': 'p1', 'type': 'Triage'}) and the named record is unchanged, so the scalar is 0; on the outage it returns `failed_platform`, so those trials are unscored and leave the training group instead of teaching the policy that an unreadable store is worth 0. + +The proof JSON beside this file holds every rollout, receipt id, and flag. Scripted policies, no GPU, not a trained policy, not a Production Seal. diff --git a/openadapt_evals/reward/proof.py b/openadapt_evals/reward/proof.py index ac8c95b..20da8e6 100644 --- a/openadapt_evals/reward/proof.py +++ b/openadapt_evals/reward/proof.py @@ -2,27 +2,34 @@ Scripted policies produce rollouts against the in-memory MockMed store from ``openadapt_evals.extradup``: the gold CREATE (``control``) and each ExtraDup -family (``dup``, ``extra``, ``omit``, ``unsubmit``, ``claim``), plus one -``oracle_outage`` condition where the write is correct but the system of -record cannot be read. Every condition runs at least three trials on a fixed +family (``dup``, ``extra``, ``omit``, ``unsubmit``, ``claim``), plus +``identity_swap`` (a correct-looking note written onto the wrong +``patient_id``) and ``oracle_outage`` (the write is correct but the system of +record cannot be read). Every condition runs at least three trials on a fixed seed schedule. The seed picks the banner wording and the receipt identities; -the store logic is deterministic. +the store logic is deterministic. No model is trained and no GPU is used. Two rewards score every rollout through the same ``assess_receipt`` path the TRL and verl adapters use: * ``visual_only``: oracle tier 0. It believes the banner, the filled form, or the policy's own claim. It is ``development_only`` and never certified. -* ``certified_sor``: oracle tier 2. It is ExtraDup's ``sor_check`` (record - count, no extra field, every spec field) behind a self-signed certificate - of synthetic scope. When the oracle is unavailable it returns - ``failed_platform``, which is unscored. +* ``certified_sor``: oracle tier 2. It reads the named ``oracle.identity_keys`` + and runs ExtraDup's ``sor_check`` (record count, no extra field, every spec + field) on that record, behind a self-signed certificate of synthetic scope. + When the oracle is unavailable it returns ``failed_platform``, which is + unscored. On ``identity_swap`` the named record is unchanged, so the scalar + is 0 even though a correct-looking row landed on a decoy identity. The certificate's epsilon is not invented. A calibration pass counts the certified reward's false accepts over the gold-FAIL rollouts of this run and the certificate carries the exact one-sided 95% Clopper-Pearson upper bound on that count. It bounds this synthetic run and nothing else. +The 2026-09-01 committed files are the ExtraDup-only snapshot (no +``identity_swap``). Pass ``--conditions`` with that frozen set to regenerate +them. The default run includes ``identity_swap``. + Run:: python -m openadapt_evals.reward.proof --json out.json --markdown out.md @@ -54,7 +61,7 @@ from openadapt_evals.extradup.checkers import new_records, sor_check from openadapt_evals.extradup.gold import MOCKMED_GOLD from openadapt_evals.extradup.mutations import OPERATORS, apply -from openadapt_evals.extradup.store import MockMedStore +from openadapt_evals.extradup.store import MockMedStore, Screen from openadapt_evals.reward.devsigner import ( SYNTHETIC_SCOPE, DevelopmentSigner, @@ -69,7 +76,15 @@ DEFAULT_SEEDS: tuple[int, ...] = (101, 202, 303) MIN_TRIALS = 3 ORACLE_OUTAGE = "oracle_outage" -CONDITIONS: tuple[str, ...] = tuple(OPERATORS) + (ORACLE_OUTAGE,) +IDENTITY_SWAP = "identity_swap" +# Named identity the certified oracle reads. Matches RewardContractV1.oracle. +ORACLE_IDENTITY_KEYS: tuple[str, ...] = ("patient_id", "type") +# Decoy the identity-swap policy writes onto. Not the gold patient_id. +WRONG_PATIENT_ID = "p2" +# ExtraDup families plus oracle_outage. The 2026-09-01 committed proof. +PROOF_2026_09_01_CONDITIONS: tuple[str, ...] = tuple(OPERATORS) + (ORACLE_OUTAGE,) +# Default run: the 09-01 set plus identity_swap, still before the outage. +CONDITIONS: tuple[str, ...] = tuple(OPERATORS) + (IDENTITY_SWAP, ORACLE_OUTAGE) VISUAL = "visual_only" CERTIFIED = "certified_sor" REWARDS: tuple[str, ...] = (VISUAL, CERTIFIED) @@ -123,6 +138,7 @@ def new_count(self) -> int: class ProofRun: seeds: tuple[int, ...] trials: int + conditions: tuple[str, ...] contract: RewardContractV1 certificate: RewardCertificateV1 public_key_b64: str @@ -143,13 +159,48 @@ def gold_for(condition: str) -> str: return "PASS" if condition in {"control", ORACLE_OUTAGE} else "FAIL" +def named_oracle_identity() -> dict[str, str]: + """The record the certified oracle reads, keyed as the contract's identity_keys.""" + + return {key: str(MOCKMED_GOLD.fields[key]) for key in ORACLE_IDENTITY_KEYS} + + +def records_for_identity( + rows: Sequence[dict[str, Any]], identity: dict[str, str] +) -> list[dict[str, Any]]: + """Rows whose identity_keys match ``identity``. The named record, nothing else.""" + + return [ + dict(row) + for row in rows + if all(str(row.get(key, "")) == value for key, value in identity.items()) + ] + + +def _write_wrong_identity( + store: MockMedStore, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], Screen]: + """Gold note text, gold type, decoy patient_id. Banner still says saved.""" + + store.reset() + before = store.snapshot() + fields = dict(MOCKMED_GOLD.fields) + fields["patient_id"] = WRONG_PATIENT_ID + screen = store.write(fields) + return before, store.snapshot(), screen + + def scripted_rollout(condition: str, trial: int, seed: int) -> Rollout: if condition not in CONDITIONS: raise KeyError(f"unknown proof condition {condition!r}") rng = random.Random(f"{seed}:{condition}:{trial}") - operator = "control" if condition == ORACLE_OUTAGE else condition store = MockMedStore() - before, after, screen = apply(store, MOCKMED_GOLD, operator) + if condition == IDENTITY_SWAP: + operator = IDENTITY_SWAP + before, after, screen = _write_wrong_identity(store) + else: + operator = "control" if condition == ORACLE_OUTAGE else condition + before, after, screen = apply(store, MOCKMED_GOLD, operator) banner_text = rng.choice(BANNER_VARIANTS) if screen.banner_saved else "" return Rollout( condition=condition, @@ -170,14 +221,24 @@ def scripted_rollout(condition: str, trial: int, seed: int) -> Rollout: ) -def scripted_rollouts(seeds: Sequence[int], trials: int) -> list[Rollout]: +def scripted_rollouts( + seeds: Sequence[int], + trials: int, + conditions: Sequence[str] = CONDITIONS, +) -> list[Rollout]: if trials < MIN_TRIALS: raise ValueError(f"at least {MIN_TRIALS} trials per condition") if len(seeds) < trials: raise ValueError("the seed schedule must have one seed per trial") + selected = tuple(conditions) + unknown = [item for item in selected if item not in CONDITIONS] + if unknown: + raise KeyError(f"unknown proof condition {unknown[0]!r}") + if not selected: + raise ValueError("need at least one condition") return [ scripted_rollout(condition, trial, seeds[trial]) - for condition in CONDITIONS + for condition in selected for trial in range(trials) ] @@ -198,11 +259,19 @@ def visual_outcome(rollout: Rollout) -> RewardOutcomeV1: def certified_outcome(rollout: Rollout) -> RewardOutcomeV1: - """Tier 2. Reads the store; unavailable oracle is unscored.""" + """Tier 2. Reads the named identity; unavailable oracle is unscored. + + The worker needs ``oracle.identity_keys`` to know which record to read. + ``identity_swap`` writes a correct-looking note onto a decoy patient_id; + the named record is unchanged, so this returns ``wrong_effect``. + """ if not rollout.oracle_available: return RewardOutcomeV1.FAILED_PLATFORM - if sor_check(MOCKMED_GOLD, rollout.before, rollout.after).ok: + identity = named_oracle_identity() + named_before = records_for_identity(rollout.before, identity) + named_after = records_for_identity(rollout.after, identity) + if sor_check(MOCKMED_GOLD, named_before, named_after).ok: return RewardOutcomeV1.VERIFIED return RewardOutcomeV1.WRONG_EFFECT @@ -292,7 +361,7 @@ def build_contract(calibration: dict[str, Any]) -> RewardContractV1: ), "oracle": { "channel": "db", - "identity_keys": ["patient_id", "type"], + "identity_keys": list(ORACLE_IDENTITY_KEYS), "oracle_contract_digest": sha256_digest( b"openadapt_evals.extradup.checkers.sor_check" ), @@ -394,9 +463,14 @@ def expiry_check( # -- the run --------------------------------------------------------------------------------- -def run_proof(seeds: Sequence[int] = DEFAULT_SEEDS, trials: int = MIN_TRIALS) -> ProofRun: +def run_proof( + seeds: Sequence[int] = DEFAULT_SEEDS, + trials: int = MIN_TRIALS, + conditions: Sequence[str] | None = None, +) -> ProofRun: seeds = tuple(int(seed) for seed in seeds) - rollouts = scripted_rollouts(seeds, trials) + selected = tuple(conditions) if conditions is not None else CONDITIONS + rollouts = scripted_rollouts(seeds, trials, selected) calibration = calibrate(rollouts) contract = build_contract(calibration) signer = DevelopmentSigner(b"seeds:" + ",".join(str(seed) for seed in seeds).encode()) @@ -423,6 +497,7 @@ def run_proof(seeds: Sequence[int] = DEFAULT_SEEDS, trials: int = MIN_TRIALS) -> run = ProofRun( seeds=seeds, trials=trials, + conditions=selected, contract=contract, certificate=certificate, public_key_b64=b64encode(signer.public_key_bytes()).decode("ascii"), @@ -438,7 +513,7 @@ def run_proof(seeds: Sequence[int] = DEFAULT_SEEDS, trials: int = MIN_TRIALS) -> def tabulate(run: ProofRun) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] - for condition in CONDITIONS: + for condition in run.conditions: gold = gold_for(condition) for reward in REWARDS: episodes = [run.scored[(condition, trial, reward)] for trial in range(run.trials)] @@ -470,6 +545,39 @@ def tabulate(run: ProofRun) -> list[dict[str, Any]]: return rows +def _named_sor_reasons(item: Rollout) -> list[str]: + identity = named_oracle_identity() + named_before = records_for_identity(item.before, identity) + named_after = records_for_identity(item.after, identity) + return list(sor_check(MOCKMED_GOLD, named_before, named_after).reasons) + + +def _rollout_payload(run: ProofRun, item: Rollout) -> dict[str, Any]: + payload: dict[str, Any] = { + **{k: v for k, v in asdict(item).items() if k not in {"before", "after"}}, + "new_count": item.new_count, + "spec_count": MOCKMED_GOLD.expected_new, + "sor_reasons": _named_sor_reasons(item), + "evidence_digest": item.evidence_digest, + "rewards": { + reward: { + "receipt_id": run.receipts[(item.condition, item.trial, reward)].receipt_id, + **run.scored[(item.condition, item.trial, reward)].metadata(), + "scalar": run.scored[(item.condition, item.trial, reward)].scalar, + } + for reward in REWARDS + }, + } + if item.condition == IDENTITY_SWAP: + identity = named_oracle_identity() + named_before = records_for_identity(item.before, identity) + named_after = records_for_identity(item.after, identity) + payload["oracle_identity"] = identity + payload["written_patient_id"] = WRONG_PATIENT_ID + payload["named_new_count"] = len(new_records(named_before, named_after)) + return payload + + # -- output ------------------------------------------------------------------------------------ @@ -487,7 +595,7 @@ def to_json(run: ProofRun) -> dict[str, Any]: "schema": PROOF_SCHEMA, "seed_schedule": list(run.seeds), "trials_per_condition": run.trials, - "conditions": list(CONDITIONS), + "conditions": list(run.conditions), "policy_update": POLICY_UPDATE, "scoring": SCORING.model_dump(mode="json"), "versions": { @@ -514,24 +622,7 @@ def to_json(run: ProofRun) -> dict[str, Any]: "calibration": run.calibration, "table": run.table, "expiry_check": run.expiry_check, - "rollouts": [ - { - **{k: v for k, v in asdict(item).items() if k not in {"before", "after"}}, - "new_count": item.new_count, - "spec_count": MOCKMED_GOLD.expected_new, - "sor_reasons": list(sor_check(MOCKMED_GOLD, item.before, item.after).reasons), - "evidence_digest": item.evidence_digest, - "rewards": { - reward: { - "receipt_id": run.receipts[(item.condition, item.trial, reward)].receipt_id, - **run.scored[(item.condition, item.trial, reward)].metadata(), - "scalar": run.scored[(item.condition, item.trial, reward)].scalar, - } - for reward in REWARDS - }, - } - for item in run.rollouts - ], + "rollouts": [_rollout_payload(run, item) for item in run.rollouts], } @@ -601,7 +692,7 @@ def to_markdown(run: ProofRun, *, date: str) -> str: "", "## Calibration", "", - f"Over the {n} gold-FAIL rollouts (five ExtraDup families, {run.trials} trials each):", + f"Over the {n} gold-FAIL rollouts ({_fail_family_phrase(run)}, {run.trials} trials each):", "", f"- `{VISUAL}` paid {fa[VISUAL]} of {n}. One-sided 95% Clopper-Pearson upper bound " f"on its false-accept rate: {cp[VISUAL]}.", @@ -620,21 +711,66 @@ def to_markdown(run: ProofRun, *, date: str) -> str: "", "## Reading it", "", - "The visual reward pays every mutant: the banner says saved after a duplicate " - "CREATE, an extra field, or a dropped field, the filled form looks complete when " - "nothing was posted, and the claim condition is the policy's own report. It also " - "pays the outage condition, where the write was correct but nobody could check. " - "The certified reward pays the control and refuses the five families on the record " - "count and field set; on the outage it returns `failed_platform`, so those trials are " - "unscored and leave the training group instead of teaching the policy that an " - "unreadable store is worth 0.", + _reading_paragraph(run), "", - "The proof JSON beside this file holds every rollout, receipt id, and flag.", + "The proof JSON beside this file holds every rollout, receipt id, and flag. " + "Scripted policies, no GPU, not a trained policy, not a Production Seal.", "", ] return "\n".join(lines) +def _fail_family_phrase(run: ProofRun) -> str: + fail = [item for item in run.conditions if gold_for(item) == "FAIL"] + extradup = [item for item in fail if item in OPERATORS] + extra = [item for item in fail if item not in OPERATORS] + parts: list[str] = [] + if extradup: + noun = "family" if len(extradup) == 1 else "families" + count = {1: "one", 2: "two", 3: "three", 4: "four", 5: "five", 6: "six"}.get( + len(extradup), str(len(extradup)) + ) + parts.append(f"{count} ExtraDup {noun}") + if extra: + parts.append(" plus ".join(extra)) + return " plus ".join(parts) if parts else "no gold-FAIL families" + + +def _reading_paragraph(run: ProofRun) -> str: + if IDENTITY_SWAP in run.conditions: + mutants = ( + "The visual reward pays every mutant: the banner says saved after a duplicate " + "CREATE, an extra field, a dropped field, or a correct-looking note written onto " + f"the wrong patient_id ({WRONG_PATIENT_ID}), the filled form looks complete when " + "nothing was posted, and the claim condition is the policy's own report." + ) + else: + mutants = ( + "The visual reward pays every mutant: the banner says saved after a duplicate " + "CREATE, an extra field, or a dropped field, the filled form looks complete when " + "nothing was posted, and the claim condition is the policy's own report." + ) + outage = ( + " It also pays the outage condition, where the write was correct but nobody could check." + ) + certified = ( + " The certified reward pays the control and refuses the ExtraDup families on the record " + "count and field set" + ) + if IDENTITY_SWAP in run.conditions: + identity = named_oracle_identity() + certified += ( + f"; on identity_swap it reads the named identity_keys {list(ORACLE_IDENTITY_KEYS)} " + f"({identity}) and the named record is unchanged, so the scalar is 0" + ) + certified += ( + "; on the outage it returns `failed_platform`, so those trials are " + "unscored and leave the training group instead of teaching the policy that an " + "unreadable store is worth 0." + ) + return mutants + outage + certified + + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser( prog="python -m openadapt_evals.reward.proof", @@ -642,16 +778,26 @@ def main(argv: Sequence[str] | None = None) -> int: ) parser.add_argument("--json", type=Path, help="write the full run as JSON") parser.add_argument("--markdown", type=Path, help="write the human table as Markdown") - parser.add_argument("--date", default="2026-09-01", help="date stamped in the Markdown title") + parser.add_argument("--date", default="2026-09-02", help="date stamped in the Markdown title") parser.add_argument("--trials", type=int, default=MIN_TRIALS) parser.add_argument( "--seeds", default=",".join(str(seed) for seed in DEFAULT_SEEDS), help="comma-separated seed schedule, one seed per trial", ) + parser.add_argument( + "--conditions", + default="", + help=( + "comma-separated conditions (default: the full set, including identity_swap). " + "Pass the 2026-09-01 set to regenerate that frozen proof: " + + ",".join(PROOF_2026_09_01_CONDITIONS) + ), + ) args = parser.parse_args(argv) seeds = tuple(int(item) for item in args.seeds.split(",") if item.strip()) - run = run_proof(seeds, args.trials) + selected = tuple(item.strip() for item in args.conditions.split(",") if item.strip()) + run = run_proof(seeds, args.trials, selected or None) if args.json: args.json.write_text(json.dumps(to_json(run), indent=2, sort_keys=True) + "\n") if args.markdown: diff --git a/tests/test_reward_proof.py b/tests/test_reward_proof.py index b1ae08b..387f9ac 100644 --- a/tests/test_reward_proof.py +++ b/tests/test_reward_proof.py @@ -7,11 +7,14 @@ import pytest +from openadapt_evals.extradup.checkers import new_records +from openadapt_evals.extradup.gold import MOCKMED_GOLD from openadapt_evals.reward import proof from openadapt_evals.reward.devsigner import verify_signature DOCS = Path(__file__).resolve().parents[1] / "docs" / "reward" -COMMITTED_JSON = DOCS / "proof_2026-09-01.json" +COMMITTED_2026_09_01 = DOCS / "proof_2026-09-01.json" +COMMITTED_2026_09_02 = DOCS / "proof_2026-09-02.json" @pytest.fixture(scope="module") @@ -19,10 +22,19 @@ def run() -> proof.ProofRun: return proof.run_proof() +@pytest.fixture(scope="module") +def run_2026_09_01() -> proof.ProofRun: + return proof.run_proof(conditions=proof.PROOF_2026_09_01_CONDITIONS) + + def _strip_versions(payload: dict) -> dict: return {key: value for key, value in payload.items() if key != "versions"} +def _fail_families(conditions: tuple[str, ...]) -> tuple[str, ...]: + return tuple(item for item in conditions if proof.gold_for(item) == "FAIL") + + def test_run_is_deterministic(run: proof.ProofRun) -> None: again = proof.run_proof() assert proof.to_json(run) == proof.to_json(again) @@ -33,13 +45,15 @@ def test_table_has_every_condition_and_three_trials(run: proof.ProofRun) -> None cells = {(row["condition"], row["reward"]) for row in run.table} assert cells == {(c, r) for c in proof.CONDITIONS for r in proof.REWARDS} assert set(proof.OPERATORS) <= set(proof.CONDITIONS) + assert proof.IDENTITY_SWAP in proof.CONDITIONS assert all(row["trials"] >= proof.MIN_TRIALS for row in run.table) assert len(run.seeds) >= proof.MIN_TRIALS + assert run.conditions == proof.CONDITIONS def test_visual_pays_every_mutant_and_certified_refuses(run: proof.ProofRun) -> None: by_cell = {(row["condition"], row["reward"]): row for row in run.table} - for family in ("dup", "extra", "omit", "unsubmit", "claim"): + for family in _fail_families(run.conditions): visual = by_cell[(family, proof.VISUAL)] certified = by_cell[(family, proof.CERTIFIED)] assert visual["silent_incorrect_success_rate"] == 1.0 @@ -49,6 +63,47 @@ def test_visual_pays_every_mutant_and_certified_refuses(run: proof.ProofRun) -> control = by_cell[("control", proof.CERTIFIED)] assert control["over_refusal_rate"] == 0.0 assert control["unscored"] == 0 + assert control["paid"] == run.trials + + +def test_identity_swap_writes_the_gold_note_onto_the_wrong_patient(run: proof.ProofRun) -> None: + identity = proof.named_oracle_identity() + assert identity["patient_id"] == MOCKMED_GOLD.fields["patient_id"] + assert identity["patient_id"] != proof.WRONG_PATIENT_ID + rollouts = [item for item in run.rollouts if item.condition == proof.IDENTITY_SWAP] + assert len(rollouts) == run.trials + for item in rollouts: + written = new_records(item.before, item.after) + assert len(written) == MOCKMED_GOLD.expected_new + assert written[0]["patient_id"] == proof.WRONG_PATIENT_ID + assert written[0]["note"] == MOCKMED_GOLD.fields["note"] + assert written[0]["type"] == MOCKMED_GOLD.fields["type"] + assert item.screen["banner_saved"] is True + named_before = proof.records_for_identity(item.before, identity) + named_after = proof.records_for_identity(item.after, identity) + assert named_before == named_after + assert len(new_records(named_before, named_after)) == 0 + visual = run.scored[(item.condition, item.trial, proof.VISUAL)] + certified = run.scored[(item.condition, item.trial, proof.CERTIFIED)] + assert visual.scalar == 1.0 + assert certified.scalar == 0.0 + assert certified.unscored is False + assert "|new(M)|=0" in ";".join(proof._named_sor_reasons(item)) + + +def test_identity_swap_table_row(run: proof.ProofRun) -> None: + by_cell = {(row["condition"], row["reward"]): row for row in run.table} + visual = by_cell[(proof.IDENTITY_SWAP, proof.VISUAL)] + certified = by_cell[(proof.IDENTITY_SWAP, proof.CERTIFIED)] + assert visual["gold"] == "FAIL" + assert visual["paid"] == run.trials + assert visual["silent_incorrect_success_rate"] == 1.0 + assert visual["certified"] is False + assert certified["paid"] == 0 + assert certified["silent_incorrect_success_rate"] == 0.0 + assert certified["unscored"] == 0 + assert certified["certified"] is True + assert certified["calibration_scope"] == "synthetic" def test_outage_is_unscored_never_zero(run: proof.ProofRun) -> None: @@ -77,7 +132,8 @@ def test_certified_reward_is_certified_with_synthetic_scope(run: proof.ProofRun) def test_certificate_epsilon_is_the_clopper_pearson_bound(run: proof.ProofRun) -> None: cal = run.calibration n = cal["gold_fail_trials"] - assert n == 5 * run.trials + assert n == len(_fail_families(run.conditions)) * run.trials + assert n == 6 * run.trials assert cal["false_accepts"][proof.VISUAL] == n assert cal["false_accepts"][proof.CERTIFIED] == 0 assert cal["clopper_pearson_upper_95"][proof.VISUAL] == 1.0 @@ -115,24 +171,49 @@ def test_cli_writes_json_and_markdown(tmp_path: Path, capsys: pytest.CaptureFixt json_path = tmp_path / "proof.json" md_path = tmp_path / "proof.md" assert ( - proof.main(["--json", str(json_path), "--markdown", str(md_path), "--date", "2026-09-01"]) + proof.main(["--json", str(json_path), "--markdown", str(md_path), "--date", "2026-09-02"]) == 0 ) payload = json.loads(json_path.read_text()) assert payload["proof"] == proof.PROOF_NAME + assert proof.IDENTITY_SWAP in payload["conditions"] assert len(payload["rollouts"]) == len(proof.CONDITIONS) * proof.MIN_TRIALS + swap = next(item for item in payload["rollouts"] if item["condition"] == proof.IDENTITY_SWAP) + assert swap["written_patient_id"] == proof.WRONG_PATIENT_ID + assert swap["named_new_count"] == 0 + assert swap["oracle_identity"] == proof.named_oracle_identity() text = md_path.read_text() assert "| condition | gold | reward |" in text + assert "identity_swap" in text assert "Clopper-Pearson" in text assert "—" not in text + assert "not a Production Seal" in text assert "| condition |" in capsys.readouterr().out -def test_committed_proof_matches_a_fresh_run(run: proof.ProofRun) -> None: - """The evidence in docs/reward is the output of this code at this seed schedule.""" +def test_committed_2026_09_01_proof_matches_frozen_conditions( + run_2026_09_01: proof.ProofRun, +) -> None: + """The ExtraDup-only snapshot stays byte-identical to the #332 claim.""" + + assert COMMITTED_2026_09_01.exists() + assert proof.IDENTITY_SWAP not in run_2026_09_01.conditions + assert run_2026_09_01.conditions == proof.PROOF_2026_09_01_CONDITIONS + committed = json.loads(COMMITTED_2026_09_01.read_text()) + assert proof.IDENTITY_SWAP not in committed["conditions"] + assert committed["calibration"]["gold_fail_trials"] == 15 + assert _strip_versions(committed) == _strip_versions(proof.to_json(run_2026_09_01)) + + +def test_committed_2026_09_02_proof_matches_a_fresh_run(run: proof.ProofRun) -> None: + """The identity-swap proof in docs/reward is this code at this seed schedule.""" - assert COMMITTED_JSON.exists(), ( - "run python -m openadapt_evals.reward.proof --json docs/reward/proof_.json" + assert COMMITTED_2026_09_02.exists(), ( + "run python -m openadapt_evals.reward.proof " + "--json docs/reward/proof_2026-09-02.json " + "--markdown docs/reward/proof_2026-09-02.md --date 2026-09-02" ) - committed = json.loads(COMMITTED_JSON.read_text()) + committed = json.loads(COMMITTED_2026_09_02.read_text()) + assert proof.IDENTITY_SWAP in committed["conditions"] + assert committed["calibration"]["gold_fail_trials"] == 18 assert _strip_versions(committed) == _strip_versions(proof.to_json(run)) From 95f40ff53a61865e0f0c37de0e5938eba0e5e794 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 2 Sep 2026 12:40:43 -0400 Subject: [PATCH 2/3] fix: amend M-freeze hashes after identity-swap proof The freeze pins proof.py. Identity-swap changes that file. Update the hash and pin proof_2026-09-02.json. ExtraDup operators and the 09-01 certificate digest stay the same. --- ..._CERTIFIED_REWARD_RL_PILOT_2026_09_02.json | 171 +++++++++++++++--- 1 file changed, 144 insertions(+), 27 deletions(-) diff --git a/docs/preregistrations/M_FREEZE_CERTIFIED_REWARD_RL_PILOT_2026_09_02.json b/docs/preregistrations/M_FREEZE_CERTIFIED_REWARD_RL_PILOT_2026_09_02.json index 03cd2d5..4cabb18 100644 --- a/docs/preregistrations/M_FREEZE_CERTIFIED_REWARD_RL_PILOT_2026_09_02.json +++ b/docs/preregistrations/M_FREEZE_CERTIFIED_REWARD_RL_PILOT_2026_09_02.json @@ -21,7 +21,9 @@ "scope": { "phase": "Phase-1/pilot", "kill_criterion": 3, - "environments": ["mockmed"], + "environments": [ + "mockmed" + ], "openemr_in_this_pilot": false, "calibration_scope": "synthetic", "production_acceptance": false, @@ -80,12 +82,22 @@ }, "seed_schedule": { "K": 3, - "seeds": [101, 202, 303], + "seeds": [ + 101, + 202, + 303 + ], "source": "openadapt_evals.reward.proof.DEFAULT_SEEDS", "rule": "No best-seed selection. Every arm times every seed reports. An abort or refusal records a reason and still occupies its cell." }, "extradup_mutants": { - "operators": ["dup", "extra", "omit", "unsubmit", "claim"], + "operators": [ + "dup", + "extra", + "omit", + "unsubmit", + "claim" + ], "control": "control", "source": "openadapt_evals.extradup.mutations.MUTANTS", "in_training_dataset": false, @@ -104,30 +116,121 @@ "train_num_tasks": 8, "holdout_generation_seed": 20260825, "holdout_num_tasks": 8, - "train_patient_ids": ["p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8"], - "holdout_patient_ids": ["h1", "h2", "h3", "h4", "h5", "h6", "h7", "h8"], + "train_patient_ids": [ + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", + "p7", + "p8" + ], + "holdout_patient_ids": [ + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "h7", + "h8" + ], "train_gold": [ - {"patient_id": "p1", "type": "Triage", "note": "Follow-up in 2 weeks; BP recheck."}, - {"patient_id": "p2", "type": "Intake", "note": "Post-op check; wound clean."}, - {"patient_id": "p3", "type": "Triage", "note": "Fasting glucose ordered."}, - {"patient_id": "p4", "type": "Intake", "note": "Post-op check; wound clean."}, - {"patient_id": "p5", "type": "Consult", "note": "Post-op check; wound clean."}, - {"patient_id": "p6", "type": "Consult", "note": "Vaccination due; schedule next visit."}, - {"patient_id": "p7", "type": "Follow-up", "note": "Vaccination due; schedule next visit."}, - {"patient_id": "p8", "type": "Follow-up", "note": "Fasting glucose ordered."} + { + "patient_id": "p1", + "type": "Triage", + "note": "Follow-up in 2 weeks; BP recheck." + }, + { + "patient_id": "p2", + "type": "Intake", + "note": "Post-op check; wound clean." + }, + { + "patient_id": "p3", + "type": "Triage", + "note": "Fasting glucose ordered." + }, + { + "patient_id": "p4", + "type": "Intake", + "note": "Post-op check; wound clean." + }, + { + "patient_id": "p5", + "type": "Consult", + "note": "Post-op check; wound clean." + }, + { + "patient_id": "p6", + "type": "Consult", + "note": "Vaccination due; schedule next visit." + }, + { + "patient_id": "p7", + "type": "Follow-up", + "note": "Vaccination due; schedule next visit." + }, + { + "patient_id": "p8", + "type": "Follow-up", + "note": "Fasting glucose ordered." + } ], "holdout_gold": [ - {"patient_id": "h1", "type": "Triage", "note": "Fasting glucose ordered."}, - {"patient_id": "h2", "type": "Follow-up", "note": "Fasting glucose ordered."}, - {"patient_id": "h3", "type": "Intake", "note": "Follow-up in 2 weeks; BP recheck."}, - {"patient_id": "h4", "type": "Triage", "note": "Refill requested; review labs first."}, - {"patient_id": "h5", "type": "Consult", "note": "Allergy noted; update chart."}, - {"patient_id": "h6", "type": "Consult", "note": "Follow-up in 2 weeks; BP recheck."}, - {"patient_id": "h7", "type": "Triage", "note": "Fasting glucose ordered."}, - {"patient_id": "h8", "type": "Follow-up", "note": "Refill requested; review labs first."} + { + "patient_id": "h1", + "type": "Triage", + "note": "Fasting glucose ordered." + }, + { + "patient_id": "h2", + "type": "Follow-up", + "note": "Fasting glucose ordered." + }, + { + "patient_id": "h3", + "type": "Intake", + "note": "Follow-up in 2 weeks; BP recheck." + }, + { + "patient_id": "h4", + "type": "Triage", + "note": "Refill requested; review labs first." + }, + { + "patient_id": "h5", + "type": "Consult", + "note": "Allergy noted; update chart." + }, + { + "patient_id": "h6", + "type": "Consult", + "note": "Follow-up in 2 weeks; BP recheck." + }, + { + "patient_id": "h7", + "type": "Triage", + "note": "Fasting glucose ordered." + }, + { + "patient_id": "h8", + "type": "Follow-up", + "note": "Refill requested; review labs first." + } + ], + "train_operators": [ + "control" + ], + "holdout_operators": [ + "control", + "dup", + "extra", + "omit", + "unsubmit", + "claim" ], - "train_operators": ["control"], - "holdout_operators": ["control", "dup", "extra", "omit", "unsubmit", "claim"], "holdout_mutant_host": "h1" }, "arms": { @@ -182,7 +285,7 @@ "missing_cells": "Every arm times every seed occupies a cell. A refusal, abort, or expired certificate records a reason and does not free the seed for a replacement run.", "minimum_detectable_effect": { "contrast": "holdout_silent_incorrect_success(visual_only) minus holdout_silent_incorrect_success(certified_sor), mean across the three seeds", - "threshold": 0.10, + "threshold": 0.1, "rule": "If the absolute contrast is below 0.10, the full Phase-2 grid does not run. The pilot publishes as a calibration null. That sentence is a kill rule, not a result." } }, @@ -222,13 +325,27 @@ }, { "path": "openadapt_evals/reward/proof.py", - "sha256": "sha256:cb2ba25272dccf2968f33d9629739520d09d062a0d4ea10bc9c35f66e80d868b", - "git_blob": "ac8c95bca2421d583b00261e2d763e3d4758ee7d" + "sha256": "sha256:7f87d60b9250110c5f10bad3e4b4f0f80209711b211d9ed41ed1d2cd2bff4466", + "git_blob": "20da8e6168ad03220aa1a21498d40a0c5f9f7626" }, { "path": "docs/reward/proof_2026-09-01.json", "sha256": "sha256:8eaaa34f8daf76dfeb45bab15d5ef17155b8d267aa97f6393e757ff6469bf1d3", "git_blob": "b11ce9c32693db6c553a7bf46a0f9e1f837ba4ef" + }, + { + "path": "docs/reward/proof_2026-09-02.json", + "sha256": "sha256:9be90c3ab1d6931484838abc48e16ba2d84bee676a702ad23f25f35ad538305d", + "git_blob": "289a65729c1452846e3dd7abcb5ba514fde49d23" } - ] + ], + "identity_swap": { + "in_extradup_mutants": false, + "in_training_dataset": false, + "in_training_reward": false, + "eval_only": true, + "proof_path": "docs/reward/proof_2026-09-02.json", + "note": "Wrong-record write. Named identity is p1; write lands on p2. Not an ExtraDup operator." + }, + "amendment_note": "2026-09-02 identity-swap: proof.py hash and proof_2026-09-02.json added. ExtraDup operators and the 2026-09-01 certificate digest are unchanged." } From a1223734ee9babc2a15b9b7c516d457dc9800874 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 2 Sep 2026 12:55:08 -0400 Subject: [PATCH 3/3] fix: keep ExtraDup kill-scan on the frozen operator set identity_swap is a 09-02 proof family, not an ExtraDup operator. score_proof uses the 09-01 conditions so SIS stays 15/15. --- openadapt_evals/extradup/kill_scan.py | 5 +++-- tests/test_extradup_kill_scan.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/openadapt_evals/extradup/kill_scan.py b/openadapt_evals/extradup/kill_scan.py index df3312e..7692c41 100644 --- a/openadapt_evals/extradup/kill_scan.py +++ b/openadapt_evals/extradup/kill_scan.py @@ -157,13 +157,14 @@ def _score_rows(name: str, rows: Sequence[Mapping[str, Any]]) -> TargetScore: def score_proof(target: str = "both") -> tuple[list[TargetScore], dict[str, Any]]: from openadapt_evals.reward import proof - run = proof.run_proof() + # ExtraDup operators only. identity_swap lives on the 09-02 proof, not this corpus. + run = proof.run_proof(conditions=proof.PROOF_2026_09_01_CONDITIONS) wanted = BUILTIN_TARGETS if target == "both" else (target,) scores: list[TargetScore] = [] for name in wanted: rows = [] for item in run.rollouts: - if item.condition == proof.ORACLE_OUTAGE: + if item.condition not in OPERATORS: continue scored = run.scored[(item.condition, item.trial, name)] paid = scored.scalar is not None and scored.scalar > 0 diff --git a/tests/test_extradup_kill_scan.py b/tests/test_extradup_kill_scan.py index c0ff4ff..83956c0 100644 --- a/tests/test_extradup_kill_scan.py +++ b/tests/test_extradup_kill_scan.py @@ -114,9 +114,12 @@ def test_builtin_kill_scan_matches_committed_proof( assert "control honest-write: 3/3" in out payload = json.loads(summary.read_text()) by_name = {row["name"]: row for row in payload["targets"]} + assert by_name["visual_only"]["gold_fail_trials"] == 15 assert by_name["visual_only"]["gold_fail_paid"] == 15 + assert by_name["certified_sor"]["gold_fail_trials"] == 15 assert by_name["certified_sor"]["gold_fail_paid"] == 0 assert by_name["certified_sor"]["control_paid"] == 3 + assert "identity_swap" not in out assert payload["execute_seal"] is False assert payload["production_seal"] is False assert payload["training_mix"] is False