diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py index 197a2f57..d4a008d1 100644 --- a/skillopt_sleep/config.py +++ b/skillopt_sleep/config.py @@ -214,13 +214,20 @@ def _load_file(path: str) -> Dict[str, Any]: def load_config(**overrides: Any) -> SleepConfig: data = dict(DEFAULTS) + user_keys: set[str] = set() path = _user_config_path() if path: try: - data.update(_load_file(path) or {}) + file_data = _load_file(path) or {} + user_keys.update(file_data.keys()) + data.update(file_data) except Exception: pass - data.update({k: v for k, v in overrides.items() if v is not None}) + for key, value in overrides.items(): + if value is not None: + data[key] = value + user_keys.add(key) if data.get("projects") == "invoked" and not data.get("invoked_project"): data["invoked_project"] = os.getcwd() + data["_user_config_keys"] = sorted(user_keys) return SleepConfig(data=data) diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index c6d2a563..f1d7bf02 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -21,12 +21,13 @@ from skillopt_sleep import evidence from skillopt_sleep.backend import Backend, CursorBackendError, build_backend -from skillopt_sleep.config import SleepConfig, load_config +from skillopt_sleep.config import DEFAULTS, SleepConfig, load_config from skillopt_sleep.dream import dream_consolidate from skillopt_sleep.evidence import EvidenceLog from skillopt_sleep.harvest_sources import harvest_for_config from skillopt_sleep.memory import ensure_skill_scaffold from skillopt_sleep.mine import group_tasks_by_skill_hint, mine +from skillopt_sleep.replay import aggregate_scores, replay_batch from skillopt_sleep.multi_skill import ( SKIPPED, GroupConsolidation, @@ -53,6 +54,41 @@ ) +def _resolve_split_fractions(cfg: SleepConfig) -> tuple[float, float]: + """Return ``(val_fraction, test_fraction)`` for tonight's split. + + ``holdout_fraction`` is the documented legacy alias for ``val_fraction``. + ``load_config`` records which keys came from the user's file or explicit + overrides in ``_user_config_keys``; the alias applies only when the user + set ``holdout_fraction`` and did not set ``val_fraction``. An explicit + ``val_fraction`` (including ``0.0``) always wins over the alias. + + Raises ``ValueError`` when fractions are out of range or their sum is >= 1. + """ + user_keys = set(cfg.get("_user_config_keys") or ()) + + if "val_fraction" in user_keys: + val = float(cfg.get("val_fraction")) + else: + val = float(DEFAULTS["val_fraction"]) + if "holdout_fraction" in user_keys: + val = float(cfg.get("holdout_fraction")) + + if "test_fraction" in user_keys: + test = float(cfg.get("test_fraction")) + else: + test = float(DEFAULTS["test_fraction"]) + + for name, value in (("val_fraction", val), ("test_fraction", test)): + if not 0.0 <= value <= 1.0: + raise ValueError(f"{name} must be between 0 and 1 inclusive, got {value}") + if val + test >= 1.0: + raise ValueError( + f"val_fraction + test_fraction must be < 1 (got {val} + {test})" + ) + return val, test + + # ── Model-swap detection (F16) ─────────────────────────────── def _make_model_key(cfg: SleepConfig) -> str: """Stable string identifying the backend object(s) actually used. @@ -658,7 +694,7 @@ def run_sleep_cycle( "backend", "model", "optimizer_backend", "optimizer_model", "target_backend", "target_model", "gate_mode", "gate_metric", "gate_mixed_weight", "gate_no_regression", "edit_budget", - "holdout_fraction", + "holdout_fraction", "val_fraction", "test_fraction", "dream_rollouts", "dream_factor", "recall_k", "max_tasks_per_night", "lookback_hours", "llm_mine", "evolve_skill", "evolve_memory")} @@ -758,12 +794,14 @@ def run_sleep_cycle( f"mine start: max_tasks={max_tasks} candidate_limit={candidate_limit} " f"llm_mine={llm_miner is not None} target_filter={target_filter}", ) + val_fraction, test_fraction = _resolve_split_fractions(cfg) try: tasks = mine( digests, max_tasks=max_tasks, candidate_limit=candidate_limit, - holdout_fraction=cfg.get("holdout_fraction", 0.34), + val_fraction=val_fraction, + test_fraction=test_fraction, seed=cfg.get("seed", 42), llm_miner=llm_miner, target_skill_text=raw_skill if target_filter else "", @@ -854,6 +892,24 @@ def run_sleep_cycle( report.unmatched_edits = result.unmatched_edits report.gate_trials = redact_secrets(getattr(result, "gate_trials", [])) + # ── held-out test measure (write-only; the gate never reads it) ────── + # consolidate() holds the test split out entirely and documents that the + # caller scores it; in the nightly, this is that caller. Scored on the + # night's FINAL documents (post-gate-decision), mirroring how the + # experiment harness reports its per-night test score. With the default + # test_fraction=0.0 no test task exists and this block never runs, so + # legacy nights are bit-for-bit unchanged and cost nothing extra. + test_tasks = [t for t in tasks if t.split == "test"] + if test_tasks and ev is not None: + final_skill = result.new_skill if result.accepted else skill + final_memory = result.new_memory if result.accepted else memory + test_pairs = replay_batch(backend, test_tasks, final_skill, final_memory) + test_hard, test_soft = aggregate_scores(test_pairs) + ev.log("test", "held_out_score", night=night, + n_test=len(test_tasks), + hard=round(test_hard, 4), soft=round(test_soft, 4), + accepted=result.accepted, gate_action=result.gate_action) + # ── 4b. optional per-skill group reporting ─────────────────────────── # Off by default. When enabled, tonight's tasks are grouped by their skill # hint and each group is consolidated independently so the report carries a diff --git a/skillopt_sleep/dream.py b/skillopt_sleep/dream.py index 245f7e89..9906e07d 100644 --- a/skillopt_sleep/dream.py +++ b/skillopt_sleep/dream.py @@ -59,19 +59,38 @@ def _tokens(text: str) -> set: return {w for w in re.findall(r"[a-z0-9]+", (text or "").lower()) if len(w) > 2} -def recall_similar(new_tasks: List[TaskRecord], history: List[TaskRecord], - k: int) -> List[TaskRecord]: +def _normalize_split(value: str) -> str: + return {"replay": "train", "holdout": "val"}.get(value, value) + + +def recall_similar( + new_tasks: List[TaskRecord], + history: List[TaskRecord], + k: int, + *, + exclude_ids: Optional[set[str]] = None, +) -> List[TaskRecord]: """Return the ``k`` historical tasks most lexically similar to any of tonight's ``new_tasks`` (max Jaccard token overlap). Recalled tasks are returned as training material (split='train'); deterministic, stdlib-only. + + Archived val/test tasks are never recalled, and ``exclude_ids`` blocks + tonight's held-out ids (and their ``derived_from`` sources) from re-entering + the training pool. """ if not history or k <= 0 or not new_tasks: return [] + blocked = set(exclude_ids or ()) + for t in new_tasks: + blocked.add(t.id) + if t.derived_from: + blocked.add(t.derived_from) new_tok = [_tokens(t.intent) for t in new_tasks] - new_ids = {t.id for t in new_tasks} scored = [] for h in history: - if h.id in new_ids: + if h.id in blocked: + continue + if _normalize_split(h.split) in ("val", "test"): continue ht = _tokens(h.intent) if not ht: @@ -127,7 +146,15 @@ def dream_consolidate( train = [t for t in tasks if t.split == "train"] enlarged = list(tasks) if recall_k > 0 and history_tasks: - enlarged += recall_similar(train, history_tasks, recall_k) + held_out_ids = { + t.id for t in tasks if _normalize_split(t.split) in ("val", "test") + } + for t in tasks: + if t.derived_from: + held_out_ids.add(t.derived_from) + enlarged += recall_similar( + train, history_tasks, recall_k, exclude_ids=held_out_ids, + ) if dream_factor > 0: seed = [t for t in enlarged if t.split == "train" and t.origin != "dream"] enlarged += dream_augment(seed, factor=dream_factor) diff --git a/skillopt_sleep/mine.py b/skillopt_sleep/mine.py index 69a1be25..f435519f 100644 --- a/skillopt_sleep/mine.py +++ b/skillopt_sleep/mine.py @@ -295,6 +295,17 @@ def assign_splits( if holdout_fraction is not None: val_fraction = holdout_fraction + for name, value in (("val_fraction", val_fraction), ("test_fraction", test_fraction)): + if not 0.0 <= value <= 1.0: + raise ValueError( + f"{name} must be between 0 and 1 inclusive, got {value}" + ) + if val_fraction + test_fraction >= 1.0: + raise ValueError( + f"val_fraction + test_fraction must be < 1 " + f"(got {val_fraction} + {test_fraction})" + ) + dream = [t for t in tasks if t.origin == "dream"] real = [t for t in tasks if t.origin != "dream"] @@ -304,8 +315,21 @@ def assign_splits( val_cut = int(round(val_fraction * 100)) test_cut = val_cut + int(round(test_fraction * 100)) + + def _stable_key(task: TaskRecord) -> tuple[int, str]: + bucket = int(hashlib.sha256((str(seed) + task.id).encode()).hexdigest(), 16) + return bucket, task.id + + def _promote_one(*, to: str, from_splits: set[str]) -> None: + """Promote one real task using hash order; never demote hash-assigned test.""" + candidates = [t for t in real if t.split in from_splits] + if not candidates: + return + candidates.sort(key=_stable_key) + candidates[0].split = to + for t in real: - bucket = int(hashlib.sha256((str(seed) + t.id).encode()).hexdigest(), 16) % 100 + bucket = _stable_key(t)[0] % 100 if bucket < val_cut: t.split = "val" elif bucket < test_cut: @@ -313,19 +337,13 @@ def assign_splits( else: t.split = "train" - # guarantee val (the gate) is non-empty when we have >=2 real tasks - real_splits = {t.split for t in real} - if len(real) >= 2 and "val" not in real_splits: - real[-1].split = "val" - # guarantee a train pool exists (dream or real) when possible + # Guarantee val (the gate) is non-empty when we have >=2 real tasks. + # Only promote from train so hash-assigned test tasks stay untouched. + if len(real) >= 2 and not any(t.split == "val" for t in real): + _promote_one(to="val", from_splits={"train"}) + # Guarantee a train pool exists when possible; never borrow from test. if not any(t.split == "train" for t in tasks) and len(real) >= 2: - real[0].split = "train" - # if test was requested but ended up empty with >=3 real tasks, carve one - if test_fraction > 0 and len(real) >= 3 and not any(t.split == "test" for t in real): - for t in real: - if t.split == "train": - t.split = "test" - break + _promote_one(to="train", from_splits={"val"}) return tasks @@ -339,13 +357,20 @@ def mine( *, max_tasks: int = 40, candidate_limit: int = 0, - holdout_fraction: float = 0.34, + val_fraction: float = 0.34, + test_fraction: float = 0.0, + holdout_fraction: float | None = None, # legacy alias for val_fraction seed: int = 42, llm_miner: Optional[Callable[[List[SessionDigest]], List[TaskRecord]]] = None, target_skill_text: str = "", target_skill_path: str = "", ) -> List[TaskRecord]: - """Top-level miner. Uses ``llm_miner`` if provided, else heuristic.""" + """Top-level miner. Uses ``llm_miner`` if provided, else heuristic. + + Split knobs mirror ``assign_splits``: ``val_fraction``/``test_fraction`` + are the real controls; ``holdout_fraction`` remains the legacy alias and, + when passed, overrides ``val_fraction`` (same contract as assign_splits). + """ candidate_limit = candidate_limit or max_tasks tasks: List[TaskRecord] = [] if llm_miner is not None: @@ -361,5 +386,11 @@ def mine( if target_skill_text or target_skill_path: tasks = filter_tasks_for_target(tasks, target_skill_text, target_skill_path) tasks = tasks[:max_tasks] - tasks = assign_splits(tasks, holdout_fraction=holdout_fraction, seed=seed) + tasks = assign_splits( + tasks, + val_fraction=val_fraction, + test_fraction=test_fraction, + holdout_fraction=holdout_fraction, + seed=seed, + ) return tasks diff --git a/tests/test_split_hardening_2x3.py b/tests/test_split_hardening_2x3.py new file mode 100644 index 00000000..d32cc36d --- /dev/null +++ b/tests/test_split_hardening_2x3.py @@ -0,0 +1,258 @@ +"""Two hardening passes x three approaches for EXC-001 split/recall hygiene. + +Pass 1 (unit / adversarial): + A recall edge vectors + B assign_splits invariants (docstring claims) + C fraction boundary validation + +Pass 2 (integration / provenance): + A dream_consolidate recall envelope + B nightly cycle with preloaded archive + recall_k + C on-disk config provenance for alias precedence +""" +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from unittest import mock + +from skillopt_sleep.backend import build_backend +from skillopt_sleep.config import load_config +from skillopt_sleep.cycle import _resolve_split_fractions, run_sleep_cycle +from skillopt_sleep.consolidate import consolidate +from skillopt_sleep.dream import dream_consolidate, recall_similar +from skillopt_sleep.mine import assign_splits +from skillopt_sleep.state import SleepState +from skillopt_sleep.types import TaskRecord + + +def _task(task_id: str, intent: str, *, split: str = "train", origin: str = "real"): + return TaskRecord( + id=task_id, + project="/repo/example", + intent=intent, + reference_kind="exact", + reference=f"answer-{task_id}", + split=split, + origin=origin, + ) + + +class Pass1ApproachARecallEdgeVectors(unittest.TestCase): + """Pass 1 / approach A: adversarial recall vectors.""" + + def test_legacy_holdout_archived_task_not_recalled(self): + new = _task("n1", "validate login form") + archived = _task("old", "validate login form fields", split="holdout") + self.assertEqual(recall_similar([new], [archived], k=1), []) + + def test_archived_val_not_recalled(self): + new = _task("n1", "validate login form") + archived = _task("old", "validate login form fields", split="val") + self.assertEqual(recall_similar([new], [archived], k=1), []) + + def test_unrelated_archived_train_can_still_be_recalled(self): + new = _task("n1", "validate login form") + archived = _task( + "old", + "validate login form fields", + split="replay", + ) + recalled = recall_similar([new], [archived], k=1) + self.assertEqual(len(recalled), 1) + self.assertEqual(recalled[0].split, "train") + self.assertTrue(recalled[0].id.startswith("recall:")) + + def test_zero_similarity_returns_empty(self): + new = _task("n1", "alpha beta gamma") + archived = _task("old", "delta epsilon zeta", split="train") + self.assertEqual(recall_similar([new], [archived], k=3), []) + + +class Pass1ApproachBAssignSplitsInvariants(unittest.TestCase): + """Pass 1 / approach B: docstring counting/ordering claims.""" + + def test_dream_tasks_always_train_even_with_high_test_fraction(self): + real = [_task(f"r{i}", f"real task {i}") for i in range(4)] + dream = [_task("d0", "dream variant", origin="dream")] + out = assign_splits( + real + dream, + val_fraction=0.34, + test_fraction=0.10, + seed=42, + ) + for t in out: + if t.origin == "dream": + self.assertEqual(t.split, "train") + + def test_real_tasks_have_exactly_one_split_label(self): + tasks = assign_splits( + [_task(f"t{i}", f"task {i}") for i in range(12)], + val_fraction=0.34, + test_fraction=0.10, + seed=7, + ) + for t in tasks: + if t.origin != "dream": + self.assertIn(t.split, {"train", "val", "test"}) + + def test_hash_assigned_test_not_demoted_for_val_top_up(self): + """val top-up promotes from train only; hash test labels stay test.""" + tasks = assign_splits( + [_task(f"t{i}", f"task {i}") for i in range(6)], + val_fraction=0.01, + test_fraction=0.50, + seed=42, + ) + test_ids = {t.id for t in tasks if t.split == "test"} + again = assign_splits( + [_task(f"t{i}", f"task {i}") for i in range(7)], + val_fraction=0.01, + test_fraction=0.50, + seed=42, + ) + for t in again: + if t.id in test_ids: + self.assertEqual(t.split, "test") + + +class Pass1ApproachCFractionBoundaries(unittest.TestCase): + """Pass 1 / approach C: reject invalid fraction knobs early.""" + + def test_assign_splits_rejects_negative_test_fraction(self): + with self.assertRaises(ValueError): + assign_splits([_task("t0", "x")], test_fraction=-0.1) + + def test_assign_splits_rejects_fraction_sum_ge_one(self): + with self.assertRaises(ValueError): + assign_splits([_task("t0", "x")], val_fraction=0.6, test_fraction=0.5) + + def test_mine_forwards_invalid_fractions_to_assign_splits(self): + from skillopt_sleep.mine import mine + + with self.assertRaises(ValueError): + mine([], llm_miner=lambda d: [_task("t0", "x")], test_fraction=1.5) + + +class Pass2ApproachADreamConsolidateEnvelope(unittest.TestCase): + """Pass 2 / approach A: recall enlarges train only.""" + + def test_recall_rows_are_train_split_only(self): + backend = build_backend(backend="mock") + tonight = assign_splits( + [_task("t0", "validate login form", split="train"), + _task("t1", "validate login form fields", split="val"), + _task("t2", "validate login form errors", split="test")], + val_fraction=0.34, + test_fraction=0.10, + seed=42, + ) + archive = [ + _task("arch-test", "validate login form fields", split="test"), + _task("arch-train", "validate login form helper", split="train"), + ] + seen: list[TaskRecord] = [] + + def _capture(backend, tasks, skill, memory, **kwargs): + seen.extend(tasks) + return consolidate(backend, tasks, skill, memory, **kwargs) + + with mock.patch("skillopt_sleep.dream.consolidate", side_effect=_capture): + dream_consolidate( + backend, + tonight, + skill="# skill", + memory="", + history_tasks=archive, + recall_k=2, + dream_rollouts=1, + dream_factor=0, + gate_mode="off", + ) + recalled = [t for t in seen if t.id.startswith("recall:")] + self.assertGreater(len(recalled), 0, "expected at least one recalled row") + for t in recalled: + self.assertEqual(t.split, "train") + self.assertFalse( + any(t.id == "recall:arch-test" for t in recalled), + "archived test rows must not be recalled", + ) + + +class Pass2ApproachBCycleArchiveRecall(unittest.TestCase): + """Pass 2 / approach B: nightly path with archive + recall_k.""" + + def test_archived_test_never_recalled_through_cycle(self): + with tempfile.TemporaryDirectory() as tmp: + state_dir = os.path.join(tmp, "state") + state_path = os.path.join(state_dir, "state.json") + state = SleepState.load(state_path) + state.add_to_archive([ + _task( + "leak-me", + "validate login form fields", + split="test", + ).to_dict(), + ]) + state.save() + + cfg = load_config( + invoked_project=tmp, + projects="invoked", + backend="mock", + state_dir=state_dir, + claude_home=os.path.join(tmp, ".claude"), + recall_k=2, + test_fraction=0.0, + ) + seed = assign_splits( + [_task("n1", "validate login form", split="train"), + _task("n2", "validate login form helper", split="val")], + val_fraction=0.34, + test_fraction=0.0, + seed=42, + ) + with mock.patch( + "skillopt_sleep.dream.recall_similar", + wraps=recall_similar, + ) as spy: + run_sleep_cycle(cfg, seed_tasks=seed, dry_run=True) + self.assertGreater(spy.call_count, 0) + for _args, kwargs in spy.call_args_list: + history = _args[1] + recalled = recall_similar(*_args, **kwargs) + for row in recalled: + self.assertEqual(row.split, "train") + for row in history: + if row.id == "leak-me": + self.assertEqual(row.split, "test") + self.assertEqual( + [r for r in recalled if r.derived_from == "leak-me"], + [], + ) + + +class Pass2ApproachCConfigFileProvenance(unittest.TestCase): + """Pass 2 / approach C: user file keys beat alias guessing.""" + + def test_on_disk_zero_val_beats_holdout_alias(self): + with tempfile.TemporaryDirectory() as cfg_dir: + cfg_path = os.path.join(cfg_dir, "config.json") + with open(cfg_path, "w", encoding="utf-8") as fh: + json.dump( + {"val_fraction": 0.0, "holdout_fraction": 0.5}, + fh, + ) + with mock.patch( + "skillopt_sleep.config._user_config_path", + return_value=cfg_path, + ): + val, test = _resolve_split_fractions(load_config()) + self.assertEqual(val, 0.0) + self.assertEqual(test, 0.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_split_wiring.py b/tests/test_split_wiring.py new file mode 100644 index 00000000..751def74 --- /dev/null +++ b/tests/test_split_wiring.py @@ -0,0 +1,201 @@ +"""val_fraction / test_fraction must actually flow from config to the splits. + +The config documents three knobs (`holdout_fraction` as a legacy alias of +`val_fraction`, plus `test_fraction`), and ``assign_splits`` implements all +three -- but the nightly path only ever forwarded ``holdout_fraction``, so +``test_fraction`` was dead config: no untouched test split could exist and no +held-out test score was ever recorded. These tests pin the wiring end to end. +""" +from __future__ import annotations + +import json +import os +import tempfile +import unittest + +from skillopt_sleep.config import DEFAULTS, load_config +from skillopt_sleep.cycle import _resolve_split_fractions, run_sleep_cycle +from skillopt_sleep.mine import assign_splits, mine +from skillopt_sleep.types import TaskRecord + + +def _mk_tasks(n): + # Realistic mined tasks: exact-reference judged, unique stable ids. + return [ + TaskRecord( + id=f"t{i:03d}", project="/repo/example", + intent=f"do the recurring thing number {i}", + reference_kind="exact", reference=f"answer {i}", + ) + for i in range(n) + ] + + +class TestMineForwardsFractions(unittest.TestCase): + def test_test_fraction_reaches_assign_splits(self): + tasks = mine( + [], llm_miner=lambda digests: _mk_tasks(60), + max_tasks=60, val_fraction=0.2, test_fraction=0.3, seed=7, + ) + splits = {t.split for t in tasks} + self.assertIn("test", splits, "test_fraction did not reach assign_splits") + self.assertIn("val", splits) + self.assertIn("train", splits) + + def test_default_call_is_two_way_like_before(self): + tasks = mine([], llm_miner=lambda digests: _mk_tasks(60), max_tasks=60) + self.assertEqual({t.split for t in tasks} - {"train", "val"}, set(), + "defaults must reproduce the legacy two-way split") + + def test_legacy_holdout_alias_still_wins_when_passed(self): + a = mine([], llm_miner=lambda d: _mk_tasks(60), max_tasks=60, + holdout_fraction=0.6, seed=7) + b = mine([], llm_miner=lambda d: _mk_tasks(60), max_tasks=60, + val_fraction=0.6, seed=7) + self.assertEqual([t.split for t in a], [t.split for t in b]) + + +class TestConfigAliasPrecedence(unittest.TestCase): + def _cfg(self, **over): + return load_config(invoked_project="/tmp/x", projects="invoked", **over) + + def test_defaults_resolve_to_legacy_behavior(self): + val, test = _resolve_split_fractions(self._cfg()) + self.assertEqual(val, DEFAULTS["val_fraction"]) + self.assertEqual(test, DEFAULTS["test_fraction"]) + + def test_legacy_config_holdout_only_still_wins(self): + val, _ = _resolve_split_fractions(self._cfg(holdout_fraction=0.2)) + self.assertEqual(val, 0.2) + + def test_user_val_fraction_beats_stale_alias_default(self): + val, _ = _resolve_split_fractions(self._cfg(val_fraction=0.5)) + self.assertEqual(val, 0.5) + + def test_explicit_val_fraction_beats_explicit_alias(self): + val, _ = _resolve_split_fractions( + self._cfg(val_fraction=0.5, holdout_fraction=0.2)) + self.assertEqual(val, 0.5) + + def test_test_fraction_flows(self): + _, test = _resolve_split_fractions(self._cfg(test_fraction=0.25)) + self.assertEqual(test, 0.25) + + def test_explicit_zero_val_fraction_is_preserved(self): + val, _ = _resolve_split_fractions(self._cfg(val_fraction=0.0)) + self.assertEqual(val, 0.0) + + def test_explicit_val_at_default_beats_alias(self): + val, _ = _resolve_split_fractions( + self._cfg(val_fraction=0.34, holdout_fraction=0.5)) + self.assertEqual(val, 0.34) + + def test_invalid_fraction_sum_raises(self): + with self.assertRaises(ValueError): + _resolve_split_fractions(self._cfg(val_fraction=0.6, test_fraction=0.5)) + + +class TestSplitStabilityAcrossNights(unittest.TestCase): + def test_appending_tasks_does_not_reassign_existing_test(self): + first = assign_splits( + _mk_tasks(4), val_fraction=0.34, test_fraction=0.10, seed=42, + ) + splits_first = {t.id: t.split for t in first} + second = assign_splits( + _mk_tasks(5), val_fraction=0.34, test_fraction=0.10, seed=42, + ) + splits_second = {t.id: t.split for t in second} + for task_id, split in splits_first.items(): + self.assertEqual( + splits_second[task_id], split, + f"{task_id} changed split when a new task was appended", + ) + + def test_maintainer_repro_seed_42_t1_does_not_reassign_t4(self): + """Hash splits are id-stable; appending t1 must not move t4's split.""" + base = [TaskRecord(id=f"t{i}", project="/p", intent=f"task {i}") for i in (0, 3, 4)] + with_t1 = base + [TaskRecord(id="t1", project="/p", intent="task 1")] + first = assign_splits(list(base), val_fraction=0.34, test_fraction=0.10, seed=42) + second = assign_splits(list(with_t1), val_fraction=0.34, test_fraction=0.10, seed=42) + self.assertEqual(next(t for t in second if t.id == "t1").split, "test") + self.assertEqual( + next(t for t in first if t.id == "t4").split, + next(t for t in second if t.id == "t4").split, + ) + + +class TestRecallSplitHygiene(unittest.TestCase): + def test_archived_test_tasks_are_never_recalled(self): + from skillopt_sleep.dream import recall_similar + + new = TaskRecord(id="n1", project="/p", intent="validate login form") + archived_test = TaskRecord( + id="same-id", project="/p", intent="validate login form fields", + split="test", + ) + recalled = recall_similar([new], [archived_test], k=1) + self.assertEqual(recalled, []) + + def test_recall_honors_exclude_ids_for_tonights_held_out(self): + from skillopt_sleep.dream import recall_similar + + held = TaskRecord( + id="held-out", project="/p", intent="validate login form fields", + split="train", + ) + new_train = TaskRecord( + id="n1", project="/p", intent="validate login form", split="train", + ) + recalled = recall_similar( + [new_train], [held], k=1, exclude_ids={"held-out"}, + ) + self.assertEqual(recalled, []) + + +class TestHeldOutScoreEvidence(unittest.TestCase): + """A night with test-split tasks writes a write-only held-out score row.""" + + def _run_night(self, tasks): + with tempfile.TemporaryDirectory() as tmp: + cfg = load_config( + invoked_project=tmp, projects="invoked", backend="mock", + state_dir=os.path.join(tmp, "state"), + claude_home=os.path.join(tmp, ".claude"), + ) + run_sleep_cycle(cfg, seed_tasks=tasks, dry_run=True) + ev_dir = os.path.join(cfg.state_dir, "evidence") + rows = [] + for name in sorted(os.listdir(ev_dir)): + with open(os.path.join(ev_dir, name), encoding="utf-8") as fh: + rows += [json.loads(line) for line in fh if line.strip()] + return rows + + def _seed(self, with_test): + tasks = _mk_tasks(6) + for i, t in enumerate(tasks): + t.split = "train" if i < 3 else ("val" if i < 5 else + ("test" if with_test else "val")) + return tasks + + def test_score_row_written_when_test_tasks_exist(self): + rows = self._run_night(self._seed(with_test=True)) + score_rows = [r for r in rows + if r.get("stage") == "test" + and r.get("event") == "held_out_score"] + self.assertEqual(len(score_rows), 1) + row = score_rows[0] + self.assertEqual(row["n_test"], 1) + self.assertIn("hard", row) + self.assertIn("soft", row) + self.assertIn("accepted", row) + + def test_no_score_row_without_test_tasks(self): + rows = self._run_night(self._seed(with_test=False)) + self.assertEqual( + [r for r in rows if r.get("stage") == "test"], [], + "no test tasks => no held-out row => bit-for-bit legacy nights", + ) + + +if __name__ == "__main__": + unittest.main()