feat(eval): capture the detail of every failing run, not just the winning one

packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py

@@ -70,6 +70,12 @@ def _build_run_dict(report: EvalReport) -> dict:
                 "conversation_id": item.conversation_id,
                 "response_id": item.response_id,
                 "reasoning": item.reasoning_steps,
+                # Beside `detail`, never merged into it: `detail` keeps its exact meaning
+                # (the winning run), so every existing consumer of this report is
+                # unaffected. This is what a partial pass costs you today -- a 1-of-3 shows
+                # only the attempt that worked -- and each entry carries the ids of its own
+                # run, which the top-level pair above cannot.
+                "failed_runs": item.failed_runs,
             }
             for item in report.items
         },

packages/gooddata-eval/src/gooddata_eval/core/runner.py

@@ -45,6 +45,18 @@ class ItemReport:
     conversation_id: str | None = None
     response_id: str | None = None
     reasoning_steps: list[str] = field(default_factory=list)
+    # One entry per run that did NOT pass, in run order. best_detail describes the winning
+    # run, so on a 1-of-3 item every visible verdict belongs to the attempt that worked and
+    # the two that failed leave no trace at all -- their `detail` is computed here and then
+    # dropped. That makes a partial pass undiagnosable after the fact: the only recourse is
+    # re-running the question and hoping it fails the same way.
+    #
+    # Failing runs only, deliberately. A fully-passing item adds nothing, so the cost tracks
+    # how broken the corpus is and shrinks as it improves. Each entry also carries its OWN
+    # conversation_id/response_id: the report's top-level pair is overwritten every run and
+    # ends up describing the LAST one, which is not necessarily the run best_detail is
+    # about, so those ids cannot be used to pull the trace for a specific failure.
+    failed_runs: list[dict] = field(default_factory=list)
     # Per-phase breakdown of what the item's time was spent on. Additive to latency_s,
     # which remains the item's own critical path. langfuse_latency_s is
     # deliberately NOT part of that path -- trace linking runs off it (see
@@ -175,6 +187,32 @@ def avg_quality_score(self) -> float:
 RunCallback = Callable[[int, int, bool, float], None]
 
 
+def _failed_run_record(run_index: int, evaluation: ItemEvaluation, chat_result: ChatResult, latency: float) -> dict:
+    """Everything needed to diagnose ONE failing run, without re-running it.
+
+    `detail` is the evaluator's own verdict for this attempt, opaque here -- the runner
+    never inspects its shape, so this works for every test kind and for kinds added later.
+
+    `stream_ended` separates a stalled turn from a wrong answer. A stall leaves the gated
+    checks False even though none of them ran, which reads as a content failure in every
+    downstream rate; this records the difference at the source instead of leaving consumers
+    to infer it.
+    """
+    return {
+        "run_index": run_index,
+        "passed": False,
+        "error": evaluation.error,
+        "detail": evaluation.detail,
+        "conversation_id": getattr(chat_result, "conversation_id", None),
+        "response_id": getattr(chat_result, "response_id", None),
+        "stream_ended": getattr(chat_result, "stream_ended", None),
+        "turn_wall_clock_sec": getattr(chat_result, "turn_wall_clock_sec", None),
+        "latency_s": round(latency, 3),
+        "reasoning_step_count": getattr(chat_result, "reasoning_step_count", 0),
+        "reasoning_steps": list(getattr(chat_result, "reasoning_steps", None) or []),
+    }
+
+
 def _run_one_item(
     item: DatasetItem, backend: ChatBackend, runs: int, on_run_done: RunCallback | None = None
 ) -> ItemReport:
@@ -212,6 +250,8 @@ def _run_one_item(
             if evaluation.passed:
                 report.pass_at_k = True
                 report.runs_passed += 1
+            else:
+                report.failed_runs.append(_failed_run_record(run_index, evaluation, chat_result, latency))
             if on_run_done is not None:
                 on_run_done(run_index, runs, evaluation.passed, latency)
     except Exception as e:  # agent/network/parse failure for this item

packages/gooddata-eval/tests/test_reporting.py

@@ -442,3 +442,49 @@ def test_a_failed_item_says_when_a_criterion_went_ungraded():
     out = _rendered(report)
 
     assert "did not pass strict checks; 1 criterion(s) ungraded" in out
+
+
+def test_json_report_carries_failed_runs_beside_the_winning_detail():
+    """`detail` keeps its exact meaning -- the winning run -- so every existing consumer of
+    this report is unaffected; the failing attempts arrive alongside it rather than
+    replacing it."""
+    report = EvalReport(model="gpt-5.2")
+    report.items.append(
+        ItemReport(
+            id="i1",
+            dataset_name="d",
+            test_kind="agentic_dashboard_summary",
+            question="q",
+            pass_at_k=True,
+            runs=2,
+            runs_passed=1,
+            best_detail={"rubric_0": True},
+            conversation_id="conv-2",
+            failed_runs=[
+                {
+                    "run_index": 1,
+                    "passed": False,
+                    "error": None,
+                    "detail": {"rubric_0": False},
+                    "conversation_id": "conv-1",
+                    "response_id": "resp-1",
+                    "stream_ended": True,
+                    "reasoning_steps": ["why it went wrong"],
+                }
+            ],
+        )
+    )
+
+    item = build_json_report(report)["items"]["i1"]
+    assert item["detail"] == {"rubric_0": True}
+    assert item["conversation_id"] == "conv-2"
+    assert [r["detail"] for r in item["failed_runs"]] == [{"rubric_0": False}]
+    assert item["failed_runs"][0]["conversation_id"] == "conv-1"
+
+
+def test_json_report_failed_runs_is_empty_for_a_clean_item():
+    report = EvalReport(model="gpt-5.2")
+    report.items.append(
+        ItemReport(id="i1", dataset_name="d", test_kind="visualization", question="q", pass_at_k=True, runs=2)
+    )
+    assert build_json_report(report)["items"]["i1"]["failed_runs"] == []

packages/gooddata-eval/tests/test_runner.py

@@ -427,3 +427,86 @@ def test_best_detail_describes_a_graded_run_when_there_is_one():
     report, _ = _run_scripted([_ungraded(), _graded(False)], runs=2)
 
     assert report.items[0].best_detail == {"judge_passed": False}
+
+
+def _chat_with_ids(conversation_id: str, response_id: str, *, stream_ended: bool = True) -> ChatResult:
+    return ChatResult.model_validate(
+        {
+            "textResponse": "which metric?",
+            "conversationId": conversation_id,
+            "responseId": response_id,
+            "streamEnded": stream_ended,
+            "reasoningSteps": [f"thinking in {conversation_id}"],
+            "reasoningStepCount": 1,
+        }
+    )
+
+
+def test_a_partial_pass_keeps_the_detail_of_the_run_that_failed():
+    """The gap this closes: best_detail describes the winning run, so a 1-of-2 item used to
+    expose only the attempt that worked and the failure left no trace to diagnose."""
+    report, _ = _run_scripted([_graded(False), _graded(True)], runs=2)
+
+    item = report.items[0]
+    assert item.pass_at_k is True and item.runs_passed == 1
+    assert item.best_detail == {"judge_passed": True}, "unchanged: still the winning run"
+    assert [r["detail"] for r in item.failed_runs] == [{"judge_passed": False}]
+    assert item.failed_runs[0]["run_index"] == 1
+
+
+def test_a_fully_passing_item_records_no_failed_runs():
+    """Failing runs only -- the cost tracks how broken the corpus is, not how large it is."""
+    report, _ = _run_scripted([_graded(True), _graded(True)], runs=2)
+
+    assert report.items[0].pass_power_k is True
+    assert report.items[0].failed_runs == []
+
+
+def test_every_failing_run_is_recorded_in_run_order():
+    report, _ = _run_scripted([_graded(False), _graded(True), _graded(False)], runs=3)
+
+    assert [r["run_index"] for r in report.items[0].failed_runs] == [1, 3]
+
+
+def test_a_failed_run_carries_the_ids_of_its_own_conversation():
+    """The report's top-level pair is overwritten every run and ends up describing the LAST
+    one, which need not be the run best_detail is about. Pulling the trace for a specific
+    failure needs that failure's own ids."""
+    backend = _FakeBackend([_chat_with_ids("conv-1", "resp-1"), _chat_with_ids("conv-2", "resp-2")])
+    with patch(
+        "gooddata_eval.core.runner.get_evaluator",
+        return_value=_ScriptedEvaluator([_graded(False), _graded(True)]),
+    ):
+        report = run_items([_item()], backend, runs=2)
+
+    item = report.items[0]
+    assert item.conversation_id == "conv-2", "top-level still describes the last run"
+    assert item.failed_runs[0]["conversation_id"] == "conv-1"
+    assert item.failed_runs[0]["response_id"] == "resp-1"
+    assert item.failed_runs[0]["reasoning_steps"] == ["thinking in conv-1"]
+
+
+def test_a_failed_run_records_whether_the_turn_actually_finished():
+    """A stall leaves the evaluator's gated checks False even though none of them ran, which
+    reads as a content failure downstream. stream_ended records the difference at source."""
+    backend = _FakeBackend([_chat_with_ids("conv-1", "resp-1", stream_ended=False)])
+    with patch(
+        "gooddata_eval.core.runner.get_evaluator",
+        return_value=_ScriptedEvaluator([_graded(False)]),
+    ):
+        report = run_items([_item()], backend, runs=1)
+
+    failed = report.items[0].failed_runs[0]
+    assert failed["stream_ended"] is False
+    assert failed["reasoning_step_count"] == 1
+    assert failed["latency_s"] >= 0
+
+
+def test_an_ungraded_run_is_recorded_as_a_failure_with_its_judge_error():
+    """Never a pass, so it belongs here -- and its judge error is the only thing that
+    explains why pass_power_k is False on an item whose graded runs all passed."""
+    report, _ = _run_scripted([_graded(True), _ungraded()], runs=2)
+
+    item = report.items[0]
+    assert item.pass_at_k is True and item.pass_power_k is False
+    assert [r["error"] for r in item.failed_runs] == ["empty body"]