diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py index 72b3dc349..55d922864 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py @@ -70,6 +70,12 @@ def _build_run_dict(report: EvalReport) -> dict: "conversation_id": item.conversation_id, "response_id": item.response_id, "reasoning": item.reasoning_steps, + # Beside `detail`, never merged into it: `detail` keeps its exact meaning + # (the winning run), so every existing consumer of this report is + # unaffected. This is what a partial pass costs you today -- a 1-of-3 shows + # only the attempt that worked -- and each entry carries the ids of its own + # run, which the top-level pair above cannot. + "failed_runs": item.failed_runs, } for item in report.items }, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index 039361d2a..7e62dc499 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -45,6 +45,18 @@ class ItemReport: conversation_id: str | None = None response_id: str | None = None reasoning_steps: list[str] = field(default_factory=list) + # One entry per run that did NOT pass, in run order. best_detail describes the winning + # run, so on a 1-of-3 item every visible verdict belongs to the attempt that worked and + # the two that failed leave no trace at all -- their `detail` is computed here and then + # dropped. That makes a partial pass undiagnosable after the fact: the only recourse is + # re-running the question and hoping it fails the same way. + # + # Failing runs only, deliberately. A fully-passing item adds nothing, so the cost tracks + # how broken the corpus is and shrinks as it improves. Each entry also carries its OWN + # conversation_id/response_id: the report's top-level pair is overwritten every run and + # ends up describing the LAST one, which is not necessarily the run best_detail is + # about, so those ids cannot be used to pull the trace for a specific failure. + failed_runs: list[dict] = field(default_factory=list) # Per-phase breakdown of what the item's time was spent on. Additive to latency_s, # which remains the item's own critical path. langfuse_latency_s is # deliberately NOT part of that path -- trace linking runs off it (see @@ -175,6 +187,32 @@ def avg_quality_score(self) -> float: RunCallback = Callable[[int, int, bool, float], None] +def _failed_run_record(run_index: int, evaluation: ItemEvaluation, chat_result: ChatResult, latency: float) -> dict: + """Everything needed to diagnose ONE failing run, without re-running it. + + `detail` is the evaluator's own verdict for this attempt, opaque here -- the runner + never inspects its shape, so this works for every test kind and for kinds added later. + + `stream_ended` separates a stalled turn from a wrong answer. A stall leaves the gated + checks False even though none of them ran, which reads as a content failure in every + downstream rate; this records the difference at the source instead of leaving consumers + to infer it. + """ + return { + "run_index": run_index, + "passed": False, + "error": evaluation.error, + "detail": evaluation.detail, + "conversation_id": getattr(chat_result, "conversation_id", None), + "response_id": getattr(chat_result, "response_id", None), + "stream_ended": getattr(chat_result, "stream_ended", None), + "turn_wall_clock_sec": getattr(chat_result, "turn_wall_clock_sec", None), + "latency_s": round(latency, 3), + "reasoning_step_count": getattr(chat_result, "reasoning_step_count", 0), + "reasoning_steps": list(getattr(chat_result, "reasoning_steps", None) or []), + } + + def _run_one_item( item: DatasetItem, backend: ChatBackend, runs: int, on_run_done: RunCallback | None = None ) -> ItemReport: @@ -212,6 +250,8 @@ def _run_one_item( if evaluation.passed: report.pass_at_k = True report.runs_passed += 1 + else: + report.failed_runs.append(_failed_run_record(run_index, evaluation, chat_result, latency)) if on_run_done is not None: on_run_done(run_index, runs, evaluation.passed, latency) except Exception as e: # agent/network/parse failure for this item diff --git a/packages/gooddata-eval/tests/test_reporting.py b/packages/gooddata-eval/tests/test_reporting.py index 7f582618b..8662b62c9 100644 --- a/packages/gooddata-eval/tests/test_reporting.py +++ b/packages/gooddata-eval/tests/test_reporting.py @@ -442,3 +442,49 @@ def test_a_failed_item_says_when_a_criterion_went_ungraded(): out = _rendered(report) assert "did not pass strict checks; 1 criterion(s) ungraded" in out + + +def test_json_report_carries_failed_runs_beside_the_winning_detail(): + """`detail` keeps its exact meaning -- the winning run -- so every existing consumer of + this report is unaffected; the failing attempts arrive alongside it rather than + replacing it.""" + report = EvalReport(model="gpt-5.2") + report.items.append( + ItemReport( + id="i1", + dataset_name="d", + test_kind="agentic_dashboard_summary", + question="q", + pass_at_k=True, + runs=2, + runs_passed=1, + best_detail={"rubric_0": True}, + conversation_id="conv-2", + failed_runs=[ + { + "run_index": 1, + "passed": False, + "error": None, + "detail": {"rubric_0": False}, + "conversation_id": "conv-1", + "response_id": "resp-1", + "stream_ended": True, + "reasoning_steps": ["why it went wrong"], + } + ], + ) + ) + + item = build_json_report(report)["items"]["i1"] + assert item["detail"] == {"rubric_0": True} + assert item["conversation_id"] == "conv-2" + assert [r["detail"] for r in item["failed_runs"]] == [{"rubric_0": False}] + assert item["failed_runs"][0]["conversation_id"] == "conv-1" + + +def test_json_report_failed_runs_is_empty_for_a_clean_item(): + report = EvalReport(model="gpt-5.2") + report.items.append( + ItemReport(id="i1", dataset_name="d", test_kind="visualization", question="q", pass_at_k=True, runs=2) + ) + assert build_json_report(report)["items"]["i1"]["failed_runs"] == [] diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py index 64c5c3567..c661b0672 100644 --- a/packages/gooddata-eval/tests/test_runner.py +++ b/packages/gooddata-eval/tests/test_runner.py @@ -427,3 +427,86 @@ def test_best_detail_describes_a_graded_run_when_there_is_one(): report, _ = _run_scripted([_ungraded(), _graded(False)], runs=2) assert report.items[0].best_detail == {"judge_passed": False} + + +def _chat_with_ids(conversation_id: str, response_id: str, *, stream_ended: bool = True) -> ChatResult: + return ChatResult.model_validate( + { + "textResponse": "which metric?", + "conversationId": conversation_id, + "responseId": response_id, + "streamEnded": stream_ended, + "reasoningSteps": [f"thinking in {conversation_id}"], + "reasoningStepCount": 1, + } + ) + + +def test_a_partial_pass_keeps_the_detail_of_the_run_that_failed(): + """The gap this closes: best_detail describes the winning run, so a 1-of-2 item used to + expose only the attempt that worked and the failure left no trace to diagnose.""" + report, _ = _run_scripted([_graded(False), _graded(True)], runs=2) + + item = report.items[0] + assert item.pass_at_k is True and item.runs_passed == 1 + assert item.best_detail == {"judge_passed": True}, "unchanged: still the winning run" + assert [r["detail"] for r in item.failed_runs] == [{"judge_passed": False}] + assert item.failed_runs[0]["run_index"] == 1 + + +def test_a_fully_passing_item_records_no_failed_runs(): + """Failing runs only -- the cost tracks how broken the corpus is, not how large it is.""" + report, _ = _run_scripted([_graded(True), _graded(True)], runs=2) + + assert report.items[0].pass_power_k is True + assert report.items[0].failed_runs == [] + + +def test_every_failing_run_is_recorded_in_run_order(): + report, _ = _run_scripted([_graded(False), _graded(True), _graded(False)], runs=3) + + assert [r["run_index"] for r in report.items[0].failed_runs] == [1, 3] + + +def test_a_failed_run_carries_the_ids_of_its_own_conversation(): + """The report's top-level pair is overwritten every run and ends up describing the LAST + one, which need not be the run best_detail is about. Pulling the trace for a specific + failure needs that failure's own ids.""" + backend = _FakeBackend([_chat_with_ids("conv-1", "resp-1"), _chat_with_ids("conv-2", "resp-2")]) + with patch( + "gooddata_eval.core.runner.get_evaluator", + return_value=_ScriptedEvaluator([_graded(False), _graded(True)]), + ): + report = run_items([_item()], backend, runs=2) + + item = report.items[0] + assert item.conversation_id == "conv-2", "top-level still describes the last run" + assert item.failed_runs[0]["conversation_id"] == "conv-1" + assert item.failed_runs[0]["response_id"] == "resp-1" + assert item.failed_runs[0]["reasoning_steps"] == ["thinking in conv-1"] + + +def test_a_failed_run_records_whether_the_turn_actually_finished(): + """A stall leaves the evaluator's gated checks False even though none of them ran, which + reads as a content failure downstream. stream_ended records the difference at source.""" + backend = _FakeBackend([_chat_with_ids("conv-1", "resp-1", stream_ended=False)]) + with patch( + "gooddata_eval.core.runner.get_evaluator", + return_value=_ScriptedEvaluator([_graded(False)]), + ): + report = run_items([_item()], backend, runs=1) + + failed = report.items[0].failed_runs[0] + assert failed["stream_ended"] is False + assert failed["reasoning_step_count"] == 1 + assert failed["latency_s"] >= 0 + + +def test_an_ungraded_run_is_recorded_as_a_failure_with_its_judge_error(): + """Never a pass, so it belongs here -- and its judge error is the only thing that + explains why pass_power_k is False on an item whose graded runs all passed.""" + report, _ = _run_scripted([_graded(True), _ungraded()], runs=2) + + item = report.items[0] + assert item.pass_at_k is True and item.pass_power_k is False + assert [r["error"] for r in item.failed_runs] == ["empty body"]