diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 1b5b18bb2..8b07ed58d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -477,6 +477,8 @@ class AlertRunResult: alert_id: str | None eval: AlertEvaluation actual_alert_arguments: dict + total_turns: int = 0 + total_steps: int = 0 reasoning_steps: list[str] = field(default_factory=list) response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) @@ -676,9 +678,13 @@ def _run_once(conv_id: str) -> AlertRunResult: # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply. conversation_history: list = [] current_question = question + turns = 0 + steps = 0 for _iteration in range(max_iterations): chat_result = client.send_message(conv_id, current_question) + turns += 1 + steps += chat_result.reasoning_step_count reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( @@ -728,6 +734,8 @@ def _run_once(conv_id: str) -> AlertRunResult: alert_id=alert_id, eval=ev, actual_alert_arguments=actual_args, + total_turns=turns, + total_steps=steps, reasoning_steps=reasoning_steps, response_id=response_id, tool_call_events=all_tool_call_events, @@ -851,6 +859,8 @@ def _write_scores(ctx: RunTraceContext) -> None: with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") + ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") + ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 6965cbb82..e1dab4470 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -336,6 +336,7 @@ class ConversationResult: full_skill_coverage: bool conversation_success: bool total_clarification_turns: int + total_steps: int = 0 reasoning_steps: list[str] = field(default_factory=list) response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) @@ -365,6 +366,7 @@ def run_agentic_conversation( turn_results: list[TurnResult] = [] turn_outputs: dict[str, dict] = {} total_clarification_turns = 0 + total_steps = 0 conversation_id: str = "" owns_conversation = False # Metrics created during this conversation, deleted after it completes so they do @@ -434,6 +436,7 @@ def run_agentic_conversation( for _iter in range(max_clarification_turns + 1): chat_result = client.send_message(conversation_id, current_message) final_result = chat_result + total_steps += chat_result.reasoning_step_count turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( chat_result, turn_offset=turn_offset, @@ -522,6 +525,7 @@ def run_agentic_conversation( full_skill_coverage=full_skill_coverage, conversation_success=conversation_success, total_clarification_turns=total_clarification_turns, + total_steps=total_steps, reasoning_steps=reasoning_steps, response_id=response_id, tool_call_events=conversation_tool_call_events, @@ -611,6 +615,22 @@ def _write_scores(ctx: RunTraceContext) -> None: value=float(result.full_skill_coverage), data_type="BOOLEAN", ) + # One turn per fixture turn, plus every simulated-user round the agent triggered. + # The clarification count alone hides how much of the conversation the fixture + # asked for, so the comparison needs the total. + ctx.score( + tid, + name="turns", + value=len(result.turn_results) + result.total_clarification_turns, + data_type="NUMERIC", + ) + ctx.score(tid, name="steps", value=result.total_steps, data_type="NUMERIC") + ctx.score( + tid, + name="clarification_turns", + value=result.total_clarification_turns, + data_type="NUMERIC", + ) for tr in result.turn_results: ctx.score( tid, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 3898e80d9..7d7910f62 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -185,6 +185,8 @@ class KdaRunResult: # Wall-clock time of the turn that called create (None if create never happened) -- # not any earlier disambiguation turn. See run_agentic_kda_skill's _run_once. turn_wall_clock_sec: float | None = None + total_turns: int = 0 + total_steps: int = 0 reasoning_steps: list[str] = field(default_factory=list) response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) @@ -276,6 +278,9 @@ def _accumulate(result: ChatResult) -> None: all_tool_call_events.extend(result.tool_call_events or []) all_reasoning_step_events.extend(result.reasoning_step_events or []) + turns = 0 + steps = 0 + for iteration in range(max_iterations): try: chat_result = client.send_message(conv_id, current_question) @@ -291,6 +296,8 @@ def _accumulate(result: ChatResult) -> None: turn_wall_clock_sec = partial.turn_wall_clock_sec turn_completed = False break + turns += 1 + steps += chat_result.reasoning_step_count reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id _accumulate(chat_result) @@ -330,6 +337,8 @@ def _accumulate(result: ChatResult) -> None: actual_create_args=create_args, actual_execute_result=execute_result, turn_wall_clock_sec=turn_wall_clock_sec, + total_turns=turns, + total_steps=steps, reasoning_steps=reasoning_steps, response_id=response_id, tool_call_events=all_tool_call_events, @@ -442,6 +451,8 @@ def _write_scores(ctx: RunTraceContext) -> None: for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") ctx.score(tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN") + ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") + ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC") if turn_wall_clock_sec is not None: # combo_report.py reads this score directly -- no trace re-resolution needed. ctx.score( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 6ad68cd59..8c0e777c7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -162,7 +162,8 @@ class MetricRunResult: metric_created: bool actual_maql: str maql_correct: bool - total_turns: float + total_turns: int + total_steps: int = 0 reasoning_steps: list[str] = field(default_factory=list) response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) @@ -253,6 +254,7 @@ def _execute_single_metric_run( metric_result: dict | None = None created_metric_ids: list[str] = [] turns = 0 + steps = 0 current_question = question reasoning_steps: list[str] = [] response_id: str | None = None @@ -280,6 +282,7 @@ def _execute_single_metric_run( ) all_tool_call_events.extend(chat_result.tool_call_events or []) all_reasoning_step_events.extend(chat_result.reasoning_step_events or []) + steps += chat_result.reasoning_step_count for metric_id in _extract_created_metric_ids(chat_result.tool_call_events or []): if metric_id not in created_metric_ids: created_metric_ids.append(metric_id) @@ -330,7 +333,8 @@ def _execute_single_metric_run( metric_created=metric_created, actual_maql=actual_maql, maql_correct=maql_correct, - total_turns=float(turns), + total_turns=turns, + total_steps=steps, reasoning_steps=reasoning_steps, response_id=response_id, tool_call_events=all_tool_call_events, @@ -466,6 +470,8 @@ def _write_scores(ctx: RunTraceContext) -> None: ) as tid: ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN") ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN") + ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") + ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 3b680debc..8a26d586e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -59,8 +59,8 @@ class RunResult: actual_output: CreatedVisualization | None eval_result: EvaluationResult best_expected: CreatedVisualization - total_turns: float - total_steps: float + total_turns: int + total_steps: int reasoning_steps: list[str] = field(default_factory=list) response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) @@ -186,8 +186,8 @@ def _execute_single_run( max_iterations: int = _DEFAULT_MAX_ITERATIONS, ) -> RunResult: """Drive one full multi-turn conversation and evaluate the result.""" - total_turns = 0.0 - total_steps = 0.0 + total_turns = 0 + total_steps = 0 all_tool_call_events: list[ToolCallEvent] = [] all_reasoning_step_events: list[ReasoningStepEvent] = [] reasoning_steps: list[str] = [] @@ -200,8 +200,8 @@ def _execute_single_run( current_result = client.send_message(conversation_id, question) for iteration in range(max_iterations): - total_turns += 1.0 - total_steps += float(current_result.reasoning_step_count) + total_turns += 1 + total_steps += current_result.reasoning_step_count turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( current_result, turn_offset=turn_offset, diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index cf812609d..3d6c091f3 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -996,3 +996,67 @@ def test_every_gen_ai_interval_is_accepted(): assert AnomalyDetectionGranularity.parse(value.lower()) is AnomalyDetectionGranularity(value) assert AnomalyDetectionGranularity.parse(None) is None assert AnomalyDetectionGranularity.parse(" ") is None + + +def test_run_agentic_alert_skill_counts_the_turns_and_reasoning_steps_it_used(): + """QA-29110: the effort comparison reads these. A refusal still took a turn, and the turn + count is what separates a wrong answer from a run max_iterations cut short.""" + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _no_alert_chat_result() + mock_client._base = "http://host/api/v1/actions/workspaces/ws1/ai" + mock_client._auth = {"Authorization": "Bearer tok"} + + with _patched(mock_client, simulated_reply="Yes please"): + summary = run_agentic_alert_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create alert", + expected_output={"operator": "GREATER_THAN", "threshold": 100}, + k=1, + max_iterations=2, + ) + + # _no_alert_chat_result has no tool calls and non-empty text, so the run replies once and + # stops at max_iterations: 2 turns, 1 reasoning step each. + assert summary.best.total_turns == 2 + assert summary.best.total_steps == 2 + + +def test_alert_skill_writes_the_turn_and_step_counts_to_langfuse(): + """The counters exist to reach Langfuse; asserting only the dataclass would pass even if + the scores were never written.""" + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _no_alert_chat_result() + mock_client._base = "http://host/api/v1/actions/workspaces/ws1/ai" + mock_client._auth = {"Authorization": "Bearer tok"} + captured = {} + + def _capture(_submit, _identity, **kwargs): + captured["write_scores"] = kwargs["write_scores"] + + with ( + _patched(mock_client), + patch("gooddata_eval.core.agentic.alert_skill.submit_trace_scoring", _capture), + pytest.raises(AlertSkillAssertionError), + ): + evaluate_agentic_alert_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create alert", + expected_output={"operator": "GREATER_THAN", "threshold": 100}, + k=1, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + + ctx = MagicMock() + captured["write_scores"](ctx) + scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list} + + assert scores["turns"] == 1 + assert scores["steps"] == 1 diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index dd39f9996..c56d3dadf 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -1025,3 +1025,107 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_ ], "latency_breakdown": [], } + + +def test_run_agentic_conversation_sums_the_reasoning_steps_of_every_turn(): + """QA-29110: the effort comparison reads `steps`. A clarification round is part of the + work the effort setting changes, so its steps count with the rest.""" + proposal_turn = ChatResult.model_validate( + { + "text_response": None, + "alertProposals": [{"cta": "Should I create this alert?", "recipients": [{"email": "a@b.com"}]}], + "reasoningStepCount": 2, + "toolCallEvents": [ + {"functionName": "set_skills", "functionArguments": '{"skills": ["alert"]}', "result": None}, + {"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None}, + ], + } + ) + created_turn = ChatResult.model_validate( + { + "text_response": "Alert created.", + "reasoningStepCount": 3, + "toolCallEvents": [ + {"functionName": "create_metric_alert", "functionArguments": "{}", "result": '{"id": "alert-1"}'} + ], + } + ) + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [proposal_turn, created_turn] + + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"), + patch( + "gooddata_eval.core.agentic.conversation._get_sim_user_response", + return_value="Yes, please create it.", + ), + ): + result = run_agentic_conversation( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + fixture=_alert_turn_fixture(), + ) + + assert result.total_steps == 5 + assert result.total_clarification_turns == 1 + + +def test_conversation_writes_the_turn_step_and_clarification_counts_to_langfuse(): + """`turns` is not the clarification count: it is one per fixture turn plus every + simulated-user round, so a test has to pin the sum rather than either half.""" + proposal_turn = ChatResult.model_validate( + { + "text_response": None, + "alertProposals": [{"cta": "Should I create this alert?", "recipients": [{"email": "a@b.com"}]}], + "reasoningStepCount": 2, + "toolCallEvents": [ + {"functionName": "set_skills", "functionArguments": '{"skills": ["alert"]}', "result": None}, + {"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None}, + ], + } + ) + created_turn = ChatResult.model_validate( + { + "text_response": "Alert created.", + "reasoningStepCount": 3, + "toolCallEvents": [ + {"functionName": "create_metric_alert", "functionArguments": "{}", "result": '{"id": "alert-1"}'} + ], + } + ) + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [proposal_turn, created_turn] + captured = {} + + def _capture(_submit, _identity, **kwargs): + captured["write_scores"] = kwargs["write_scores"] + + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"), + patch("gooddata_eval.core.agentic.conversation.submit_trace_scoring", _capture), + patch( + "gooddata_eval.core.agentic.conversation._get_sim_user_response", + return_value="Yes, please create it.", + ), + ): + evaluate_agentic_conversation( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + fixture=_alert_turn_fixture(), + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + + ctx = MagicMock() + captured["write_scores"](ctx) + scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list} + + assert scores["clarification_turns"] == 1 + assert scores["turns"] == 2 # 1 fixture turn + 1 clarification round + assert scores["steps"] == 5 diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index 6e493f4db..9a41dfd4d 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -1195,3 +1195,88 @@ def test_evaluate_agentic_kda_skill_preserves_reasoning_from_a_chat_error_partia assert exc_info.value.reasoning_steps == ["analyzing before cutoff"] assert exc_info.value.response_id == "resp-3" + + +def test_run_agentic_kda_skill_counts_the_turns_and_reasoning_steps_it_used(): + """QA-29110: the effort comparison reads these. A binary pass/fail cannot separate two + efforts on a nightly's sample, while the reasoning-step count moves with the effort.""" + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + # Turn 1 asks for clarification, turn 2 runs the analysis: 2 turns, 1 step each. + mock_client.send_message.side_effect = [ + _no_kda_chat_result("Which metric did you mean?"), + _kda_chat_result(success=True), + ] + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", return_value="Revenue"), + ): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove the change?", + expected_output=_EXPECTED, + k=1, + max_iterations=2, + ) + + assert summary.best.total_turns == 2 + assert summary.best.total_steps == 2 + + +def test_run_agentic_kda_skill_reports_no_turns_when_the_first_send_fails(): + """A run that never got a reply must not report a turn it did not take.""" + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = RuntimeError("stream died") + + with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove the change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + ) + + assert summary.best.total_turns == 0 + assert summary.best.total_steps == 0 + + +def test_kda_skill_writes_the_turn_and_step_counts_to_langfuse(): + """The counters exist to reach Langfuse; asserting only the dataclass would pass even if + the scores were never written.""" + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _kda_chat_result(success=True) + captured = {} + + def _capture(_submit, _identity, **kwargs): + captured["write_scores"] = kwargs["write_scores"] + + with ( + patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.kda_skill.submit_trace_scoring", _capture), + ): + evaluate_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove the change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + + ctx = MagicMock() + captured["write_scores"](ctx) + scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list} + + assert scores["turns"] == 1 + assert scores["steps"] == 1 diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index bc36fe4f0..1633355eb 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -582,7 +582,7 @@ def test_run_agentic_metric_skill_fails_the_run_when_the_simulated_reply_cannot_ assert summary.pass_at_k is False assert summary.best.metric_created is False - assert summary.best.total_turns == 1.0 + assert summary.best.total_turns == 1 mock_client.close.assert_called_once() mock_sim.assert_called_once_with( "Which brand field should I count?", [{"maql": "SELECT {metric/foo}"}], "Create metric foo" @@ -764,3 +764,90 @@ def test_no_timer_output_by_default(monkeypatch, capsys): assert "[timer]" not in capsys.readouterr().out # Silenced, not un-measured. assert summary.run_results[0].timings.agent_s == 3.0 + + +def test_run_agentic_metric_skill_counts_the_turns_and_reasoning_steps_it_used(): + """QA-29110: the effort comparison reads these. A clarification round is part of the work + the effort setting changes, so its steps count with the rest.""" + clarify_turn = ChatResult.model_validate( + {"textResponse": "Which foo?", "toolCallEvents": [], "reasoningStepCount": 2} + ) + created_turn = ChatResult.model_validate( + { + "textResponse": "done", + "reasoningStepCount": 3, + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": '{"data": {"maql": "SELECT {metric/foo}"}}', + } + ], + } + ) + mock_client = _client() + mock_client.send_message.side_effect = [clarify_turn, created_turn] + + with _patched(mock_client, simulated_reply="It's foo"): + summary = run_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=2, + ) + + assert summary.best.total_turns == 2 + assert summary.best.total_steps == 5 + + +def test_metric_skill_writes_the_turn_and_step_counts_to_langfuse(): + """The counters exist to reach Langfuse; asserting only the dataclass would pass even if + the scores were never written.""" + mock_client = _client() + mock_client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": "done", + "reasoningStepCount": 4, + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": '{"data": {"maql": "SELECT {metric/foo}"}}', + } + ], + } + ) + captured = {} + + def _capture(_submit, _identity, **kwargs): + captured["write_scores"] = kwargs["write_scores"] + + with ( + _patched(mock_client), + patch("gooddata_eval.core.agentic.metric_skill.submit_trace_scoring", _capture), + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + + ctx = MagicMock() + captured["write_scores"](ctx) + scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list} + + assert scores["turns"] == 1 + assert scores["steps"] == 4 + # `==` does not separate 1 from 1.0, so the counts need their type pinned separately: + # they are counts, and a float reads as though a fraction of a turn were possible. + assert isinstance(scores["turns"], int) + assert isinstance(scores["steps"], int) diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index 766313d74..69e1b762b 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -71,8 +71,8 @@ def test_execute_single_run_viz_on_first_turn(): assert result.eval_result.visualization_created is True assert result.eval_result.strict_pass is True - assert result.total_turns == 1.0 - assert result.total_steps == 2.0 + assert result.total_turns == 1 + assert result.total_steps == 2 assert result.conversation_id == "conv-1" client.send_message.assert_called_once_with("conv-1", "Show revenue") @@ -93,7 +93,7 @@ def test_execute_single_run_clarification_then_viz(monkeypatch): result = _execute_single_run(client, "conv-1", "Show me a chart", [_expected()]) assert result.eval_result.visualization_created is True - assert result.total_turns == 2.0 + assert result.total_turns == 2 assert client.send_message.call_count == 2 assert client.send_message.call_args_list[1] == call("conv-1", "Revenue please") @@ -111,7 +111,7 @@ def test_execute_single_run_no_viz_no_text(): result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) assert result.eval_result.visualization_created is False - assert result.total_turns == 1.0 + assert result.total_turns == 1 def test_execute_single_run_max_iterations_stops_loop(monkeypatch):