[bot] Merge master/b145e1a3 into rel/dev

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -477,6 +477,8 @@ class AlertRunResult:
     alert_id: str | None
     eval: AlertEvaluation
     actual_alert_arguments: dict
+    total_turns: int = 0
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -676,9 +678,13 @@ def _run_once(conv_id: str) -> AlertRunResult:
             # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply.
             conversation_history: list = []
             current_question = question
+            turns = 0
+            steps = 0
 
             for _iteration in range(max_iterations):
                 chat_result = client.send_message(conv_id, current_question)
+                turns += 1
+                steps += chat_result.reasoning_step_count
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
                 response_id = chat_result.response_id or response_id
                 turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
@@ -728,6 +734,8 @@ def _run_once(conv_id: str) -> AlertRunResult:
                 alert_id=alert_id,
                 eval=ev,
                 actual_alert_arguments=actual_args,
+                total_turns=turns,
+                total_steps=steps,
                 reasoning_steps=reasoning_steps,
                 response_id=response_id,
                 tool_call_events=all_tool_call_events,
@@ -851,6 +859,8 @@ def _write_scores(ctx: RunTraceContext) -> None:
                 with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k)
                     ctx.quality(
                         tid,

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -336,6 +336,7 @@ class ConversationResult:
     full_skill_coverage: bool
     conversation_success: bool
     total_clarification_turns: int
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -365,6 +366,7 @@ def run_agentic_conversation(
     turn_results: list[TurnResult] = []
     turn_outputs: dict[str, dict] = {}
     total_clarification_turns = 0
+    total_steps = 0
     conversation_id: str = ""
     owns_conversation = False
     # Metrics created during this conversation, deleted after it completes so they do
@@ -434,6 +436,7 @@ def run_agentic_conversation(
             for _iter in range(max_clarification_turns + 1):
                 chat_result = client.send_message(conversation_id, current_message)
                 final_result = chat_result
+                total_steps += chat_result.reasoning_step_count
                 turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
                     chat_result,
                     turn_offset=turn_offset,
@@ -522,6 +525,7 @@ def run_agentic_conversation(
         full_skill_coverage=full_skill_coverage,
         conversation_success=conversation_success,
         total_clarification_turns=total_clarification_turns,
+        total_steps=total_steps,
         reasoning_steps=reasoning_steps,
         response_id=response_id,
         tool_call_events=conversation_tool_call_events,
@@ -611,6 +615,22 @@ def _write_scores(ctx: RunTraceContext) -> None:
                     value=float(result.full_skill_coverage),
                     data_type="BOOLEAN",
                 )
+                # One turn per fixture turn, plus every simulated-user round the agent triggered.
+                # The clarification count alone hides how much of the conversation the fixture
+                # asked for, so the comparison needs the total.
+                ctx.score(
+                    tid,
+                    name="turns",
+                    value=len(result.turn_results) + result.total_clarification_turns,
+                    data_type="NUMERIC",
+                )
+                ctx.score(tid, name="steps", value=result.total_steps, data_type="NUMERIC")
+                ctx.score(
+                    tid,
+                    name="clarification_turns",
+                    value=result.total_clarification_turns,
+                    data_type="NUMERIC",
+                )
                 for tr in result.turn_results:
                     ctx.score(
                         tid,

packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py

@@ -185,6 +185,8 @@ class KdaRunResult:
     # Wall-clock time of the turn that called create (None if create never happened) --
     # not any earlier disambiguation turn. See run_agentic_kda_skill's _run_once.
     turn_wall_clock_sec: float | None = None
+    total_turns: int = 0
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -276,6 +278,9 @@ def _accumulate(result: ChatResult) -> None:
             all_tool_call_events.extend(result.tool_call_events or [])
             all_reasoning_step_events.extend(result.reasoning_step_events or [])
 
+        turns = 0
+        steps = 0
+
         for iteration in range(max_iterations):
             try:
                 chat_result = client.send_message(conv_id, current_question)
@@ -291,6 +296,8 @@ def _accumulate(result: ChatResult) -> None:
                         turn_wall_clock_sec = partial.turn_wall_clock_sec
                 turn_completed = False
                 break
+            turns += 1
+            steps += chat_result.reasoning_step_count
             reasoning_steps.extend(chat_result.reasoning_steps or [])
             response_id = chat_result.response_id or response_id
             _accumulate(chat_result)
@@ -330,6 +337,8 @@ def _accumulate(result: ChatResult) -> None:
             actual_create_args=create_args,
             actual_execute_result=execute_result,
             turn_wall_clock_sec=turn_wall_clock_sec,
+            total_turns=turns,
+            total_steps=steps,
             reasoning_steps=reasoning_steps,
             response_id=response_id,
             tool_call_events=all_tool_call_events,
@@ -442,6 +451,8 @@ def _write_scores(ctx: RunTraceContext) -> None:
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
                     ctx.score(tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     if turn_wall_clock_sec is not None:
                         # combo_report.py reads this score directly -- no trace re-resolution needed.
                         ctx.score(

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -162,7 +162,8 @@ class MetricRunResult:
     metric_created: bool
     actual_maql: str
     maql_correct: bool
-    total_turns: float
+    total_turns: int
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -253,6 +254,7 @@ def _execute_single_metric_run(
     metric_result: dict | None = None
     created_metric_ids: list[str] = []
     turns = 0
+    steps = 0
     current_question = question
     reasoning_steps: list[str] = []
     response_id: str | None = None
@@ -280,6 +282,7 @@ def _execute_single_metric_run(
             )
             all_tool_call_events.extend(chat_result.tool_call_events or [])
             all_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
+            steps += chat_result.reasoning_step_count
             for metric_id in _extract_created_metric_ids(chat_result.tool_call_events or []):
                 if metric_id not in created_metric_ids:
                     created_metric_ids.append(metric_id)
@@ -330,7 +333,8 @@ def _execute_single_metric_run(
             metric_created=metric_created,
             actual_maql=actual_maql,
             maql_correct=maql_correct,
-            total_turns=float(turns),
+            total_turns=turns,
+            total_steps=steps,
             reasoning_steps=reasoning_steps,
             response_id=response_id,
             tool_call_events=all_tool_call_events,
@@ -466,6 +470,8 @@ def _write_scores(ctx: RunTraceContext) -> None:
                 ) as tid:
                     ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN")
                     ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k)
                     ctx.quality(
                         tid,

packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py

@@ -59,8 +59,8 @@ class RunResult:
     actual_output: CreatedVisualization | None
     eval_result: EvaluationResult
     best_expected: CreatedVisualization
-    total_turns: float
-    total_steps: float
+    total_turns: int
+    total_steps: int
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -186,8 +186,8 @@ def _execute_single_run(
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
 ) -> RunResult:
     """Drive one full multi-turn conversation and evaluate the result."""
-    total_turns = 0.0
-    total_steps = 0.0
+    total_turns = 0
+    total_steps = 0
     all_tool_call_events: list[ToolCallEvent] = []
     all_reasoning_step_events: list[ReasoningStepEvent] = []
     reasoning_steps: list[str] = []
@@ -200,8 +200,8 @@ def _execute_single_run(
     current_result = client.send_message(conversation_id, question)
 
     for iteration in range(max_iterations):
-        total_turns += 1.0
-        total_steps += float(current_result.reasoning_step_count)
+        total_turns += 1
+        total_steps += current_result.reasoning_step_count
         turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
             current_result,
             turn_offset=turn_offset,

packages/gooddata-eval/tests/test_agentic_alert_skill.py

@@ -996,3 +996,67 @@ def test_every_gen_ai_interval_is_accepted():
         assert AnomalyDetectionGranularity.parse(value.lower()) is AnomalyDetectionGranularity(value)
     assert AnomalyDetectionGranularity.parse(None) is None
     assert AnomalyDetectionGranularity.parse("  ") is None
+
+
+def test_run_agentic_alert_skill_counts_the_turns_and_reasoning_steps_it_used():
+    """QA-29110: the effort comparison reads these. A refusal still took a turn, and the turn
+    count is what separates a wrong answer from a run max_iterations cut short."""
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.return_value = _no_alert_chat_result()
+    mock_client._base = "http://host/api/v1/actions/workspaces/ws1/ai"
+    mock_client._auth = {"Authorization": "Bearer tok"}
+
+    with _patched(mock_client, simulated_reply="Yes please"):
+        summary = run_agentic_alert_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="Create alert",
+            expected_output={"operator": "GREATER_THAN", "threshold": 100},
+            k=1,
+            max_iterations=2,
+        )
+
+    # _no_alert_chat_result has no tool calls and non-empty text, so the run replies once and
+    # stops at max_iterations: 2 turns, 1 reasoning step each.
+    assert summary.best.total_turns == 2
+    assert summary.best.total_steps == 2
+
+
+def test_alert_skill_writes_the_turn_and_step_counts_to_langfuse():
+    """The counters exist to reach Langfuse; asserting only the dataclass would pass even if
+    the scores were never written."""
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.return_value = _no_alert_chat_result()
+    mock_client._base = "http://host/api/v1/actions/workspaces/ws1/ai"
+    mock_client._auth = {"Authorization": "Bearer tok"}
+    captured = {}
+
+    def _capture(_submit, _identity, **kwargs):
+        captured["write_scores"] = kwargs["write_scores"]
+
+    with (
+        _patched(mock_client),
+        patch("gooddata_eval.core.agentic.alert_skill.submit_trace_scoring", _capture),
+        pytest.raises(AlertSkillAssertionError),
+    ):
+        evaluate_agentic_alert_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="Create alert",
+            expected_output={"operator": "GREATER_THAN", "threshold": 100},
+            k=1,
+            max_iterations=1,
+            langfuse=MagicMock(),
+            dataset_item_id="item-1",
+        )
+
+    ctx = MagicMock()
+    captured["write_scores"](ctx)
+    scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list}
+
+    assert scores["turns"] == 1
+    assert scores["steps"] == 1

packages/gooddata-eval/tests/test_agentic_conversation.py

@@ -1025,3 +1025,107 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_
         ],
         "latency_breakdown": [],
     }
+
+
+def test_run_agentic_conversation_sums_the_reasoning_steps_of_every_turn():
+    """QA-29110: the effort comparison reads `steps`. A clarification round is part of the
+    work the effort setting changes, so its steps count with the rest."""
+    proposal_turn = ChatResult.model_validate(
+        {
+            "text_response": None,
+            "alertProposals": [{"cta": "Should I create this alert?", "recipients": [{"email": "a@b.com"}]}],
+            "reasoningStepCount": 2,
+            "toolCallEvents": [
+                {"functionName": "set_skills", "functionArguments": '{"skills": ["alert"]}', "result": None},
+                {"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None},
+            ],
+        }
+    )
+    created_turn = ChatResult.model_validate(
+        {
+            "text_response": "Alert created.",
+            "reasoningStepCount": 3,
+            "toolCallEvents": [
+                {"functionName": "create_metric_alert", "functionArguments": "{}", "result": '{"id": "alert-1"}'}
+            ],
+        }
+    )
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.side_effect = [proposal_turn, created_turn]
+
+    with (
+        patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client),
+        patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"),
+        patch(
+            "gooddata_eval.core.agentic.conversation._get_sim_user_response",
+            return_value="Yes, please create it.",
+        ),
+    ):
+        result = run_agentic_conversation(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            fixture=_alert_turn_fixture(),
+        )
+
+    assert result.total_steps == 5
+    assert result.total_clarification_turns == 1
+
+
+def test_conversation_writes_the_turn_step_and_clarification_counts_to_langfuse():
+    """`turns` is not the clarification count: it is one per fixture turn plus every
+    simulated-user round, so a test has to pin the sum rather than either half."""
+    proposal_turn = ChatResult.model_validate(
+        {
+            "text_response": None,
+            "alertProposals": [{"cta": "Should I create this alert?", "recipients": [{"email": "a@b.com"}]}],
+            "reasoningStepCount": 2,
+            "toolCallEvents": [
+                {"functionName": "set_skills", "functionArguments": '{"skills": ["alert"]}', "result": None},
+                {"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None},
+            ],
+        }
+    )
+    created_turn = ChatResult.model_validate(
+        {
+            "text_response": "Alert created.",
+            "reasoningStepCount": 3,
+            "toolCallEvents": [
+                {"functionName": "create_metric_alert", "functionArguments": "{}", "result": '{"id": "alert-1"}'}
+            ],
+        }
+    )
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.side_effect = [proposal_turn, created_turn]
+    captured = {}
+
+    def _capture(_submit, _identity, **kwargs):
+        captured["write_scores"] = kwargs["write_scores"]
+
+    with (
+        patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client),
+        patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"),
+        patch("gooddata_eval.core.agentic.conversation.submit_trace_scoring", _capture),
+        patch(
+            "gooddata_eval.core.agentic.conversation._get_sim_user_response",
+            return_value="Yes, please create it.",
+        ),
+    ):
+        evaluate_agentic_conversation(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            fixture=_alert_turn_fixture(),
+            langfuse=MagicMock(),
+            dataset_item_id="item-1",
+        )
+
+    ctx = MagicMock()
+    captured["write_scores"](ctx)
+    scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list}
+
+    assert scores["clarification_turns"] == 1
+    assert scores["turns"] == 2  # 1 fixture turn + 1 clarification round
+    assert scores["steps"] == 5