fix(gooddata-eval): detect KDA clarifying questions followed by an option list

_is_asking_kda_clarification only matched text ending literally on "?", to avoid
false-positiving on a final answer that merely quotes a question elsewhere. That
missed a real, common response shape: a clarifying question immediately followed
by a bullet/numbered list of the options being offered (e.g. "Which metric?\n-
metric A\n- metric B") -- the message doesn't end on "?" itself, so the run gave up
after turn 1 instead of ever nudging the simulated user to pick one, scoring a
genuinely-ambiguous case as triggered=False.

Found via a real CI trace (QA-28800, gpt56luna_openai / globalmart): the chatbot
asked to disambiguate between two "Total Net Revenue" metrics -- one of which was
the expected answer -- but kda_disambiguated stayed False and the session never
got a second turn, confirming the simulated-reply path was never reached.

Fix: still match text ending on "?"; additionally match a "?" followed only by
list-marker lines (nothing else after it) -- real prose after the list still means
a final answer that merely enumerated something, not a still-open request.

Also fixed: generate_simulated_kda_response only ever knew about measure
candidates, even when the agent's clarifying question was about the PERIOD to
compare instead (e.g. "Which period would you like to compare?") -- it had nothing
period-specific to answer with. Now also builds a period_hint from
expected_output's Date Attribute/Analyzed Period/Reference Period and passes it
alongside the measure candidates, so the reply answers whichever the agent
actually asked about.

Tests: one end-to-end run_agentic_kda_skill test per shape -- the real captured
question+option-list response, and a period-clarification response verifying the
period hint reaches generate_simulated_kda_response correctly.

JIRA: QA-28800

Author: FrankHuynh

packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py

@@ -20,31 +20,47 @@
 _DEFAULT_MAX_ITERATIONS = 3
 
 
+_LIST_ITEM_RE = re.compile(r"^\s*([-*]|\d+[.)])\s+\S")
+
+
 def _is_asking_kda_clarification(text: str) -> bool:
     """True if ``text`` reads as the agent asking for input, not a final answer.
 
     KDA-specific, not shared with metric_skill.py/conversation.py -- each skill's
     disambiguation heuristic has already drifted independently. Requires the text to
-    end on "?" (a "?" anywhere also matches a final answer that merely quotes one).
+    end on "?", or on "?" followed only by a bullet/numbered list of the options being
+    offered (e.g. "Which metric?\n- a\n- b") -- a "?" anywhere else also matches a final
+    answer that merely quotes one.
     """
     if not text:
         return False
     t = text.strip().lower()
     if t.endswith("?"):
         return True
+    if "?" in t:
+        tail = t.rsplit("?", 1)[1]
+        lines = [ln for ln in tail.splitlines() if ln.strip()]
+        if lines and all(_LIST_ITEM_RE.match(ln) for ln in lines):
+            return True
     # "To clarify, ..." means "in other words" (a final answer), not a request for one --
     # strip it first so "clarif" below only matches genuine clarification requests.
     t = re.sub(r"^(just )?to clarify,?\s*", "", t)
     return "could you" in t or "please provide" in t or "clarif" in t
 
 
-def generate_simulated_kda_response(agent_message: str, measure_candidates: dict | list[dict] | None) -> str:
+def generate_simulated_kda_response(
+    agent_message: str,
+    measure_candidates: dict | list[dict] | None,
+    period_hint: str | None = None,
+) -> str:
     """Generate a user reply to keep the KDA-skill conversation going (gpt-4o-mini).
 
     Used only when the agent asks a clarifying question instead of triggering KDA
-    directly. Picks *any* candidate from ``measure_candidates`` -- scope only needs KDA
-    to trigger, not the resulting measure to be exactly right. Always OpenAI regardless
-    of the combo's own provider -- this is test-harness plumbing, not the system under test.
+    directly -- the question may be about which measure to use, which period to
+    compare, or both, so both are given as reference and the reply answers whichever
+    was actually asked. Scope only needs KDA to trigger, not the resulting
+    measure/period to be exactly right. Always OpenAI regardless of the combo's own
+    provider -- this is test-harness plumbing, not the system under test.
     """
     try:
         from openai import OpenAI  # noqa: PLC0415
@@ -61,11 +77,14 @@ def generate_simulated_kda_response(agent_message: str, measure_candidates: dict
         f"{c.get('type')} '{c.get('id')}'" + (f" (aggregation {c['aggregation']})" if c.get("aggregation") else "")
         for c in candidates
     )
+    reference = f"an acceptable metric/fact is {candidate_desc}"
+    if period_hint:
+        reference += f"; the intended time period is {period_hint}"
     prompt = (
         f"You are simulating a user in a conversation with a BI assistant that runs key driver "
-        f"analysis. The assistant said: '{agent_message}'. "
-        f"The user is happy to proceed with any of the following: {candidate_desc}. "
-        f"Reply briefly as the user, picking whichever of those the assistant offered."
+        f"analysis. The assistant asked: '{agent_message}'. "
+        f"For reference, {reference}. "
+        f"Reply briefly as the user, answering whichever of those the assistant actually asked about."
     )
     response = client.chat.completions.create(
         model="gpt-4o-mini",
@@ -216,8 +235,15 @@ def _run_once(conv_id: str) -> KdaRunResult:
                 break
             if _is_asking_kda_clarification(response_text):
                 measure_candidates = expected_output.get("Measure") if isinstance(expected_output, dict) else None
+                period_hint = None
+                if isinstance(expected_output, dict):
+                    date_attr = expected_output.get("Date Attribute")
+                    analyzed = expected_output.get("Analyzed Period")
+                    reference_period = expected_output.get("Reference Period")
+                    if date_attr and analyzed and reference_period:
+                        period_hint = f"{date_attr}, comparing {analyzed} to {reference_period}"
                 try:
-                    current_question = generate_simulated_kda_response(response_text, measure_candidates)
+                    current_question = generate_simulated_kda_response(response_text, measure_candidates, period_hint)
                     disambiguated = True
                 except Exception as exc:  # noqa: BLE001 -- safety net, not the assertion; end only this run
                     _log.warning("Simulated KDA user reply failed for conversation %s: %s", conv_id, exc)

packages/gooddata-eval/tests/test_agentic_kda_skill.py

@@ -427,6 +427,90 @@ def test_run_agentic_kda_skill_marks_disambiguated_after_a_simulated_reply():
     assert summary.best.evaluation.triggered is True
 
 
+def test_run_agentic_kda_skill_disambiguates_on_question_followed_by_option_list():
+    # Regression (QA-28800): the real captured response ends with a bullet list of
+    # candidate metrics, not literally on "?" -- before the _is_asking_kda_clarification
+    # fix, this run gave up after turn 1 (triggered=False) instead of ever nudging the
+    # simulated user to pick one.
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.side_effect = [
+        _no_kda_chat_result(
+            'I found two different "Total Net Revenue" metrics in your data model. '
+            "Which one should I analyze for the 2024 vs 2023 drop?\n\n"
+            "- {metric/metric_l1_sql_net_sales_summary_net_revenue}\n"
+            "- {metric/metric_l1_total_net_revenue}"
+        ),
+        _kda_chat_result(success=True),
+    ]
+
+    with (
+        patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client),
+        patch(
+            "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response",
+            return_value="Use metric_l1_sql_net_sales_summary_net_revenue.",
+        ) as mock_simulate,
+    ):
+        summary = run_agentic_kda_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="Why did Total Net Revenue of Net Sales Summary drop in 2024 compared to 2023?",
+            expected_output=_EXPECTED,
+            k=1,
+            max_iterations=2,
+        )
+
+    mock_simulate.assert_called_once()
+    assert summary.best.evaluation.disambiguated is True
+    assert summary.best.evaluation.triggered is True
+    assert mock_client.send_message.call_count == 2
+
+
+def test_run_agentic_kda_skill_disambiguates_on_period_clarification():
+    # generate_simulated_kda_response used to only know about measure candidates -- if the
+    # agent asked about the PERIOD instead, it had nothing period-specific to answer with.
+    # Verify the period hint built from expected_output's Date Attribute/Analyzed
+    # Period/Reference Period reaches the simulated-reply call.
+    expected_output = {
+        "Measure": {"type": "metric", "id": "revenue"},
+        "Date Attribute": "transaction_date.quarter",
+        "Analyzed Period": "2026-2",
+        "Reference Period": "2026-1",
+    }
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.side_effect = [
+        _no_kda_chat_result("Which period would you like to compare?"),
+        _kda_chat_result(success=True),
+    ]
+
+    with (
+        patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client),
+        patch(
+            "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response",
+            return_value="Compare 2026-2 to 2026-1.",
+        ) as mock_simulate,
+    ):
+        summary = run_agentic_kda_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="Why did revenue drop?",
+            expected_output=expected_output,
+            k=1,
+            max_iterations=2,
+        )
+
+    mock_simulate.assert_called_once_with(
+        "Which period would you like to compare?",
+        {"type": "metric", "id": "revenue"},
+        "transaction_date.quarter, comparing 2026-2 to 2026-1",
+    )
+    assert summary.best.evaluation.disambiguated is True
+    assert summary.best.evaluation.triggered is True
+
+
 def test_run_agentic_kda_skill_disambiguates_when_expected_output_is_not_a_dict():
     # DatasetItem.expected_output on the gdc-nas side allows str/list, not just dict.
     # expected_output.get("Measure") would raise AttributeError on those shapes, silently
@@ -457,7 +541,7 @@ def test_run_agentic_kda_skill_disambiguates_when_expected_output_is_not_a_dict(
             max_iterations=2,
         )
 
-    mock_generate.assert_called_once_with("Could you clarify which measure?", None)
+    mock_generate.assert_called_once_with("Could you clarify which measure?", None, None)
     assert summary.best.evaluation.disambiguated is True
     assert summary.best.evaluation.triggered is True