fix(gooddata-eval): three false-negative sources in eval scoring

QA-29230: capture every multipart response part, not just text/visualization/
alertProposal, and render them into what the judge and the simulated user see.
QA-29226: normalize whitespace around MAQL punctuation before comparing, in one
shared module the agentic and non-agentic metric comparators both use.
QA-29225: read the fixture's anomaly granularity and stop telling the simulated
user to refuse one, which deadlocked ANOMALY alert items.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: myhoai

packages/gooddata-eval/src/gooddata_eval/core/agentic/_catalog.py

@@ -28,6 +28,8 @@ class CatalogMetricAlert:
     """List of recipient email addresses."""
     filters: list | str | None = None
     """Attribute filters applied to the alert condition."""
+    granularity: str | None = None
+    """Detection interval for an ANOMALY alert (DAY/WEEK/MONTH/...). Not a date filter."""
 
     @classmethod
     def from_dict(cls, d: dict) -> CatalogMetricAlert:
@@ -46,4 +48,5 @@ def from_dict(cls, d: dict) -> CatalogMetricAlert:
             metric_id=d.get("metric_id"),
             recipients=recipients,
             filters=d.get("filters"),
+            granularity=d.get("granularity"),
         )

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -21,6 +21,7 @@
     submit_trace_scoring,
     utc_now,
 )
+from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import (
@@ -258,20 +259,33 @@ def generate_simulated_alert_response(
         )
     elif filters == []:
         filters_rule = (
-            "5. Your alert must have NO filters and NO date/time window — it evaluates over all time. "
-            "If the agent asks which time period each check should cover, or offers a choice such as "
-            "'last Day / Week / Month', do NOT pick one: reply that you want no date filter at all, "
-            "all time. Never invent a period, a granularity or an 'evaluate each run on a X basis' "
-            "instruction the goal did not ask for.\n"
+            "5. Your alert must have NO filters and NO date/time window on the metric — it evaluates "
+            "over all time. If the agent asks which time period each check should cover, or offers a "
+            "choice such as 'last Day / Week / Month', do NOT pick one: reply that you want no date "
+            "filter at all, all time.\n"
         )
     else:
         filters_rule = (
             "5. Ask only for the filters your original request implies — do not invent an evaluation "
-            "period, granularity or date window that was not requested. If the agent offers a choice "
+            "period or date window that was not requested. If the agent offers a choice "
             "such as 'last Day / Week / Month' that your request never mentioned, say you do not want "
             "a date window.\n"
         )
 
+    if operator == "ANOMALY":
+        granularity = expected.granularity or "day"
+        anomaly_rule = (
+            "7. This is an ANOMALY alert. Anomaly detection REQUIRES a time granularity, and that "
+            f"granularity is NOT a date filter. State it in your first reply and repeat it whenever "
+            f"asked: use {granularity} granularity. Rule 5 constrains filters on the metric only — it "
+            "never applies to this detection interval, so never refuse to give one.\n"
+        )
+    else:
+        anomaly_rule = (
+            "7. Do not invent an evaluation period, a granularity or an 'evaluate each run on a X "
+            "basis' instruction your goal never asked for.\n"
+        )
+
     original_request = f'Your original request to the agent was: "{question}"\n' if question else ""
 
     system_prompt = (
@@ -295,8 +309,7 @@ def generate_simulated_alert_response(
         "   Do not wait for the agent to ask — state it alongside the metric and condition answers.\n"
         + filters_rule
         + f"6. Proactively state how often you want to be alerted in your first reply: {trigger_request}. "
-        "   Repeat it if the agent proposes a different cadence.\n"
-        "Reply concisely and directly."
+        "   Repeat it if the agent proposes a different cadence.\n" + anomaly_rule + "Reply concisely and directly."
     )
 
     messages: list = [{"role": "system", "content": system_prompt}]
@@ -435,6 +448,8 @@ def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
 
     filters = _normalize_expected_filters(expected)
 
+    granularity = _case_insensitive_get(expected, "granularity", "detection granularity")
+
     return CatalogMetricAlert(
         operator=operator,
         threshold=threshold,
@@ -444,6 +459,7 @@ def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
         metric_id=metric_id,
         recipients=recipients,
         filters=filters,
+        granularity=str(granularity).strip() if granularity else None,
     )
 
 
@@ -542,6 +558,8 @@ def _run_once(conv_id: str) -> AlertRunResult:
                 response_text = (chat_result.text_response or "").strip()
                 if not response_text and chat_result.alert_proposals:
                     response_text = render_alert_proposal(chat_result.alert_proposals[-1])
+                if not response_text:
+                    response_text = render_answer_text(chat_result)
                 # Stop if agent gave a completely empty response (stuck)
                 if not response_text and not chat_result.tool_call_events:
                     break

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -22,6 +22,7 @@
 )
 from gooddata_eval.core.agentic.alert_skill import render_alert_proposal
 from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids, _extract_metric_result
+from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import (
@@ -214,7 +215,7 @@ def _check_output_correct(turn: TurnDefinition, chat_result: ChatResult) -> bool
 
     Returns None when expected_output is absent (presence check only).
     """
-    from gooddata_eval.core.agentic.metric_skill import _normalize_maql  # noqa: PLC0415
+    from gooddata_eval.core.evaluators._maql import normalize_maql  # noqa: PLC0415
 
     otype = turn.expected_output_type
     expected = turn.expected_output
@@ -256,7 +257,7 @@ def _check_output_correct(turn: TurnDefinition, chat_result: ChatResult) -> bool
         metric_result = _extract_metric_result(chat_result.tool_call_events or [])
         if not metric_result:
             return False
-        return _normalize_maql(metric_result.get("maql", "")) == _normalize_maql(expected.get("maql", ""))
+        return normalize_maql(metric_result.get("maql", "")) == normalize_maql(expected.get("maql", ""))
 
     return None
 
@@ -451,6 +452,8 @@ def run_agentic_conversation(
                 response_text = (chat_result.text_response or "").strip()
                 if not response_text and chat_result.alert_proposals:
                     response_text = render_alert_proposal(chat_result.alert_proposals[-1])
+                if not response_text:
+                    response_text = render_answer_text(chat_result)
                 if not response_text and not chat_result.tool_call_events:
                     break
                 if clarification_turns >= max_clarification_turns:

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -15,6 +15,7 @@
     submit_trace_scoring,
     utc_now,
 )
+from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run
@@ -112,7 +113,7 @@ def _run_single_general_question(
     item_started = time.monotonic()
     agent_started = time.monotonic()
     chat_result = client.send_message(conversation_id, question, user_context=user_context)
-    actual_output = (chat_result.text_response or "").strip()
+    actual_output = render_answer_text(chat_result)
     agent_elapsed = time.monotonic() - agent_started
     log_timer(
         f"[timer] general_question {conversation_id} GoodData response complete after "

packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py

@@ -14,6 +14,7 @@
     submit_trace_scoring,
     utc_now,
 )
+from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run
@@ -107,7 +108,7 @@ def _run_single_guardrail(
     and the remaining K-1) cannot drift -- they had already duplicated the whole body once.
     """
     chat_result = client.send_message(conversation_id, question)
-    actual_output = (chat_result.text_response or "").strip()
+    actual_output = render_answer_text(chat_result)
     verdict = score_run(judge, input=question, expected_output=expected_output, actual_output=actual_output)
     return GuardrailResult(
         conversation_id=conversation_id,

packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py

@@ -16,6 +16,7 @@
     submit_trace_scoring,
     utc_now,
 )
+from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import (
@@ -286,7 +287,7 @@ def _accumulate(result: ChatResult) -> None:
             response_id = chat_result.response_id or response_id
             _accumulate(chat_result)
             create_args, execute_result = _extract_kda_calls(chat_result.tool_call_events or [])
-            response_text = (chat_result.text_response or "").strip()
+            response_text = render_answer_text(chat_result)
             turn_completed = chat_result.stream_ended and bool(response_text)
             if create_args is not None:
                 # This turn's own time -- the turn that called create, not any earlier

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -20,8 +20,10 @@
     submit_trace_scoring,
     utc_now,
 )
+from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.evaluators._maql import normalize_maql
 from gooddata_eval.core.models import (
     AgenticAssertionError,
     AgenticEvalOutcome,
@@ -40,76 +42,16 @@
 _DEFAULT_K = 1
 _DEFAULT_MAX_ITERATIONS = 7
 
-_IFNULL_RE = re.compile(r"IFNULL\s*\([^,]+,\s*0\)", re.IGNORECASE)
-_SELECT_WRAP_RE = re.compile(r"^\s*\(\s*SELECT\s*\{([^}]+)\}\s*\)\s*$", re.IGNORECASE)
-_INNER_SELECT_RE = re.compile(r"\(\s*SELECT\s*\{([^}]+)\}\s*\)", re.IGNORECASE)
-# Matches whichever comes first: a {type/id} identifier reference or a quoted string
-# literal -- both are case-sensitive data and must survive casefolding untouched.
-# Everything else in MAQL (keywords, operators, numbers, punctuation) carries no
-# case-sensitive meaning, per the MAQL reference (SELECT/BY/WHERE/FOR PREVIOUS/etc.
-# are case-insensitive; only {..} identifiers and quoted literal values are not).
-# Feeds _normalize_maql, the scoring comparator (_best_maql_match) -- do not widen this
-# to handle \X escapes without confirming MAQL literals actually support backslash
-# escaping (unconfirmed; see PR #1760 review). A wrong guess here silently changes
-# maql_correct for the whole eval dataset, not just a hint. _no_where_clause_hint()
-# below has its own, separately-scoped regex for that reason.
-_PROTECTED_RE = re.compile(r"\{[^}]*\}|\"[^\"]*\"|'[^']*'")
-
-
-def _strip_outer_parens(s: str) -> str:
-    """Strip one balanced layer of outer () if they wrap the entire expression."""
-    if not (s.startswith("(") and s.endswith(")")):
-        return s
-    depth = 0
-    for i, ch in enumerate(s):
-        if ch == "(":
-            depth += 1
-        elif ch == ")":
-            depth -= 1
-            if depth == 0 and i < len(s) - 1:
-                return s  # Closing paren found before end — not a simple outer wrapper
-    return s[1:-1].strip()
-
-
-def _casefold_outside_protected(s: str) -> str:
-    """Lowercase MAQL keywords/operators while preserving case-sensitive {type/id}
-    identifiers and quoted string literal values (e.g. WHERE {label/x} = "Active")."""
-    parts = []
-    last = 0
-    for m in _PROTECTED_RE.finditer(s):
-        parts.append(s[last : m.start()].lower())
-        parts.append(m.group(0))
-        last = m.end()
-    parts.append(s[last:].lower())
-    return "".join(parts)
-
-
-def _normalize_maql(maql: str) -> str:
-    """Semantic normalisation: strip whitespace, unwrap IFNULL/SELECT wrappers, casefold keywords."""
-    if not maql:
-        return ""
-    m = maql.strip()
-    m = _IFNULL_RE.sub(
-        lambda mo: _strip_outer_parens(mo.group(0).split(",")[0].strip()[len("IFNULL(") :].strip()),
-        m,
-    )
-    m = _SELECT_WRAP_RE.sub(r"{\1}", m)
-    m = _INNER_SELECT_RE.sub(r"{\1}", m)
-    m = re.sub(r"\{\s+", "{", m)
-    m = re.sub(r"\s+\}", "}", m)
-    m = re.sub(r"\s+", " ", m)
-    return _casefold_outside_protected(m.strip())
-
 
 def _best_maql_match(actual_maql: str, expected_outputs: list[dict]) -> tuple[bool, str]:
     """Try actual MAQL against every candidate; return (matched, best_expected_maql).
 
     First match wins. First candidate is used for error reporting when none match.
     """
-    normalized_actual = _normalize_maql(actual_maql)
+    normalized_actual = normalize_maql(actual_maql)
     for candidate in expected_outputs:
         expected_maql = candidate.get("maql", "")
-        if normalized_actual == _normalize_maql(expected_maql):
+        if normalized_actual == normalize_maql(expected_maql):
             return True, expected_maql
     return False, expected_outputs[0].get("maql", "") if expected_outputs else ""
 
@@ -122,9 +64,9 @@ class SimulatedResponseError(RuntimeError):
     """
 
 
-# Separate from _PROTECTED_RE on purpose: this one only feeds a same-turn LLM-prompt hint
-# (see _no_where_clause_hint), never the scoring comparator, so it can afford to consume
-# \X escape sequences inside quoted literals without risking maql_correct semantics.
+# Separate from evaluators._maql._PROTECTED_RE on purpose: this one only feeds a same-turn
+# LLM-prompt hint (see _no_where_clause_hint), never the scoring comparator, so it can afford
+# to consume \X escape sequences inside quoted literals without risking maql_correct semantics.
 _HINT_PROTECTED_RE = re.compile(r"\{[^}]*\}|\"(?:[^\"\\]|\\.)*\"|'(?:[^'\\]|\\.)*'")
 
 
@@ -342,6 +284,8 @@ def _execute_single_metric_run(
                 metric_result = candidate
                 break
             response_text = (chat_result.text_response or "").strip()
+            if not response_text:
+                response_text = render_answer_text(chat_result)
             if not response_text and not chat_result.tool_call_events:
                 break
             if _iteration >= max_iterations - 1:

packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py

@@ -19,6 +19,7 @@
     submit_trace_scoring,
     utc_now,
 )
+from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators.visualization import (
@@ -207,12 +208,13 @@ def _execute_single_run(
         viz_produced = bool(current_result.created_visualizations and current_result.created_visualizations.objects)
         if viz_produced:
             break
-        if not current_result.text_response:
+        response_text = render_answer_text(current_result)
+        if not response_text:
             break
         if iteration >= max_iterations - 1:
             break
 
-        follow_up = generate_simulated_response(current_result.text_response, simulated_response_guide)
+        follow_up = generate_simulated_response(response_text, simulated_response_guide)
         current_result = client.send_message(conversation_id, follow_up)
 
     skill_activated = _check_visualization_skill_activated(all_tool_call_events)

packages/gooddata-eval/src/gooddata_eval/core/chat/render.py

@@ -0,0 +1,47 @@
+# (C) 2026 GoodData Corporation
+"""Render a chat turn's non-text parts into the prose an evaluator can read."""
+
+import json
+
+from gooddata_eval.core.models import ChatResult
+
+_MAX_UNHANDLED_PART_CHARS = 2000
+
+
+def render_search_results(part: dict) -> str:
+    """Render one ``searchResults`` part as the list of objects it resolved."""
+    objects = part.get("objects") or []
+    if not objects:
+        return ""
+    lines = []
+    for obj in objects:
+        title = str(obj.get("title") or "").strip()
+        obj_id = str(obj.get("id") or "").strip()
+        obj_type = str(obj.get("type") or "").strip()
+        label = f"{title} ({obj_type}/{obj_id})" if obj_id else title
+        description = str(obj.get("description") or "").strip()
+        lines.append(f"- {label}: {description}" if description else f"- {label}")
+    requested = str(part.get("requestedObjectType") or "object").strip()
+    return f"Search results ({len(objects)} {requested}):\n" + "\n".join(lines)
+
+
+def render_unhandled_part(part: dict) -> str:
+    """Best-effort rendering of a part gd-eval does not model, truncated to a sane size."""
+    ptype = str(part.get("type") or "unknown")
+    body = json.dumps({k: v for k, v in part.items() if k != "type"}, ensure_ascii=False)
+    if len(body) > _MAX_UNHANDLED_PART_CHARS:
+        body = body[:_MAX_UNHANDLED_PART_CHARS] + "… (truncated)"
+    return f"[{ptype}]\n{body}"
+
+
+def render_answer_text(result: ChatResult) -> str:
+    """Everything the agent said this turn: prose plus the content-bearing parts.
+
+    Alert proposals are excluded -- ``agentic.alert_skill.render_alert_proposal`` renders
+    those, and importing it here would close a cycle. Returns "" for a turn that produced
+    nothing, so callers can keep using falsiness as their "the agent is stuck" signal.
+    """
+    chunks = [(result.text_response or "").strip()]
+    chunks += [render_search_results(part) for part in result.search_results]
+    chunks += [render_unhandled_part(part) for part in result.unhandled_parts]
+    return "\n\n".join(chunk for chunk in chunks if chunk)