chore: Gate Managed Agent evaluation to supported metrics.

agentplatform/_genai/_evals_common.py

@@ -1564,13 +1564,57 @@ def _build_interaction_id_dataset(
     return types.EvaluationDataset(eval_cases=eval_cases)
 
 
+# Metrics supported for Managed Agent evaluation.
+_MANAGED_AGENT_SUPPORTED_METRICS = frozenset({
+    "safety_v1",
+    "final_response_quality_v1",
+    "multi_turn_task_success_v1",
+})
+
+
 def _has_interactions_data_source(
     eval_cases: list[types.EvalCase],
 ) -> bool:
     """Returns True if any EvalCase has interactions_data_source set."""
     return any(case.interactions_data_source is not None for case in eval_cases)
 
 
+def _validate_managed_agent_metrics(
+    agent: Optional[str],
+    metrics: list[types.Metric],
+) -> None:
+    """Validates metrics are supported for Managed Agent evaluation.
+
+    When the ``agent`` parameter is a Gemini Agent resource name
+    (``projects/{p}/locations/{l}/agents/{id}``), only a subset of
+    metrics are supported for Preview. This function raises ValueError
+    if any unsupported metrics are requested.
+
+    Args:
+        agent: The agent resource name, or None.
+        metrics: The list of metrics to validate.
+
+    Raises:
+        ValueError: If any metric is not in the supported set.
+    """
+    if not agent or not _is_gemini_agent_resource(agent):
+        return
+
+    unsupported = []
+    for metric in metrics:
+        name = getattr(metric, "name", None)
+        if name:
+            name_lower = name.lower()
+            if name_lower not in _MANAGED_AGENT_SUPPORTED_METRICS:
+                unsupported.append(name_lower)
+    if unsupported:
+        raise ValueError(
+            f"Metrics {unsupported} are not supported for Managed Agent"
+            " evaluation. Supported metrics:"
+            f" {sorted(_MANAGED_AGENT_SUPPORTED_METRICS)}."
+        )
+
+
 def _resolve_interactions_to_eval_cases(
     api_client: BaseApiClient,
     eval_cases: list[types.EvalCase],
@@ -3037,6 +3081,9 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
 
     resolved_metrics = _resolve_metrics(metrics, api_client)
 
+    # Validate metrics are supported for Managed Agent evaluation.
+    _validate_managed_agent_metrics(agent, resolved_metrics)
+
     evaluation_run_config = _evals_metric_handlers.EvaluationRunConfig(
         evals_module=evals.Evals(api_client_=api_client),
         dataset=processed_eval_dataset,

agentplatform/_genai/evals.py

@@ -3382,6 +3382,11 @@ def create_evaluation_run(
 
         if isinstance(dataset, types.EvaluationDataset):
             _evals_utils._validate_dataset_agent_data(dataset, inference_configs)
+        # Validate metrics are supported for Managed Agent evaluation.
+        _evals_common._validate_managed_agent_metrics(
+            agent,
+            _evals_common._resolve_metrics(metrics, self._api_client),
+        )
         resolved_dataset = _evals_common._resolve_dataset(
             self._api_client, dataset, dest, parsed_agent_info
         )
@@ -5661,6 +5666,11 @@ async def create_evaluation_run(
 
         if isinstance(dataset, types.EvaluationDataset):
             _evals_utils._validate_dataset_agent_data(dataset, inference_configs)
+        # Validate metrics are supported for Managed Agent evaluation.
+        _evals_common._validate_managed_agent_metrics(
+            agent,
+            _evals_common._resolve_metrics(metrics, self._api_client),
+        )
         resolved_dataset = _evals_common._resolve_dataset(
             self._api_client, dataset, dest, parsed_agent_info
         )

tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py

@@ -744,7 +744,7 @@ def test_create_eval_run_with_interactions_data_source(mock_uuid4, client):
         display_name="test_interactions_data_source",
         dataset=types.EvaluationDataset(eval_cases=[eval_case]),
         dest=GCS_DEST,
-        metrics=[GENERAL_QUALITY_METRIC],
+        metrics=[FINAL_RESPONSE_QUALITY_METRIC],
     )
     assert isinstance(evaluation_run, types.EvaluationRun)
     assert evaluation_run.state == types.EvaluationRunState.PENDING
@@ -927,7 +927,7 @@ def test_create_eval_run_with_gemini_agent(client):
         display_name="test_gemini_agent",
         dataset=types.EvaluationRunDataSource(evaluation_set=eval_set),
         dest=GCS_DEST,
-        metrics=[GENERAL_QUALITY_METRIC],
+        metrics=[FINAL_RESPONSE_QUALITY_METRIC],
         agent_info=types.evals.AgentInfo(name="gemini-agent"),
         agent=gemini_agent,
         user_simulator_config=types.evals.UserSimulatorConfig(max_turn=3),

tests/unit/agentplatform/genai/test_evals.py

@@ -11767,6 +11767,70 @@ def test_non_sandbox_tool_before_user_input_not_merged(self):
         assert len(result.turns) == 2
 
 
+class TestValidateManagedAgentMetrics:
+    """Tests for _validate_managed_agent_metrics."""
+
+    MANAGED_AGENT = "projects/p/locations/global/agents/my-agent"
+    NON_MANAGED_AGENT = "projects/p/locations/global/reasoningEngines/123"
+
+    def _make_metric(self, name):
+        return agentplatform_genai_types.Metric(name=name)
+
+    def test_supported_metric_passes(self):
+        _evals_common._validate_managed_agent_metrics(
+            self.MANAGED_AGENT,
+            [self._make_metric("safety_v1")],
+        )
+
+    def test_multiple_supported_metrics_pass(self):
+        _evals_common._validate_managed_agent_metrics(
+            self.MANAGED_AGENT,
+            [
+                self._make_metric("safety_v1"),
+                self._make_metric("final_response_quality_v1"),
+                self._make_metric("multi_turn_task_success_v1"),
+            ],
+        )
+
+    def test_unsupported_metric_raises(self):
+        with pytest.raises(ValueError, match="not supported for Managed Agent"):
+            _evals_common._validate_managed_agent_metrics(
+                self.MANAGED_AGENT,
+                [self._make_metric("hallucination_v1")],
+            )
+
+    def test_unsupported_metric_lists_supported(self):
+        with pytest.raises(ValueError, match="multi_turn_task_success_v1"):
+            _evals_common._validate_managed_agent_metrics(
+                self.MANAGED_AGENT,
+                [self._make_metric("multi_turn_trajectory_quality_v1")],
+            )
+
+    def test_non_managed_agent_allows_any_metric(self):
+        # Non-Managed Agent (reasoning engine) should allow any metric.
+        _evals_common._validate_managed_agent_metrics(
+            self.NON_MANAGED_AGENT,
+            [self._make_metric("hallucination_v1")],
+        )
+
+    def test_no_agent_allows_any_metric(self):
+        # No agent parameter should allow any metric.
+        _evals_common._validate_managed_agent_metrics(
+            None,
+            [self._make_metric("hallucination_v1")],
+        )
+
+    def test_mixed_supported_and_unsupported_raises(self):
+        with pytest.raises(ValueError, match="not supported"):
+            _evals_common._validate_managed_agent_metrics(
+                self.MANAGED_AGENT,
+                [
+                    self._make_metric("safety_v1"),
+                    self._make_metric("hallucination_v1"),
+                ],
+            )
+
+
 class TestMergeTextPartsInAgentData:
     """Tests for _merge_text_parts_in_agent_data."""