fix(pydantic_ai): capture Anthropic/Google reasoning tokens

py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_run_anthropic_reasoning_tokens.yaml

@@ -0,0 +1,68 @@
+interactions:
+- request:
+    body: '{"max_tokens":2048,"messages":[{"role":"user","content":[{"text":"What
+      is 17 * 23? Think it through, then give the number.","type":"text"}]}],"model":"claude-haiku-4-5-20251001","stream":false,"thinking":{"type":"enabled","budget_tokens":1024}}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      anthropic-version:
+      - '2023-06-01'
+      connection:
+      - keep-alive
+      content-length:
+      - '244'
+      content-type:
+      - application/json
+      host:
+      - api.anthropic.com
+      user-agent:
+      - pydantic-ai/2.36.0
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - async:asyncio
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.2.0
+      x-stainless-read-timeout:
+      - '600'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.15
+      x-stainless-timeout:
+      - '600'
+    method: POST
+    uri: https://api.anthropic.com/v1/messages?beta=true
+  response:
+    body:
+      string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CeorTzoNmBZLscy4jhvVS","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"17
+        * 23\n\nLet me break this down:\n17 * 23 = 17 * (20 + 3)\n= 17 * 20 + 17 *
+        3\n= 340 + 51\n= 391\n\nLet me verify: \n17 * 23\n= (20 - 3) * 23\n= 20 *
+        23 - 3 * 23\n= 460 - 69\n= 391\n\nYes, 391 is correct.","signature":"Er0DCpoBCBEYAipAfQOcVMh+fN1kqKgyYniNFmLoXQndf7+wItJBZdo15tWSaYQloTdiBYEPUjjfVb5BWuTi7Y2SC01sXMYmfZe+sTIZY2xhdWRlLWhhaWt1LTQtNS0yMDI1MTAwMTgAQgh0aGlua2luZ1okMjc3OTY2NjgtNzM1MS00MGFjLWFjYzQtMDI0YWVlODk5NWE1qAG7i/rUBhIMwNYhm9izIza3IBM+Ggw3Yl78vAm4cQvUy1IiMC+l0Z2l9ca6Ed9mQVMPsQkx8I/TEWhrLmvfeEZFefFnfDsmfJz79ZHIg+KGNzVXtirPAaJFPsSgcn6FdRadilHlhlgKCcvbQuyaHEDd4iYQb/UVnFK5xwpyuFJDw9qoWmghP5LmkAdbKfj/iA+2mnJDE69ZVd//dHtVu6Y58bH9B3Tbwku46gf/0rru7m6dd6k0DztvAqT5wSQ4KUV4iGdID1hNlo+Qj3QXYAXrJkQCnWPjlVdqCXH2z02uR2B2fSC/KDCcUQ2lIJBYCNmYPVBi8BmJ4X8yC8/XoewSy26hKUMM3oLhnQ5k28UB1HMe2i/F0Uur7oeExShBTJXLWfPnkhgB"},{"type":"text","text":"Let
+        me work through this:\n\n17 * 23 = 17 * (20 + 3)\n= (17 * 20) + (17 * 3)\n=
+        340 + 51\n= **391**"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":54,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":186,"output_tokens_details":{"thinking_tokens":125},"service_tier":"standard","inference_geo":"not_available"}}'
+    headers:
+      connection:
+      - keep-alive
+      content-type:
+      - application/json
+      date:
+      - Mon, 07 Sep 2026 09:36:59 GMT
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains
+      transfer-encoding:
+      - chunked
+      vary:
+      - accept-encoding
+    status:
+      code: 200
+      message: OK
+version: 1

py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py

@@ -3,6 +3,7 @@
 # pyright: reportUnknownParameterType=false
 # pyright: reportPrivateUsage=false
 import inspect
+import os
 import time
 
 import pytest
@@ -2212,6 +2213,80 @@ def calc(a: int, b: int) -> int:
     assert "b" in tool["parameters"]["properties"]
 
 
+@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path"])
+@pytest.mark.asyncio
+async def test_agent_run_anthropic_reasoning_tokens(memory_logger):
+    """Real Anthropic extended-thinking response through pydantic-ai's own usage mapping.
+
+    pydantic-ai stashes the reasoning count under ``details["thinking_tokens"]`` for
+    Anthropic (vs ``reasoning_tokens`` for OpenAI); the extractor must surface it as
+    ``completion_reasoning_tokens``. This exercises the real provider mapping via a
+    checked-in cassette; ``test_reasoning_tokens_extraction_provider_keys`` is the
+    supplemental synthetic coverage.
+    """
+    if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "latest":
+        pytest.skip("Anthropic extended-thinking usage requires the latest pydantic-ai cassette")
+
+    import anthropic
+    from pydantic_ai.models.anthropic import AnthropicModel, AnthropicModelSettings
+    from pydantic_ai.providers.anthropic import AnthropicProvider
+
+    assert not memory_logger.pop()
+
+    model = AnthropicModel(
+        "claude-haiku-4-5-20251001",
+        provider=AnthropicProvider(anthropic_client=anthropic.AsyncAnthropic()),
+    )
+    agent = Agent(
+        model,
+        model_settings=AnthropicModelSettings(
+            max_tokens=2048,
+            anthropic_thinking={"type": "enabled", "budget_tokens": 1024},
+        ),
+    )
+
+    result = await agent.run("What is 17 * 23? Think it through, then give the number.")
+    assert "391" in str(result.output)
+
+    spans = memory_logger.pop()
+    chat_span = next((s for s in spans if "chat" in s["span_attributes"]["name"]), None)
+    assert chat_span is not None, "chat span not found"
+    assert chat_span["metadata"]["provider"] == "anthropic"
+    # pylint: disable=unsupported-membership-test,unsubscriptable-object
+    assert chat_span["metrics"]["completion_reasoning_tokens"] > 0
+    # pylint: enable=unsupported-membership-test,unsubscriptable-object
+
+
+@pytest.mark.parametrize(
+    "details_key",
+    [
+        "reasoning_tokens",  # OpenAI
+        "thinking_tokens",  # Anthropic
+        "thoughts_tokens",  # Google
+    ],
+)
+def test_reasoning_tokens_extraction_provider_keys(details_key):
+    """pydantic_ai stashes the reasoning-token count under a provider-specific key:
+    OpenAI "reasoning_tokens", Anthropic "thinking_tokens", Google "thoughts_tokens".
+    All three must surface as `completion_reasoning_tokens` (previously only OpenAI's
+    key was read, silently dropping Anthropic/Google reasoning).
+    """
+    from types import SimpleNamespace
+
+    from braintrust.integrations.pydantic_ai.tracing import _extract_response_metrics
+    from pydantic_ai.usage import RequestUsage
+
+    usage = RequestUsage(input_tokens=10, output_tokens=20, details={details_key: 128})
+    response = SimpleNamespace(parts=[], usage=usage)
+
+    metrics = _extract_response_metrics(response, start_time=1.0, end_time=2.0)
+
+    assert metrics is not None
+    # pylint: disable=unsupported-membership-test,unsubscriptable-object
+    assert metrics["completion_reasoning_tokens"] == 128.0
+    # pylint: enable=unsupported-membership-test,unsubscriptable-object
+
+
 def test_explicit_toolsets_kwarg_in_input():
     """Test that explicitly passed toolsets kwarg goes to input (not just metadata)."""
     from braintrust.integrations.pydantic_ai.tracing import _build_agent_input_and_metadata

py/src/braintrust/integrations/pydantic_ai/tracing.py

@@ -1392,9 +1392,20 @@ def _extract_response_metrics(
             metrics["completion_audio_tokens"] = float(usage.output_audio_tokens)
 
         # RequestUsage.details is a dict; providers stash reasoning_tokens/cached_tokens here.
+        # The reasoning-token count lives under a provider-specific key: OpenAI uses
+        # "reasoning_tokens", Anthropic "thinking_tokens", Google "thoughts_tokens". All three
+        # are the same normalized quantity (a subset of the output tokens), so read whichever
+        # is present.
         details = getattr(usage, "details", None)
         if isinstance(details, dict):
-            reasoning = details.get("reasoning_tokens")
+            reasoning = next(
+                (
+                    details[key]
+                    for key in ("reasoning_tokens", "thinking_tokens", "thoughts_tokens")
+                    if details.get(key) is not None
+                ),
+                None,
+            )
             if reasoning is not None:
                 metrics["completion_reasoning_tokens"] = float(reasoning)
             cached = details.get("cached_tokens")