From 5a9872a99136f1c6e715a1d93634842bc2625a12 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 12:39:08 +0200 Subject: [PATCH 1/7] fix(boto3): trace the complete client-call lifecycle --- sentry_sdk/consts.py | 12 + sentry_sdk/integrations/boto3/_client.py | 103 +++++-- sentry_sdk/integrations/boto3/_context.py | 44 +++ .../integrations/boto3/_instrumentation.py | 203 +++++++------ sentry_sdk/integrations/stdlib.py | 12 +- tests/integrations/boto3/test_client.py | 268 +++++++++++++++++- tests/integrations/boto3/test_s3.py | 47 +-- 7 files changed, 567 insertions(+), 122 deletions(-) create mode 100644 sentry_sdk/integrations/boto3/_context.py diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index a1613bfbd7..2f5d81a0c5 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -1164,6 +1164,18 @@ class SPANDATA: Example: "prod" """ + SENTRY_OP = "sentry.op" + """ + The operation of a span. + Example: "http.client" + """ + + SENTRY_ORIGIN = "sentry.origin" + """ + The origin of the instrumentation (e.g. span, log, etc.) + Example: "auto.http.otel.fastify" + """ + SENTRY_RELEASE = "sentry.release" """ The Sentry release. diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index b5803b6e80..33d9899055 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -1,44 +1,109 @@ -from functools import partial +from contextlib import contextmanager from typing import TYPE_CHECKING -from sentry_sdk.integrations import DidNotEnable, _check_minimum_version +import sentry_sdk +from sentry_sdk.integrations import DidNotEnable +from sentry_sdk.integrations.boto3._context import AwsCallContext from sentry_sdk.integrations.boto3._instrumentation import ( - _sentry_after_call, - _sentry_after_call_error, + _finish_span, + _instrument_streaming_body, _sentry_before_sign, _sentry_request_created, + _start_client_span, ) -from sentry_sdk.utils import parse_version +from sentry_sdk.traces import NoOpStreamedSpan, StreamedSpan +from sentry_sdk.utils import capture_internal_exceptions if TYPE_CHECKING: - from typing import Any + from typing import Any, Iterator, Optional, Union + + from sentry_sdk.tracing import Span try: - from botocore import __version__ as BOTOCORE_VERSION from botocore.client import BaseClient except ImportError: - raise DidNotEnable("botocore is not installed") + raise DidNotEnable("botocore not installed") + + +@contextmanager +def _activate_client_span(span: "StreamedSpan") -> "Iterator[StreamedSpan]": + """Temporarily activate an inactive boto span without ending it.""" + if isinstance(span, NoOpStreamedSpan): + yield span + return + + scope = sentry_sdk.get_current_scope() + previous_span = scope.streamed_span + scope.streamed_span = span + try: + yield span + finally: + scope.streamed_span = previous_span def _patch_botocore_client() -> None: from sentry_sdk.integrations.boto3 import Boto3Integration - version = parse_version(BOTOCORE_VERSION) - _check_minimum_version(Boto3Integration, version, "botocore") - orig_init = BaseClient.__init__ + orig_make_api_call = BaseClient._make_api_call # type: ignore def sentry_patched_init(self: "BaseClient", *args: "Any", **kwargs: "Any") -> None: orig_init(self, *args, **kwargs) meta = self.meta - service_id = meta.service_model.service_id - meta.events.register( - "request-created", - partial(_sentry_request_created, service_id=service_id), - ) - # run after other `before-sign` handlers, allowing it to see and preserve existing baggage. + meta.events.register("request-created", _sentry_request_created) + # run after other `before-sign` handlers so existing baggage is preserved. meta.events.register_last("before-sign", _sentry_before_sign) - meta.events.register("after-call", _sentry_after_call) - meta.events.register("after-call-error", _sentry_after_call_error) + + def sentry_patched_make_api_call( + self: "BaseClient", operation_name: str, api_params: "Any" + ) -> "Any": + """ + Track a single API call, including retries, serialization, and endpoint + resolution. For streaming responses, keep the span open until the + response body is consumed or closed. + https://github.com/boto/botocore/blob/develop/botocore/client.py + https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span + """ + client = sentry_sdk.get_client() + if client.get_integration(Boto3Integration) is None: + return orig_make_api_call(self, operation_name, api_params) + + ctx = AwsCallContext(operation_name) + + # add optional metadata to context. + with capture_internal_exceptions(): + ctx.add_metadata(self) + + span: "Optional[Union[Span, StreamedSpan]]" = None + with capture_internal_exceptions(): + span = _start_client_span(ctx) + + if span is None: + return orig_make_api_call(self, operation_name, api_params) + + # activate without finishing; a streaming response may outlive the call. + span_ctx = ( + _activate_client_span(span) if isinstance(span, StreamedSpan) else span + ) + + try: + with span_ctx: + parsed = orig_make_api_call(self, operation_name, api_params) + except BaseException as error: + # finish `StreamedSpan` explicitly; static spans are finished by + # their context manager. + if isinstance(span, StreamedSpan): + _finish_span(span, error) + raise + + streaming_body_instrumented = False + with capture_internal_exceptions(): + streaming_body_instrumented = _instrument_streaming_body(span, parsed) + + # `StreamingBody`s finish their span when consumed or closed. + if isinstance(span, StreamedSpan) and not streaming_body_instrumented: + _finish_span(span) + return parsed BaseClient.__init__ = sentry_patched_init # type: ignore + BaseClient._make_api_call = sentry_patched_make_api_call # type: ignore diff --git a/sentry_sdk/integrations/boto3/_context.py b/sentry_sdk/integrations/boto3/_context.py new file mode 100644 index 0000000000..38f7e0d76a --- /dev/null +++ b/sentry_sdk/integrations/boto3/_context.py @@ -0,0 +1,44 @@ +from typing import TYPE_CHECKING + +from sentry_sdk.integrations import DidNotEnable +from sentry_sdk.utils import capture_internal_exceptions + +if TYPE_CHECKING: + from typing import Any, Optional + +try: + from botocore.client import BaseClient +except ImportError: + raise DidNotEnable("botocore not installed") + + +class AwsCallContext: + __slots__ = ( + "service_id", + "service_id_hyphenized", + "operation_name", + ) + + def __init__(self, operation_name: str) -> None: + self.operation_name: str = operation_name + self.service_id: "Optional[str]" = None + self.service_id_hyphenized: "Optional[str]" = None + + def add_metadata(self, client: "BaseClient") -> None: + def _get_attr(obj: "Any", name: str) -> "Any": + if obj is None: + return None + + with capture_internal_exceptions(): + return getattr(obj, name) + + client_meta = _get_attr(client, "meta") + service_model = _get_attr(client_meta, "service_model") + + # modeled AWS service identity used in span names, e.g. `API Gateway`. + service_id = _get_attr(service_model, "service_id") + if service_id is not None: + with capture_internal_exceptions(): + self.service_id = str(service_id) + with capture_internal_exceptions(): + self.service_id_hyphenized = service_id.hyphenize() diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 5dc90ce5d7..d13b7d2864 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -19,29 +19,94 @@ ) if TYPE_CHECKING: - from typing import Any, Dict, Optional, Type, Union - - from botocore.model import ServiceId + from typing import Any, Dict, Optional, Union + from sentry_sdk._types import Attributes + from sentry_sdk.integrations.boto3._context import AwsCallContext try: from botocore.awsrequest import AWSRequest from botocore.response import StreamingBody except ImportError: - raise DidNotEnable("botocore is not installed") + raise DidNotEnable("botocore not installed") -def _sentry_request_created( - service_id: "ServiceId", request: "AWSRequest", operation_name: str, **kwargs: "Any" -) -> None: +def _start_client_span( + ctx: "AwsCallContext", +) -> "Optional[Union[Span, StreamedSpan]]": from sentry_sdk.integrations.boto3 import Boto3Integration - description = "aws.%s.%s" % (service_id.hyphenize(), operation_name) - client = sentry_sdk.get_client() if client.get_integration(Boto3Integration) is None: + return None + + # Use unknown if `service_id_hyphenized` is unavailable so a span name can + # still be created, e.g. "aws.unknown.GetObject". + service_name = ctx.service_id_hyphenized or "unknown" + span_name = "aws.%s.%s" % (service_name, ctx.operation_name) + + if has_span_streaming_enabled(client.options): + if sentry_sdk.traces.get_current_span() is None: + return None + + attributes: "Attributes" = { + SPANDATA.SENTRY_OP: OP.HTTP_CLIENT, + SPANDATA.SENTRY_ORIGIN: ORIGIN, + } + if ctx.service_id: + attributes[SPANDATA.RPC_METHOD] = "%s/%s" % ( + ctx.service_id, + ctx.operation_name, + ) + return sentry_sdk.traces.start_span( + name=span_name, + attributes=attributes, + # `StreamingBody` responses outlive `_make_api_call()`. `_activate_client_span()` + # activates this span only while the call itself runs. + active=False, + ) + + span = sentry_sdk.start_span( + name=span_name, + op=OP.HTTP_CLIENT, + origin=ORIGIN, + ) + with capture_internal_exceptions(): + if ctx.service_id_hyphenized: + span.set_tag("aws.service_id", ctx.service_id_hyphenized) + span.set_tag("aws.operation_name", ctx.operation_name) + return span + + +def _set_request_attributes( + span: "Union[Span, StreamedSpan]", + request: "AWSRequest", +) -> None: + client = sentry_sdk.get_client() + + parsed_url = None + if request.url is not None: + with capture_internal_exceptions(): + parsed_url = parse_url(request.url, sanitize=False) + + if isinstance(span, StreamedSpan): + span.set_attributes(get_url_attributes(client, parsed_url)) + if request.method is not None: + span.set_attribute(SPANDATA.HTTP_REQUEST_METHOD, request.method) return + if parsed_url is not None: + span.set_data("aws.request.url", parsed_url.url) + span.set_data(SPANDATA.HTTP_QUERY, parsed_url.query) + span.set_data(SPANDATA.HTTP_FRAGMENT, parsed_url.fragment) + + if request.method is not None: + span.set_data(SPANDATA.HTTP_METHOD, request.method) + + +def _add_request_breadcrumb(request: "AWSRequest") -> None: + client = sentry_sdk.get_client() + parsed_url = None if request.url is not None: with capture_internal_exceptions(): @@ -49,39 +114,12 @@ def _sentry_request_created( breadcrumb: "dict[str, Any]" = {} - is_span_streaming_enabled = has_span_streaming_enabled(client.options) - span: "Union[Span, StreamedSpan, None]" = None - if is_span_streaming_enabled: - url_attributes = get_url_attributes(client, parsed_url) - breadcrumb.update(url_attributes) - + if has_span_streaming_enabled(client.options): + breadcrumb.update(get_url_attributes(client, parsed_url)) if request.method is not None: breadcrumb[SPANDATA.HTTP_REQUEST_METHOD] = request.method - - if sentry_sdk.traces.get_current_span() is not None: - span = sentry_sdk.traces.start_span( - name=description, - attributes={ - "sentry.op": OP.HTTP_CLIENT, - "sentry.origin": ORIGIN, - SPANDATA.RPC_METHOD: f"{service_id}/{operation_name}", - }, - ) - span.set_attributes(url_attributes) - - if request.method is not None: - span.set_attribute(SPANDATA.HTTP_REQUEST_METHOD, request.method) else: - span = sentry_sdk.start_span( - op=OP.HTTP_CLIENT, - name=description, - origin=ORIGIN, - ) - - if parsed_url: - span.set_data("aws.request.url", parsed_url.url) - span.set_data(SPANDATA.HTTP_QUERY, parsed_url.query) - span.set_data(SPANDATA.HTTP_FRAGMENT, parsed_url.fragment) + if parsed_url is not None: breadcrumb.update( { "aws.request.url": parsed_url.url, @@ -90,21 +128,45 @@ def _sentry_request_created( } ) - span.set_tag("aws.service_id", service_id.hyphenize()) - span.set_tag("aws.operation_name", operation_name) if request.method is not None: - span.set_data(SPANDATA.HTTP_METHOD, request.method) breadcrumb[SPANDATA.HTTP_METHOD] = request.method - # We do it in order for subsequent http calls/retries be - # attached to this span. - span.__enter__() - add_http_breadcrumb(None, breadcrumb) - if span is not None: - # request.context is an open-ended data-structure - # where we can add anything useful in request life cycle. + +def _sentry_request_created( + request: "AWSRequest", operation_name: str, **kwargs: "Any" +) -> None: + """ + Enrich a single `AWSRequest` attempt. Botocore creates a fresh + `AWSRequest` on every retry. + https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L178-L202 + """ + from sentry_sdk.integrations.boto3 import Boto3Integration + + client = sentry_sdk.get_client() + if client.get_integration(Boto3Integration) is None: + return + + with capture_internal_exceptions(): + _add_request_breadcrumb(request) + + span = ( + sentry_sdk.traces.get_current_span() + if has_span_streaming_enabled(client.options) + else sentry_sdk.get_current_span() + ) + if span is None: + return + + # An ignored streamed span is not activated; avoid enriching its parent. + if isinstance(span, StreamedSpan) and ( + span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) != ORIGIN + ): + return + + _set_request_attributes(span, request) + # Each attempt has a fresh `request.context`; carry the active client span. request.context["_sentrysdk_span"] = span @@ -118,8 +180,9 @@ def _sentry_before_sign( return with capture_internal_exceptions(): - # presigned requests are executed later by another caller. Adding propagation - # headers here would make those headers part of the signature, requiring the caller to reproduce the same values. + # Presigned requests are executed later by another caller. Adding propagation + # headers here would make those headers part of the signature, requiring the + # caller to reproduce the same values. if isinstance(signature_version, str) and signature_version.endswith( ("-query", "-presign-post") ): @@ -140,24 +203,23 @@ def _replace_header(request: "AWSRequest", key: str, value: str) -> None: del request.headers[key] request.headers[key] = value - # use span associated with this botocore request + # Use the span associated with this botocore request. span = request.context.get("_sentrysdk_span") - headers = sentry_sdk.get_current_scope().iter_trace_propagation_headers( span=span ) for header_name, header_value in headers: if header_name != BAGGAGE_HEADER_NAME: - # normal headers (e.g. `sentry-trace`) are non-shared, so replace stale values + # Normal headers (e.g. `sentry-trace`) are non-shared, so replace + # stale values. _replace_header(request, header_name, header_value) continue - # merge existing `baggage` values under single header + # Preserve third-party baggage and replace stale `sentry-*` values. existing_values = request.headers.get_all(BAGGAGE_HEADER_NAME, []) combined_baggage = { BAGGAGE_HEADER_NAME: ",".join(str(value) for value in existing_values) } - # preserve third-party baggage, replace stale `sentry-*` values add_sentry_baggage_to_headers(combined_baggage, header_value) _replace_header( request, BAGGAGE_HEADER_NAME, combined_baggage[BAGGAGE_HEADER_NAME] @@ -228,6 +290,8 @@ def finish_span(error: "Optional[BaseException]" = None) -> None: finished = True _finish_span(streaming_span, error) + if isinstance(span, StreamedSpan): + _finish_span(span, error) def content_length_reached() -> bool: content_length = getattr(body, "_content_length", None) @@ -287,30 +351,3 @@ def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: raise return True - - -def _sentry_after_call( - context: "Dict[str, Any]", parsed: "Dict[str, Any]", **kwargs: "Any" -) -> None: - span: "Optional[Union[Span, StreamedSpan]]" = context.pop("_sentrysdk_span", None) - - # Span could be absent if the integration is disabled. - if span is None: - return - - span.__exit__(None, None, None) - - with capture_internal_exceptions(): - _instrument_streaming_body(span, parsed) - - -def _sentry_after_call_error( - context: "Dict[str, Any]", exception: "Type[BaseException]", **kwargs: "Any" -) -> None: - span: "Optional[Union[Span, StreamedSpan]]" = context.pop("_sentrysdk_span", None) - - # Span could be absent if the integration is disabled. - if span is None: - return - - span.__exit__(type(exception), exception, None) diff --git a/sentry_sdk/integrations/stdlib.py b/sentry_sdk/integrations/stdlib.py index 02f8b245f7..b9d0450ac2 100644 --- a/sentry_sdk/integrations/stdlib.py +++ b/sentry_sdk/integrations/stdlib.py @@ -288,7 +288,14 @@ def putrequest( breadcrumb[SPANDATA.HTTP_REQUEST_METHOD] = method breadcrumb.update(url_attributes) - if sentry_sdk.traces.get_current_span() is not None: + parent_span = sentry_sdk.traces.get_current_span() + if parent_span is not None: + is_inactive_boto3_span = ( + client.get_integration("boto3") is not None + and parent_span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) + == getattr(client.get_integration("boto3"), "origin", None) + and not getattr(parent_span, "active", True) + ) span = sentry_sdk.traces.start_span( name="%s %s" % ( @@ -300,6 +307,9 @@ def putrequest( "sentry.op": OP.HTTP_CLIENT, SPANDATA.HTTP_REQUEST_METHOD: method, }, + # boto3 integration owns span's lifecycle; keep child inactive so it + # can't restore boto3 span later on. + active=not is_inactive_boto3_span, ) for key, value in url_attributes.items(): diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 6c81ba9313..0ad81d38bd 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -1,11 +1,17 @@ +from http.server import BaseHTTPRequestHandler, HTTPServer +from threading import Thread + import boto3 import pytest from botocore.awsrequest import AWSResponse from botocore.config import Config +from botocore.exceptions import ClientError, EndpointConnectionError +from botocore.response import StreamingBody import sentry_sdk -from sentry_sdk.consts import OP +from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration +from sentry_sdk.integrations.stdlib import StdlibIntegration from tests.integrations.boto3.aws_mock import Body session = boto3.Session( # type: ignore[attr-defined] @@ -15,11 +21,159 @@ ) +@pytest.fixture +def streaming_s3_server(): + class StreamingS3Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Length", "1") + self.send_header("Content-Type", "application/octet-stream") + self.end_headers() + self.wfile.write(b"x") + self.wfile.flush() + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), StreamingS3Handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join() + + def test_public_api(): assert Boto3Integration.__module__ == "sentry_sdk.integrations.boto3" assert Boto3Integration.identifier == "boto3" +@pytest.mark.parametrize( + "consume", + ["read_exact", "context"], +) +def test_streaming_span_order_and_scope( + sentry_init, + capture_items, + streaming_s3_server, + consume, +): + sentry_init( + traces_sample_rate=1.0, + trace_lifecycle="stream", + default_integrations=False, + integrations=[Boto3Integration(), StdlibIntegration()], + server_name="", + ) + server = streaming_s3_server + client = session.client( + "s3", + endpoint_url="http://127.0.0.1:%s" % server.server_port, + config=Config( + retries={"total_max_attempts": 1, "mode": "standard"}, + s3={"addressing_style": "path"}, + ), + ) + items = capture_items("span") + + with sentry_sdk.traces.start_span(name="parent") as parent: # type: ignore[attr-defined] + body = client.get_object(Bucket="bucket", Key="key")["Body"] + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + + if consume == "read_exact": + assert body.read(1) == b"x" + elif consume == "context": + if not hasattr(body, "__enter__"): + body.close() + pytest.skip("`StreamingBody` context manager is unavailable.") + with body as raw_stream: + assert raw_stream.read() == b"x" + + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + + probe = sentry_sdk.traces.start_span(name="probe") # type: ignore[attr-defined] + assert probe._parent_span_id == parent.span_id + probe.end() + + body.close() + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + + sentry_sdk.flush() + spans = [item.payload for item in items] + client_spans = [ + span + for span in spans + if span["name"] == "aws.s3.GetObject" + and span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == Boto3Integration.origin + and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT + ] + http_spans = [ + span + for span in spans + if span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == "auto.http.stdlib.httplib" + ] + stream_spans = [ + span + for span in spans + if span["name"] == "aws.s3.GetObject" + and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT_STREAM + ] + assert len(client_spans) == 1 + assert len(http_spans) == 1 + assert len(stream_spans) == 1 + client_span = client_spans[0] + http_span = http_spans[0] + stream_span = stream_spans[0] + + assert http_span["parent_span_id"] == client_span["span_id"] + assert stream_span["parent_span_id"] == client_span["span_id"] + assert client_span["start_timestamp"] <= http_span["start_timestamp"] + assert http_span["start_timestamp"] <= stream_span["start_timestamp"] + assert http_span["end_timestamp"] <= stream_span["end_timestamp"] + assert stream_span["end_timestamp"] <= client_span["end_timestamp"] + + +def test_non_body_stream_does_not_delay_client_span(sentry_init, capture_items): + sentry_init( + traces_sample_rate=1.0, + trace_lifecycle="stream", + integrations=[Boto3Integration()], + server_name="", + ) + client = session.client("lambda") + + def respond(request, **kwargs): + return AWSResponse( + request.url, + 200, + {"content-length": "1"}, + Body(b"x"), + ) + + client.meta.events.register("before-send", respond) + items = capture_items("span") + + with sentry_sdk.traces.start_span(name="parent") as parent: # type: ignore[attr-defined] + response = client.invoke(FunctionName="function") + assert isinstance(response["Payload"], StreamingBody) + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + + sentry_sdk.flush() + spans = [item.payload for item in items] + boto_spans = [ + span + for span in spans + if span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == Boto3Integration.origin + ] + assert len(boto_spans) == 1 + assert boto_spans[0]["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT + response["Payload"].close() + + @pytest.fixture def client_factory(sentry_init, monkeypatch, span_streaming): sentry_init( @@ -45,6 +199,25 @@ def make_client(service_name="s3", attempt_count=1, **client_kwargs): return make_client +def _mock_responses(client, status_codes): + request_span_ids = [] + + def record_request(request, **kwargs): + span = request.context.get("_sentrysdk_span") + assert span is not None + request_span_ids.append(span.span_id) + + def respond(request, **kwargs): + # `request_created` runs before `before_send`, so use zero-based index for current + # attempt; `min(..., len(status_codes) - 1)` clamps to last status to avoid `IndexError`. + response_index = min(len(request_span_ids) - 1, len(status_codes) - 1) + return AWSResponse(request.url, status_codes[response_index], {}, Body(b"")) + + client.meta.events.register("request-created", record_request) + client.meta.events.register("before-send", respond) + return request_span_ids + + def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streaming): items = capture_items() @@ -57,7 +230,7 @@ def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streami item.payload for item in items if item.type == "span" - and item.payload["attributes"].get("sentry.origin") + and item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) == Boto3Integration.origin ] else: @@ -73,7 +246,9 @@ def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streami spans_by_op = {} for span in spans: - op = span["attributes"].get("sentry.op") if span_streaming else span["op"] + op = ( + span["attributes"].get(SPANDATA.SENTRY_OP) if span_streaming else span["op"] + ) spans_by_op.setdefault(op, []).append(span) return spans_by_op @@ -89,6 +264,91 @@ def _assert_one_failed_span(spans, span_streaming): _assert_span_finished(spans[0], span_streaming) +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_retry_attempts_share_one_client_span( + capture_items, + client_factory, + span_streaming, +): + attempt_count = 3 + client = client_factory(attempt_count=attempt_count) + request_span_ids = _mock_responses(client, [500] * (attempt_count - 1) + [200]) + + spans_by_op = _capture_boto3_spans_by_op( + lambda: client.head_object(Bucket="bucket", Key="foo"), + capture_items, + span_streaming, + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + assert len(request_span_ids) == attempt_count + # all `AWSRequest` instances created during retries reference the same client span. + assert len(set(request_span_ids)) == 1 + assert len(client_spans) == 1 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_retries_exhausted_has_one_failed_client_span( + capture_items, + client_factory, + span_streaming, +): + client = client_factory(attempt_count=2) + request_span_ids = _mock_responses(client, [500]) + + def attempt_failed_head_object_call(): + with pytest.raises(ClientError): + client.head_object(Bucket="bucket", Key="foo.pdf") + + spans_by_op = _capture_boto3_spans_by_op( + attempt_failed_head_object_call, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + assert len(request_span_ids) == 2 + assert len(set(request_span_ids)) == 1 + _assert_one_failed_span(client_spans, span_streaming) + + +@pytest.mark.parametrize( + "event_name", + [ + pytest.param("before-parameter-build"), + pytest.param("before-send"), + ], +) +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_call_exception_is_unchanged_and_finishes_span( + capture_items, + client_factory, + span_streaming, + event_name, +): + client = client_factory() + if event_name == "before-send": + original_exception = EndpointConnectionError( + endpoint_url="https://s3.eu-north-1.amazonaws.com" + ) + else: + original_exception = ValueError("parameter processing failed") + + def raise_original_exception(**kwargs): + raise original_exception + + client.meta.events.register(event_name, raise_original_exception) + + def invoke_failing_client_method(): + with pytest.raises(type(original_exception)) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + _assert_one_failed_span(client_spans, span_streaming) + + @pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_body_read_failure_finishes_stream_span( capture_items, @@ -131,4 +391,6 @@ def invoke_client_method_and_read_body(): stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) assert len(client_spans) == 1 + if span_streaming: + _assert_one_failed_span(client_spans, span_streaming=True) _assert_one_failed_span(stream_spans, span_streaming) diff --git a/tests/integrations/boto3/test_s3.py b/tests/integrations/boto3/test_s3.py index 910c520534..dcd38dab9b 100644 --- a/tests/integrations/boto3/test_s3.py +++ b/tests/integrations/boto3/test_s3.py @@ -110,9 +110,19 @@ def test_streaming( spans = [item.payload for item in items] assert len(spans) == 3 - span1 = spans[0] - assert span1["attributes"]["sentry.op"] == "http.client" - assert span1["name"] == "aws.s3.GetObject" + stream_span, client_span, parent_span = spans + assert stream_span["attributes"]["sentry.op"] == "http.client.stream" + assert stream_span["name"] == "aws.s3.GetObject" + assert stream_span["parent_span_id"] == client_span["span_id"] + + assert client_span["attributes"]["sentry.op"] == "http.client" + assert client_span["name"] == "aws.s3.GetObject" + assert client_span["parent_span_id"] == parent_span["span_id"] + + assert parent_span["name"] == "custom parent" + assert parent_span["start_timestamp"] <= client_span["start_timestamp"] + assert client_span["start_timestamp"] <= stream_span["start_timestamp"] + assert stream_span["end_timestamp"] <= client_span["end_timestamp"] expected_attrs = { "http.request.method": "GET", @@ -131,17 +141,12 @@ def test_streaming( } if send_default_pii: expected_attrs["url.full"] = "https://bucket.s3.amazonaws.com/foo.pdf" - assert span1["attributes"] == ApproxDict(expected_attrs) + assert client_span["attributes"] == ApproxDict(expected_attrs) - assert "url.fragment" not in span1["attributes"] - assert "url.query" not in span1["attributes"] + assert "url.fragment" not in client_span["attributes"] + assert "url.query" not in client_span["attributes"] if not send_default_pii: - assert "url.full" not in span1["attributes"] - - span2 = spans[1] - assert span2["attributes"]["sentry.op"] == "http.client.stream" - assert span2["name"] == "aws.s3.GetObject" - assert span2["parent_span_id"] == span1["span_id"] + assert "url.full" not in client_span["attributes"] else: events = capture_events() @@ -207,10 +212,20 @@ def test_streaming_close( sentry_sdk.flush() spans = [item.payload for item in items] assert len(spans) == 3 - span1 = spans[0] - assert span1["attributes"]["sentry.op"] == "http.client" - span2 = spans[1] - assert span2["attributes"]["sentry.op"] == "http.client.stream" + + stream_span, client_span, parent_span = spans + assert stream_span["attributes"]["sentry.op"] == "http.client.stream" + assert stream_span["name"] == "aws.s3.GetObject" + assert stream_span["parent_span_id"] == client_span["span_id"] + + assert client_span["attributes"]["sentry.op"] == "http.client" + assert client_span["name"] == "aws.s3.GetObject" + assert client_span["parent_span_id"] == parent_span["span_id"] + + assert parent_span["name"] == "custom parent" + assert parent_span["start_timestamp"] <= client_span["start_timestamp"] + assert client_span["start_timestamp"] <= stream_span["start_timestamp"] + assert stream_span["end_timestamp"] <= client_span["end_timestamp"] else: events = capture_events() From 7a747d9d811bb327b82a862ff625782504a5b8a9 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 13:04:41 +0200 Subject: [PATCH 2/7] fix merging issues --- .../integrations/boto3/_instrumentation.py | 278 +++++++++--------- tests/integrations/boto3/test_client.py | 8 +- 2 files changed, 144 insertions(+), 142 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index d13b7d2864..37b9046d31 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -40,8 +40,8 @@ def _start_client_span( if client.get_integration(Boto3Integration) is None: return None - # Use unknown if `service_id_hyphenized` is unavailable so a span name can - # still be created, e.g. "aws.unknown.GetObject". + # use unknown if `service_id_hyphenized` so span name can still be created. + # e.g. "aws.unkown.GetObject" service_name = ctx.service_id_hyphenized or "unknown" span_name = "aws.%s.%s" % (service_name, ctx.operation_name) @@ -78,6 +78,133 @@ def _start_client_span( return span +def _finish_span( + span: "Union[Span, StreamedSpan]", + error: "Optional[BaseException]" = None, +) -> None: + with capture_internal_exceptions(): + if not isinstance(span, StreamedSpan): + if error is not None: + span.set_status(SPANSTATUS.INTERNAL_ERROR) + span.finish() + return + + if error is None: + span.end() + else: + span.__exit__(type(error), error, error.__traceback__) + + +def _instrument_streaming_body( + span: "Union[Span, StreamedSpan]", parsed: "Dict[str, Any]" +) -> bool: + if isinstance(span, NoOpStreamedSpan): + return False + + body = parsed.get("Body") + if not isinstance(body, StreamingBody): + return False + + streaming_span: "Union[Span, StreamedSpan]" + if isinstance(span, StreamedSpan): + streaming_span = sentry_sdk.traces.start_span( + name=span.name, + # `parent_span` is set explicitly to the boto span. + parent_span=span, + # avoid making the streaming span the current span on the scope since the application might + # keep `StreamingBody` open before reading it. Otherwise: 1. when the streamingspan ends it + # could restore the parent span on the scope, breaking the parent-child relation of newly + # created spans; 2. newly created spans would be attached to the streaming span. + active=False, + attributes={ + "sentry.op": OP.HTTP_CLIENT_STREAM, + "sentry.origin": ORIGIN, + }, + ) + else: + streaming_span = span.start_child( + op=OP.HTTP_CLIENT_STREAM, + name=span.description, + origin=ORIGIN, + ) + + orig_read = body.read + orig_close = body.close + raw_stream = body._raw_stream # type: ignore[attr-defined] + orig_raw_close = raw_stream.close + finished = False + read_in_progress = False + + def finish_span(error: "Optional[BaseException]" = None) -> None: + nonlocal finished + if finished: + return + + finished = True + _finish_span(streaming_span, error) + if isinstance(span, StreamedSpan): + _finish_span(span, error) + + def content_length_reached() -> bool: + content_length = getattr(body, "_content_length", None) + amount_read = getattr(body, "_amount_read", None) + return ( + content_length is not None + and amount_read is not None + and amount_read >= int(content_length) + ) + + def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: + nonlocal read_in_progress + read_in_progress = True + try: + read_return_value = orig_read(*args, **kwargs) + with capture_internal_exceptions(): + amount_of_bytes_requested = args[0] if args else kwargs.get("amt") + if ( + amount_of_bytes_requested is None + or amount_of_bytes_requested < 0 + or (amount_of_bytes_requested > 0 and not read_return_value) + or content_length_reached() + ): + finish_span() + return read_return_value + except BaseException as error: + finish_span(error) + raise + finally: + read_in_progress = False + + def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: + try: + orig_close(*args, **kwargs) + finish_span() + except BaseException as error: + finish_span(error) + raise + + def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: + try: + orig_raw_close(*args, **kwargs) + if not read_in_progress: + finish_span() + except BaseException as error: + finish_span(error) + raise + + try: + # StreamingBody.__exit__ closes `_raw_stream` directly, bypassing + # StreamingBody.close(), so both levels need to be instrumented. + raw_stream.close = sentry_raw_stream_close + body.read = sentry_streaming_body_read # type: ignore + body.close = sentry_streaming_body_close # type: ignore + except Exception: + finish_span() + raise + + return True + + def _set_request_attributes( span: "Union[Span, StreamedSpan]", request: "AWSRequest", @@ -138,8 +265,8 @@ def _sentry_request_created( request: "AWSRequest", operation_name: str, **kwargs: "Any" ) -> None: """ - Enrich a single `AWSRequest` attempt. Botocore creates a fresh - `AWSRequest` on every retry. + Enrich a single `AWSRequest` attempt. Botocore creates a + fresh `AWSRequest` on every retry. https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L178-L202 """ from sentry_sdk.integrations.boto3 import Boto3Integration @@ -159,14 +286,14 @@ def _sentry_request_created( if span is None: return - # An ignored streamed span is not activated; avoid enriching its parent. + # an ignored streamed span is not activated; avoid enriching its parent. if isinstance(span, StreamedSpan) and ( span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) != ORIGIN ): return _set_request_attributes(span, request) - # Each attempt has a fresh `request.context`; carry the active client span. + # each attempt has a fresh `request.context`; carry the active client span. request.context["_sentrysdk_span"] = span @@ -180,9 +307,8 @@ def _sentry_before_sign( return with capture_internal_exceptions(): - # Presigned requests are executed later by another caller. Adding propagation - # headers here would make those headers part of the signature, requiring the - # caller to reproduce the same values. + # presigned requests are executed later by another caller. Adding propagation + # headers here would make those headers part of the signature, requiring the caller to reproduce the same values. if isinstance(signature_version, str) and signature_version.endswith( ("-query", "-presign-post") ): @@ -203,19 +329,18 @@ def _replace_header(request: "AWSRequest", key: str, value: str) -> None: del request.headers[key] request.headers[key] = value - # Use the span associated with this botocore request. + # use span associated with this botocore request span = request.context.get("_sentrysdk_span") headers = sentry_sdk.get_current_scope().iter_trace_propagation_headers( span=span ) for header_name, header_value in headers: if header_name != BAGGAGE_HEADER_NAME: - # Normal headers (e.g. `sentry-trace`) are non-shared, so replace - # stale values. + # normal headers (e.g. `sentry-trace`) are non-shared, so replace stale values _replace_header(request, header_name, header_value) continue - # Preserve third-party baggage and replace stale `sentry-*` values. + # merge existing `baggage` values under single header existing_values = request.headers.get_all(BAGGAGE_HEADER_NAME, []) combined_baggage = { BAGGAGE_HEADER_NAME: ",".join(str(value) for value in existing_values) @@ -224,130 +349,3 @@ def _replace_header(request: "AWSRequest", key: str, value: str) -> None: _replace_header( request, BAGGAGE_HEADER_NAME, combined_baggage[BAGGAGE_HEADER_NAME] ) - - -def _finish_span( - span: "Union[Span, StreamedSpan]", - error: "Optional[BaseException]" = None, -) -> None: - with capture_internal_exceptions(): - if not isinstance(span, StreamedSpan): - if error is not None: - span.set_status(SPANSTATUS.INTERNAL_ERROR) - span.finish() - return - - if error is None: - span.end() - else: - span.__exit__(type(error), error, error.__traceback__) - - -def _instrument_streaming_body( - span: "Union[Span, StreamedSpan]", parsed: "Dict[str, Any]" -) -> bool: - if isinstance(span, NoOpStreamedSpan): - return False - - body = parsed.get("Body") - if not isinstance(body, StreamingBody): - return False - - streaming_span: "Union[Span, StreamedSpan]" - if isinstance(span, StreamedSpan): - streaming_span = sentry_sdk.traces.start_span( - name=span.name, - # `parent_span` is set explicitly to the boto span. - parent_span=span, - # avoid making the streaming span the current span on the scope since the application might - # keep `StreamingBody` open before reading it. Otherwise: 1. when the streamingspan ends it - # could restore the parent span on the scope, breaking the parent-child relation of newly - # created spans; 2. newly created spans would be attached to the streaming span. - active=False, - attributes={ - "sentry.op": OP.HTTP_CLIENT_STREAM, - "sentry.origin": ORIGIN, - }, - ) - else: - streaming_span = span.start_child( - op=OP.HTTP_CLIENT_STREAM, - name=span.description, - origin=ORIGIN, - ) - - orig_read = body.read - orig_close = body.close - raw_stream = body._raw_stream # type: ignore[attr-defined] - orig_raw_close = raw_stream.close - finished = False - read_in_progress = False - - def finish_span(error: "Optional[BaseException]" = None) -> None: - nonlocal finished - if finished: - return - - finished = True - _finish_span(streaming_span, error) - if isinstance(span, StreamedSpan): - _finish_span(span, error) - - def content_length_reached() -> bool: - content_length = getattr(body, "_content_length", None) - amount_read = getattr(body, "_amount_read", None) - return ( - content_length is not None - and amount_read is not None - and amount_read >= int(content_length) - ) - - def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: - nonlocal read_in_progress - read_in_progress = True - try: - read_return_value = orig_read(*args, **kwargs) - with capture_internal_exceptions(): - amount_of_bytes_requested = args[0] if args else kwargs.get("amt") - if ( - amount_of_bytes_requested is None - or amount_of_bytes_requested < 0 - or (amount_of_bytes_requested > 0 and not read_return_value) - or content_length_reached() - ): - finish_span() - return read_return_value - except BaseException as error: - finish_span(error) - raise - finally: - read_in_progress = False - - def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: - try: - orig_close(*args, **kwargs) - finish_span() - except BaseException as error: - finish_span(error) - raise - - def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: - try: - orig_raw_close(*args, **kwargs) - if not read_in_progress: - finish_span() - except BaseException as error: - finish_span(error) - raise - - try: - # StreamingBody.__exit__ closes `_raw_stream` directly, bypassing - # StreamingBody.close(), so both levels need to be instrumented. - raw_stream.close = sentry_raw_stream_close - body.read = sentry_streaming_body_read # type: ignore - body.close = sentry_streaming_body_close # type: ignore - except Exception: - finish_span() - raise - - return True diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 0ad81d38bd..7e5b02d6b9 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -54,7 +54,7 @@ def test_public_api(): @pytest.mark.parametrize( "consume", - ["read_exact", "context"], + ["read", "read_exact", "context", "close"], ) def test_streaming_span_order_and_scope( sentry_init, @@ -84,7 +84,9 @@ def test_streaming_span_order_and_scope( body = client.get_object(Bucket="bucket", Key="key")["Body"] assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] - if consume == "read_exact": + if consume == "read": + assert body.read() == b"x" + elif consume == "read_exact": assert body.read(1) == b"x" elif consume == "context": if not hasattr(body, "__enter__"): @@ -92,6 +94,8 @@ def test_streaming_span_order_and_scope( pytest.skip("`StreamingBody` context manager is unavailable.") with body as raw_stream: assert raw_stream.read() == b"x" + else: + body.close() assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] From 39d11db095b8e22b2f68697e37115338dca0e07f Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 13:20:24 +0200 Subject: [PATCH 3/7] review changes --- sentry_sdk/integrations/boto3/_instrumentation.py | 14 +++++--------- sentry_sdk/integrations/stdlib.py | 4 +++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 37b9046d31..7f7d8f072b 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -43,7 +43,7 @@ def _start_client_span( # use unknown if `service_id_hyphenized` so span name can still be created. # e.g. "aws.unkown.GetObject" service_name = ctx.service_id_hyphenized or "unknown" - span_name = "aws.%s.%s" % (service_name, ctx.operation_name) + span_name = f"aws.{service_name}.{ctx.operation_name}" if has_span_streaming_enabled(client.options): if sentry_sdk.traces.get_current_span() is None: @@ -54,10 +54,7 @@ def _start_client_span( SPANDATA.SENTRY_ORIGIN: ORIGIN, } if ctx.service_id: - attributes[SPANDATA.RPC_METHOD] = "%s/%s" % ( - ctx.service_id, - ctx.operation_name, - ) + attributes[SPANDATA.RPC_METHOD] = f"{ctx.service_id}/{ctx.operation_name}" return sentry_sdk.traces.start_span( name=span_name, attributes=attributes, @@ -287,10 +284,9 @@ def _sentry_request_created( return # an ignored streamed span is not activated; avoid enriching its parent. - if isinstance(span, StreamedSpan) and ( - span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) != ORIGIN - ): - return + if isinstance(span, StreamedSpan): + if not (span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) == ORIGIN): + return _set_request_attributes(span, request) # each attempt has a fresh `request.context`; carry the active client span. diff --git a/sentry_sdk/integrations/stdlib.py b/sentry_sdk/integrations/stdlib.py index b9d0450ac2..35157fdd97 100644 --- a/sentry_sdk/integrations/stdlib.py +++ b/sentry_sdk/integrations/stdlib.py @@ -296,6 +296,7 @@ def putrequest( == getattr(client.get_integration("boto3"), "origin", None) and not getattr(parent_span, "active", True) ) + # fmt: off span = sentry_sdk.traces.start_span( name="%s %s" % ( @@ -309,8 +310,9 @@ def putrequest( }, # boto3 integration owns span's lifecycle; keep child inactive so it # can't restore boto3 span later on. - active=not is_inactive_boto3_span, + active = not is_inactive_boto3_span, ) + # fmt: on for key, value in url_attributes.items(): span.set_attribute(key, value) From 8d55f74ed908118b6612dc2452664c579f8e34b0 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 13:54:19 +0200 Subject: [PATCH 4/7] add test for when `StreamingBody` instrumentation setup fails --- .../integrations/boto3/_instrumentation.py | 9 +-- tests/integrations/boto3/test_client.py | 71 +++++++++++++++---- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 7f7d8f072b..5d4efd12fc 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -125,10 +125,6 @@ def _instrument_streaming_body( origin=ORIGIN, ) - orig_read = body.read - orig_close = body.close - raw_stream = body._raw_stream # type: ignore[attr-defined] - orig_raw_close = raw_stream.close finished = False read_in_progress = False @@ -190,6 +186,11 @@ def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: raise try: + orig_read = body.read + orig_close = body.close + raw_stream = body._raw_stream # type: ignore[attr-defined] + orig_raw_close = raw_stream.close + # StreamingBody.__exit__ closes `_raw_stream` directly, bypassing # StreamingBody.close(), so both levels need to be instrumented. raw_stream.close = sentry_raw_stream_close diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 7e5b02d6b9..d17960306d 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -11,6 +11,8 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration +from sentry_sdk.integrations.boto3._instrumentation import _instrument_streaming_body +from sentry_sdk.integrations.boto3.consts import ORIGIN from sentry_sdk.integrations.stdlib import StdlibIntegration from tests.integrations.boto3.aws_mock import Body @@ -47,11 +49,6 @@ def log_message(self, *args): thread.join() -def test_public_api(): - assert Boto3Integration.__module__ == "sentry_sdk.integrations.boto3" - assert Boto3Integration.identifier == "boto3" - - @pytest.mark.parametrize( "consume", ["read", "read_exact", "context", "close"], @@ -112,7 +109,7 @@ def test_streaming_span_order_and_scope( span for span in spans if span["name"] == "aws.s3.GetObject" - and span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == Boto3Integration.origin + and span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT ] http_spans = [ @@ -141,6 +138,57 @@ def test_streaming_span_order_and_scope( assert stream_span["end_timestamp"] <= client_span["end_timestamp"] +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_streaming_body_instrumentation_setup_failure_finishes_stream_span( + sentry_init, + capture_items, + span_streaming, +): + sentry_init( + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", + integrations=[Boto3Integration()], + server_name="", + ) + + class _RawStreamLookupFailingBody(StreamingBody): + @property + def _raw_stream(self): + raise RuntimeError("raw stream lookup failed") + + @_raw_stream.setter + def _raw_stream(self, raw_stream): + self._raw_stream_value = raw_stream + + body = _RawStreamLookupFailingBody(Body(b"x"), "1") + + def invoke(): + if not span_streaming: + with sentry_sdk.start_span( + name="client", op=OP.HTTP_CLIENT, origin=ORIGIN + ) as span: + with pytest.raises(RuntimeError, match="raw stream lookup failed"): + _instrument_streaming_body(span, {"Body": body}) + return + + span = sentry_sdk.traces.start_span( # type: ignore[attr-defined] + name="client", + attributes={ + SPANDATA.SENTRY_OP: OP.HTTP_CLIENT, + SPANDATA.SENTRY_ORIGIN: ORIGIN, + }, + active=False, + ) + with pytest.raises(RuntimeError, match="raw stream lookup failed"): + _instrument_streaming_body(span, {"Body": body}) + + spans_by_op = _capture_boto3_spans_by_op(invoke, capture_items, span_streaming) + stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) + + assert len(stream_spans) == 1 + _assert_span_finished(stream_spans[0], span_streaming) + + def test_non_body_stream_does_not_delay_client_span(sentry_init, capture_items): sentry_init( traces_sample_rate=1.0, @@ -171,7 +219,7 @@ def respond(request, **kwargs): boto_spans = [ span for span in spans - if span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == Boto3Integration.origin + if span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN ] assert len(boto_spans) == 1 assert boto_spans[0]["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT @@ -234,19 +282,14 @@ def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streami item.payload for item in items if item.type == "span" - and item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) - == Boto3Integration.origin + and item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN ] else: with sentry_sdk.start_transaction(): invoke_client_method() transaction = next(item.payload for item in items if item.type == "transaction") - spans = [ - span - for span in transaction["spans"] - if span["origin"] == Boto3Integration.origin - ] + spans = [span for span in transaction["spans"] if span["origin"] == ORIGIN] spans_by_op = {} for span in spans: From a8108f3f14e899f1091ac5197409d2eac1aec7ff Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 14:20:31 +0200 Subject: [PATCH 5/7] fix(boto3): finish streaming spans before legacy boto spans --- sentry_sdk/integrations/boto3/_client.py | 28 +++--- .../integrations/boto3/_instrumentation.py | 3 +- tests/integrations/boto3/test_client.py | 88 ++++++++++++++----- 3 files changed, 84 insertions(+), 35 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 33d9899055..04f7db217d 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -26,19 +26,30 @@ @contextmanager -def _activate_client_span(span: "StreamedSpan") -> "Iterator[StreamedSpan]": +def _activate_client_span( + span: "Union[Span, StreamedSpan]", +) -> "Iterator[Union[Span, StreamedSpan]]": """Temporarily activate an inactive boto span without ending it.""" if isinstance(span, NoOpStreamedSpan): yield span return scope = sentry_sdk.get_current_scope() - previous_span = scope.streamed_span + if not isinstance(span, StreamedSpan): + previous_span = scope.span + scope.span = span + try: + yield span + finally: + scope.span = previous_span + return + + previous_streamed_span = scope.streamed_span scope.streamed_span = span try: yield span finally: - scope.streamed_span = previous_span + scope.streamed_span = previous_streamed_span def _patch_botocore_client() -> None: @@ -82,18 +93,13 @@ def sentry_patched_make_api_call( return orig_make_api_call(self, operation_name, api_params) # activate without finishing; a streaming response may outlive the call. - span_ctx = ( - _activate_client_span(span) if isinstance(span, StreamedSpan) else span - ) + span_ctx = _activate_client_span(span) try: with span_ctx: parsed = orig_make_api_call(self, operation_name, api_params) except BaseException as error: - # finish `StreamedSpan` explicitly; static spans are finished by - # their context manager. - if isinstance(span, StreamedSpan): - _finish_span(span, error) + _finish_span(span, error) raise streaming_body_instrumented = False @@ -101,7 +107,7 @@ def sentry_patched_make_api_call( streaming_body_instrumented = _instrument_streaming_body(span, parsed) # `StreamingBody`s finish their span when consumed or closed. - if isinstance(span, StreamedSpan) and not streaming_body_instrumented: + if not streaming_body_instrumented: _finish_span(span) return parsed diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 5d4efd12fc..a6ba6b18e7 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -135,8 +135,7 @@ def finish_span(error: "Optional[BaseException]" = None) -> None: finished = True _finish_span(streaming_span, error) - if isinstance(span, StreamedSpan): - _finish_span(span, error) + _finish_span(span, error) def content_length_reached() -> bool: content_length = getattr(body, "_content_length", None) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index d17960306d..5d687e16b1 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -14,6 +14,8 @@ from sentry_sdk.integrations.boto3._instrumentation import _instrument_streaming_body from sentry_sdk.integrations.boto3.consts import ORIGIN from sentry_sdk.integrations.stdlib import StdlibIntegration +from sentry_sdk.traces import StreamedSpan +from sentry_sdk.tracing import Span from tests.integrations.boto3.aws_mock import Body session = boto3.Session( # type: ignore[attr-defined] @@ -53,15 +55,17 @@ def log_message(self, *args): "consume", ["read", "read_exact", "context", "close"], ) +@pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_span_order_and_scope( sentry_init, capture_items, streaming_s3_server, consume, + span_streaming, ): sentry_init( traces_sample_rate=1.0, - trace_lifecycle="stream", + trace_lifecycle="stream" if span_streaming else "static", default_integrations=False, integrations=[Boto3Integration(), StdlibIntegration()], server_name="", @@ -75,11 +79,31 @@ def test_streaming_span_order_and_scope( s3={"addressing_style": "path"}, ), ) - items = capture_items("span") + request_client_spans = [] - with sentry_sdk.traces.start_span(name="parent") as parent: # type: ignore[attr-defined] + def record_client_span(request, **kwargs): + request_client_spans.append(request.context["_sentrysdk_span"]) + + client.meta.events.register("request-created", record_client_span) + items = capture_items() + + parent_context = ( + sentry_sdk.traces.start_span(name="parent") # type: ignore[attr-defined] + if span_streaming + else sentry_sdk.start_transaction(name="parent") + ) + with parent_context as parent: body = client.get_object(Bucket="bucket", Key="key")["Body"] - assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + assert len(request_client_spans) == 1 + request_client_span = request_client_spans[0] + if span_streaming: + assert isinstance(request_client_span, StreamedSpan) + assert request_client_span.end_timestamp is None + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + else: + assert isinstance(request_client_span, Span) + assert not isinstance(request_client_span, StreamedSpan) + assert request_client_span.timestamp is None if consume == "read": assert body.read() == b"x" @@ -94,34 +118,54 @@ def test_streaming_span_order_and_scope( else: body.close() - assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + if span_streaming: + assert request_client_span.end_timestamp is not None + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] - probe = sentry_sdk.traces.start_span(name="probe") # type: ignore[attr-defined] - assert probe._parent_span_id == parent.span_id - probe.end() + probe = sentry_sdk.traces.start_span(name="probe") # type: ignore[attr-defined] + assert probe._parent_span_id == parent.span_id + probe.end() - body.close() - assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + body.close() + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + else: + assert request_client_span.timestamp is not None sentry_sdk.flush() - spans = [item.payload for item in items] + if span_streaming: + spans = [item.payload for item in items] + else: + transaction = next(item.payload for item in items if item.type == "transaction") + spans = transaction["spans"] client_spans = [ span for span in spans - if span["name"] == "aws.s3.GetObject" - and span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN - and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT + if span.get("name", span.get("description")) == "aws.s3.GetObject" + and ( + span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN + and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT + if span_streaming + else span["origin"] == ORIGIN and span["op"] == OP.HTTP_CLIENT + ) ] http_spans = [ span for span in spans - if span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == "auto.http.stdlib.httplib" + if ( + span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == "auto.http.stdlib.httplib" + if span_streaming + else span["origin"] == "auto.http.stdlib.httplib" + ) ] stream_spans = [ span for span in spans - if span["name"] == "aws.s3.GetObject" - and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT_STREAM + if span.get("name", span.get("description")) == "aws.s3.GetObject" + and ( + span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT_STREAM + if span_streaming + else span["op"] == OP.HTTP_CLIENT_STREAM + ) ] assert len(client_spans) == 1 assert len(http_spans) == 1 @@ -132,10 +176,12 @@ def test_streaming_span_order_and_scope( assert http_span["parent_span_id"] == client_span["span_id"] assert stream_span["parent_span_id"] == client_span["span_id"] + assert client_span["span_id"] == request_client_span.span_id + end_timestamp = "end_timestamp" if span_streaming else "timestamp" assert client_span["start_timestamp"] <= http_span["start_timestamp"] assert http_span["start_timestamp"] <= stream_span["start_timestamp"] - assert http_span["end_timestamp"] <= stream_span["end_timestamp"] - assert stream_span["end_timestamp"] <= client_span["end_timestamp"] + assert http_span[end_timestamp] <= stream_span[end_timestamp] + assert stream_span[end_timestamp] <= client_span[end_timestamp] @pytest.mark.parametrize("span_streaming", [True, False]) @@ -437,7 +483,5 @@ def invoke_client_method_and_read_body(): client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) - assert len(client_spans) == 1 - if span_streaming: - _assert_one_failed_span(client_spans, span_streaming=True) + _assert_one_failed_span(client_spans, span_streaming) _assert_one_failed_span(stream_spans, span_streaming) From af30a9082053362f650694a1bcc1bf7c6f30d1a9 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 14:25:18 +0200 Subject: [PATCH 6/7] add permalink --- sentry_sdk/integrations/boto3/_instrumentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index a6ba6b18e7..67642dce5e 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -264,7 +264,7 @@ def _sentry_request_created( """ Enrich a single `AWSRequest` attempt. Botocore creates a fresh `AWSRequest` on every retry. - https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L178-L202 + https://github.com/boto/botocore/blob/f9195c79ea2bf46350dd320d2a0bf3db7da0b460/botocore/endpoint.py#L178-L202 """ from sentry_sdk.integrations.boto3 import Boto3Integration From 3e4fcfd1cf761f779bef7d97e1bf853733fd108e Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 14:40:09 +0200 Subject: [PATCH 7/7] add some more comments --- sentry_sdk/integrations/boto3/_client.py | 14 +++++++++++++- sentry_sdk/integrations/boto3/_instrumentation.py | 13 ++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 04f7db217d..2163233e1d 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -29,7 +29,19 @@ def _activate_client_span( span: "Union[Span, StreamedSpan]", ) -> "Iterator[Union[Span, StreamedSpan]]": - """Temporarily activate an inactive boto span without ending it.""" + """ + Activate the boto span temporarily during `_make_api_call()` without ending it. + + Botocore returns a `StreamingBody` before its bytes are consumed. Using the + context manager would finish it as soon as `_make_api_call()` returns, so + restore the caller's span here and let the `StreamingBody` wrapper finish + the boto span when body is consumed/closed. + + faulty: desired: + boto3 [_make_api_call] boto3 [_make_api_call------] + http [request] http [request] + stream [read] stream [read] + """ if isinstance(span, NoOpStreamedSpan): yield span return diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 67642dce5e..f12e4c393a 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -106,12 +106,11 @@ def _instrument_streaming_body( if isinstance(span, StreamedSpan): streaming_span = sentry_sdk.traces.start_span( name=span.name, - # `parent_span` is set explicitly to the boto span. + # keep stream span under the boto span after `_make_api_call()` returns. parent_span=span, - # avoid making the streaming span the current span on the scope since the application might - # keep `StreamingBody` open before reading it. Otherwise: 1. when the streamingspan ends it - # could restore the parent span on the scope, breaking the parent-child relation of newly - # created spans; 2. newly created spans would be attached to the streaming span. + # the body may outlive the api call, so keep it inactive. Otherwise it + # 1. could restore the already-finished boto span when it ends; 2. make + # unrelated new spans attach to the stream span since it's the current span. active=False, attributes={ "sentry.op": OP.HTTP_CLIENT_STREAM, @@ -134,6 +133,7 @@ def finish_span(error: "Optional[BaseException]" = None) -> None: return finished = True + # finish stream span before boto span, and only once across read/close. _finish_span(streaming_span, error) _finish_span(span, error) @@ -153,6 +153,7 @@ def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: read_return_value = orig_read(*args, **kwargs) with capture_internal_exceptions(): amount_of_bytes_requested = args[0] if args else kwargs.get("amt") + # detect read-to-end, eof, or the known content length being consumed. if ( amount_of_bytes_requested is None or amount_of_bytes_requested < 0 @@ -190,8 +191,6 @@ def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: raw_stream = body._raw_stream # type: ignore[attr-defined] orig_raw_close = raw_stream.close - # StreamingBody.__exit__ closes `_raw_stream` directly, bypassing - # StreamingBody.close(), so both levels need to be instrumented. raw_stream.close = sentry_raw_stream_close body.read = sentry_streaming_body_read # type: ignore body.close = sentry_streaming_body_close # type: ignore