Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix: detect Task mismatch before OTel context detach in Anthropic streaming (#1370)
Ronak Bhalgamie9c2eeda2c59
diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.pyindex 09c74d5b..2b1ac2ee 100644--- a/sdk/python/src/openlit/__helpers.py+++ b/sdk/python/src/openlit/__helpers.py@@ -20,6 +20,7 @@ from opentelemetry.sdk.resources import (TELEMETRY_SDK_NAME,DEPLOYMENT_ENVIRONMENT,)+from opentelemetry import context as context_apifrom opentelemetry.trace import Status, StatusCodefrom opentelemetry._logs import LogRecordfrom openlit.semcov import SemanticConvention@@ -501,6 +502,65 @@ def handle_exception(span, e):span.set_attribute(SemanticConvention.ERROR_TYPE, error_type)+def safe_detach(token, attaching_task=None):+ """Detach an OTel context token only when it is safe to do so.++ A streaming wrapper's `__aexit__`/`__anext__` can run inside a *different*+ asyncio Task than the one that called `attach()` - e.g. when the caller+ abandons an async generator mid-iteration and asyncio's asyncgen GC hook+ later drives its `aclose()` in a fresh finalizer Task. `Token.reset()`+ requires the exact Context it was created in, so detaching from the wrong+ Task always raises ValueError - and there is nothing correct to do at+ that point: the attaching Task's Context object (and the stack frame we'd+ need to pop the token from) is not reachable from here. Skip the detach+ instead of attempting and swallowing the error, so this is a deliberate+ no-op rather than exception-driven control flow. The attaching Task's+ context reverts on its own once that Task completes, since contextvars+ are scoped per-Task and never shared across Tasks.++ Pass ``attaching_task`` (the Task active when ``attach()`` was called) to+ short-circuit async cross-Task exits before any detach is attempted.++ Important: ``opentelemetry.context.detach()`` catches all exceptions and+ logs them at ERROR (``Failed to detach context``), so it never re-raises+ ``ValueError`` to callers. This helper therefore detaches via the runtime+ context (``_RUNTIME_CONTEXT``) when available, so a genuine Context+ mismatch can be handled at DEBUG without polluting application logs.+ Sync callers may omit ``attaching_task`` and still avoid the ERROR path.+ """+ if attaching_task is not None:+ try:+ current_task = asyncio.current_task()+ except RuntimeError:+ current_task = None+ if current_task is not attaching_task:+ logger.debug(+ "safe_detach: skipping detach - called from Task %r but "+ "attach() ran in Task %r (likely an abandoned/GC-finalized "+ "stream); the attaching Task's context will revert on its "+ "own once that Task completes.",+ current_task,+ attaching_task,+ )+ return++ # Public context_api.detach() swallows exceptions and logs ERROR; use the+ # runtime Context so we can treat cross-Context Tokens as a quiet no-op.+ runtime = getattr(context_api, "_RUNTIME_CONTEXT", None)+ if runtime is not None:+ try:+ runtime.detach(token)+ except ValueError:+ logger.debug(+ "safe_detach: skipped detach; token was created in a "+ "different Context (attaching_task=%r).",+ attaching_task,+ )+ return++ context_api.detach(token)++def calculate_ttft(timestamps: List[float], start_time: float) -> float:"""Calculate the time to the first tokens.diff --git a/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/anthropic.pyindex 93b251b7..842d0bc5 100644--- a/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py+++ b/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py@@ -10,6 +10,7 @@ from openlit.__helpers import (set_server_address_and_port,record_completion_metrics,is_framework_llm_active,+ safe_detach,)from openlit.instrumentation.anthropic.utils import (process_chunk,@@ -137,10 +138,10 @@ def messages(awaited_wrapped = wrapped(*args, **kwargs)except Exception as e:handle_exception(span, e)- context_api.detach(token)+ safe_detach(token)span.end()raise- context_api.detach(token)+ safe_detach(token)return TracedSyncStream(awaited_wrapped,@@ -372,7 +373,8 @@ def messages_stream(Detaches the context and handles any exceptions inside the 'with' block."""if self._token:- context_api.detach(self._token)+ safe_detach(self._token)+ self._token = Noneif exc_type:handle_exception(self._span, exc_val)diff --git a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.pyindex de9a0c99..96c46014 100644--- a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py+++ b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py@@ -2,6 +2,7 @@Module for monitoring Anthropic API calls."""+import asyncioimport timefrom opentelemetry import trace as trace_api, context as context_apifrom opentelemetry.trace import SpanKind@@ -10,6 +11,7 @@ from openlit.__helpers import (set_server_address_and_port,record_completion_metrics,is_framework_llm_active,+ safe_detach,)from openlit.instrumentation.anthropic.utils import (process_chunk,@@ -133,14 +135,15 @@ def async_messages(span = tracer.start_span(span_name, kind=SpanKind.CLIENT)ctx = trace_api.set_span_in_context(span)token = context_api.attach(ctx)+ attaching_task = asyncio.current_task()try:awaited_wrapped = await wrapped(*args, **kwargs)except Exception as e:handle_exception(span, e)- context_api.detach(token)+ safe_detach(token, attaching_task)span.end()raise- context_api.detach(token)+ safe_detach(token, attaching_task)return TracedAsyncStream(awaited_wrapped,@@ -347,6 +350,7 @@ def async_messages_stream(self._server_port = server_portself._event_provider = event_providerself._token = None+ self._attaching_task = Noneasync def __aenter__(self):"""@@ -354,6 +358,7 @@ def async_messages_stream("""ctx = trace_api.set_span_in_context(self._span)self._token = context_api.attach(ctx)+ self._attaching_task = asyncio.current_task()stream = await self._original_manager.__aenter__()@@ -372,7 +377,9 @@ def async_messages_stream(Detaches context and handles exceptions occurring inside the 'async with' block."""if self._token:- context_api.detach(self._token)+ safe_detach(self._token, self._attaching_task)+ self._token = None+ self._attaching_task = Noneif exc_type:handle_exception(self._span, exc_val)diff --git a/sdk/python/tests/test_context_detach.py b/sdk/python/tests/test_context_detach.pynew file mode 100644index 00000000..507a88f5--- /dev/null+++ b/sdk/python/tests/test_context_detach.py@@ -0,0 +1,167 @@+"""+Regression tests for the "Failed to detach context" / ValueError raised when+an OTel context Token created in one asyncio Task/Context is detached from a+different one - e.g. a streaming Anthropic response abandoned mid-iteration+whose `__aexit__` is later driven by asyncio's asyncgen GC finalizer in a+fresh Task. See openlit/__helpers.py's `safe_detach` and+TracedAsyncMessageStreamManager in+openlit/instrumentation/anthropic/async_anthropic.py.+"""++import asyncio+import contextvars+import logging++import pytest+from opentelemetry import trace as trace_api, context as context_api++from openlit.__helpers import safe_detach+from openlit.instrumentation.anthropic.async_anthropic import async_messages_stream+++def test_safe_detach_cross_context_without_task_hint_does_not_error_log(caplog):+ """+ Without attaching_task, a bare Context mismatch must not raise and must+ not emit the public API's ERROR ``Failed to detach context`` log.+ """+ token = context_api.attach(context_api.set_value("k", "v"))++ with caplog.at_level(logging.ERROR, logger="opentelemetry.context"):+ contextvars.Context().run(lambda: safe_detach(token))++ assert not any(+ "Failed to detach context" in record.getMessage() for record in caplog.records+ )++ # The token was never actually detached in the original context - clean+ # it up so "k" doesn't leak into later tests.+ context_api.detach(token)+++def test_safe_detach_attaching_task_hint_with_no_running_loop_skips_detach():+ """+ A non-None attaching_task with no running event loop makes+ asyncio.current_task() raise RuntimeError. There is no way to prove+ we're in the attaching Task in that case, so this is treated like a+ foreign-Task mismatch: skip, don't attempt the detach.+ """+ token = context_api.attach(context_api.set_value("k", "v"))+ fake_task = object() # no real Task exists outside a running loop++ safe_detach(token, fake_task)++ # Skipped, not detached.+ assert context_api.get_value("k") == "v"++ context_api.detach(token)+++def test_safe_detach_same_task_still_detaches():+ """The common, well-behaved case must be unaffected: same Task -> real detach happens."""++ async def main():+ before = context_api.get_value("k")+ token = context_api.attach(context_api.set_value("k", "v"))+ assert context_api.get_value("k") == "v"++ safe_detach(token, asyncio.current_task())++ assert context_api.get_value("k") == before++ asyncio.run(main())+++def test_safe_detach_skips_foreign_task_without_error_log(caplog):+ """+ Foreign Task -> the mismatch is detected up front and detach is skipped+ entirely (not attempted-then-caught). Must not emit ERROR from+ ``opentelemetry.context``.+ """++ async def main():+ token_holder = {}++ async def attach_only():+ token_holder["token"] = context_api.attach(+ context_api.set_value("span", "abandoned-stream")+ )+ token_holder["task"] = asyncio.current_task()++ await asyncio.create_task(attach_only())++ with caplog.at_level(logging.ERROR, logger="opentelemetry.context"):+ # Runs in main()'s task, which never attached anything - mirrors+ # the asyncgen finalizer Task in the real bug.+ safe_detach(token_holder["token"], token_holder["task"])++ assert not any(+ "Failed to detach context" in record.getMessage()+ for record in caplog.records+ )++ asyncio.run(main())+++def _make_stream_manager():+ """Builds a TracedAsyncMessageStreamManager wired to a no-op fake Anthropic stream."""+ tracer = trace_api.get_tracer(__name__)++ class FakeAnthropicStream:+ """Immediately-exhausted stand-in for the real Anthropic stream object."""++ def __aiter__(self):+ return self++ async def __anext__(self):+ raise StopAsyncIteration++ class FakeOriginalManager:+ """Stand-in for Anthropic's own (un-instrumented) stream context manager."""++ async def __aenter__(self):+ return FakeAnthropicStream()++ async def __aexit__(self, exc_type, exc_val, exc_tb):+ return False++ def fake_wrapped(*args, **kwargs):+ return FakeOriginalManager()++ wrapper = async_messages_stream(+ version="test",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )++ return wrapper(fake_wrapped, None, (), {"model": "claude-3-5-sonnet-latest"})+++async def test_stream_manager_aexit_in_foreign_task_does_not_raise(caplog):+ """+ End-to-end: enter the manager in one Task, exit it (as the asyncgen+ finalizer would) from a different Task. Must not raise and must not+ emit ERROR ``Failed to detach context``.+ """+ manager = _make_stream_manager()++ # pylint: disable=unnecessary-dunder-call+ # Entering/exiting manually (not via `async with`) so each half can run in+ # a different Task, mirroring the asyncgen finalizer split in production.+ await manager.__aenter__()++ async def exit_from_foreign_task():+ await manager.__aexit__(None, None, None)++ with caplog.at_level(logging.ERROR, logger="opentelemetry.context"):+ await asyncio.create_task(exit_from_foreign_task())++ assert not any(+ "Failed to detach context" in record.getMessage() for record in caplog.records+ )+ assert manager._token is None