Skip to content

Build and run agents you can see, understand and trust.

fix(tracing): avoid otel context detach error on cross-task stream close (#2077)

天晟c5d26cd2465c
diff --git a/src/agentscope/middleware/_tracing/_trace.py b/src/agentscope/middleware/_tracing/_trace.py
index d8c3fb3af..8a9ac7ef9 100644
--- a/src/agentscope/middleware/_tracing/_trace.py
+++ b/src/agentscope/middleware/_tracing/_trace.py
@@ -13,6 +13,7 @@ from typing import (
import aioitertools
+from opentelemetry import context as otel_context
from opentelemetry import trace as otel_trace
from opentelemetry.trace import StatusCode
@@ -154,91 +155,103 @@ class TracingMiddleware(MiddlewareBase):
)
span_name = _get_agent_span_name(request_attributes)
- with tracer.start_as_current_span(
+ span = tracer.start_span(
name=span_name,
attributes={
**request_attributes,
**common_attrs,
},
- end_on_exit=False,
- ) as span:
- # Synthetic execute_tool spans for externally executed tools
- event_arg = input_kwargs.get("inputs")
- if isinstance(event_arg, ExternalExecutionResultEvent):
- for result in event_arg.execution_results:
- tool_attrs: dict[str, Any] = {
- SpanAttributes.GEN_AI_OPERATION_NAME: (
- OperationNameValues.EXECUTE_TOOL
- ),
- SpanAttributes.GEN_AI_TOOL_CALL_ID: result.id,
- SpanAttributes.GEN_AI_TOOL_NAME: result.name,
- SpanAttributes.AGENTSCOPE_IS_EXTERNAL_EXECUTION: (
- True
- ),
- **common_attrs,
- }
- if result.output is not None:
- tool_attrs[
- SpanAttributes.GEN_AI_TOOL_CALL_RESULT
- ] = _serialize_to_str(result.output)
- with tracer.start_as_current_span(
- name=(
- f"{OperationNameValues.EXECUTE_TOOL}"
- f" {result.name}"
- ),
- attributes=tool_attrs,
- ):
- pass
-
- has_error = False
- error_exc: BaseException | None = None
- last_msg: Msg | None = None
- hitl_pending: list[str] = []
- external_pending: list[str] = []
- observed_reply_id: str | None = None
-
- try:
- async for item in next_handler(**input_kwargs):
- if isinstance(item, ReplyStartEvent):
- observed_reply_id = item.reply_id
- elif isinstance(item, RequireUserConfirmEvent):
- hitl_pending.extend(t.name for t in item.tool_calls)
- elif isinstance(item, RequireExternalExecutionEvent):
- external_pending.extend(
- t.name for t in item.tool_calls
- )
- if isinstance(item, Msg):
- last_msg = item
- yield item
- except BaseException as e:
- has_error = True
- error_exc = e
- raise
- finally:
- reply_id = observed_reply_id or agent.state.reply_id
- if reply_id:
- span.set_attribute(
- SpanAttributes.AGENTSCOPE_REPLY_ID,
- reply_id,
- )
- if hitl_pending:
- span.set_attribute(
- SpanAttributes.AGENTSCOPE_HITL_PENDING_TOOLS,
- json.dumps(hitl_pending, ensure_ascii=False),
- )
- if external_pending:
- span.set_attribute(
- SpanAttributes.AGENTSCOPE_EXTERNAL_EXECUTION_PENDING_TOOLS, # noqa
- json.dumps(external_pending, ensure_ascii=False),
+ )
+ # Keep the reply span as parent for downstream spans, but do NOT hold
+ # it as the current OTel context across ``yield`` boundaries. The
+ # context is only attached while advancing ``next_handler`` and is
+ # detached again within the same ``__anext__`` step, so the token is
+ # always detached in the context it was created in even when this
+ # async generator is closed from another asyncio task (issue #2076).
+ span_context = otel_trace.set_span_in_context(span)
+
+ # Synthetic execute_tool spans for externally executed tools
+ event_arg = input_kwargs.get("inputs")
+ if isinstance(event_arg, ExternalExecutionResultEvent):
+ for result in event_arg.execution_results:
+ tool_attrs: dict[str, Any] = {
+ SpanAttributes.GEN_AI_OPERATION_NAME: (
+ OperationNameValues.EXECUTE_TOOL
+ ),
+ SpanAttributes.GEN_AI_TOOL_CALL_ID: result.id,
+ SpanAttributes.GEN_AI_TOOL_NAME: result.name,
+ SpanAttributes.AGENTSCOPE_IS_EXTERNAL_EXECUTION: (True),
+ **common_attrs,
+ }
+ if result.output is not None:
+ tool_attrs[
+ SpanAttributes.GEN_AI_TOOL_CALL_RESULT
+ ] = _serialize_to_str(result.output)
+ with tracer.start_as_current_span(
+ name=(
+ f"{OperationNameValues.EXECUTE_TOOL}" f" {result.name}"
+ ),
+ attributes=tool_attrs,
+ context=span_context,
+ ):
+ pass
+
+ has_error = False
+ error_exc: BaseException | None = None
+ last_msg: Msg | None = None
+ hitl_pending: list[str] = []
+ external_pending: list[str] = []
+ observed_reply_id: str | None = None
+
+ gen = next_handler(**input_kwargs)
+ try:
+ while True:
+ token = otel_context.attach(span_context)
+ try:
+ item = await anext(gen)
+ except StopAsyncIteration:
+ break
+ finally:
+ otel_context.detach(token)
+
+ if isinstance(item, ReplyStartEvent):
+ observed_reply_id = item.reply_id
+ elif isinstance(item, RequireUserConfirmEvent):
+ hitl_pending.extend(t.name for t in item.tool_calls)
+ elif isinstance(item, RequireExternalExecutionEvent):
+ external_pending.extend(t.name for t in item.tool_calls)
+ if isinstance(item, Msg):
+ last_msg = item
+ yield item
+ except BaseException as e:
+ has_error = True
+ error_exc = e
+ raise
+ finally:
+ reply_id = observed_reply_id or agent.state.reply_id
+ if reply_id:
+ span.set_attribute(
+ SpanAttributes.AGENTSCOPE_REPLY_ID,
+ reply_id,
+ )
+ if hitl_pending:
+ span.set_attribute(
+ SpanAttributes.AGENTSCOPE_HITL_PENDING_TOOLS,
+ json.dumps(hitl_pending, ensure_ascii=False),
+ )
+ if external_pending:
+ span.set_attribute(
+ SpanAttributes.AGENTSCOPE_EXTERNAL_EXECUTION_PENDING_TOOLS, # noqa
+ json.dumps(external_pending, ensure_ascii=False),
+ )
+ if has_error and error_exc is not None:
+ _set_span_error_status(span, error_exc)
+ else:
+ if last_msg is not None:
+ span.set_attributes(
+ _get_agent_response_attributes(last_msg),
)
- if has_error and error_exc is not None:
- _set_span_error_status(span, error_exc)
- else:
- if last_msg is not None:
- span.set_attributes(
- _get_agent_response_attributes(last_msg),
- )
- _set_span_success_status(span)
+ _set_span_success_status(span)
# ------------------------------------------------------------------
# on_model_call
@@ -321,28 +334,41 @@ class TracingMiddleware(MiddlewareBase):
)
span_name = _get_tool_span_name(request_attributes)
- with tracer.start_as_current_span(
+ span = tracer.start_span(
name=span_name,
attributes={
**request_attributes,
**_get_common_attributes(agent.state.session_id),
},
- end_on_exit=False,
- ) as span:
- has_error = False
- last_item = None
- try:
- async for item in next_handler(**input_kwargs):
- last_item = item
- yield item
- except BaseException as e:
- has_error = True
- _set_span_error_status(span, e)
- raise
- finally:
- if not has_error:
- if last_item is not None:
- span.set_attributes(
- _get_tool_response_attributes(last_item),
- )
- _set_span_success_status(span)
+ )
+ # See ``on_reply``: keep the span as parent for nested spans without
+ # holding it as the current context across ``yield`` boundaries, so a
+ # cross-task generator close does not raise OTel detach errors
+ # (issue #2076).
+ span_context = otel_trace.set_span_in_context(span)
+
+ has_error = False
+ last_item = None
+ gen = next_handler(**input_kwargs)
+ try:
+ while True:
+ token = otel_context.attach(span_context)
+ try:
+ item = await anext(gen)
+ except StopAsyncIteration:
+ break
+ finally:
+ otel_context.detach(token)
+ last_item = item
+ yield item
+ except BaseException as e:
+ has_error = True
+ _set_span_error_status(span, e)
+ raise
+ finally:
+ if not has_error:
+ if last_item is not None:
+ span.set_attributes(
+ _get_tool_response_attributes(last_item),
+ )
+ _set_span_success_status(span)
diff --git a/tests/tracing_test.py b/tests/tracing_test.py
index 47ed5999e..fefa14e0e 100644
--- a/tests/tracing_test.py
+++ b/tests/tracing_test.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
"""Unit tests for the tracing module using an in-memory OTel exporter."""
+import asyncio
import json
from typing import Any
from unittest.async_case import IsolatedAsyncioTestCase
@@ -847,3 +848,65 @@ class TracingTest(IsolatedAsyncioTestCase):
span_attrs.get("agentscope.agent.incoming_event_type"),
"external_execution_result",
)
+
+ # -----------------------------------------------------------------------
+ # Tests: streaming close from another asyncio task (issue #2076)
+ # -----------------------------------------------------------------------
+
+ async def _drive_then_close_from_other_task(self, gen: Any) -> None:
+ """Advance an async generator once, then close it from a *different*
+ asyncio task, reproducing the cross-context close in issue #2076."""
+ await anext(gen)
+
+ async def _close() -> None:
+ await gen.aclose()
+
+ await asyncio.create_task(_close())
+
+ async def test_on_reply_close_from_other_task_no_detach_error(
+ self,
+ ) -> None:
+ """on_reply must not emit OTel 'Failed to detach context' errors when
+ its stream is closed from another asyncio task (issue #2076)."""
+ middleware = TracingMiddleware()
+
+ async def next_handler(**_kwargs: Any) -> Any:
+ yield "chunk-1"
+ await asyncio.Event().wait() # suspend at the yield boundary
+
+ gen = middleware.on_reply(
+ self.agent,
+ {"inputs": UserMsg(name="user", content="hi")},
+ next_handler,
+ )
+ with self.assertNoLogs("opentelemetry.context", level="ERROR"):
+ await self._drive_then_close_from_other_task(gen)
+
+ async def test_on_acting_close_from_other_task_no_detach_error(
+ self,
+ ) -> None:
+ """on_acting must not emit OTel 'Failed to detach context' errors when
+ its stream is closed from another asyncio task (issue #2076)."""
+ middleware = TracingMiddleware()
+
+ async def next_handler(**_kwargs: Any) -> Any:
+ yield ToolResultBlock(
+ id="t1",
+ name="get_weather",
+ output="ok",
+ state=ToolResultState.SUCCESS,
+ )
+ await asyncio.Event().wait() # suspend at the yield boundary
+
+ tool_call = ToolCallBlock(
+ id="t1",
+ name="get_weather",
+ input=json.dumps({"city": "Hangzhou"}),
+ )
+ gen = middleware.on_acting(
+ self.agent,
+ {"tool_call": tool_call},
+ next_handler,
+ )
+ with self.assertNoLogs("opentelemetry.context", level="ERROR"):
+ await self._drive_then_close_from_other_task(gen)