Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(tracing): avoid otel context detach error on cross-task stream close (#2077)
天晟c5d26cd2465c
diff --git a/src/agentscope/middleware/_tracing/_trace.py b/src/agentscope/middleware/_tracing/_trace.pyindex d8c3fb3af..8a9ac7ef9 100644--- a/src/agentscope/middleware/_tracing/_trace.py+++ b/src/agentscope/middleware/_tracing/_trace.py@@ -13,6 +13,7 @@ from typing import (import aioitertools+from opentelemetry import context as otel_contextfrom opentelemetry import trace as otel_tracefrom opentelemetry.trace import StatusCode@@ -154,91 +155,103 @@ class TracingMiddleware(MiddlewareBase):)span_name = _get_agent_span_name(request_attributes)- with tracer.start_as_current_span(+ span = tracer.start_span(name=span_name,attributes={**request_attributes,**common_attrs,},- end_on_exit=False,- ) as span:- # Synthetic execute_tool spans for externally executed tools- event_arg = input_kwargs.get("inputs")- if isinstance(event_arg, ExternalExecutionResultEvent):- for result in event_arg.execution_results:- tool_attrs: dict[str, Any] = {- SpanAttributes.GEN_AI_OPERATION_NAME: (- OperationNameValues.EXECUTE_TOOL- ),- SpanAttributes.GEN_AI_TOOL_CALL_ID: result.id,- SpanAttributes.GEN_AI_TOOL_NAME: result.name,- SpanAttributes.AGENTSCOPE_IS_EXTERNAL_EXECUTION: (- True- ),- **common_attrs,- }- if result.output is not None:- tool_attrs[- SpanAttributes.GEN_AI_TOOL_CALL_RESULT- ] = _serialize_to_str(result.output)- with tracer.start_as_current_span(- name=(- f"{OperationNameValues.EXECUTE_TOOL}"- f" {result.name}"- ),- attributes=tool_attrs,- ):- pass-- has_error = False- error_exc: BaseException | None = None- last_msg: Msg | None = None- hitl_pending: list[str] = []- external_pending: list[str] = []- observed_reply_id: str | None = None-- try:- async for item in next_handler(**input_kwargs):- if isinstance(item, ReplyStartEvent):- observed_reply_id = item.reply_id- elif isinstance(item, RequireUserConfirmEvent):- hitl_pending.extend(t.name for t in item.tool_calls)- elif isinstance(item, RequireExternalExecutionEvent):- external_pending.extend(- t.name for t in item.tool_calls- )- if isinstance(item, Msg):- last_msg = item- yield item- except BaseException as e:- has_error = True- error_exc = e- raise- finally:- reply_id = observed_reply_id or agent.state.reply_id- if reply_id:- span.set_attribute(- SpanAttributes.AGENTSCOPE_REPLY_ID,- reply_id,- )- if hitl_pending:- span.set_attribute(- SpanAttributes.AGENTSCOPE_HITL_PENDING_TOOLS,- json.dumps(hitl_pending, ensure_ascii=False),- )- if external_pending:- span.set_attribute(- SpanAttributes.AGENTSCOPE_EXTERNAL_EXECUTION_PENDING_TOOLS, # noqa- json.dumps(external_pending, ensure_ascii=False),+ )+ # Keep the reply span as parent for downstream spans, but do NOT hold+ # it as the current OTel context across ``yield`` boundaries. The+ # context is only attached while advancing ``next_handler`` and is+ # detached again within the same ``__anext__`` step, so the token is+ # always detached in the context it was created in even when this+ # async generator is closed from another asyncio task (issue #2076).+ span_context = otel_trace.set_span_in_context(span)++ # Synthetic execute_tool spans for externally executed tools+ event_arg = input_kwargs.get("inputs")+ if isinstance(event_arg, ExternalExecutionResultEvent):+ for result in event_arg.execution_results:+ tool_attrs: dict[str, Any] = {+ SpanAttributes.GEN_AI_OPERATION_NAME: (+ OperationNameValues.EXECUTE_TOOL+ ),+ SpanAttributes.GEN_AI_TOOL_CALL_ID: result.id,+ SpanAttributes.GEN_AI_TOOL_NAME: result.name,+ SpanAttributes.AGENTSCOPE_IS_EXTERNAL_EXECUTION: (True),+ **common_attrs,+ }+ if result.output is not None:+ tool_attrs[+ SpanAttributes.GEN_AI_TOOL_CALL_RESULT+ ] = _serialize_to_str(result.output)+ with tracer.start_as_current_span(+ name=(+ f"{OperationNameValues.EXECUTE_TOOL}" f" {result.name}"+ ),+ attributes=tool_attrs,+ context=span_context,+ ):+ pass++ has_error = False+ error_exc: BaseException | None = None+ last_msg: Msg | None = None+ hitl_pending: list[str] = []+ external_pending: list[str] = []+ observed_reply_id: str | None = None++ gen = next_handler(**input_kwargs)+ try:+ while True:+ token = otel_context.attach(span_context)+ try:+ item = await anext(gen)+ except StopAsyncIteration:+ break+ finally:+ otel_context.detach(token)++ if isinstance(item, ReplyStartEvent):+ observed_reply_id = item.reply_id+ elif isinstance(item, RequireUserConfirmEvent):+ hitl_pending.extend(t.name for t in item.tool_calls)+ elif isinstance(item, RequireExternalExecutionEvent):+ external_pending.extend(t.name for t in item.tool_calls)+ if isinstance(item, Msg):+ last_msg = item+ yield item+ except BaseException as e:+ has_error = True+ error_exc = e+ raise+ finally:+ reply_id = observed_reply_id or agent.state.reply_id+ if reply_id:+ span.set_attribute(+ SpanAttributes.AGENTSCOPE_REPLY_ID,+ reply_id,+ )+ if hitl_pending:+ span.set_attribute(+ SpanAttributes.AGENTSCOPE_HITL_PENDING_TOOLS,+ json.dumps(hitl_pending, ensure_ascii=False),+ )+ if external_pending:+ span.set_attribute(+ SpanAttributes.AGENTSCOPE_EXTERNAL_EXECUTION_PENDING_TOOLS, # noqa+ json.dumps(external_pending, ensure_ascii=False),+ )+ if has_error and error_exc is not None:+ _set_span_error_status(span, error_exc)+ else:+ if last_msg is not None:+ span.set_attributes(+ _get_agent_response_attributes(last_msg),)- if has_error and error_exc is not None:- _set_span_error_status(span, error_exc)- else:- if last_msg is not None:- span.set_attributes(- _get_agent_response_attributes(last_msg),- )- _set_span_success_status(span)+ _set_span_success_status(span)# ------------------------------------------------------------------# on_model_call@@ -321,28 +334,41 @@ class TracingMiddleware(MiddlewareBase):)span_name = _get_tool_span_name(request_attributes)- with tracer.start_as_current_span(+ span = tracer.start_span(name=span_name,attributes={**request_attributes,**_get_common_attributes(agent.state.session_id),},- end_on_exit=False,- ) as span:- has_error = False- last_item = None- try:- async for item in next_handler(**input_kwargs):- last_item = item- yield item- except BaseException as e:- has_error = True- _set_span_error_status(span, e)- raise- finally:- if not has_error:- if last_item is not None:- span.set_attributes(- _get_tool_response_attributes(last_item),- )- _set_span_success_status(span)+ )+ # See ``on_reply``: keep the span as parent for nested spans without+ # holding it as the current context across ``yield`` boundaries, so a+ # cross-task generator close does not raise OTel detach errors+ # (issue #2076).+ span_context = otel_trace.set_span_in_context(span)++ has_error = False+ last_item = None+ gen = next_handler(**input_kwargs)+ try:+ while True:+ token = otel_context.attach(span_context)+ try:+ item = await anext(gen)+ except StopAsyncIteration:+ break+ finally:+ otel_context.detach(token)+ last_item = item+ yield item+ except BaseException as e:+ has_error = True+ _set_span_error_status(span, e)+ raise+ finally:+ if not has_error:+ if last_item is not None:+ span.set_attributes(+ _get_tool_response_attributes(last_item),+ )+ _set_span_success_status(span)diff --git a/tests/tracing_test.py b/tests/tracing_test.pyindex 47ed5999e..fefa14e0e 100644--- a/tests/tracing_test.py+++ b/tests/tracing_test.py@@ -1,5 +1,6 @@# -*- coding: utf-8 -*-"""Unit tests for the tracing module using an in-memory OTel exporter."""+import asyncioimport jsonfrom typing import Anyfrom unittest.async_case import IsolatedAsyncioTestCase@@ -847,3 +848,65 @@ class TracingTest(IsolatedAsyncioTestCase):span_attrs.get("agentscope.agent.incoming_event_type"),"external_execution_result",)++ # -----------------------------------------------------------------------+ # Tests: streaming close from another asyncio task (issue #2076)+ # -----------------------------------------------------------------------++ async def _drive_then_close_from_other_task(self, gen: Any) -> None:+ """Advance an async generator once, then close it from a *different*+ asyncio task, reproducing the cross-context close in issue #2076."""+ await anext(gen)++ async def _close() -> None:+ await gen.aclose()++ await asyncio.create_task(_close())++ async def test_on_reply_close_from_other_task_no_detach_error(+ self,+ ) -> None:+ """on_reply must not emit OTel 'Failed to detach context' errors when+ its stream is closed from another asyncio task (issue #2076)."""+ middleware = TracingMiddleware()++ async def next_handler(**_kwargs: Any) -> Any:+ yield "chunk-1"+ await asyncio.Event().wait() # suspend at the yield boundary++ gen = middleware.on_reply(+ self.agent,+ {"inputs": UserMsg(name="user", content="hi")},+ next_handler,+ )+ with self.assertNoLogs("opentelemetry.context", level="ERROR"):+ await self._drive_then_close_from_other_task(gen)++ async def test_on_acting_close_from_other_task_no_detach_error(+ self,+ ) -> None:+ """on_acting must not emit OTel 'Failed to detach context' errors when+ its stream is closed from another asyncio task (issue #2076)."""+ middleware = TracingMiddleware()++ async def next_handler(**_kwargs: Any) -> Any:+ yield ToolResultBlock(+ id="t1",+ name="get_weather",+ output="ok",+ state=ToolResultState.SUCCESS,+ )+ await asyncio.Event().wait() # suspend at the yield boundary++ tool_call = ToolCallBlock(+ id="t1",+ name="get_weather",+ input=json.dumps({"city": "Hangzhou"}),+ )+ gen = middleware.on_acting(+ self.agent,+ {"tool_call": tool_call},+ next_handler,+ )+ with self.assertNoLogs("opentelemetry.context", level="ERROR"):+ await self._drive_then_close_from_other_task(gen)