Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(tracing): isolate Celery contexts and guard unsampled tool spans (#45049)
zyssyz123a141540ea8e1
diff --git a/api/extensions/otel/celery_sqlcommenter.py b/api/extensions/otel/celery_sqlcommenter.pyindex 7a92f12b9c6..c9cd6cdb04c 100644--- a/api/extensions/otel/celery_sqlcommenter.py+++ b/api/extensions/otel/celery_sqlcommenter.py@@ -10,12 +10,15 @@ by opentelemetry.instrumentation.sqlcommenter_utils._add_framework_tags() when tSQLAlchemy instrumentor appends comments to SQL statements."""-import contextlibimport loggingfrom typing import Any, TypedDict+from celery.app.task import Taskfrom celery.signals import task_postrun, task_prerunfrom opentelemetry import context+from opentelemetry.instrumentation.celery import CeleryInstrumentor+from opentelemetry.metrics import MeterProvider+from opentelemetry.trace import TracerProviderfrom opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagatorlogger = logging.getLogger(__name__)@@ -79,7 +82,7 @@ def _get_traceparent() -> str | None:def _on_task_prerun(*args: object, **kwargs: object) -> None:task = kwargs.get("task")- if not task:+ if not isinstance(task, Task):returntags = _build_celery_sqlcommenter_tags(task)@@ -89,33 +92,41 @@ def _on_task_prerun(*args: object, **kwargs: object) -> None:current = context.get_current()new_ctx = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, tags, current)token = context.attach(new_ctx)- setattr(task, _TOKEN_ATTR, token)+ # Celery reuses one Task instance across concurrent invocations. Its request+ # is execution-local, so another invocation cannot overwrite this token.+ setattr(task.request, _TOKEN_ATTR, token)def _on_task_postrun(*args: object, **kwargs: object) -> None:task = kwargs.get("task")- if not task:+ if not isinstance(task, Task):return- token = getattr(task, _TOKEN_ATTR, None)+ request = task.request+ token = getattr(request, _TOKEN_ATTR, None)if token is None:return- try:- context.detach(token)- except Exception:- logger.debug("Failed to detach SQL commenter context", exc_info=True)- finally:- with contextlib.suppress(AttributeError):- delattr(task, _TOKEN_ATTR)+ delattr(request, _TOKEN_ATTR)+ context.detach(token)-def setup_celery_sqlcommenter() -> None:+def setup_celery_sqlcommenter(*, tracer_provider: TracerProvider, meter_provider: MeterProvider) -> None:"""- Connect Celery task_prerun and task_postrun handlers to inject SQL comment- context for worker queries. Call this from init_celery_worker after- CeleryInstrumentor().instrument() so our handlers run after the OTEL- instrumentor's and the trace context is already attached.+ Install Celery instrumentation with SQL comments inside its activation scope.++ Celery invokes signal receivers in registration order. The SQL context must+ attach after OTel's task span is active, then detach before OTel restores its+ parent. Registering both handlers after instrument() would restore an ended+ task span during postrun instead of leaving the caller's context intact."""- task_prerun.connect(_on_task_prerun, weak=False)+ instrumentor = CeleryInstrumentor()+ if instrumentor.is_instrumented_by_opentelemetry:+ # Auto-instrumentation may already own the signal order and providers.+ # Keep its spans intact rather than add a context we cannot unwind first.+ logger.warning("Celery is already instrumented; skipping SQL commenter signal registration")+ return+task_postrun.connect(_on_task_postrun, weak=False)+ instrumentor.instrument(tracer_provider=tracer_provider, meter_provider=meter_provider)+ task_prerun.connect(_on_task_prerun, weak=False)diff --git a/api/extensions/otel/runtime.py b/api/extensions/otel/runtime.pyindex 47751c8600d..321d84744d8 100644--- a/api/extensions/otel/runtime.py+++ b/api/extensions/otel/runtime.py@@ -87,7 +87,6 @@ def set_identity_span_attributes(*, tenant_id: str | None, user_id: str) -> None@worker_init.connect(weak=False)def init_celery_worker(*args: object, **kwargs: object):if dify_config.ENABLE_OTEL:- from opentelemetry.instrumentation.celery import CeleryInstrumentorfrom opentelemetry.metrics import get_meter_providerfrom opentelemetry.trace import get_tracer_provider@@ -97,8 +96,7 @@ def init_celery_worker(*args: object, **kwargs: object):metric_provider = get_meter_provider()if dify_config.DEBUG:logger.info("Initializing OpenTelemetry for Celery worker")- CeleryInstrumentor().instrument(tracer_provider=tracer_provider, meter_provider=metric_provider)- setup_celery_sqlcommenter()+ setup_celery_sqlcommenter(tracer_provider=tracer_provider, meter_provider=metric_provider)def is_instrument_flag_enabled() -> bool:diff --git a/api/tests/unit_tests/extensions/otel/test_celery_sqlcommenter.py b/api/tests/unit_tests/extensions/otel/test_celery_sqlcommenter.pyindex f2f2d57672a..1619fa6bf9a 100644--- a/api/tests/unit_tests/extensions/otel/test_celery_sqlcommenter.py+++ b/api/tests/unit_tests/extensions/otel/test_celery_sqlcommenter.py@@ -1,172 +1,307 @@-"""Tests for Celery SQL comment context injection."""+"""SQL comment scope through real Celery requests and OTel task instrumentation."""-from unittest.mock import MagicMock, patch+from __future__ import annotations-from opentelemetry import context+from collections.abc import Callable, Iterator+from concurrent.futures import ThreadPoolExecutor+from dataclasses import dataclass+from threading import Event+from typing import Protocol, cast+from uuid import uuid4+import pytest+from celery import Celery, Task+from celery.app.task import Context+from celery.result import EagerResult+from celery.signals import task_postrun, task_prerun+from opentelemetry import context, trace+from opentelemetry.instrumentation.celery import CeleryInstrumentor+from opentelemetry.sdk.metrics import MeterProvider+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from opentelemetry.sdk.trace.sampling import ALWAYS_OFF, ALWAYS_ON-class TestBuildCelerySqlcommenterTags:- """Tests for _build_celery_sqlcommenter_tags."""+from core.app.apps.exc import GenerateTaskStoppedError+from extensions.otel.celery_sqlcommenter import (+ _SQLCOMMENTER_CONTEXT_KEY,+ _TOKEN_ATTR,+ CelerySqlcommenterTagsDict,+ _build_celery_sqlcommenter_tags,+ _on_task_postrun,+ _on_task_prerun,+ setup_celery_sqlcommenter,+)- def test_includes_framework_and_task_name(self) -> None:- """Tags include celery framework version and task name."""- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags- task = MagicMock()- task.name = "tasks.async_workflow_tasks.execute_workflow_team"- task.request = MagicMock()- task.request.retries = 0- task.request.delivery_info = dict[str, object]()+class _DisconnectableSignal(Protocol):+ """Celery's signal stubs omit the public disconnect method."""- with patch("extensions.otel.celery_sqlcommenter._get_traceparent", return_value=None):- tags = _build_celery_sqlcommenter_tags(task)+ def disconnect(self, receiver: Callable[..., None]) -> bool: ...- assert "framework" in tags- assert tags["framework"].startswith("celery:")- assert tags["task_name"] == "tasks.async_workflow_tasks.execute_workflow_team"- def test_includes_celery_retries_when_nonzero(self) -> None:- """celery_retries is included when retries > 0."""- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags+@dataclass(frozen=True)+class _Invocation:+ task_object: int+ request: Context+ span_context: trace.SpanContext+ tags: CelerySqlcommenterTagsDict- task = MagicMock()- task.name = "tasks.my_task"- task.request = MagicMock()- task.request.retries = 3- task.request.delivery_info = dict[str, object]()- with patch("extensions.otel.celery_sqlcommenter._get_traceparent", return_value=None):- tags = _build_celery_sqlcommenter_tags(task)+def celery_task() -> Iterator[Task[..., object]]:+ app = Celery("sqlcommenter-unit-test", broker="memory://", set_as_current=False)- assert tags["celery_retries"] == 3+ @app.task(name=f"sqlcommenter.{uuid4()}")+ def unused() -> None:+ pass- def test_omits_celery_retries_when_zero(self) -> None:- """celery_retries is omitted when retries is 0."""- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags+ task = app.tasks[unused.name]+ task.push_request(id=str(uuid4()), retries=0, delivery_info={})+ try:+ yield task+ finally:+ task.pop_request()+ app.close()- task = MagicMock()- task.name = "tasks.my_task"- task.request = MagicMock()- task.request.retries = 0- task.request.delivery_info = dict[str, object]()- with patch("extensions.otel.celery_sqlcommenter._get_traceparent", return_value=None):- tags = _build_celery_sqlcommenter_tags(task)+def test_build_tags_includes_celery_framework_and_task_name(celery_task: Task[..., object]) -> None:+ tags = _build_celery_sqlcommenter_tags(celery_task)+ assert tags["framework"].startswith("celery:")+ assert tags["task_name"] == celery_task.name++[email protected]("retries", [0, 3])+def test_build_tags_includes_only_nonzero_retries(celery_task: Task[..., object], retries: int) -> None:+ celery_task.request.retries = retries++ tags = _build_celery_sqlcommenter_tags(celery_task)++ if retries:+ assert tags["celery_retries"] == retries+ else:assert "celery_retries" not in tags- def test_includes_routing_key_from_delivery_info(self) -> None:- """routing_key is included when present in delivery_info."""- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags- task = MagicMock()- task.name = "tasks.my_task"- task.request = MagicMock()- task.request.retries = 0- task.request.delivery_info = {"routing_key": "workflow_based_app_execution"}+def test_build_tags_includes_request_routing_key(celery_task: Task[..., object]) -> None:+ celery_task.request.delivery_info = {"routing_key": "workflow_based_app_execution"}- with patch("extensions.otel.celery_sqlcommenter._get_traceparent", return_value=None):- tags = _build_celery_sqlcommenter_tags(task)+ assert _build_celery_sqlcommenter_tags(celery_task)["routing_key"] == "workflow_based_app_execution"- assert tags["routing_key"] == "workflow_based_app_execution"- def test_includes_traceparent_when_available(self) -> None:- """traceparent is included when injectable from current context."""- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags+def test_build_tags_includes_current_traceparent(celery_task: Task[..., object]) -> None:+ span_context = trace.SpanContext(trace_id=0x123, span_id=0x456, is_remote=True, trace_flags=trace.TraceFlags(1))+ span = trace.NonRecordingSpan(span_context)- task = MagicMock()- task.name = "tasks.my_task"- task.request = MagicMock()- task.request.retries = 0- task.request.delivery_info = dict[str, object]()+ with trace.use_span(span):+ tags = _build_celery_sqlcommenter_tags(celery_task)- traceparent = "00-5db86c23fa8d05b67db315694b518684-737bbf30cdcda066-00"- with patch(- "extensions.otel.celery_sqlcommenter._get_traceparent",- return_value=traceparent,- ):- tags = _build_celery_sqlcommenter_tags(task)+ assert tags["traceparent"] == f"00-{span_context.trace_id:032x}-{span_context.span_id:016x}-01"- assert tags["traceparent"] == traceparent- def test_handles_task_without_request(self) -> None:- """Gracefully handles task without request attribute."""- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags+def test_handlers_restore_context_and_remove_request_token(celery_task: Task[..., object]) -> None:+ previous_context = context.get_current()- task = MagicMock()- task.name = "tasks.my_task"- del task.request+ _on_task_prerun(task=celery_task)+ assert hasattr(celery_task.request, _TOKEN_ATTR)+ assert not hasattr(celery_task, _TOKEN_ATTR)+ tags = context.get_value(_SQLCOMMENTER_CONTEXT_KEY)+ assert isinstance(tags, dict)+ assert tags["task_name"] == celery_task.name- with patch("extensions.otel.celery_sqlcommenter._get_traceparent", return_value=None):- tags = _build_celery_sqlcommenter_tags(task)+ _on_task_postrun(task=celery_task)+ _on_task_postrun(task=celery_task)- assert "framework" in tags- assert "task_name" in tags+ assert context.get_current() is previous_context+ assert not hasattr(celery_task.request, _TOKEN_ATTR)-class TestTaskPrerunPostrunHandlers:- """Tests for task_prerun and task_postrun signal handlers."""+def test_handlers_skip_missing_task_or_request_token(celery_task: Task[..., object]) -> None:+ previous_context = context.get_current()- def test_prerun_sets_context_postrun_detaches(self) -> None:- """task_prerun attaches SQLCOMMENTER context; task_postrun detaches it."""- from extensions.otel.celery_sqlcommenter import (- _SQLCOMMENTER_CONTEXT_KEY,- _TOKEN_ATTR,- _on_task_postrun,- _on_task_prerun,- )+ _on_task_prerun()+ _on_task_postrun()+ _on_task_postrun(task=celery_task)- clean_ctx = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, None)- token = context.attach(clean_ctx)- try:- task = MagicMock()- task.name = "tasks.async_workflow_tasks.execute_workflow_team"- task.request = MagicMock()- task.request.retries = 1- task.request.delivery_info = {"routing_key": "workflow_based_app_execution"}-- with patch(- "extensions.otel.celery_sqlcommenter._get_traceparent",- return_value="00-abc123-def456-00",- ):- _on_task_prerun(task=task)-- tags = context.get_value(_SQLCOMMENTER_CONTEXT_KEY)- assert isinstance(tags, dict)- assert tags["framework"].startswith("celery:")- assert tags["task_name"] == "tasks.async_workflow_tasks.execute_workflow_team"- assert tags["celery_retries"] == 1- assert tags["routing_key"] == "workflow_based_app_execution"- assert tags["traceparent"] == "00-abc123-def456-00"- assert hasattr(task, _TOKEN_ATTR)-- _on_task_postrun(task=task)-- tags_after = context.get_value(_SQLCOMMENTER_CONTEXT_KEY)- assert tags_after is None- assert not hasattr(task, _TOKEN_ATTR)- finally:- context.detach(token)+ assert context.get_current() is previous_context- def test_prerun_skips_when_no_task(self) -> None:- """prerun does nothing when task is missing from kwargs."""- from extensions.otel.celery_sqlcommenter import (- _SQLCOMMENTER_CONTEXT_KEY,- _on_task_prerun,- )- clean_ctx = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, None)- token = context.attach(clean_ctx)[email protected]("sampled", [False, True])[email protected]("outcome", ["success", "failure", "retry", "cancel"])[email protected]("finish_order", [(0, 1), (1, 0)])+def test_overlapping_task_invocations_restore_otel_parent(+ sampled: bool,+ outcome: str,+ finish_order: tuple[int, int],+ caplog: pytest.LogCaptureFixture,+) -> None:+ """Real eager task tracing includes signals, exception and retry handling.++ Two threads execute the same registered Task singleton. Each Celery request+ has its own token and incoming parent. Cancellation models the Workflow's+ cooperative stop returning through Celery postrun, not killing a process.+ """+ provider = TracerProvider(sampler=ALWAYS_ON if sampled else ALWAYS_OFF)+ exporter = InMemorySpanExporter()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ meter_provider = MeterProvider()+ setup_celery_sqlcommenter(tracer_provider=provider, meter_provider=meter_provider)+ app = Celery("sqlcommenter-concurrency-test", broker="memory://", set_as_current=False)+ started = [Event(), Event()]+ release = [Event(), Event()]+ attempts: dict[int, list[_Invocation]] = {0: [], 1: []}+ restored: dict[int, bool] = {}+ parents: dict[int, trace.SpanContext] = {}+ name = f"sqlcommenter.concurrent.{uuid4()}"++ @app.task(bind=True, name=name, max_retries=1)+ def execute_task(task: Task[..., object], marker: int) -> dict[str, str]:+ span = trace.get_current_span()+ attempts[marker].append(+ _Invocation(+ task_object=id(task),+ request=task.request,+ span_context=span.get_span_context(),+ tags=cast(CelerySqlcommenterTagsDict, context.get_value(_SQLCOMMENTER_CONTEXT_KEY)),+ )+ )+ if task.request.retries == 0:+ started[marker].set()+ assert release[marker].wait(timeout=10), "concurrent invocation was never released"+ if outcome == "failure":+ raise ValueError(f"synthetic failure {marker}")+ if outcome == "retry" and task.request.retries == 0:+ raise task.retry(countdown=0)+ if outcome == "cancel":+ try:+ raise GenerateTaskStoppedError("synthetic cooperative stop")+ except GenerateTaskStoppedError:+ return {"status": "stopped"}+ return {"status": "succeeded"}++ task = app.tasks[name]++ def invoke(marker: int) -> EagerResult[object]:+ parent = provider.get_tracer(__name__).start_span(f"caller-{marker}")+ parents[marker] = parent.get_span_context()+ baseline = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, {"caller": marker}, trace.set_span_in_context(parent))+ token = context.attach(baseline)try:- _on_task_prerun()- tags = context.get_value(_SQLCOMMENTER_CONTEXT_KEY)- assert tags is None+ result = task.apply(+ kwargs={"marker": marker}, task_id=str(uuid4()), throw=False, routing_key=f"route-{marker}"+ )+ restored[marker] = context.get_current() is baseline and trace.get_current_span() is parent+ assert context.get_value(_SQLCOMMENTER_CONTEXT_KEY) == {"caller": marker}+ return resultfinally:context.detach(token)-- def test_postrun_skips_when_no_token(self) -> None:- """postrun does nothing when task has no token (e.g. prerun was skipped)."""- from extensions.otel.celery_sqlcommenter import _on_task_postrun-- task = MagicMock()- _on_task_postrun(task=task)+ parent.end()++ try:+ with ThreadPoolExecutor(max_workers=2) as executor:+ futures = [executor.submit(invoke, 0)]+ try:+ assert started[0].wait(timeout=10)+ futures.append(executor.submit(invoke, 1))+ assert started[1].wait(timeout=10)+ for marker in finish_order:+ release[marker].set()+ result = futures[marker].result(timeout=10)+ if outcome == "failure":+ assert result.state == "FAILURE"+ assert isinstance(result.result, ValueError)+ assert str(result.result) == f"synthetic failure {marker}"+ else:+ assert result.state == "SUCCESS"+ assert result.result == {"status": "stopped" if outcome == "cancel" else "succeeded"}+ finally:+ for event in release:+ event.set()++ assert restored == {0: True, 1: True}+ assert not hasattr(task, _TOKEN_ATTR)+ assert parents[0].trace_id != parents[1].trace_id+ expected_attempts = 2 if outcome == "retry" else 1+ for marker, invocations in attempts.items():+ assert len(invocations) == expected_attempts+ for index, invocation in enumerate(invocations):+ span_context = invocation.span_context+ tags = invocation.tags+ assert invocation.task_object == id(task)+ assert not hasattr(invocation.request, _TOKEN_ATTR)+ assert span_context.trace_id == parents[marker].trace_id+ assert span_context.trace_flags.sampled is sampled+ assert tags["routing_key"] == f"route-{marker}"+ assert tags["traceparent"] == (+ f"00-{span_context.trace_id:032x}-{span_context.span_id:016x}-{int(span_context.trace_flags):02x}"+ )+ if index:+ assert tags["celery_retries"] == index+ else:+ assert "celery_retries" not in tags+ task_spans = [+ span+ for span in exporter.get_finished_spans()+ if span.attributes is not None and span.attributes.get("celery.task_name") == name+ ]+ assert len(task_spans) == (2 * expected_attempts if sampled else 0)+ for span in task_spans:+ assert span.parent is not None+ assert span.context is not None+ parent = next(parent for parent in parents.values() if parent.trace_id == span.context.trace_id)+ assert span.parent.span_id == parent.span_id+ assert "Failed to detach context" not in caplog.text+ finally:+ cast(_DisconnectableSignal, task_prerun).disconnect(_on_task_prerun)+ cast(_DisconnectableSignal, task_postrun).disconnect(_on_task_postrun)+ CeleryInstrumentor().uninstrument()+ app.close()+ provider.shutdown()+ meter_provider.shutdown()+++def test_external_instrumentation_keeps_its_provider_and_context(caplog: pytest.LogCaptureFixture) -> None:+ """An existing instrumentor has no public signal-ordering API to nest under."""+ external_provider = TracerProvider()+ external_exporter = InMemorySpanExporter()+ external_provider.add_span_processor(SimpleSpanProcessor(external_exporter))+ unused_provider = TracerProvider()+ unused_exporter = InMemorySpanExporter()+ unused_provider.add_span_processor(SimpleSpanProcessor(unused_exporter))+ meter_provider = MeterProvider()+ instrumentor = CeleryInstrumentor()+ instrumentor.instrument(tracer_provider=external_provider, meter_provider=meter_provider)+ setup_celery_sqlcommenter(tracer_provider=unused_provider, meter_provider=meter_provider)+ app = Celery("sqlcommenter-external-instrumentation", broker="memory://", set_as_current=False)+ baseline_tags = {"outer": "external-owner"}++ @app.task(name=f"sqlcommenter.external.{uuid4()}")+ def execute_task() -> None:+ assert context.get_value(_SQLCOMMENTER_CONTEXT_KEY) == baseline_tags+ assert not hasattr(app.tasks[execute_task.name].request, _TOKEN_ATTR)++ parent = external_provider.get_tracer(__name__).start_span("external-parent")+ baseline = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, baseline_tags, trace.set_span_in_context(parent))+ token = context.attach(baseline)+ try:+ result = execute_task.apply(throw=True)++ assert result.state == "SUCCESS"+ assert context.get_current() is baseline+ assert trace.get_current_span() is parent+ spans = external_exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].parent is not None+ assert spans[0].parent.span_id == parent.get_span_context().span_id+ assert not unused_exporter.get_finished_spans()+ assert "skipping SQL commenter signal registration" in caplog.text+ assert "Failed to detach context" not in caplog.text+ finally:+ context.detach(token)+ parent.end()+ instrumentor.uninstrument()+ app.close()+ external_provider.shutdown()+ unused_provider.shutdown()+ meter_provider.shutdown()diff --git a/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py b/dify-agent/src/dify_agent/layers/dify_core_tools/layer.pyindex b2a4b232a93..3cdc8d1c21c 100644--- a/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py+++ b/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py@@ -123,10 +123,13 @@ def _mark_tool_span_failed(ctx: RunContext[Deps], error: DifyCoreToolsClientErroand the instrumentation's content policy controls whether that text is traced."""span = get_current_span()+ # Logfire wraps unsampled NonRecordingSpan as ReadableSpan, so check recording+ # before accessing SDK attributes that the underlying span does not expose.+ if not span.is_recording():+ returnattributes = span.attributes if isinstance(span, ReadableSpan) else Noneif (- not span.is_recording()- or not attributes+ not attributesor attributes.get("gen_ai.operation.name") != "execute_tool"or ctx.tool_call_id is Noneor attributes.get("gen_ai.tool.call.id") != ctx.tool_call_iddiff --git a/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.pyindex dc0f7716253..1a6fc7ca6a2 100644--- a/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py+++ b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py@@ -6,10 +6,18 @@ from collections.abc import Iteratorimport httpximport logfireimport pytest-from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace import ReadableSpan, TracerProviderfrom opentelemetry.sdk.trace.export import SimpleSpanProcessorfrom opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter-from opentelemetry.trace import StatusCode+from opentelemetry.trace import (+ INVALID_SPAN,+ NonRecordingSpan,+ SpanContext,+ StatusCode,+ TraceFlags,+ set_span_in_context,+ use_span,+)from pydantic_ai import Agentfrom pydantic_ai.messages import ModelMessage, ModelRequest, ModelResponse, TextPart, ToolCallPart, ToolReturnPartfrom pydantic_ai.models.function import AgentInfo, FunctionModel@@ -186,3 +194,63 @@ async def test_uninstrumented_tool_failure_does_not_mark_an_unrelated_ambient_spassert spans[0].status.status_code is not StatusCode.ERRORfinally:provider.shutdown()++[email protected]("ambient_parent", ["none", "otel-unsampled", "logfire-unsampled"])[email protected]("outcome", ["success", "invocation", "timeout", "configuration"])+async def test_uninstrumented_tool_preserves_observations_under_nonrecording_parents(+ ambient_parent: str,+ outcome: str,+) -> None:+ platform_client = None+ parent = INVALID_SPAN+ if ambient_parent != "none":+ parent = NonRecordingSpan(SpanContext(trace_id=1, span_id=2, is_remote=True, trace_flags=TraceFlags(0)))+ if ambient_parent == "logfire-unsampled":+ platform_client = logfire.configure(+ local=True,+ send_to_logfire=False,+ console=False,+ metrics=False,+ inspect_arguments=False,+ )+ parent = (+ platform_client.config.get_tracer_provider()+ .get_tracer("platform")+ .start_span("Unsampled ambient", context=set_span_in_context(parent))+ )+ # Logfire advertises ReadableSpan while delegating to a non-recording+ # OTel span that has no attributes property.+ assert isinstance(parent, ReadableSpan)+ assert isinstance(parent.span, NonRecordingSpan)+ assert not parent.is_recording()+ calls: list[str] = []++ def handle(request: httpx.Request) -> httpx.Response:+ calls.append(request.url.path)+ if outcome == "timeout":+ raise httpx.ReadTimeout("private-transport-content", request=request)+ if outcome == "success":+ return httpx.Response(200, json={"observation": "successful observation"})+ return httpx.Response(422, json={"code": "agent_tool_invoke_failed", "message": "handled failure"})++ try:+ async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as client:+ agent = Agent(FunctionModel(tool_model), deps_type=Deps, toolsets=[Toolset("core_tools")])+ agent.instrument = False+ with use_span(parent, end_on_exit=True):+ result = await agent.run(+ "Call the tool", deps=dependencies(client, missing_context=outcome == "configuration")+ )+ expected = {+ "success": "successful observation",+ "invocation": "tool invoke error: handled failure",+ "timeout": "Tool is temporarily unavailable. Please continue without it if possible.",+ "configuration": "Tool is unavailable because required execution context is missing.",+ }+ assert result.output == f"Handled: {expected[outcome]}"+ assert len(calls) == (0 if outcome == "configuration" else 1)+ finally:+ if platform_client is not None:+ platform_client.shutdown()