Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(tracing): isolate Celery contexts and guard unsampled tool spans (#45049)

zyssyz123a141540ea8e1
diff --git a/api/extensions/otel/celery_sqlcommenter.py b/api/extensions/otel/celery_sqlcommenter.py
index 7a92f12b9c6..c9cd6cdb04c 100644
--- a/api/extensions/otel/celery_sqlcommenter.py
+++ b/api/extensions/otel/celery_sqlcommenter.py
@@ -10,12 +10,15 @@ by opentelemetry.instrumentation.sqlcommenter_utils._add_framework_tags() when t
SQLAlchemy instrumentor appends comments to SQL statements.
"""
-import contextlib
import logging
from typing import Any, TypedDict
+from celery.app.task import Task
from celery.signals import task_postrun, task_prerun
from opentelemetry import context
+from opentelemetry.instrumentation.celery import CeleryInstrumentor
+from opentelemetry.metrics import MeterProvider
+from opentelemetry.trace import TracerProvider
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
logger = logging.getLogger(__name__)
@@ -79,7 +82,7 @@ def _get_traceparent() -> str | None:
def _on_task_prerun(*args: object, **kwargs: object) -> None:
task = kwargs.get("task")
- if not task:
+ if not isinstance(task, Task):
return
tags = _build_celery_sqlcommenter_tags(task)
@@ -89,33 +92,41 @@ def _on_task_prerun(*args: object, **kwargs: object) -> None:
current = context.get_current()
new_ctx = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, tags, current)
token = context.attach(new_ctx)
- setattr(task, _TOKEN_ATTR, token)
+ # Celery reuses one Task instance across concurrent invocations. Its request
+ # is execution-local, so another invocation cannot overwrite this token.
+ setattr(task.request, _TOKEN_ATTR, token)
def _on_task_postrun(*args: object, **kwargs: object) -> None:
task = kwargs.get("task")
- if not task:
+ if not isinstance(task, Task):
return
- token = getattr(task, _TOKEN_ATTR, None)
+ request = task.request
+ token = getattr(request, _TOKEN_ATTR, None)
if token is None:
return
- try:
- context.detach(token)
- except Exception:
- logger.debug("Failed to detach SQL commenter context", exc_info=True)
- finally:
- with contextlib.suppress(AttributeError):
- delattr(task, _TOKEN_ATTR)
+ delattr(request, _TOKEN_ATTR)
+ context.detach(token)
-def setup_celery_sqlcommenter() -> None:
+def setup_celery_sqlcommenter(*, tracer_provider: TracerProvider, meter_provider: MeterProvider) -> None:
"""
- Connect Celery task_prerun and task_postrun handlers to inject SQL comment
- context for worker queries. Call this from init_celery_worker after
- CeleryInstrumentor().instrument() so our handlers run after the OTEL
- instrumentor's and the trace context is already attached.
+ Install Celery instrumentation with SQL comments inside its activation scope.
+
+ Celery invokes signal receivers in registration order. The SQL context must
+ attach after OTel's task span is active, then detach before OTel restores its
+ parent. Registering both handlers after instrument() would restore an ended
+ task span during postrun instead of leaving the caller's context intact.
"""
- task_prerun.connect(_on_task_prerun, weak=False)
+ instrumentor = CeleryInstrumentor()
+ if instrumentor.is_instrumented_by_opentelemetry:
+ # Auto-instrumentation may already own the signal order and providers.
+ # Keep its spans intact rather than add a context we cannot unwind first.
+ logger.warning("Celery is already instrumented; skipping SQL commenter signal registration")
+ return
+
task_postrun.connect(_on_task_postrun, weak=False)
+ instrumentor.instrument(tracer_provider=tracer_provider, meter_provider=meter_provider)
+ task_prerun.connect(_on_task_prerun, weak=False)
diff --git a/api/extensions/otel/runtime.py b/api/extensions/otel/runtime.py
index 47751c8600d..321d84744d8 100644
--- a/api/extensions/otel/runtime.py
+++ b/api/extensions/otel/runtime.py
@@ -87,7 +87,6 @@ def set_identity_span_attributes(*, tenant_id: str | None, user_id: str) -> None
@worker_init.connect(weak=False)
def init_celery_worker(*args: object, **kwargs: object):
if dify_config.ENABLE_OTEL:
- from opentelemetry.instrumentation.celery import CeleryInstrumentor
from opentelemetry.metrics import get_meter_provider
from opentelemetry.trace import get_tracer_provider
@@ -97,8 +96,7 @@ def init_celery_worker(*args: object, **kwargs: object):
metric_provider = get_meter_provider()
if dify_config.DEBUG:
logger.info("Initializing OpenTelemetry for Celery worker")
- CeleryInstrumentor().instrument(tracer_provider=tracer_provider, meter_provider=metric_provider)
- setup_celery_sqlcommenter()
+ setup_celery_sqlcommenter(tracer_provider=tracer_provider, meter_provider=metric_provider)
def is_instrument_flag_enabled() -> bool:
diff --git a/api/tests/unit_tests/extensions/otel/test_celery_sqlcommenter.py b/api/tests/unit_tests/extensions/otel/test_celery_sqlcommenter.py
index f2f2d57672a..1619fa6bf9a 100644
--- a/api/tests/unit_tests/extensions/otel/test_celery_sqlcommenter.py
+++ b/api/tests/unit_tests/extensions/otel/test_celery_sqlcommenter.py
@@ -1,172 +1,307 @@
-"""Tests for Celery SQL comment context injection."""
+"""SQL comment scope through real Celery requests and OTel task instrumentation."""
-from unittest.mock import MagicMock, patch
+from __future__ import annotations
-from opentelemetry import context
+from collections.abc import Callable, Iterator
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from threading import Event
+from typing import Protocol, cast
+from uuid import uuid4
+import pytest
+from celery import Celery, Task
+from celery.app.task import Context
+from celery.result import EagerResult
+from celery.signals import task_postrun, task_prerun
+from opentelemetry import context, trace
+from opentelemetry.instrumentation.celery import CeleryInstrumentor
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.sdk.trace.sampling import ALWAYS_OFF, ALWAYS_ON
-class TestBuildCelerySqlcommenterTags:
- """Tests for _build_celery_sqlcommenter_tags."""
+from core.app.apps.exc import GenerateTaskStoppedError
+from extensions.otel.celery_sqlcommenter import (
+ _SQLCOMMENTER_CONTEXT_KEY,
+ _TOKEN_ATTR,
+ CelerySqlcommenterTagsDict,
+ _build_celery_sqlcommenter_tags,
+ _on_task_postrun,
+ _on_task_prerun,
+ setup_celery_sqlcommenter,
+)
- def test_includes_framework_and_task_name(self) -> None:
- """Tags include celery framework version and task name."""
- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags
- task = MagicMock()
- task.name = "tasks.async_workflow_tasks.execute_workflow_team"
- task.request = MagicMock()
- task.request.retries = 0
- task.request.delivery_info = dict[str, object]()
+class _DisconnectableSignal(Protocol):
+ """Celery's signal stubs omit the public disconnect method."""
- with patch("extensions.otel.celery_sqlcommenter._get_traceparent", return_value=None):
- tags = _build_celery_sqlcommenter_tags(task)
+ def disconnect(self, receiver: Callable[..., None]) -> bool: ...
- assert "framework" in tags
- assert tags["framework"].startswith("celery:")
- assert tags["task_name"] == "tasks.async_workflow_tasks.execute_workflow_team"
- def test_includes_celery_retries_when_nonzero(self) -> None:
- """celery_retries is included when retries > 0."""
- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags
+@dataclass(frozen=True)
+class _Invocation:
+ task_object: int
+ request: Context
+ span_context: trace.SpanContext
+ tags: CelerySqlcommenterTagsDict
- task = MagicMock()
- task.name = "tasks.my_task"
- task.request = MagicMock()
- task.request.retries = 3
- task.request.delivery_info = dict[str, object]()
- with patch("extensions.otel.celery_sqlcommenter._get_traceparent", return_value=None):
- tags = _build_celery_sqlcommenter_tags(task)
+def celery_task() -> Iterator[Task[..., object]]:
+ app = Celery("sqlcommenter-unit-test", broker="memory://", set_as_current=False)
- assert tags["celery_retries"] == 3
+ @app.task(name=f"sqlcommenter.{uuid4()}")
+ def unused() -> None:
+ pass
- def test_omits_celery_retries_when_zero(self) -> None:
- """celery_retries is omitted when retries is 0."""
- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags
+ task = app.tasks[unused.name]
+ task.push_request(id=str(uuid4()), retries=0, delivery_info={})
+ try:
+ yield task
+ finally:
+ task.pop_request()
+ app.close()
- task = MagicMock()
- task.name = "tasks.my_task"
- task.request = MagicMock()
- task.request.retries = 0
- task.request.delivery_info = dict[str, object]()
- with patch("extensions.otel.celery_sqlcommenter._get_traceparent", return_value=None):
- tags = _build_celery_sqlcommenter_tags(task)
+def test_build_tags_includes_celery_framework_and_task_name(celery_task: Task[..., object]) -> None:
+ tags = _build_celery_sqlcommenter_tags(celery_task)
+ assert tags["framework"].startswith("celery:")
+ assert tags["task_name"] == celery_task.name
+
+
[email protected]("retries", [0, 3])
+def test_build_tags_includes_only_nonzero_retries(celery_task: Task[..., object], retries: int) -> None:
+ celery_task.request.retries = retries
+
+ tags = _build_celery_sqlcommenter_tags(celery_task)
+
+ if retries:
+ assert tags["celery_retries"] == retries
+ else:
assert "celery_retries" not in tags
- def test_includes_routing_key_from_delivery_info(self) -> None:
- """routing_key is included when present in delivery_info."""
- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags
- task = MagicMock()
- task.name = "tasks.my_task"
- task.request = MagicMock()
- task.request.retries = 0
- task.request.delivery_info = {"routing_key": "workflow_based_app_execution"}
+def test_build_tags_includes_request_routing_key(celery_task: Task[..., object]) -> None:
+ celery_task.request.delivery_info = {"routing_key": "workflow_based_app_execution"}
- with patch("extensions.otel.celery_sqlcommenter._get_traceparent", return_value=None):
- tags = _build_celery_sqlcommenter_tags(task)
+ assert _build_celery_sqlcommenter_tags(celery_task)["routing_key"] == "workflow_based_app_execution"
- assert tags["routing_key"] == "workflow_based_app_execution"
- def test_includes_traceparent_when_available(self) -> None:
- """traceparent is included when injectable from current context."""
- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags
+def test_build_tags_includes_current_traceparent(celery_task: Task[..., object]) -> None:
+ span_context = trace.SpanContext(trace_id=0x123, span_id=0x456, is_remote=True, trace_flags=trace.TraceFlags(1))
+ span = trace.NonRecordingSpan(span_context)
- task = MagicMock()
- task.name = "tasks.my_task"
- task.request = MagicMock()
- task.request.retries = 0
- task.request.delivery_info = dict[str, object]()
+ with trace.use_span(span):
+ tags = _build_celery_sqlcommenter_tags(celery_task)
- traceparent = "00-5db86c23fa8d05b67db315694b518684-737bbf30cdcda066-00"
- with patch(
- "extensions.otel.celery_sqlcommenter._get_traceparent",
- return_value=traceparent,
- ):
- tags = _build_celery_sqlcommenter_tags(task)
+ assert tags["traceparent"] == f"00-{span_context.trace_id:032x}-{span_context.span_id:016x}-01"
- assert tags["traceparent"] == traceparent
- def test_handles_task_without_request(self) -> None:
- """Gracefully handles task without request attribute."""
- from extensions.otel.celery_sqlcommenter import _build_celery_sqlcommenter_tags
+def test_handlers_restore_context_and_remove_request_token(celery_task: Task[..., object]) -> None:
+ previous_context = context.get_current()
- task = MagicMock()
- task.name = "tasks.my_task"
- del task.request
+ _on_task_prerun(task=celery_task)
+ assert hasattr(celery_task.request, _TOKEN_ATTR)
+ assert not hasattr(celery_task, _TOKEN_ATTR)
+ tags = context.get_value(_SQLCOMMENTER_CONTEXT_KEY)
+ assert isinstance(tags, dict)
+ assert tags["task_name"] == celery_task.name
- with patch("extensions.otel.celery_sqlcommenter._get_traceparent", return_value=None):
- tags = _build_celery_sqlcommenter_tags(task)
+ _on_task_postrun(task=celery_task)
+ _on_task_postrun(task=celery_task)
- assert "framework" in tags
- assert "task_name" in tags
+ assert context.get_current() is previous_context
+ assert not hasattr(celery_task.request, _TOKEN_ATTR)
-class TestTaskPrerunPostrunHandlers:
- """Tests for task_prerun and task_postrun signal handlers."""
+def test_handlers_skip_missing_task_or_request_token(celery_task: Task[..., object]) -> None:
+ previous_context = context.get_current()
- def test_prerun_sets_context_postrun_detaches(self) -> None:
- """task_prerun attaches SQLCOMMENTER context; task_postrun detaches it."""
- from extensions.otel.celery_sqlcommenter import (
- _SQLCOMMENTER_CONTEXT_KEY,
- _TOKEN_ATTR,
- _on_task_postrun,
- _on_task_prerun,
- )
+ _on_task_prerun()
+ _on_task_postrun()
+ _on_task_postrun(task=celery_task)
- clean_ctx = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, None)
- token = context.attach(clean_ctx)
- try:
- task = MagicMock()
- task.name = "tasks.async_workflow_tasks.execute_workflow_team"
- task.request = MagicMock()
- task.request.retries = 1
- task.request.delivery_info = {"routing_key": "workflow_based_app_execution"}
-
- with patch(
- "extensions.otel.celery_sqlcommenter._get_traceparent",
- return_value="00-abc123-def456-00",
- ):
- _on_task_prerun(task=task)
-
- tags = context.get_value(_SQLCOMMENTER_CONTEXT_KEY)
- assert isinstance(tags, dict)
- assert tags["framework"].startswith("celery:")
- assert tags["task_name"] == "tasks.async_workflow_tasks.execute_workflow_team"
- assert tags["celery_retries"] == 1
- assert tags["routing_key"] == "workflow_based_app_execution"
- assert tags["traceparent"] == "00-abc123-def456-00"
- assert hasattr(task, _TOKEN_ATTR)
-
- _on_task_postrun(task=task)
-
- tags_after = context.get_value(_SQLCOMMENTER_CONTEXT_KEY)
- assert tags_after is None
- assert not hasattr(task, _TOKEN_ATTR)
- finally:
- context.detach(token)
+ assert context.get_current() is previous_context
- def test_prerun_skips_when_no_task(self) -> None:
- """prerun does nothing when task is missing from kwargs."""
- from extensions.otel.celery_sqlcommenter import (
- _SQLCOMMENTER_CONTEXT_KEY,
- _on_task_prerun,
- )
- clean_ctx = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, None)
- token = context.attach(clean_ctx)
[email protected]("sampled", [False, True])
[email protected]("outcome", ["success", "failure", "retry", "cancel"])
[email protected]("finish_order", [(0, 1), (1, 0)])
+def test_overlapping_task_invocations_restore_otel_parent(
+ sampled: bool,
+ outcome: str,
+ finish_order: tuple[int, int],
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """Real eager task tracing includes signals, exception and retry handling.
+
+ Two threads execute the same registered Task singleton. Each Celery request
+ has its own token and incoming parent. Cancellation models the Workflow's
+ cooperative stop returning through Celery postrun, not killing a process.
+ """
+ provider = TracerProvider(sampler=ALWAYS_ON if sampled else ALWAYS_OFF)
+ exporter = InMemorySpanExporter()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ meter_provider = MeterProvider()
+ setup_celery_sqlcommenter(tracer_provider=provider, meter_provider=meter_provider)
+ app = Celery("sqlcommenter-concurrency-test", broker="memory://", set_as_current=False)
+ started = [Event(), Event()]
+ release = [Event(), Event()]
+ attempts: dict[int, list[_Invocation]] = {0: [], 1: []}
+ restored: dict[int, bool] = {}
+ parents: dict[int, trace.SpanContext] = {}
+ name = f"sqlcommenter.concurrent.{uuid4()}"
+
+ @app.task(bind=True, name=name, max_retries=1)
+ def execute_task(task: Task[..., object], marker: int) -> dict[str, str]:
+ span = trace.get_current_span()
+ attempts[marker].append(
+ _Invocation(
+ task_object=id(task),
+ request=task.request,
+ span_context=span.get_span_context(),
+ tags=cast(CelerySqlcommenterTagsDict, context.get_value(_SQLCOMMENTER_CONTEXT_KEY)),
+ )
+ )
+ if task.request.retries == 0:
+ started[marker].set()
+ assert release[marker].wait(timeout=10), "concurrent invocation was never released"
+ if outcome == "failure":
+ raise ValueError(f"synthetic failure {marker}")
+ if outcome == "retry" and task.request.retries == 0:
+ raise task.retry(countdown=0)
+ if outcome == "cancel":
+ try:
+ raise GenerateTaskStoppedError("synthetic cooperative stop")
+ except GenerateTaskStoppedError:
+ return {"status": "stopped"}
+ return {"status": "succeeded"}
+
+ task = app.tasks[name]
+
+ def invoke(marker: int) -> EagerResult[object]:
+ parent = provider.get_tracer(__name__).start_span(f"caller-{marker}")
+ parents[marker] = parent.get_span_context()
+ baseline = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, {"caller": marker}, trace.set_span_in_context(parent))
+ token = context.attach(baseline)
try:
- _on_task_prerun()
- tags = context.get_value(_SQLCOMMENTER_CONTEXT_KEY)
- assert tags is None
+ result = task.apply(
+ kwargs={"marker": marker}, task_id=str(uuid4()), throw=False, routing_key=f"route-{marker}"
+ )
+ restored[marker] = context.get_current() is baseline and trace.get_current_span() is parent
+ assert context.get_value(_SQLCOMMENTER_CONTEXT_KEY) == {"caller": marker}
+ return result
finally:
context.detach(token)
-
- def test_postrun_skips_when_no_token(self) -> None:
- """postrun does nothing when task has no token (e.g. prerun was skipped)."""
- from extensions.otel.celery_sqlcommenter import _on_task_postrun
-
- task = MagicMock()
- _on_task_postrun(task=task)
+ parent.end()
+
+ try:
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ futures = [executor.submit(invoke, 0)]
+ try:
+ assert started[0].wait(timeout=10)
+ futures.append(executor.submit(invoke, 1))
+ assert started[1].wait(timeout=10)
+ for marker in finish_order:
+ release[marker].set()
+ result = futures[marker].result(timeout=10)
+ if outcome == "failure":
+ assert result.state == "FAILURE"
+ assert isinstance(result.result, ValueError)
+ assert str(result.result) == f"synthetic failure {marker}"
+ else:
+ assert result.state == "SUCCESS"
+ assert result.result == {"status": "stopped" if outcome == "cancel" else "succeeded"}
+ finally:
+ for event in release:
+ event.set()
+
+ assert restored == {0: True, 1: True}
+ assert not hasattr(task, _TOKEN_ATTR)
+ assert parents[0].trace_id != parents[1].trace_id
+ expected_attempts = 2 if outcome == "retry" else 1
+ for marker, invocations in attempts.items():
+ assert len(invocations) == expected_attempts
+ for index, invocation in enumerate(invocations):
+ span_context = invocation.span_context
+ tags = invocation.tags
+ assert invocation.task_object == id(task)
+ assert not hasattr(invocation.request, _TOKEN_ATTR)
+ assert span_context.trace_id == parents[marker].trace_id
+ assert span_context.trace_flags.sampled is sampled
+ assert tags["routing_key"] == f"route-{marker}"
+ assert tags["traceparent"] == (
+ f"00-{span_context.trace_id:032x}-{span_context.span_id:016x}-{int(span_context.trace_flags):02x}"
+ )
+ if index:
+ assert tags["celery_retries"] == index
+ else:
+ assert "celery_retries" not in tags
+ task_spans = [
+ span
+ for span in exporter.get_finished_spans()
+ if span.attributes is not None and span.attributes.get("celery.task_name") == name
+ ]
+ assert len(task_spans) == (2 * expected_attempts if sampled else 0)
+ for span in task_spans:
+ assert span.parent is not None
+ assert span.context is not None
+ parent = next(parent for parent in parents.values() if parent.trace_id == span.context.trace_id)
+ assert span.parent.span_id == parent.span_id
+ assert "Failed to detach context" not in caplog.text
+ finally:
+ cast(_DisconnectableSignal, task_prerun).disconnect(_on_task_prerun)
+ cast(_DisconnectableSignal, task_postrun).disconnect(_on_task_postrun)
+ CeleryInstrumentor().uninstrument()
+ app.close()
+ provider.shutdown()
+ meter_provider.shutdown()
+
+
+def test_external_instrumentation_keeps_its_provider_and_context(caplog: pytest.LogCaptureFixture) -> None:
+ """An existing instrumentor has no public signal-ordering API to nest under."""
+ external_provider = TracerProvider()
+ external_exporter = InMemorySpanExporter()
+ external_provider.add_span_processor(SimpleSpanProcessor(external_exporter))
+ unused_provider = TracerProvider()
+ unused_exporter = InMemorySpanExporter()
+ unused_provider.add_span_processor(SimpleSpanProcessor(unused_exporter))
+ meter_provider = MeterProvider()
+ instrumentor = CeleryInstrumentor()
+ instrumentor.instrument(tracer_provider=external_provider, meter_provider=meter_provider)
+ setup_celery_sqlcommenter(tracer_provider=unused_provider, meter_provider=meter_provider)
+ app = Celery("sqlcommenter-external-instrumentation", broker="memory://", set_as_current=False)
+ baseline_tags = {"outer": "external-owner"}
+
+ @app.task(name=f"sqlcommenter.external.{uuid4()}")
+ def execute_task() -> None:
+ assert context.get_value(_SQLCOMMENTER_CONTEXT_KEY) == baseline_tags
+ assert not hasattr(app.tasks[execute_task.name].request, _TOKEN_ATTR)
+
+ parent = external_provider.get_tracer(__name__).start_span("external-parent")
+ baseline = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, baseline_tags, trace.set_span_in_context(parent))
+ token = context.attach(baseline)
+ try:
+ result = execute_task.apply(throw=True)
+
+ assert result.state == "SUCCESS"
+ assert context.get_current() is baseline
+ assert trace.get_current_span() is parent
+ spans = external_exporter.get_finished_spans()
+ assert len(spans) == 1
+ assert spans[0].parent is not None
+ assert spans[0].parent.span_id == parent.get_span_context().span_id
+ assert not unused_exporter.get_finished_spans()
+ assert "skipping SQL commenter signal registration" in caplog.text
+ assert "Failed to detach context" not in caplog.text
+ finally:
+ context.detach(token)
+ parent.end()
+ instrumentor.uninstrument()
+ app.close()
+ external_provider.shutdown()
+ unused_provider.shutdown()
+ meter_provider.shutdown()
diff --git a/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py b/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py
index b2a4b232a93..3cdc8d1c21c 100644
--- a/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py
+++ b/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py
@@ -123,10 +123,13 @@ def _mark_tool_span_failed(ctx: RunContext[Deps], error: DifyCoreToolsClientErro
and the instrumentation's content policy controls whether that text is traced.
"""
span = get_current_span()
+ # Logfire wraps unsampled NonRecordingSpan as ReadableSpan, so check recording
+ # before accessing SDK attributes that the underlying span does not expose.
+ if not span.is_recording():
+ return
attributes = span.attributes if isinstance(span, ReadableSpan) else None
if (
- not span.is_recording()
- or not attributes
+ not attributes
or attributes.get("gen_ai.operation.name") != "execute_tool"
or ctx.tool_call_id is None
or attributes.get("gen_ai.tool.call.id") != ctx.tool_call_id
diff --git a/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py
index dc0f7716253..1a6fc7ca6a2 100644
--- a/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py
+++ b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py
@@ -6,10 +6,18 @@ from collections.abc import Iterator
import httpx
import logfire
import pytest
-from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
-from opentelemetry.trace import StatusCode
+from opentelemetry.trace import (
+ INVALID_SPAN,
+ NonRecordingSpan,
+ SpanContext,
+ StatusCode,
+ TraceFlags,
+ set_span_in_context,
+ use_span,
+)
from pydantic_ai import Agent
from pydantic_ai.messages import ModelMessage, ModelRequest, ModelResponse, TextPart, ToolCallPart, ToolReturnPart
from pydantic_ai.models.function import AgentInfo, FunctionModel
@@ -186,3 +194,63 @@ async def test_uninstrumented_tool_failure_does_not_mark_an_unrelated_ambient_sp
assert spans[0].status.status_code is not StatusCode.ERROR
finally:
provider.shutdown()
+
+
[email protected]("ambient_parent", ["none", "otel-unsampled", "logfire-unsampled"])
[email protected]("outcome", ["success", "invocation", "timeout", "configuration"])
+async def test_uninstrumented_tool_preserves_observations_under_nonrecording_parents(
+ ambient_parent: str,
+ outcome: str,
+) -> None:
+ platform_client = None
+ parent = INVALID_SPAN
+ if ambient_parent != "none":
+ parent = NonRecordingSpan(SpanContext(trace_id=1, span_id=2, is_remote=True, trace_flags=TraceFlags(0)))
+ if ambient_parent == "logfire-unsampled":
+ platform_client = logfire.configure(
+ local=True,
+ send_to_logfire=False,
+ console=False,
+ metrics=False,
+ inspect_arguments=False,
+ )
+ parent = (
+ platform_client.config.get_tracer_provider()
+ .get_tracer("platform")
+ .start_span("Unsampled ambient", context=set_span_in_context(parent))
+ )
+ # Logfire advertises ReadableSpan while delegating to a non-recording
+ # OTel span that has no attributes property.
+ assert isinstance(parent, ReadableSpan)
+ assert isinstance(parent.span, NonRecordingSpan)
+ assert not parent.is_recording()
+ calls: list[str] = []
+
+ def handle(request: httpx.Request) -> httpx.Response:
+ calls.append(request.url.path)
+ if outcome == "timeout":
+ raise httpx.ReadTimeout("private-transport-content", request=request)
+ if outcome == "success":
+ return httpx.Response(200, json={"observation": "successful observation"})
+ return httpx.Response(422, json={"code": "agent_tool_invoke_failed", "message": "handled failure"})
+
+ try:
+ async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as client:
+ agent = Agent(FunctionModel(tool_model), deps_type=Deps, toolsets=[Toolset("core_tools")])
+ agent.instrument = False
+ with use_span(parent, end_on_exit=True):
+ result = await agent.run(
+ "Call the tool", deps=dependencies(client, missing_context=outcome == "configuration")
+ )
+ expected = {
+ "success": "successful observation",
+ "invocation": "tool invoke error: handled failure",
+ "timeout": "Tool is temporarily unavailable. Please continue without it if possible.",
+ "configuration": "Tool is unavailable because required execution context is missing.",
+ }
+ assert result.output == f"Handled: {expected[outcome]}"
+ assert len(calls) == (0 if outcome == "configuration" else 1)
+ finally:
+ if platform_client is not None:
+ platform_client.shutdown()