Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
from __future__ import annotations
import asyncio
import logging
from collections.abc import Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, Literal, override
from opentelemetry.context import Context
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.trace import (
INVALID_SPAN,
Link,
NonRecordingSpan,
Span,
SpanContext,
SpanKind,
Status,
StatusCode,
TraceFlags,
Tracer,
TracerProvider,
get_current_span,
set_span_in_context,
)
from opentelemetry.util.types import Attributes
from dify_agent.layers.execution_context.configs import ApplicationTracingContext, DifyExecutionContextLayerConfig
from dify_agent.runtime.application_trace_exporter import (
ApplicationBatchSpanProcessor,
DIFY_TRACING_INVOCATION_ID_ATTRIBUTE,
DIFY_TRACING_RUN_ID_ATTRIBUTE,
JSONSpanExporter,
)
if TYPE_CHECKING:
from logfire import Logfire
from pydantic_ai import Agent
DIFY_TRACE_ID_ATTRIBUTE: Final[str] = "dify.trace_id"
DIFY_TENANT_ID_ATTRIBUTE: Final[str] = "dify.tenant_id"
DIFY_APP_ID_ATTRIBUTE: Final[str] = "dify.app_id"
DIFY_AGENT_ID_ATTRIBUTE: Final[str] = "dify.agent_id"
DIFY_INVOKE_FROM_ATTRIBUTE: Final[str] = "dify.invoke_from"
DIFY_CONVERSATION_ID_ATTRIBUTE: Final[str] = "dify.conversation.id"
DIFY_WORKFLOW_ID_ATTRIBUTE: Final[str] = "dify.workflow.id"
DIFY_WORKFLOW_RUN_ID_ATTRIBUTE: Final[str] = "dify.workflow.run_id"
DIFY_NODE_ID_ATTRIBUTE: Final[str] = "dify.node.id"
DIFY_NODE_EXECUTION_ID_ATTRIBUTE: Final[str] = "dify.node.execution_id"
GEN_AI_USER_ID_ATTRIBUTE: Final[str] = "gen_ai.user.id"
logger = logging.getLogger(__name__)
def dify_run_attributes(execution_context: DifyExecutionContextLayerConfig) -> tuple[tuple[str, str], ...]:
return tuple(
(key, value)
for key, value in (
(DIFY_TRACE_ID_ATTRIBUTE, execution_context.trace_id),
(DIFY_TENANT_ID_ATTRIBUTE, execution_context.tenant_id),
(DIFY_APP_ID_ATTRIBUTE, execution_context.app_id),
(DIFY_AGENT_ID_ATTRIBUTE, execution_context.agent_id),
(GEN_AI_USER_ID_ATTRIBUTE, execution_context.user_id),
(DIFY_INVOKE_FROM_ATTRIBUTE, execution_context.invoke_from),
(DIFY_CONVERSATION_ID_ATTRIBUTE, execution_context.conversation_id),
(DIFY_WORKFLOW_ID_ATTRIBUTE, execution_context.workflow_id),
(DIFY_WORKFLOW_RUN_ID_ATTRIBUTE, execution_context.workflow_run_id),
(DIFY_NODE_ID_ATTRIBUTE, execution_context.node_id),
(DIFY_NODE_EXECUTION_ID_ATTRIBUTE, execution_context.node_execution_id),
)
if value
)
@dataclass(frozen=True)
class IsolatedTracerProvider(TracerProvider):
client: Logfire
preserve_external_parent: bool = False
@override
def get_tracer(
self,
instrumenting_module_name: str,
instrumenting_library_version: str | None = None,
schema_url: str | None = None,
attributes: Attributes = None,
) -> Tracer:
delegate = self.client.config.get_tracer_provider().get_tracer(
instrumenting_module_name,
instrumenting_library_version,
schema_url=schema_url,
attributes=attributes,
)
return _IsolatedTracer(delegate, self.client, self.preserve_external_parent)
@dataclass(frozen=True)
class _IsolatedTracer(Tracer):
delegate: Tracer
client: Logfire
preserve_external_parent: bool
def _parent_context(self, context: Context | None) -> Context | None:
parent = get_current_span(context)
if not parent.get_span_context().is_valid:
return context
# Logfire's ReadableSpan wrapper can delegate to a NonRecordingSpan,
# which propagates a valid context but has no SDK resource.
resource = getattr(parent, "resource", None) if isinstance(parent, ReadableSpan) else None
if resource is not None:
instance_id = self.client.resource_attributes.get("service.instance.id")
if instance_id is not None and resource.attributes.get("service.instance.id") == instance_id:
return context
elif self.preserve_external_parent:
return context
return set_span_in_context(INVALID_SPAN, context)
@override
def start_span(
self,
name: str,
context: Context | None = None,
kind: SpanKind = SpanKind.INTERNAL,
attributes: Attributes = None,
links: Sequence[Link] | None = None,
start_time: int | None = None,
record_exception: bool = True,
set_status_on_exception: bool = True,
) -> Span:
return self.delegate.start_span(
name,
context=self._parent_context(context),
kind=kind,
attributes=attributes,
links=links,
start_time=start_time,
record_exception=record_exception,
set_status_on_exception=set_status_on_exception,
)
@override
def start_as_current_span(
self,
name: str,
context: Context | None = None,
kind: SpanKind = SpanKind.INTERNAL,
attributes: Attributes = None,
links: Sequence[Link] | None = None,
start_time: int | None = None,
record_exception: bool = True,
set_status_on_exception: bool = True,
end_on_exit: bool = True,
):
return self.delegate.start_as_current_span(
name,
context=self._parent_context(context),
kind=kind,
attributes=attributes,
links=links,
start_time=start_time,
record_exception=record_exception,
set_status_on_exception=set_status_on_exception,
end_on_exit=end_on_exit,
)
@dataclass(frozen=True)
class RunScopedTracerProvider(TracerProvider):
"""Stamp one run's fixed attributes on every span a delegate provider creates."""
delegate: TracerProvider
attributes: tuple[tuple[str, str], ...]
@override
def get_tracer(
self,
instrumenting_module_name: str,
instrumenting_library_version: str | None = None,
schema_url: str | None = None,
attributes: Attributes = None,
) -> Tracer:
delegate = self.delegate.get_tracer(
instrumenting_module_name,
instrumenting_library_version,
schema_url=schema_url,
attributes=attributes,
)
return _RunScopedTracer(delegate, self.attributes)
@dataclass(frozen=True)
class _RunScopedTracer(Tracer):
delegate: Tracer
attributes: tuple[tuple[str, str], ...]
def _run_attributes(self, attributes: Attributes) -> Attributes:
return {**(attributes or {}), **dict(self.attributes)}
def _parent_context(self, context: Context | None) -> Context | None:
return context
@override
def start_span(
self,
name: str,
context: Context | None = None,
kind: SpanKind = SpanKind.INTERNAL,
attributes: Attributes = None,
links: Sequence[Link] | None = None,
start_time: int | None = None,
record_exception: bool = True,
set_status_on_exception: bool = True,
) -> Span:
return self.delegate.start_span(
name,
context=self._parent_context(context),
kind=kind,
attributes=self._run_attributes(attributes),
links=links,
start_time=start_time,
record_exception=record_exception,
set_status_on_exception=set_status_on_exception,
)
@override
def start_as_current_span(
self,
name: str,
context: Context | None = None,
kind: SpanKind = SpanKind.INTERNAL,
attributes: Attributes = None,
links: Sequence[Link] | None = None,
start_time: int | None = None,
record_exception: bool = True,
set_status_on_exception: bool = True,
end_on_exit: bool = True,
):
return self.delegate.start_as_current_span(
name,
context=self._parent_context(context),
kind=kind,
attributes=self._run_attributes(attributes),
links=links,
start_time=start_time,
record_exception=record_exception,
set_status_on_exception=set_status_on_exception,
end_on_exit=end_on_exit,
)
@dataclass(frozen=True, slots=True)
class AgentObservability:
client: Logfire
include_content: bool = False
trace_context_mode: Literal["isolated", "shared"] = "isolated"
def instrument(
self,
agent: Agent[Any, Any],
*,
execution_context: DifyExecutionContextLayerConfig | None = None,
include_content: bool | None = None,
) -> None:
"""Instrument one run's agent and attribute its spans to its Dify context.
``execution_context`` is the run's Dify identity and correlation context.
Its Data Push-aligned attributes are attached to every span the
instrumented agent produces; see ``dify_run_attributes``. Runs without an
execution context are still instrumented, just without Dify attribution.
"""
self.client.instrument_pydantic_ai(
agent,
include_content=self.include_content if include_content is None else include_content,
include_binary_content=False,
tracer_provider=self._tracer_provider(execution_context),
)
def _tracer_provider(self, execution_context: DifyExecutionContextLayerConfig | None) -> TracerProvider:
provider: TracerProvider = (
IsolatedTracerProvider(self.client)
if self.trace_context_mode == "isolated"
else self.client.config.get_tracer_provider()
)
if execution_context is None:
return provider
run_attributes = dify_run_attributes(execution_context)
if not run_attributes:
return provider
return RunScopedTracerProvider(provider, run_attributes)
async def aclose(self) -> None:
"""Flush and tear down this instance's own export pipeline.
The owning scope closes the instance; shutdown is blocking, so it runs off
the event loop. This only affects the Agent pipeline and leaves the
platform Logfire instance running.
"""
await asyncio.to_thread(self.client.shutdown, timeout_millis=5000)
@dataclass(frozen=True)
class _ApplicationRunTracer(_RunScopedTracer):
tracing: ApplicationTracingContext
@override
def _parent_context(self, context: Context | None) -> Context:
current = get_current_span(context)
# Preserve only this invocation's descendants. Platform spans, a previous
# invocation, or caller-supplied ambient context must not choose the root.
if isinstance(current, ReadableSpan):
# Logfire also wraps non-recording spans as ReadableSpan. Without
# attributes they cannot prove invocation ownership; use the API parent.
attributes = getattr(current, "attributes", None) or {}
current_context = current.get_span_context()
if (
attributes.get(DIFY_TRACING_RUN_ID_ATTRIBUTE) == str(self.tracing.run_id)
and attributes.get(DIFY_TRACING_INVOCATION_ID_ATTRIBUTE) == str(self.tracing.invocation_id)
and current_context is not None
and current_context.trace_id == int(self.tracing.trace_id, 16)
):
return context if context is not None else set_span_in_context(current)
parent = SpanContext(
trace_id=int(self.tracing.trace_id, 16),
span_id=int(self.tracing.parent_span_id, 16),
is_remote=True,
trace_flags=TraceFlags(TraceFlags.SAMPLED if self.tracing.sampled else TraceFlags.DEFAULT),
)
return set_span_in_context(NonRecordingSpan(parent), Context())
@dataclass(frozen=True)
class _ApplicationRunTracerProvider(RunScopedTracerProvider):
tracing: ApplicationTracingContext
@override
def get_tracer(
self,
instrumenting_module_name: str,
instrumenting_library_version: str | None = None,
schema_url: str | None = None,
attributes: Attributes = None,
) -> Tracer:
delegate = self.delegate.get_tracer(
instrumenting_module_name,
instrumenting_library_version,
schema_url=schema_url,
attributes=attributes,
)
return _ApplicationRunTracer(delegate, self.attributes, self.tracing)
@dataclass(frozen=True, slots=True, kw_only=True)
class ApplicationAgentObservability(AgentObservability):
"""Per-invocation policy over one process-owned internal export pipeline."""
exporter: JSONSpanExporter
processor: ApplicationBatchSpanProcessor
@override
def instrument(
self,
agent: Agent[Any, Any],
*,
execution_context: DifyExecutionContextLayerConfig | None = None,
include_content: bool | None = None,
) -> None:
tracing = execution_context.tracing if execution_context else None
if tracing is None or not tracing.sampled:
agent.instrument = False
return
# The immutable API policy always wins over process defaults/callers.
AgentObservability.instrument(
self, agent, execution_context=execution_context, include_content=tracing.capture_content
)
@override
def _tracer_provider(self, execution_context: DifyExecutionContextLayerConfig | None) -> TracerProvider:
if execution_context is None or execution_context.tracing is None:
raise ValueError("Application tracing requires API-issued execution context")
tracing = execution_context.tracing
attributes = dict(dify_run_attributes(execution_context))
attributes.update(
{
DIFY_TRACING_RUN_ID_ATTRIBUTE: str(tracing.run_id),
DIFY_TRACING_INVOCATION_ID_ATTRIBUTE: str(tracing.invocation_id),
DIFY_TRACE_ID_ATTRIBUTE: tracing.trace_id,
"gen_ai.session.id": tracing.session_id,
}
)
return _ApplicationRunTracerProvider(
self.client.config.get_tracer_provider(), tuple(attributes.items()), tracing
)
async def finish_invocation(self, tracing: ApplicationTracingContext) -> bool:
"""Flush ended spans before the terminal event exposes the completed run."""
def flush_and_finish() -> bool:
if not self.processor.force_flush(timeout_millis=30000):
self.exporter.mark_incomplete((tracing.run_id, tracing.invocation_id))
return False
try:
return self.exporter.finish_invocation(tracing.run_id, tracing.invocation_id)
finally:
self.processor.release_invocation((tracing.run_id, tracing.invocation_id))
try:
# The worker may continue its bounded retries after this deadline;
# collection must not hold the Agent's terminal event indefinitely.
return await asyncio.wait_for(asyncio.to_thread(flush_and_finish), timeout=5)
except TimeoutError:
logger.warning("Application trace finalization timed out; invocation_id=%s", tracing.invocation_id)
return False
except Exception:
logger.warning(
"Application trace finalization failed; invocation_id=%s", tracing.invocation_id, exc_info=True
)
return False
def record_unstarted_invocation(
self,
execution_context: DifyExecutionContextLayerConfig,
*,
start_time_ns: int,
end_time_ns: int,
error: BaseException | None,
cancelled: bool,
) -> None:
"""Represent setup failure/cancellation only when no native run span began.
Successful invocations keep the native tree. An unexplained empty
collector stays incomplete rather than being invented as success.
"""
tracing = execution_context.tracing
if tracing is None or not tracing.sampled or (error is None and not cancelled):
return
if self.processor.invocation_started((tracing.run_id, tracing.invocation_id)):
return
error_type = type(error).__name__ if error is not None else "CancelledError"
message = "Agent cancelled before execution" if cancelled else error_type
if not cancelled and error is not None and tracing.capture_content:
message = str(error) or error_type
try:
tracer = self._tracer_provider(execution_context).get_tracer("dify-agent.initialization")
span = tracer.start_span(
"invoke_agent dify-agent",
start_time=start_time_ns,
attributes={
"gen_ai.operation.name": "invoke_agent",
"dify.agent.execution_state": "cancelled" if cancelled else "initialization_failed",
"error.type": error_type,
},
)
span.set_status(Status(StatusCode.ERROR, message))
span.end(end_time=max(start_time_ns, end_time_ns))
except Exception:
self.exporter.mark_incomplete((tracing.run_id, tracing.invocation_id))
logger.warning("Unable to record Agent initialization failure; invocation_id=%s", tracing.invocation_id)
__all__ = [
"DIFY_AGENT_ID_ATTRIBUTE",
"DIFY_APP_ID_ATTRIBUTE",
"DIFY_CONVERSATION_ID_ATTRIBUTE",
"DIFY_INVOKE_FROM_ATTRIBUTE",
"DIFY_NODE_EXECUTION_ID_ATTRIBUTE",
"DIFY_NODE_ID_ATTRIBUTE",
"DIFY_TENANT_ID_ATTRIBUTE",
"DIFY_TRACE_ID_ATTRIBUTE",
"DIFY_WORKFLOW_ID_ATTRIBUTE",
"DIFY_WORKFLOW_RUN_ID_ATTRIBUTE",
"GEN_AI_USER_ID_ATTRIBUTE",
"AgentObservability",
"IsolatedTracerProvider",
"RunScopedTracerProvider",
"dify_run_attributes",
]