Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

17.8 KB
Raw Blame
from __future__ import annotations

import asyncio
import logging
from collections.abc import Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, Literal, override

from opentelemetry.context import Context
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.trace import (
    INVALID_SPAN,
    Link,
    NonRecordingSpan,
    Span,
    SpanContext,
    SpanKind,
    Status,
    StatusCode,
    TraceFlags,
    Tracer,
    TracerProvider,
    get_current_span,
    set_span_in_context,
)
from opentelemetry.util.types import Attributes

from dify_agent.layers.execution_context.configs import ApplicationTracingContext, DifyExecutionContextLayerConfig
from dify_agent.runtime.application_trace_exporter import (
    ApplicationBatchSpanProcessor,
    DIFY_TRACING_INVOCATION_ID_ATTRIBUTE,
    DIFY_TRACING_RUN_ID_ATTRIBUTE,
    JSONSpanExporter,
)

if TYPE_CHECKING:
    from logfire import Logfire
    from pydantic_ai import Agent

DIFY_TRACE_ID_ATTRIBUTE: Final[str] = "dify.trace_id"
DIFY_TENANT_ID_ATTRIBUTE: Final[str] = "dify.tenant_id"
DIFY_APP_ID_ATTRIBUTE: Final[str] = "dify.app_id"
DIFY_AGENT_ID_ATTRIBUTE: Final[str] = "dify.agent_id"
DIFY_INVOKE_FROM_ATTRIBUTE: Final[str] = "dify.invoke_from"
DIFY_CONVERSATION_ID_ATTRIBUTE: Final[str] = "dify.conversation.id"
DIFY_WORKFLOW_ID_ATTRIBUTE: Final[str] = "dify.workflow.id"
DIFY_WORKFLOW_RUN_ID_ATTRIBUTE: Final[str] = "dify.workflow.run_id"
DIFY_NODE_ID_ATTRIBUTE: Final[str] = "dify.node.id"
DIFY_NODE_EXECUTION_ID_ATTRIBUTE: Final[str] = "dify.node.execution_id"
GEN_AI_USER_ID_ATTRIBUTE: Final[str] = "gen_ai.user.id"
logger = logging.getLogger(__name__)


def dify_run_attributes(execution_context: DifyExecutionContextLayerConfig) -> tuple[tuple[str, str], ...]:
    return tuple(
        (key, value)
        for key, value in (
            (DIFY_TRACE_ID_ATTRIBUTE, execution_context.trace_id),
            (DIFY_TENANT_ID_ATTRIBUTE, execution_context.tenant_id),
            (DIFY_APP_ID_ATTRIBUTE, execution_context.app_id),
            (DIFY_AGENT_ID_ATTRIBUTE, execution_context.agent_id),
            (GEN_AI_USER_ID_ATTRIBUTE, execution_context.user_id),
            (DIFY_INVOKE_FROM_ATTRIBUTE, execution_context.invoke_from),
            (DIFY_CONVERSATION_ID_ATTRIBUTE, execution_context.conversation_id),
            (DIFY_WORKFLOW_ID_ATTRIBUTE, execution_context.workflow_id),
            (DIFY_WORKFLOW_RUN_ID_ATTRIBUTE, execution_context.workflow_run_id),
            (DIFY_NODE_ID_ATTRIBUTE, execution_context.node_id),
            (DIFY_NODE_EXECUTION_ID_ATTRIBUTE, execution_context.node_execution_id),
        )
        if value
    )


@dataclass(frozen=True)
class IsolatedTracerProvider(TracerProvider):
    client: Logfire
    preserve_external_parent: bool = False

    @override
    def get_tracer(
        self,
        instrumenting_module_name: str,
        instrumenting_library_version: str | None = None,
        schema_url: str | None = None,
        attributes: Attributes = None,
    ) -> Tracer:
        delegate = self.client.config.get_tracer_provider().get_tracer(
            instrumenting_module_name,
            instrumenting_library_version,
            schema_url=schema_url,
            attributes=attributes,
        )
        return _IsolatedTracer(delegate, self.client, self.preserve_external_parent)


@dataclass(frozen=True)
class _IsolatedTracer(Tracer):
    delegate: Tracer
    client: Logfire
    preserve_external_parent: bool

    def _parent_context(self, context: Context | None) -> Context | None:
        parent = get_current_span(context)
        if not parent.get_span_context().is_valid:
            return context
        # Logfire's ReadableSpan wrapper can delegate to a NonRecordingSpan,
        # which propagates a valid context but has no SDK resource.
        resource = getattr(parent, "resource", None) if isinstance(parent, ReadableSpan) else None
        if resource is not None:
            instance_id = self.client.resource_attributes.get("service.instance.id")
            if instance_id is not None and resource.attributes.get("service.instance.id") == instance_id:
                return context
        elif self.preserve_external_parent:
            return context
        return set_span_in_context(INVALID_SPAN, context)

    @override
    def start_span(
        self,
        name: str,
        context: Context | None = None,
        kind: SpanKind = SpanKind.INTERNAL,
        attributes: Attributes = None,
        links: Sequence[Link] | None = None,
        start_time: int | None = None,
        record_exception: bool = True,
        set_status_on_exception: bool = True,
    ) -> Span:
        return self.delegate.start_span(
            name,
            context=self._parent_context(context),
            kind=kind,
            attributes=attributes,
            links=links,
            start_time=start_time,
            record_exception=record_exception,
            set_status_on_exception=set_status_on_exception,
        )

    @override
    def start_as_current_span(
        self,
        name: str,
        context: Context | None = None,
        kind: SpanKind = SpanKind.INTERNAL,
        attributes: Attributes = None,
        links: Sequence[Link] | None = None,
        start_time: int | None = None,
        record_exception: bool = True,
        set_status_on_exception: bool = True,
        end_on_exit: bool = True,
    ):
        return self.delegate.start_as_current_span(
            name,
            context=self._parent_context(context),
            kind=kind,
            attributes=attributes,
            links=links,
            start_time=start_time,
            record_exception=record_exception,
            set_status_on_exception=set_status_on_exception,
            end_on_exit=end_on_exit,
        )


@dataclass(frozen=True)
class RunScopedTracerProvider(TracerProvider):
    """Stamp one run's fixed attributes on every span a delegate provider creates."""

    delegate: TracerProvider
    attributes: tuple[tuple[str, str], ...]

    @override
    def get_tracer(
        self,
        instrumenting_module_name: str,
        instrumenting_library_version: str | None = None,
        schema_url: str | None = None,
        attributes: Attributes = None,
    ) -> Tracer:
        delegate = self.delegate.get_tracer(
            instrumenting_module_name,
            instrumenting_library_version,
            schema_url=schema_url,
            attributes=attributes,
        )
        return _RunScopedTracer(delegate, self.attributes)


@dataclass(frozen=True)
class _RunScopedTracer(Tracer):
    delegate: Tracer
    attributes: tuple[tuple[str, str], ...]

    def _run_attributes(self, attributes: Attributes) -> Attributes:
        return {**(attributes or {}), **dict(self.attributes)}

    def _parent_context(self, context: Context | None) -> Context | None:
        return context

    @override
    def start_span(
        self,
        name: str,
        context: Context | None = None,
        kind: SpanKind = SpanKind.INTERNAL,
        attributes: Attributes = None,
        links: Sequence[Link] | None = None,
        start_time: int | None = None,
        record_exception: bool = True,
        set_status_on_exception: bool = True,
    ) -> Span:
        return self.delegate.start_span(
            name,
            context=self._parent_context(context),
            kind=kind,
            attributes=self._run_attributes(attributes),
            links=links,
            start_time=start_time,
            record_exception=record_exception,
            set_status_on_exception=set_status_on_exception,
        )

    @override
    def start_as_current_span(
        self,
        name: str,
        context: Context | None = None,
        kind: SpanKind = SpanKind.INTERNAL,
        attributes: Attributes = None,
        links: Sequence[Link] | None = None,
        start_time: int | None = None,
        record_exception: bool = True,
        set_status_on_exception: bool = True,
        end_on_exit: bool = True,
    ):
        return self.delegate.start_as_current_span(
            name,
            context=self._parent_context(context),
            kind=kind,
            attributes=self._run_attributes(attributes),
            links=links,
            start_time=start_time,
            record_exception=record_exception,
            set_status_on_exception=set_status_on_exception,
            end_on_exit=end_on_exit,
        )


@dataclass(frozen=True, slots=True)
class AgentObservability:
    client: Logfire
    include_content: bool = False
    trace_context_mode: Literal["isolated", "shared"] = "isolated"

    def instrument(
        self,
        agent: Agent[Any, Any],
        *,
        execution_context: DifyExecutionContextLayerConfig | None = None,
        include_content: bool | None = None,
    ) -> None:
        """Instrument one run's agent and attribute its spans to its Dify context.

        ``execution_context`` is the run's Dify identity and correlation context.
        Its Data Push-aligned attributes are attached to every span the
        instrumented agent produces; see ``dify_run_attributes``. Runs without an
        execution context are still instrumented, just without Dify attribution.
        """
        self.client.instrument_pydantic_ai(
            agent,
            include_content=self.include_content if include_content is None else include_content,
            include_binary_content=False,
            tracer_provider=self._tracer_provider(execution_context),
        )

    def _tracer_provider(self, execution_context: DifyExecutionContextLayerConfig | None) -> TracerProvider:
        provider: TracerProvider = (
            IsolatedTracerProvider(self.client)
            if self.trace_context_mode == "isolated"
            else self.client.config.get_tracer_provider()
        )
        if execution_context is None:
            return provider
        run_attributes = dify_run_attributes(execution_context)
        if not run_attributes:
            return provider
        return RunScopedTracerProvider(provider, run_attributes)

    async def aclose(self) -> None:
        """Flush and tear down this instance's own export pipeline.

        The owning scope closes the instance; shutdown is blocking, so it runs off
        the event loop. This only affects the Agent pipeline and leaves the
        platform Logfire instance running.
        """
        await asyncio.to_thread(self.client.shutdown, timeout_millis=5000)


@dataclass(frozen=True)
class _ApplicationRunTracer(_RunScopedTracer):
    tracing: ApplicationTracingContext

    @override
    def _parent_context(self, context: Context | None) -> Context:
        current = get_current_span(context)
        # Preserve only this invocation's descendants. Platform spans, a previous
        # invocation, or caller-supplied ambient context must not choose the root.
        if isinstance(current, ReadableSpan):
            # Logfire also wraps non-recording spans as ReadableSpan. Without
            # attributes they cannot prove invocation ownership; use the API parent.
            attributes = getattr(current, "attributes", None) or {}
            current_context = current.get_span_context()
            if (
                attributes.get(DIFY_TRACING_RUN_ID_ATTRIBUTE) == str(self.tracing.run_id)
                and attributes.get(DIFY_TRACING_INVOCATION_ID_ATTRIBUTE) == str(self.tracing.invocation_id)
                and current_context is not None
                and current_context.trace_id == int(self.tracing.trace_id, 16)
            ):
                return context if context is not None else set_span_in_context(current)
        parent = SpanContext(
            trace_id=int(self.tracing.trace_id, 16),
            span_id=int(self.tracing.parent_span_id, 16),
            is_remote=True,
            trace_flags=TraceFlags(TraceFlags.SAMPLED if self.tracing.sampled else TraceFlags.DEFAULT),
        )
        return set_span_in_context(NonRecordingSpan(parent), Context())


@dataclass(frozen=True)
class _ApplicationRunTracerProvider(RunScopedTracerProvider):
    tracing: ApplicationTracingContext

    @override
    def get_tracer(
        self,
        instrumenting_module_name: str,
        instrumenting_library_version: str | None = None,
        schema_url: str | None = None,
        attributes: Attributes = None,
    ) -> Tracer:
        delegate = self.delegate.get_tracer(
            instrumenting_module_name,
            instrumenting_library_version,
            schema_url=schema_url,
            attributes=attributes,
        )
        return _ApplicationRunTracer(delegate, self.attributes, self.tracing)


@dataclass(frozen=True, slots=True, kw_only=True)
class ApplicationAgentObservability(AgentObservability):
    """Per-invocation policy over one process-owned internal export pipeline."""

    exporter: JSONSpanExporter
    processor: ApplicationBatchSpanProcessor

    @override
    def instrument(
        self,
        agent: Agent[Any, Any],
        *,
        execution_context: DifyExecutionContextLayerConfig | None = None,
        include_content: bool | None = None,
    ) -> None:
        tracing = execution_context.tracing if execution_context else None
        if tracing is None or not tracing.sampled:
            agent.instrument = False
            return
        # The immutable API policy always wins over process defaults/callers.
        AgentObservability.instrument(
            self, agent, execution_context=execution_context, include_content=tracing.capture_content
        )

    @override
    def _tracer_provider(self, execution_context: DifyExecutionContextLayerConfig | None) -> TracerProvider:
        if execution_context is None or execution_context.tracing is None:
            raise ValueError("Application tracing requires API-issued execution context")
        tracing = execution_context.tracing
        attributes = dict(dify_run_attributes(execution_context))
        attributes.update(
            {
                DIFY_TRACING_RUN_ID_ATTRIBUTE: str(tracing.run_id),
                DIFY_TRACING_INVOCATION_ID_ATTRIBUTE: str(tracing.invocation_id),
                DIFY_TRACE_ID_ATTRIBUTE: tracing.trace_id,
                "gen_ai.session.id": tracing.session_id,
            }
        )
        return _ApplicationRunTracerProvider(
            self.client.config.get_tracer_provider(), tuple(attributes.items()), tracing
        )

    async def finish_invocation(self, tracing: ApplicationTracingContext) -> bool:
        """Flush ended spans before the terminal event exposes the completed run."""

        def flush_and_finish() -> bool:
            if not self.processor.force_flush(timeout_millis=30000):
                self.exporter.mark_incomplete((tracing.run_id, tracing.invocation_id))
                return False
            try:
                return self.exporter.finish_invocation(tracing.run_id, tracing.invocation_id)
            finally:
                self.processor.release_invocation((tracing.run_id, tracing.invocation_id))

        try:
            # The worker may continue its bounded retries after this deadline;
            # collection must not hold the Agent's terminal event indefinitely.
            return await asyncio.wait_for(asyncio.to_thread(flush_and_finish), timeout=5)
        except TimeoutError:
            logger.warning("Application trace finalization timed out; invocation_id=%s", tracing.invocation_id)
            return False
        except Exception:
            logger.warning(
                "Application trace finalization failed; invocation_id=%s", tracing.invocation_id, exc_info=True
            )
            return False

    def record_unstarted_invocation(
        self,
        execution_context: DifyExecutionContextLayerConfig,
        *,
        start_time_ns: int,
        end_time_ns: int,
        error: BaseException | None,
        cancelled: bool,
    ) -> None:
        """Represent setup failure/cancellation only when no native run span began.

        Successful invocations keep the native tree. An unexplained empty
        collector stays incomplete rather than being invented as success.
        """
        tracing = execution_context.tracing
        if tracing is None or not tracing.sampled or (error is None and not cancelled):
            return
        if self.processor.invocation_started((tracing.run_id, tracing.invocation_id)):
            return
        error_type = type(error).__name__ if error is not None else "CancelledError"
        message = "Agent cancelled before execution" if cancelled else error_type
        if not cancelled and error is not None and tracing.capture_content:
            message = str(error) or error_type
        try:
            tracer = self._tracer_provider(execution_context).get_tracer("dify-agent.initialization")
            span = tracer.start_span(
                "invoke_agent dify-agent",
                start_time=start_time_ns,
                attributes={
                    "gen_ai.operation.name": "invoke_agent",
                    "dify.agent.execution_state": "cancelled" if cancelled else "initialization_failed",
                    "error.type": error_type,
                },
            )
            span.set_status(Status(StatusCode.ERROR, message))
            span.end(end_time=max(start_time_ns, end_time_ns))
        except Exception:
            self.exporter.mark_incomplete((tracing.run_id, tracing.invocation_id))
            logger.warning("Unable to record Agent initialization failure; invocation_id=%s", tracing.invocation_id)


__all__ = [
    "DIFY_AGENT_ID_ATTRIBUTE",
    "DIFY_APP_ID_ATTRIBUTE",
    "DIFY_CONVERSATION_ID_ATTRIBUTE",
    "DIFY_INVOKE_FROM_ATTRIBUTE",
    "DIFY_NODE_EXECUTION_ID_ATTRIBUTE",
    "DIFY_NODE_ID_ATTRIBUTE",
    "DIFY_TENANT_ID_ATTRIBUTE",
    "DIFY_TRACE_ID_ATTRIBUTE",
    "DIFY_WORKFLOW_ID_ATTRIBUTE",
    "DIFY_WORKFLOW_RUN_ID_ATTRIBUTE",
    "GEN_AI_USER_ID_ATTRIBUTE",
    "AgentObservability",
    "IsolatedTracerProvider",
    "RunScopedTracerProvider",
    "dify_run_attributes",
]