Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

60.0 KB
Raw Blame
import json
from copy import deepcopy
from dataclasses import dataclass, replace
from datetime import UTC, datetime, timedelta
from decimal import Decimal
from pathlib import Path
from typing import Literal, override
from uuid import NAMESPACE_URL, UUID, uuid4, uuid5

import pytest
from dify_agent.layers.execution_context import ApplicationTracingContext
from dify_agent.protocol.tracing import AgentTraceBatch, AgentTraceSpan
from dify_trace_langfuse.provider import LangfuseProvider
from dify_trace_sdk.egress import Egress
from dify_trace_sdk.events import (
    ConversationNamed,
    MessageCompleted,
    ModerationChecked,
    TraceCorrelation,
    TraceEvent,
    TracePayload,
    WorkflowRunCompleted,
)
from dify_trace_sdk.ids import span_id_for
from dify_trace_sdk.model import Span, SpanKind, SpanStatus, Trace
from dify_trace_sdk.testing import RecordingEgress, otlp_attributes, otlp_spans, response
from flask import Flask
from sqlalchemy import URL, Engine, create_engine, delete, event, func, select
from sqlalchemy.orm import Session, sessionmaker

from core.app.app_config.entities import WorkflowUIBasedAppConfig
from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom, WorkflowAppGenerateEntity
from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository
from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext
from domain.tracing.trace_run_agg import TraceRunState, TraceSourceKind
from extensions.tracing import application_tracing_hooks, bind_tracing
from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus
from graphon.runtime import VariablePool
from libs.datetime_utils import naive_utc_now
from models.account import Tenant, TenantAccountJoin, TenantAccountRole
from models.agent import (
    Agent,
    AgentConfigSnapshot,
    AgentScope,
    AgentSource,
    WorkflowAgentBindingType,
    WorkflowAgentNodeBinding,
)
from models.agent_config_entities import AgentSoulConfig, WorkflowNodeJobConfig
from models.application_trace import ApplicationTraceRun
from models.enums import ConversationFromSource, CreatorUserRole, WorkflowRunTriggeredFrom
from models.model import App, AppMode, Message, TraceAppConfig
from models.workflow import (
    WorkflowAppLog,
    WorkflowAppLogCreatedFrom,
    WorkflowNodeExecutionModel,
    WorkflowNodeExecutionTriggeredFrom,
    WorkflowRun,
    WorkflowType,
)
from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
from repositories.application_trace_repository import ApplicationTraceRepository
from repositories.factory import DifyAPIRepositoryFactory
from repositories.trace_record_repository import TraceRecordRepository
from services.tracing.adapters import StoredPolicyTraceScopes
from services.tracing.canonical import CanonicalTraceBuilder
from services.tracing.collection import AgentTraceSettings, AppTracingSnapshot, TraceRun, workflow_has_agent_v2
from services.tracing.delivery import ExportJob, ExportRetryQueue, TraceDeliveryService
from services.tracing.errors import TraceCollectorUnavailableError, TracingStoreUnavailableError
from services.tracing.providers import ProviderConfigCodec
from services.tracing.records import WorkflowTraceInfo
from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
from services.tracing.scopes import TraceScopes
from services.tracing.service import ApplicationTracingService
from tests.helpers.unit.model_factories import (
    make_account,
    make_app,
    make_conversation,
    make_message,
    make_tenant,
    make_workflow,
)
from tests.helpers.unit.services.tracing_ports import (
    AGENT_TRACE_SETTINGS,
    InstalledTraceProviders,
    WorkspaceBoundCipher,
)

"""SQLite ownership regressions at the handoff from trace delivery to Agent trace collection."""


AGENT_GRAPH = {
    "nodes": [{"id": "agent", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}],
    "edges": list[object](),
}


STARTED_AT = datetime(2026, 10, 15, tzinfo=UTC)


def identifier(value: str) -> str:
    return str(uuid5(NAMESPACE_URL, value))


@dataclass(frozen=True)
class Collection:
    repository: ApplicationTraceRepository
    sessions: sessionmaker[Session]
    run: TraceRun


class _Recorder:
    def __init__(self) -> None:
        self.events: list[TraceEvent] = []

    def emit(self, event: TraceEvent) -> None:
        self.events.append(event)


class _SingleAttempt(ExportRetryQueue):
    """Delivery here makes one provider attempt; it never schedules a retry."""

    @override
    def schedule(self, job: ExportJob, *, delay_seconds: int) -> None:
        raise AssertionError(f"unexpected export retry: provider={job.provider_id}")


class _FreezeFails(ApplicationTraceRepository):
    """The real ledger, except that reading the tracing snapshot a run freezes fails with ``failure``."""

    def __init__(self, sessions: sessionmaker[Session], failure: Exception) -> None:
        super().__init__(sessions)
        self._failure = failure

    @override
    def app_tracing(self, *, tenant_id: str, app_id: str) -> AppTracingSnapshot | None:
        raise self._failure


def record_source(sessions: sessionmaker[Session]) -> TraceRecordRepository:
    return TraceRecordRepository(
        sessions,
        workflow_runs=DifyAPIRepositoryFactory.create_api_workflow_run_repository(sessions),
        node_executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(sessions),
        files_url="https://files.test",
        enterprise_details=False,
    )


def tracing_service(
    sessions: sessionmaker[Session],
    traces: ApplicationTraceRepository,
    egress: Egress,
    scheduled: list[str],
    settings: AgentTraceSettings,
) -> ApplicationTracingService:
    """Agent trace collection exporting to Langfuse; scheduled exports are recorded instead of enqueued."""
    return ApplicationTracingService(
        runs=traces,
        run_query=traces,
        sources=traces,
        records=record_source(sessions),
        builder=CanonicalTraceBuilder(),
        providers=InstalledTraceProviders(LangfuseProvider),
        codec=ProviderConfigCodec(WorkspaceBoundCipher()),
        egress=egress,
        schedule_export=scheduled.append,
        settings=settings,
    )


def trace_scopes(sessions: sessionmaker[Session]) -> StoredPolicyTraceScopes:
    """Trace scopes as the composition root binds them, with events kept in memory."""
    return StoredPolicyTraceScopes(
        TraceScopes(
            emitter=_Recorder(),
            policies=SQLAlchemyAppTracingConfigRepository(session_factory=sessions),
            enterprise_enabled=lambda: False,
        )
    )


def langfuse_config(tenant_id: str, host: str) -> dict[str, str]:
    """A stored Langfuse configuration whose secrets only ``tenant_id`` decrypts."""
    cipher = WorkspaceBoundCipher()
    return {
        "public_key": cipher.encrypt(tenant_id, "public"),
        "secret_key": cipher.encrypt(tenant_id, "secret"),
        "host": host,
    }


def no_egress() -> RecordingEgress:
    """An egress that fails any request the test did not script."""
    return RecordingEgress([], grpc_target=None)


def exported() -> RecordingEgress:
    """A provider destination that accepts exactly one ingestion request."""
    return RecordingEgress([response(200, {})], grpc_target=None)


def root_metadata(destination: RecordingEgress) -> dict[str, object]:
    """The Langfuse metadata of the root observation the provider sent."""
    root = next(span for span in otlp_spans(destination) if not span.parent_span_id)
    prefix = "langfuse.observation.metadata."
    return {key.removeprefix(prefix): value for key, value in otlp_attributes(root).items() if key.startswith(prefix)}


def run_count(sessions: sessionmaker[Session]) -> int | None:
    with sessions() as session:
        return session.scalar(select(func.count()).select_from(ApplicationTraceRun))


def seed(sessions: sessionmaker[Session], name: str) -> Collection:
    """An Agent app tracing to Langfuse, one of its messages, and the run frozen when the message started."""
    app_id, tenant_id, message_id = identifier(f"app-{name}"), identifier("tenant"), identifier(f"message-{name}")
    with sessions.begin() as session:
        session.add(
            make_app(
                app_id=app_id,
                tenant_id=tenant_id,
                mode=AppMode.AGENT,
                tracing=json.dumps(
                    {"enabled": True, "tracing_provider": "langfuse", "capture_content": False, "capture_agent": True}
                ),
            )
        )
        session.add(
            make_message(
                message_id=message_id,
                app_id=app_id,
                conversation_id=identifier(f"conversation-{name}"),
                inputs={},
                query="hello",
                message={},
                answer="done",
                message_unit_price=Decimal(0),
                answer_unit_price=Decimal(0),
                currency="USD",
                from_source=ConversationFromSource.CONSOLE,
            )
        )
        session.add(
            TraceAppConfig(
                app_id=app_id,
                tracing_provider="langfuse",
                tracing_config=langfuse_config(tenant_id, "https://langfuse.test"),
            )
        )
    repository = ApplicationTraceRepository(sessions)
    run = tracing_service(sessions, repository, no_egress(), [], AGENT_TRACE_SETTINGS).start(
        tenant_id=tenant_id,
        app_id=app_id,
        source_kind="message",
        source_id=message_id,
        session_id="session-original",
        workflow_id=None,
    )
    assert run is not None
    return Collection(repository, sessions, run)


def attach_conversation(collection: Collection, mode: AppMode) -> None:
    """Message records are assembled only for a message whose conversation belongs to its app."""
    with collection.sessions.begin() as session:
        message = session.get(Message, collection.run.source_id)
        assert message is not None
        session.add(
            make_conversation(
                conversation_id=message.conversation_id,
                app_id=collection.run.app_id,
                mode=mode,
                inputs={},
                from_source=ConversationFromSource.CONSOLE,
            )
        )


def add_app_owner(collection: Collection, name: str) -> str:
    """An account of the app's workspace that creates the app."""
    owner_id = identifier(f"owner-{name}")
    with collection.sessions.begin() as session:
        if session.get(Tenant, collection.run.tenant_id) is None:
            session.add(make_tenant(tenant_id=collection.run.tenant_id))
        session.add(make_account(account_id=owner_id, email=f"{name}@example.com"))
        session.add(
            TenantAccountJoin(tenant_id=collection.run.tenant_id, account_id=owner_id, role=TenantAccountRole.OWNER)
        )
        app = session.get(App, collection.run.app_id)
        assert app is not None
        app.created_by = owner_id
    return owner_id


def trace_event(run: TraceRun, payload: TracePayload) -> TraceEvent:
    return TraceEvent(
        event_id=uuid4().hex,
        tenant_id=run.tenant_id,
        app_id=run.app_id,
        user_id=None,
        occurred_at=datetime.now(UTC),
        correlation=TraceCorrelation(
            external_trace_id=None, session_id=None, parent_workflow_run_id=None, parent_node_execution_id=None
        ),
        payload=payload,
    )


def capture(traces: ApplicationTraceRepository, run_id: str, trace: Trace) -> None:
    """Keep a business terminal in its run without scheduling the run's export."""
    run = traces.get(run_id)
    assert run is not None
    with traces.change(run_id, tenant_id=run.tenant_id) as locked:
        assert locked is not None
        locked.run.capture_terminal(trace, now=naive_utc_now())


def ingest(traces: ApplicationTraceRepository, batch: AgentTraceBatch, *, max_spans: int, max_bytes: int) -> None:
    """Store a runtime batch in its run without scheduling the run's export."""
    run = traces.get(str(batch.run_id))
    assert run is not None
    with traces.change(run.id, tenant_id=run.tenant_id) as locked:
        assert locked is not None
        locked.run.ingest(batch, max_spans=max_spans, max_bytes=max_bytes, now=naive_utc_now())


def terminal(collection: Collection) -> None:
    """Persist a root-only business terminal for the collection's run."""
    run = collection.run
    root = Span(
        span_id=run.root_span_id,
        parent_span_id=None,
        name="Agent turn",
        kind=SpanKind.CHAIN,
        start_time=datetime.fromtimestamp(0, UTC),
        end_time=datetime.fromtimestamp(3, UTC),
        status=SpanStatus.OK,
    )
    trace = Trace(
        trace_id=run.trace_id,
        external_trace_id=None,
        session_id=run.session_id,
        user_id=None,
        parent=None,
        spans=(root,),
    )
    capture(collection.repository, run.id, trace)


@dataclass(frozen=True)
class Case:
    collection: Collection
    service: ApplicationTracingService
    scheduled: list[str]


def handoff(collection: Collection, traces: ApplicationTraceRepository, egress: Egress) -> Case:
    """The Agent collector over ``traces``, exporting through ``egress``."""
    scheduled: list[str] = []
    return Case(
        collection, tracing_service(collection.sessions, traces, egress, scheduled, AGENT_TRACE_SETTINGS), scheduled
    )


def setup(sessions: sessionmaker[Session], name: str, mode: AppMode, *, snapshot: bool) -> Case:
    collection = seed(sessions, name)
    attach_conversation(collection, mode)
    with sessions.begin() as session:
        app = session.get(App, collection.run.app_id)
        assert app is not None
        app.mode = mode
        if not snapshot:
            session.execute(delete(ApplicationTraceRun).where(ApplicationTraceRun.id == collection.run.id))
    return handoff(collection, collection.repository, no_egress())


def deliver(case: Case, event: TraceEvent, destination: RecordingEgress) -> None:
    """Deliver one event the way the trace worker does, with this case's service as the Agent collector."""
    sessions = case.collection.sessions
    TraceDeliveryService(
        records=record_source(sessions),
        destinations=SQLAlchemyAppTracingConfigRepository(session_factory=sessions),
        providers=InstalledTraceProviders(LangfuseProvider),
        codec=ProviderConfigCodec(WorkspaceBoundCipher()),
        builder=CanonicalTraceBuilder(),
        egress=destination,
        retries=_SingleAttempt(),
        enterprise_sink=None,
        collector=case.service,
        max_export_attempts=1,
    ).deliver(event, enterprise_delivered=False)


def message(case: Case) -> Message:
    with case.collection.sessions() as session:
        row = session.get(Message, case.collection.run.source_id)
        assert row is not None
        return row


def message_event(case: Case) -> TraceEvent:
    row = message(case)
    return trace_event(case.collection.run, MessageCompleted(message_id=row.id, conversation_id=row.conversation_id))


def workflow_event(row: WorkflowRun, parent: WorkflowRun | None) -> TraceEvent:
    return TraceEvent(
        event_id=uuid4().hex,
        tenant_id=row.tenant_id,
        app_id=row.app_id,
        user_id=None,
        occurred_at=datetime.now(UTC),
        correlation=TraceCorrelation(
            external_trace_id=None,
            session_id=None,
            parent_workflow_run_id=parent.id if parent else None,
            parent_node_execution_id=identifier("parent-tool-node") if parent else None,
        ),
        payload=WorkflowRunCompleted(workflow_run_id=row.id, conversation_id=None, total_tokens=None),
    )


def workflow(case: Case, name: str, *, has_agent: bool, snapshot: bool) -> tuple[WorkflowRun, TraceRun | None]:
    graph = AGENT_GRAPH if has_agent else {"nodes": [{"data": {"type": "llm"}}], "edges": list[object]()}
    now = datetime(2026, 10, 15)
    with case.collection.sessions.begin() as session:
        definition = make_workflow(
            workflow_id=identifier(f"workflow-{name}"),
            tenant_id=case.collection.run.tenant_id,
            app_id=case.collection.run.app_id,
            graph=graph,
        )
        session.add(definition)
        app = session.get(App, case.collection.run.app_id)
        assert app is not None
        app.workflow_id = definition.id
        row = WorkflowRun(
            id=identifier(f"workflow-run-{name}"),
            app_id=app.id,
            tenant_id=app.tenant_id,
            workflow_id=definition.id,
            type=WorkflowType.WORKFLOW,
            version="1",
            graph=json.dumps(graph),
            inputs="{}",
            outputs="{}",
            status=WorkflowExecutionStatus.SUCCEEDED,
            triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
            created_by_role=CreatorUserRole.ACCOUNT,
            created_by=identifier("owner"),
            finished_at=now + timedelta(seconds=2),
            elapsed_time=2,
            total_tokens=0,
            total_steps=0,
        )
        row.created_at = now
        session.add(row)
    trace_run = (
        case.service.start(
            tenant_id=row.tenant_id,
            app_id=row.app_id,
            source_kind="workflow",
            source_id=row.id,
            session_id="workflow-session",
            workflow_id=None,
        )
        if snapshot
        else None
    )
    return row, trace_run


def workflow_info(row: WorkflowRun, parent: WorkflowRun | None) -> WorkflowTraceInfo:
    assert row.finished_at is not None
    metadata: dict[str, object] = {"app_id": row.app_id}
    if parent is not None:
        metadata["parent_trace_context"] = {
            "parent_workflow_run_id": parent.id,
            "parent_node_execution_id": identifier("parent-tool-node"),
        }
    return WorkflowTraceInfo(
        node_executions=[],
        conversation_id=None,
        message_started_at=None,
        workflow_app_log_id=None,
        workflow_id=row.workflow_id,
        tenant_id=row.tenant_id,
        workflow_run_id=row.id,
        workflow_run_elapsed_time=2,
        workflow_run_status="succeeded",
        workflow_run_inputs={},
        workflow_run_outputs={},
        workflow_run_version="1",
        error=None,
        total_tokens=0,
        prompt_tokens=None,
        completion_tokens=None,
        file_list=[],
        invoked_by=None,
        query="",
        metadata=metadata,
        start_time=row.created_at,
        end_time=row.finished_at,
    )


class TestApplicationTraceHandoffCases:
    @staticmethod
    def test_chatflow_message_event_is_not_exported_again(sqlite_session_factory: sessionmaker[Session]) -> None:
        case = setup(sqlite_session_factory, "chatflow", AppMode.ADVANCED_CHAT, snapshot=False)
        row, trace_run = workflow(case, "chatflow", has_agent=True, snapshot=True)
        assert trace_run is not None
        terminal(Collection(case.collection.repository, case.collection.sessions, trace_run))
        before = case.collection.repository.get(trace_run.id)
        assert before is not None
        with case.collection.sessions.begin() as session:
            stored = session.get(Message, case.collection.run.source_id)
            assert stored is not None
            stored.workflow_run_id = row.id
        destination = no_egress()
        deliver(case, message_event(case), destination)
        assert destination.requests == []
        after = case.collection.repository.get(trace_run.id)
        assert after is not None
        assert after.outer_trace == before.outer_trace
        assert case.scheduled == []
        assert run_count(case.collection.sessions) == 1

    @staticmethod
    @pytest.mark.parametrize("mode", [AppMode.AGENT, AppMode.ADVANCED_CHAT])
    def test_snapshot_failure_is_persisted_and_never_falls_back(
        sqlite_session_factory: sessionmaker[Session], mode: AppMode
    ) -> None:
        case = setup(sqlite_session_factory, "failure", mode, snapshot=False)
        failing = handoff(
            case.collection, _FreezeFails(sqlite_session_factory, TracingStoreUnavailableError()), no_egress()
        )
        context = AgentAppRuntimeBuildContext(
            dify_context=DifyRunContext(
                tenant_id=case.collection.run.tenant_id,
                app_id=case.collection.run.app_id,
                user_id=identifier("user"),
                user_from=UserFrom.ACCOUNT,
                invoke_from=InvokeFrom.DEBUGGER,
            ),
            agent_id=identifier("agent"),
            idempotency_key=case.collection.run.source_id,
            conversation_id="conversation",
            agent_config_snapshot_id=identifier("snapshot"),
            agent_soul=AgentSoulConfig(),
            user_query="query",
            binding_id=identifier("binding"),
            backend_binding_ref="backend",
        )
        assert RuntimeApplicationTracingHooks(failing.service).prepare_agent_app(context) is None
        destination = no_egress()
        deliver(failing, message_event(failing), destination)
        assert destination.requests == []
        stored = case.collection.repository.source_run(
            tenant_id=case.collection.run.tenant_id,
            app_id=case.collection.run.app_id,
            source_kind="message",
            source_id=case.collection.run.source_id,
        )
        assert stored is not None
        assert stored.state is TraceRunState.FAILED
        assert stored.error_code == "tracing_initialization_failed"
        assert stored.provider_config == {}

    @staticmethod
    @pytest.mark.parametrize("mode", [AppMode.AGENT, AppMode.ADVANCED_CHAT, AppMode.CHAT])
    @pytest.mark.parametrize("kind", ["title", "moderation", "aliased_moderation"])
    def test_auxiliary_events_resolve_owner_without_bypassing_content_policy(
        sqlite_session_factory: sessionmaker[Session], mode: AppMode, kind: str
    ) -> None:
        case = setup(sqlite_session_factory, "aux", mode, snapshot=False)
        row = message(case)
        if kind == "aliased_moderation":
            # Chatflow moderation is keyed by its workflow log, not by the message it guards.
            workflow_run_id = identifier("aliased-workflow-run")
            with case.collection.sessions.begin() as session:
                stored = session.get(Message, row.id)
                assert stored is not None
                stored.workflow_run_id = workflow_run_id
                session.add(
                    WorkflowAppLog(
                        tenant_id=case.collection.run.tenant_id,
                        app_id=case.collection.run.app_id,
                        workflow_id=identifier("aliased-workflow"),
                        workflow_run_id=workflow_run_id,
                        created_from=WorkflowAppLogCreatedFrom.WEB_APP,
                        created_by_role=CreatorUserRole.ACCOUNT,
                        created_by=identifier("owner"),
                    )
                )
        payload: TracePayload = (
            ConversationNamed(
                conversation_id=row.conversation_id,
                message_id=row.id,
                inputs="private conversation",
                name="private generated title",
                started_at=STARTED_AT,
                ended_at=STARTED_AT,
            )
            if kind == "title"
            else ModerationChecked(
                message_id=row.id,
                inputs={"query": "private prompt"},
                query="private prompt",
                flagged=False,
                action="direct_output",
                preset_response="",
                started_at=STARTED_AT,
                ended_at=STARTED_AT,
            )
        )
        destination = exported() if mode is AppMode.CHAT else no_egress()
        deliver(case, trace_event(case.collection.run, payload), destination)
        assert len(destination.requests) == (1 if mode is AppMode.CHAT else 0)
        assert case.scheduled == []
        assert run_count(case.collection.sessions) == 0

    @staticmethod
    def test_plain_nested_workflow_keeps_its_parent_context_for_the_provider(
        sqlite_session_factory: sessionmaker[Session],
    ) -> None:
        parent_case = setup(sqlite_session_factory, "plain-parent", AppMode.WORKFLOW, snapshot=False)
        parent, _ = workflow(parent_case, "plain-parent", has_agent=False, snapshot=False)
        child_case = setup(sqlite_session_factory, "plain-child", AppMode.WORKFLOW, snapshot=False)
        child, _ = workflow(child_case, "plain-child", has_agent=False, snapshot=False)
        add_app_owner(child_case.collection, "plain-child")
        event = workflow_event(child, parent)
        record = record_source(sqlite_session_factory).assemble(event)
        assert isinstance(record, WorkflowTraceInfo)
        assert child_case.service.capture_terminal(tenant_id=child.tenant_id, info=record) is False
        assert child_case.service.link_managed_parent(tenant_id=child.tenant_id, info=record) is record
        destination = exported()
        deliver(child_case, event, destination)
        metadata = root_metadata(destination)
        assert metadata["parent_workflow_run_id"] == parent.id
        assert metadata["parent_node_execution_id"] == identifier("parent-tool-node")
        assert "linked_parent_workflow_run_id" not in metadata
        assert run_count(sqlite_session_factory) == 0

    @staticmethod
    def test_plain_chatflow_auxiliary_events_keep_the_provider_route(
        sqlite_session_factory: sessionmaker[Session],
    ) -> None:
        case = setup(sqlite_session_factory, "plain-chatflow", AppMode.ADVANCED_CHAT, snapshot=False)
        row, _ = workflow(case, "plain-chatflow", has_agent=False, snapshot=False)
        with case.collection.sessions.begin() as session:
            stored = session.get(Message, case.collection.run.source_id)
            assert stored is not None
            stored.workflow_run_id = row.id
            payload = ConversationNamed(
                conversation_id=stored.conversation_id,
                message_id=stored.id,
                inputs="provider input",
                name="provider title",
                started_at=STARTED_AT,
                ended_at=STARTED_AT,
            )
        destination = exported()
        deliver(case, trace_event(case.collection.run, payload), destination)
        assert len(destination.requests) == 1

    @staticmethod
    def test_provider_exported_child_of_managed_parent_names_the_collected_parent(
        sqlite_session_factory: sessionmaker[Session],
    ) -> None:
        parent_case = setup(sqlite_session_factory, "managed-parent", AppMode.WORKFLOW, snapshot=False)
        parent, parent_trace = workflow(parent_case, "managed-parent", has_agent=True, snapshot=True)
        assert parent_trace is not None
        child_case = setup(sqlite_session_factory, "provider-child", AppMode.WORKFLOW, snapshot=False)
        child, _ = workflow(child_case, "provider-child", has_agent=False, snapshot=False)
        info = workflow_info(child, parent)
        assert child_case.service.capture_terminal(tenant_id=child.tenant_id, info=info) is False
        linked = child_case.service.link_managed_parent(tenant_id=child.tenant_id, info=info)
        assert linked.resolved_parent_context == (None, None)
        assert linked.metadata["linked_parent_workflow_run_id"] == parent.id
        assert linked.metadata["linked_parent_node_execution_id"] == identifier("parent-tool-node")
        assert linked.metadata["linked_parent_application_trace_id"] == parent_trace.id
        assert linked.metadata["linked_parent_trace_id"] == parent_trace.trace_id
        # Records are frozen: the delivered record is a copy, the original keeps its unavailable context.
        assert info.resolved_parent_context == (parent.id, identifier("parent-tool-node"))
        add_app_owner(child_case.collection, "provider-child")
        destination = exported()
        deliver(child_case, workflow_event(child, parent), destination)
        metadata = root_metadata(destination)
        assert metadata["linked_parent_trace_id"] == parent_trace.trace_id
        assert "parent_workflow_run_id" not in metadata

    @staticmethod
    def test_collector_database_failure_redelivers_without_provider_fallback(
        sqlite_session_factory: sessionmaker[Session], tmp_path: Path
    ) -> None:
        case = setup(sqlite_session_factory, "collector-unavailable", AppMode.AGENT, snapshot=True)
        # The ledger's database cannot be opened while the business records stay readable.
        unreachable = create_engine(URL.create("sqlite", database=str(tmp_path / "missing" / "ledger.sqlite3")))
        destination = no_egress()
        try:
            collector = handoff(
                case.collection, ApplicationTraceRepository(sessionmaker(bind=unreachable)), no_egress()
            )
            with pytest.raises(TraceCollectorUnavailableError):
                deliver(collector, message_event(case), destination)
        finally:
            unreachable.dispose()
        assert destination.requests == []

    @staticmethod
    def test_mixed_workflow_exports_normal_llm_and_agent_calls_without_double_usage(
        sqlite_session_factory: sessionmaker[Session],
    ) -> None:
        case = setup(sqlite_session_factory, "mixed", AppMode.WORKFLOW, snapshot=False)
        row, managed = workflow(case, "mixed", has_agent=True, snapshot=True)
        assert managed is not None
        owner = add_app_owner(case.collection, "mixed")
        agent_node_id = identifier("agent-execution")
        with case.collection.sessions.begin() as session:
            for index, kind in enumerate(("llm", "agent")):
                node_id = identifier("normal-execution") if kind == "llm" else agent_node_id
                usage: dict[str, int] = (
                    {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} if kind == "llm" else {}
                )
                node_record = WorkflowNodeExecutionModel(
                    id=node_id,
                    node_execution_id=node_id,
                    tenant_id=row.tenant_id,
                    app_id=row.app_id,
                    workflow_id=row.workflow_id,
                    workflow_run_id=row.id,
                    triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
                    index=index,
                    node_id=kind,
                    node_type=kind,
                    title=kind,
                    inputs="{}",
                    outputs="{}",
                    process_data=json.dumps(
                        {
                            "model_mode": "chat",
                            "model_name": "ordinary-model",
                            "model_provider": "openai",
                            "usage": usage,
                        }
                    )
                    if kind == "llm"
                    else "{}",
                    status=WorkflowNodeExecutionStatus.SUCCEEDED,
                    elapsed_time=1,
                    execution_metadata=json.dumps(
                        {"total_tokens": 15 if kind == "llm" else 10, "total_price": "0.02", "currency": "USD"}
                    ),
                    finished_at=row.finished_at,
                    created_by=owner,
                    created_by_role=CreatorUserRole.ACCOUNT,
                )
                node_record.created_at = row.created_at
                session.add(node_record)
        with case.collection.repository.change(managed.id, tenant_id=managed.tenant_id) as locked:
            assert locked is not None
            invocation = locked.run.open_invocation(
                agent_id=identifier("agent"), node_execution_id=agent_node_id, attempt=0, call_key="agent:0"
            )
        spans = [
            AgentTraceSpan(
                trace_id=managed.trace_id,
                span_id="a" * 16,
                parent_span_id=invocation.span_id,
                name="agent run",
                start_time_ns=1_000_000_000,
                end_time_ns=3_000_000_000,
                status="ok",
                attributes={"gen_ai.operation.name": "invoke_agent"},
            )
        ]
        spans.extend(
            AgentTraceSpan(
                trace_id=managed.trace_id,
                span_id=key * 16,
                parent_span_id="a" * 16,
                name="model call",
                start_time_ns=1_000_000_000,
                end_time_ns=2_000_000_000,
                status="ok",
                attributes={
                    "gen_ai.operation.name": "chat",
                    "gen_ai.request.model": "agent-model",
                    "gen_ai.usage.input_tokens": 3,
                    "gen_ai.usage.output_tokens": 2,
                    "dify.llm.total_price": "0.01",
                    "dify.llm.currency": "USD",
                },
            )
            for key in ("b", "c")
        )
        ingest(
            case.collection.repository,
            AgentTraceBatch(
                run_id=UUID(managed.id), invocation_id=UUID(invocation.id), batch_id=uuid4(), final=True, spans=spans
            ),
            max_spans=100,
            max_bytes=100000,
        )
        record = record_source(sqlite_session_factory).assemble(workflow_event(row, None))
        assert isinstance(record, WorkflowTraceInfo)
        assert case.service.capture_terminal(tenant_id=row.tenant_id, info=record) is True
        assert case.scheduled == [managed.id]
        destination = exported()
        handoff(case.collection, case.collection.repository, destination).service.deliver(managed.id)
        delivered = case.collection.repository.get(managed.id)
        assert delivered is not None
        assert delivered.state is TraceRunState.ACCEPTED
        assert [request.url for request in destination.requests] == ["https://langfuse.test/api/public/otel/v1/traces"]
        generations = [
            attributes
            for attributes in map(otlp_attributes, otlp_spans(destination))
            if attributes["langfuse.observation.type"] == "generation"
        ]
        usages = [json.loads(str(attributes["langfuse.observation.usage_details"])) for attributes in generations]
        costs = [json.loads(str(attributes["langfuse.observation.cost_details"])) for attributes in generations]
        assert len(usages) == 3
        assert sum(usage["total"] for usage in usages) == 25
        assert sum(cost["total"] for cost in costs) == pytest.approx(0.04)


"""Verify execution hook ownership, policy gating, and fail-open business behavior."""


@dataclass(frozen=True)
class Bridge:
    sessions: sessionmaker[Session]
    traces: ApplicationTraceRepository
    hooks: RuntimeApplicationTracingHooks

    def run(self, *, app_id: str, source_kind: TraceSourceKind, source_id: str) -> TraceRun:
        """The run frozen for a source of the bridge's workspace."""
        run = self.traces.source_run(tenant_id="tenant", app_id=app_id, source_kind=source_kind, source_id=source_id)
        assert run is not None
        return run


def bridge_hooks(
    sessions: sessionmaker[Session], traces: ApplicationTraceRepository, settings: AgentTraceSettings
) -> RuntimeApplicationTracingHooks:
    return RuntimeApplicationTracingHooks(tracing_service(sessions, traces, no_egress(), [], settings))


def run_context(app_id: str) -> DifyRunContext:
    return DifyRunContext(
        tenant_id="tenant",
        app_id=app_id,
        user_id="account",
        user_from=UserFrom.ACCOUNT,
        invoke_from=InvokeFrom.DEBUGGER,
    )


def agent_context() -> AgentAppRuntimeBuildContext:
    return AgentAppRuntimeBuildContext(
        dify_context=run_context("agent-app"),
        agent_id="agent",
        agent_config_snapshot_id="snapshot",
        agent_soul=AgentSoulConfig(),
        conversation_id="conversation",
        user_query="query",
        idempotency_key="message-id",
        binding_id="binding",
        backend_binding_ref="reference",
    )


def workflow_agent_context(preview: bool) -> WorkflowAgentRuntimeBuildContext:
    return WorkflowAgentRuntimeBuildContext(
        dify_context=run_context("workflow-app"),
        agent=Agent(
            id="referenced-agent",
            tenant_id="tenant",
            name="Agent",
            scope=AgentScope.ROSTER,
            source=AgentSource.ROSTER,
        ),
        snapshot=AgentConfigSnapshot(
            id="snapshot", tenant_id="tenant", agent_id="referenced-agent", version=1, config_snapshot=AgentSoulConfig()
        ),
        binding=WorkflowAgentNodeBinding(
            id="binding",
            tenant_id="tenant",
            app_id="workflow-app",
            workflow_id="workflow",
            workflow_version="draft",
            node_id="node",
            binding_type=WorkflowAgentBindingType.ROSTER_AGENT,
            agent_id="referenced-agent",
            node_job_config=WorkflowNodeJobConfig(),
        ),
        workflow_id="workflow",
        workflow_run_id=None if preview else "run",
        node_id="node",
        node_execution_id="node-execution",
        attempt=2,
        variable_pool=VariablePool(),
        binding_id="binding",
        backend_binding_ref="reference",
    )


def workflow_entity(extras: dict[str, object]) -> WorkflowAppGenerateEntity:
    return WorkflowAppGenerateEntity.model_construct(
        app_config=WorkflowUIBasedAppConfig(
            app_id="workflow-app", tenant_id="tenant", app_mode=AppMode.WORKFLOW, workflow_id="workflow"
        ),
        extras=extras,
    )


class TestRuntimeBridgeCases:
    @staticmethod
    @pytest.fixture
    def bridge(sqlite_session_factory: sessionmaker[Session]) -> Bridge:
        """An Agent app with one message and a workflow app with an Agent node, both tracing their content."""
        with sqlite_session_factory.begin() as session:
            for app_id, mode in (("agent-app", AppMode.AGENT), ("workflow-app", AppMode.WORKFLOW)):
                session.add(
                    make_app(
                        app_id=app_id,
                        tenant_id="tenant",
                        mode=mode,
                        tracing=json.dumps({"enabled": True, "tracing_provider": "langfuse", "capture_content": True}),
                    )
                )
                session.add(
                    TraceAppConfig(
                        app_id=app_id,
                        tracing_provider="langfuse",
                        tracing_config=langfuse_config("tenant", "https://langfuse.test"),
                    )
                )
            session.add(
                make_workflow(workflow_id="workflow", tenant_id="tenant", app_id="workflow-app", graph=AGENT_GRAPH)
            )
            session.add(
                make_conversation(
                    conversation_id="conversation",
                    app_id="agent-app",
                    mode=AppMode.AGENT,
                    inputs={},
                    from_source=ConversationFromSource.CONSOLE,
                )
            )
            session.add(
                make_message(
                    message_id="message-id",
                    app_id="agent-app",
                    conversation_id="conversation",
                    inputs={},
                    query="query",
                    message={},
                    answer="",
                    message_unit_price=Decimal(0),
                    answer_unit_price=Decimal(0),
                    currency="USD",
                    from_source=ConversationFromSource.CONSOLE,
                )
            )
        traces = ApplicationTraceRepository(sqlite_session_factory)
        return Bridge(
            sqlite_session_factory, traces, bridge_hooks(sqlite_session_factory, traces, AGENT_TRACE_SETTINGS)
        )

    @staticmethod
    def test_disabled_deployment_binds_no_hooks_and_never_freezes_a_run(bridge: Bridge) -> None:
        # The composition root binds no hooks when the deployment does not collect Agent traces.
        application = Flask(__name__)
        bind_tracing(application, scopes=trace_scopes(bridge.sessions), hooks=None)
        with application.app_context():
            assert application_tracing_hooks() is None
        disabled = bridge_hooks(bridge.sessions, bridge.traces, replace(AGENT_TRACE_SETTINGS, enabled=False))
        assert disabled.prepare_agent_app(agent_context()) is None
        assert disabled.prepare_workflow_agent(workflow_agent_context(False), None) is None
        disabled.freeze_workflow_run(
            workflow_entity({}),
            workflow_id="workflow",
            workflow_run_id="run",
            conversation_id=None,
            graph_data=AGENT_GRAPH,
        )
        assert run_count(bridge.sessions) == 0

    @staticmethod
    def test_agent_app_uses_message_identity_and_stable_conversation(bridge: Bridge) -> None:
        runtime = bridge.hooks.prepare_agent_app(agent_context())
        run = bridge.run(app_id="agent-app", source_kind="message", source_id="message-id")
        assert run.session_id == "conversation"
        [invocation] = bridge.traces.invocations(run.id)
        assert (invocation.agent_id, invocation.node_execution_id, invocation.attempt) == ("agent", None, 0)
        # The message is the call key, so a redelivered request reuses its invocation.
        assert invocation.id == str(uuid5(UUID(run.id), "message-id"))
        assert runtime == ApplicationTracingContext(
            run_id=UUID(run.id),
            invocation_id=UUID(invocation.id),
            trace_id=run.trace_id,
            parent_span_id=invocation.span_id,
            session_id="conversation",
            capture_content=True,
        )

    @staticmethod
    @pytest.mark.parametrize("preview", [False, True])
    def test_workflow_invocation_uses_owner_app_and_execution_attempt(bridge: Bridge, preview: bool) -> None:
        # Neither source row exists yet: only the workflow the hook passes lets the ledger reserve the run.
        runtime = bridge.hooks.prepare_workflow_agent(workflow_agent_context(preview), None)
        source_kind: TraceSourceKind = "node" if preview else "workflow"
        source_id = "node-execution" if preview else "run"
        run = bridge.run(app_id="workflow-app", source_kind=source_kind, source_id=source_id)
        assert run.session_id == source_id
        [invocation] = bridge.traces.invocations(run.id)
        assert (invocation.agent_id, invocation.node_execution_id, invocation.attempt) == (
            "referenced-agent",
            "node-execution",
            2,
        )
        assert invocation.id == str(uuid5(UUID(run.id), "node-execution:2"))
        assert invocation.parent_span_id == span_id_for("node-execution")
        assert runtime is not None
        assert (runtime.run_id, runtime.invocation_id) == (UUID(run.id), UUID(invocation.id))

    @staticmethod
    def test_workflow_start_prefers_explicit_trace_session(bridge: Bridge) -> None:
        bridge.hooks.freeze_workflow_run(
            workflow_entity({"trace_session_id": "external-session"}),
            workflow_id="workflow",
            workflow_run_id="run",
            conversation_id="conversation",
            graph_data={"nodes": [{"data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}]},
        )
        run = bridge.run(app_id="workflow-app", source_kind="workflow", source_id="run")
        assert run.session_id == "external-session"
        assert run.state is TraceRunState.COLLECTING

    @staticmethod
    def test_business_hook_failure_logs_type_without_sensitive_exception_message(
        bridge: Bridge, caplog: pytest.LogCaptureFixture
    ) -> None:
        failing = _FreezeFails(bridge.sessions, ValueError("password=private-secret"))
        hooks = bridge_hooks(bridge.sessions, failing, AGENT_TRACE_SETTINGS)
        assert hooks.prepare_agent_app(agent_context()) is None
        assert "ValueError" in caplog.text
        assert "private-secret" not in caplog.text

    @staticmethod
    def test_ordinary_workflow_does_not_create_managed_snapshot(bridge: Bridge) -> None:
        bridge.hooks.freeze_workflow_run(
            workflow_entity({}),
            workflow_id="workflow",
            workflow_run_id="run",
            conversation_id=None,
            graph_data={"nodes": [{"data": {"type": "llm"}}, {"data": {"type": "agent", "version": "1"}}]},
        )
        assert run_count(bridge.sessions) == 0

    @staticmethod
    @pytest.mark.parametrize(
        "graph", ["not-json", "null", "[]", {}, {"nodes": {}}, {"nodes": [None, {"data": {"type": "llm"}}]}]
    )
    def test_invalid_or_ordinary_graph_cannot_enable_agent_collection(graph: object) -> None:
        assert workflow_has_agent_v2(graph) is False

    @staticmethod
    def test_graph_string_with_agent_is_recognized() -> None:
        assert (
            workflow_has_agent_v2('{"nodes":[{"data":{"type":"agent","version":"2","agent_node_kind":"dify_agent"}}]}')
            is True
        )

    @staticmethod
    @pytest.mark.parametrize("outcome", ["succeeded", "failed", "cancelled"])
    def test_worker_terminal_handoff_preserves_outcome_without_sse(
        bridge: Bridge, outcome: Literal["succeeded", "failed", "cancelled"]
    ) -> None:
        started = tracing_service(bridge.sessions, bridge.traces, no_egress(), [], AGENT_TRACE_SETTINGS).start(
            tenant_id="tenant",
            app_id="agent-app",
            source_kind="message",
            source_id="message-id",
            session_id="conversation",
            workflow_id=None,
        )
        assert started is not None
        bridge.hooks.capture_agent_app_terminal(
            tenant_id="tenant",
            app_id="agent-app",
            message_id="message-id",
            output="answer",
            outcome=outcome,
        )
        stored = bridge.traces.get(started.id)
        assert stored is not None
        root = Trace.model_validate(stored.outer_trace).root
        assert root.attributes["message_execution_state"] == outcome
        assert root.status is (SpanStatus.OK if outcome == "succeeded" else SpanStatus.ERROR)
        assert root.outputs == "answer"

    @staticmethod
    def test_terminal_handoff_fail_open_log_redacts_sensitive_details(
        bridge: Bridge, caplog: pytest.LogCaptureFixture
    ) -> None:
        bridge.hooks.capture_agent_app_terminal(
            tenant_id="tenant",
            app_id="agent-app",
            message_id="missing-message",
            outcome="failed",
            output="private-content",
        )
        assert "tracing_source_not_found" in caplog.text
        assert "private-content" not in caplog.text

    @staticmethod
    def test_workflow_fail_open_logs_error_kind_without_content(
        bridge: Bridge, caplog: pytest.LogCaptureFixture
    ) -> None:
        failing = _FreezeFails(bridge.sessions, ValueError("private-prompt"))
        hooks = bridge_hooks(bridge.sessions, failing, AGENT_TRACE_SETTINGS)
        assert hooks.prepare_workflow_agent(workflow_agent_context(False), "conversation") is None
        hooks.freeze_workflow_run(
            workflow_entity({}),
            workflow_id="workflow",
            workflow_run_id="run",
            conversation_id="conversation",
            graph_data={"nodes": [{"data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}]},
        )
        assert "workflow_agent" in caplog.text
        assert "workflow_start" in caplog.text
        assert "ValueError" in caplog.text
        assert "private-prompt" not in caplog.text

    @staticmethod
    def test_hooks_expose_only_the_agent_collection_lifecycle(bridge: Bridge) -> None:
        assert {name for name in dir(bridge.hooks) if not name.startswith("_")} == {
            "prepare_agent_app",
            "prepare_workflow_agent",
            "freeze_workflow_run",
            "capture_agent_app_terminal",
        }
        application = Flask(__name__)
        bind_tracing(application, scopes=trace_scopes(bridge.sessions), hooks=bridge.hooks)
        with application.app_context():
            assert application_tracing_hooks() is bridge.hooks


"""Hidden Workflow Tool Agents must not invent a trace owned by the source app."""


def _workflow_tool_tracing_boundary_seed(
    sessions: sessionmaker[Session], *, root_has_agent: bool, source_enabled: bool
) -> tuple[
    ApplicationTraceRepository, RuntimeApplicationTracingHooks, WorkflowAgentRuntimeBuildContext, dict[str, str]
]:
    ids = {
        name: identifier("tool-boundary-" + name)
        for name in [
            "tenant",
            "root",
            "source",
            "root_workflow",
            "source_workflow",
            "run",
            "root_agent",
            "root_tool",
            "child",
            "actor",
        ]
    }
    root_graph: dict[str, list[dict[str, object]]] = {
        "nodes": [{"id": "tool", "data": {"type": "tool", "title": "Root tool"}}],
        "edges": [],
    }
    if root_has_agent:
        root_graph["nodes"].append(
            {
                "id": "root-agent",
                "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent", "title": "Root Agent"},
            }
        )
    now = datetime(2026, 10, 21, tzinfo=UTC)
    configs = {
        "root": langfuse_config(ids["tenant"], "https://tracing.test/root"),
        "source": {
            "api_key": WorkspaceBoundCipher().encrypt(ids["tenant"], "secret"),
            "project": "source",
            "endpoint": "https://tracing.test/source",
        },
    }
    with sessions.begin() as session:
        for app, provider, enabled, content in [
            ("root", "langfuse", True, False),
            ("source", "langsmith", source_enabled, True),
        ]:
            session.add(
                make_app(
                    app_id=ids[app],
                    tenant_id=ids["tenant"],
                    mode=AppMode.WORKFLOW,
                    tracing=json.dumps({"enabled": enabled, "tracing_provider": provider, "capture_content": content}),
                )
            )
            session.add(
                make_workflow(
                    workflow_id=ids[app + "_workflow"],
                    app_id=ids[app],
                    tenant_id=ids["tenant"],
                    graph=root_graph if app == "root" else {"nodes": [], "edges": []},
                )
            )
            session.add(TraceAppConfig(app_id=ids[app], tracing_provider=provider, tracing_config=configs[app]))
        workflow_record = WorkflowRun(
            id=ids["run"],
            tenant_id=ids["tenant"],
            app_id=ids["root"],
            workflow_id=ids["root_workflow"],
            type=WorkflowType.WORKFLOW,
            triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
            version="1",
            graph=json.dumps(root_graph),
            inputs="{}",
            status=WorkflowExecutionStatus.SUCCEEDED,
            created_by=ids["actor"],
            created_by_role=CreatorUserRole.ACCOUNT,
            finished_at=now + timedelta(seconds=3),
        )
        workflow_record.created_at = now
        session.add(workflow_record)
        for key, node_type in [("root_agent", "agent"), ("root_tool", "tool"), ("child", "agent")]:
            child = key == "child"
            app = "source" if child else "root"
            node_record = WorkflowNodeExecutionModel(
                id=ids[key],
                tenant_id=ids["tenant"],
                app_id=ids[app],
                workflow_id=ids[app + "_workflow"],
                workflow_run_id=ids["run"],
                triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_TOOL
                if child
                else WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
                triggered_from_workflow_id=ids["root_workflow"] if child else None,
                triggered_from_node_execution_id=ids["root_tool"] if child else None,
                node_execution_id=ids[key],
                index=1,
                node_id=key,
                node_type=node_type,
                title=key,
                status=WorkflowNodeExecutionStatus.SUCCEEDED,
                created_by=ids["actor"],
                created_by_role=CreatorUserRole.ACCOUNT,
                finished_at=now + timedelta(seconds=2),
                inputs="{}",
                outputs="{}",
                process_data=json.dumps(
                    {"workflow_tool_root_app_id": ids["root"], "workflow_tool_invocation_id": "tool-call"}
                )
                if child
                else "{}",
            )
            node_record.created_at = now
            session.add(node_record)
    traces = ApplicationTraceRepository(sessions)
    hooks = bridge_hooks(sessions, traces, AGENT_TRACE_SETTINGS)
    entity = workflow_entity({})
    entity.app_config = entity.app_config.model_copy(
        update={"app_id": ids["root"], "tenant_id": ids["tenant"], "workflow_id": ids["root_workflow"]}
    )
    hooks.freeze_workflow_run(
        entity,
        workflow_id=ids["root_workflow"],
        workflow_run_id=ids["run"],
        conversation_id=None,
        graph_data=root_graph,
    )
    original = workflow_agent_context(False)
    context = replace(
        original,
        dify_context=original.dify_context.model_copy(
            update={"tenant_id": ids["tenant"], "app_id": ids["source"], "workflow_tool_invocation_id": "tool-call"}
        ),
        workflow_id=ids["source_workflow"],
        workflow_run_id=ids["run"],
        node_execution_id=ids["child"],
    )
    return traces, hooks, context, ids


class TestWorkflowToolTracingBoundaryCases:
    @staticmethod
    @pytest.mark.parametrize("root_has_agent", [False, True])
    @pytest.mark.parametrize("source_enabled", [False, True])
    def test_hidden_child_never_reserves_source_policy_or_changes_root_freeze(
        sqlite_engine: Engine,
        sqlite_session_factory: sessionmaker[Session],
        root_has_agent: bool,
        source_enabled: bool,
    ) -> None:
        traces, hooks, context, ids = _workflow_tool_tracing_boundary_seed(
            sqlite_session_factory, root_has_agent=root_has_agent, source_enabled=source_enabled
        )
        with sqlite_session_factory() as session:
            before = deepcopy([dict(row) for row in session.execute(select(ApplicationTraceRun.__table__)).mappings()])
        statements: list[str] = []

        def executed(
            _connection: object,
            _cursor: object,
            statement: str,
            _parameters: object,
            _context: object,
            _executemany: object,
        ) -> None:
            statements.append(statement)

        # Marked children never consult policy or ownership, even for malformed
        # empty markers or a foreign tenant. They cannot create an orphan B run.
        event.listen(sqlite_engine, "before_cursor_execute", executed)
        try:
            for tenant in (ids["tenant"], identifier("foreign-tool-tenant")):
                for marker in ("tool-call", ""):
                    child = replace(
                        context,
                        dify_context=context.dify_context.model_copy(
                            update={
                                "tenant_id": tenant,
                                "workflow_tool_invocation_id": marker,
                            }
                        ),
                    )
                    assert hooks.prepare_workflow_agent(child, None) is None
        finally:
            event.remove(sqlite_engine, "before_cursor_execute", executed)
        assert statements == []
        with sqlite_session_factory() as session:
            observed = [
                (r.app_id, r.provider, r.capture_content, r.source_id)
                for r in session.scalars(select(ApplicationTraceRun))
            ]
            assert observed == ([(ids["root"], "langfuse", False, ids["run"])] if root_has_agent else [])
            after = [dict(row) for row in session.execute(select(ApplicationTraceRun.__table__)).mappings()]
        assert after == before
        assert (
            traces.source_run(
                tenant_id=ids["tenant"], app_id=ids["source"], source_kind="workflow", source_id=ids["run"]
            )
            is None
        )

    @staticmethod
    def test_hidden_child_skip_keeps_normal_root_agent_trace_assemblable(
        sqlite_session_factory: sessionmaker[Session],
    ) -> None:
        traces, hooks, child, ids = _workflow_tool_tracing_boundary_seed(
            sqlite_session_factory, root_has_agent=True, source_enabled=True
        )
        assert hooks.prepare_workflow_agent(child, None) is None
        root_context = replace(
            child,
            dify_context=child.dify_context.model_copy(
                update={"app_id": ids["root"], "workflow_tool_invocation_id": None}
            ),
            workflow_id=ids["root_workflow"],
            node_execution_id=ids["root_agent"],
        )
        runtime = hooks.prepare_workflow_agent(root_context, None)
        assert runtime is not None
        run = traces.get(str(runtime.run_id))
        assert run is not None
        assert run.app_id == ids["root"]
        frozen = deepcopy(run.provider_config)
        with sqlite_session_factory.begin() as session:
            owner = session.get(App, ids["root"])
            assert owner is not None
            owner.tracing = json.dumps({"enabled": True, "tracing_provider": "langsmith", "capture_content": True})
            workflow = session.get(WorkflowRun, ids["run"])
            assert workflow is not None
        assert workflow.finished_at is not None
        actor = make_account(account_id=ids["actor"])
        nodes = SQLAlchemyWorkflowNodeExecutionRepository(
            session_factory=sqlite_session_factory,
            tenant_id=ids["tenant"],
            user=actor,
            app_id=ids["root"],
            triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
        )
        info = WorkflowTraceInfo(
            node_executions=nodes.get_by_workflow_execution(ids["run"]),
            conversation_id=None,
            message_started_at=None,
            workflow_app_log_id=None,
            workflow_id=ids["root_workflow"],
            tenant_id=ids["tenant"],
            workflow_run_id=ids["run"],
            workflow_run_elapsed_time=3,
            workflow_run_status="succeeded",
            workflow_run_inputs={},
            workflow_run_outputs={},
            workflow_run_version="1",
            error=None,
            total_tokens=0,
            prompt_tokens=None,
            completion_tokens=None,
            file_list=[],
            invoked_by=None,
            query="",
            metadata={"app_id": ids["root"]},
            start_time=workflow.created_at,
            end_time=workflow.finished_at,
        )
        outer = CanonicalTraceBuilder().build(info)
        assert outer is not None
        capture(traces, run.id, outer)
        [invocation] = traces.invocations(run.id)
        ingest(
            traces,
            AgentTraceBatch(
                run_id=UUID(run.id),
                invocation_id=UUID(invocation.id),
                batch_id=uuid4(),
                final=True,
                spans=[
                    AgentTraceSpan(
                        trace_id=run.trace_id,
                        span_id="a" * 16,
                        parent_span_id=invocation.span_id,
                        name="Agent execution",
                        start_time_ns=1_000_000_000,
                        end_time_ns=2_000_000_000,
                        status="ok",
                        attributes={"gen_ai.operation.name": "invoke_agent"},
                    )
                ],
            ),
            max_spans=100,
            max_bytes=1_000_000,
        )
        destination = exported()
        tracing_service(sqlite_session_factory, traces, destination, [], AGENT_TRACE_SETTINGS).deliver(run.id)
        current = traces.get(run.id)
        assert current is not None
        # Assembly rejects any span whose parent is missing, so acceptance proves a complete tree.
        assert current.state is TraceRunState.ACCEPTED
        assert [request.url for request in destination.requests] == [
            "https://tracing.test/root/api/public/otel/v1/traces"
        ]
        sent = otlp_spans(destination)
        emitted = {span.span_id for span in sent}
        assert all(span.parent_span_id in emitted for span in sent if span.parent_span_id)
        assert current.provider_config == frozen
        assert current.provider == "langfuse"
        assert current.capture_content is False