Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
import json
import threading
from collections.abc import Callable, Generator, Iterator, Mapping, Sequence
from contextlib import contextmanager
from dataclasses import dataclass, field, replace
from datetime import UTC, datetime, timedelta
from decimal import Decimal
from typing import NoReturn, override
from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
import pytest
from aliyun.log.logclient import LogClient
from dify_agent.protocol.tracing import AgentTraceBatch, AgentTraceEvent, AgentTraceSpan
from dify_trace_langfuse.config import LangfuseConfig
from dify_trace_sdk.egress import Egress, EgressResponse
from dify_trace_sdk.errors import EgressError, ExportOutcomeUnknownError, ExportRejectedError, ExportRetryableError
from dify_trace_sdk.events import MessageCompleted, TraceCorrelation, TraceEvent, TracePayload
from dify_trace_sdk.exporting import send_export_request
from dify_trace_sdk.model import Span, SpanKind, SpanStatus, Trace
from dify_trace_sdk.provider import ProviderCapabilities, SendResult, TraceProvider
from dify_trace_sdk.testing import RecordingEgress, response
from sqlalchemy import Connection, Engine, event, func, select
from sqlalchemy.orm import Session, sessionmaker
from typing_extensions import TypedDict
from core.helper.encrypter import obfuscated_token
from domain.tracing.errors import TraceRunConflictError
from domain.tracing.trace_run_agg import TraceInvocation, TraceRunAgg, TraceRunState, TraceSourceKind
from extensions.logstore.aliyun_logstore import AliyunLogStore
from extensions.logstore.repositories.logstore_api_workflow_node_execution_repository import (
_dict_to_workflow_node_execution_model,
)
from extensions.logstore.repositories.logstore_api_workflow_run_repository import _dict_to_workflow_run
from graphon.enums import WorkflowNodeExecutionStatus
from libs.datetime_utils import naive_utc_now
from machinery.context import RequestContext
from models.account import Tenant, TenantAccountJoin, TenantAccountRole
from models.application_trace import ApplicationTraceBatch, ApplicationTraceRun
from models.enums import ConversationFromSource, CreatorUserRole
from models.model import App, AppMode, Message, TraceAppConfig
from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRun
from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
from repositories.application_trace_repository import ApplicationTraceRepository
from repositories.factory import DifyAPIRepositoryFactory
from repositories.trace_record_repository import TraceRecordRepository
from services.app_tracing_config_service import AppTracingConfigService
from services.tracing import service as service_module
from services.tracing.canonical import CanonicalTraceBuilder
from services.tracing.collection import AgentTraceSettings, TraceRun, TraceRunStatus
from services.tracing.collection_ports import LockedTraceRun
from services.tracing.errors import (
TraceCollectionConflictError,
TraceCollectionLimitError,
TraceCollectionNotFoundError,
TracingStoreUnavailableError,
)
from services.tracing.provider_configs import ProviderConfigGateway
from services.tracing.providers import ProviderConfigCodec, TraceProviderCatalog, TracingSecretCipher
from services.tracing.records import GenerateNameTraceInfo, MessageTraceInfo
from services.tracing.registry import EntryPointTraceProviderCatalog
from services.tracing.service import ApplicationTracingService
from tests.helpers.unit.config_override import apply_config_overrides
from tests.helpers.unit.model_factories import (
make_account,
make_app,
make_conversation,
make_message,
make_tenant,
make_workflow,
)
from tests.helpers.unit.services.tracing_ports import AGENT_TRACE_SETTINGS, InstalledTraceProviders
"""Exercise collection transactions and delivery state against real SQLite rows."""
def _application_trace_collection_identifier(value: str) -> str:
return str(uuid5(NAMESPACE_URL, value))
@dataclass
class _ApplicationTraceCollectionCollection:
repository: ApplicationTraceRepository
sessions: sessionmaker[Session]
run: TraceRun
@contextmanager
def change(self) -> Generator[TraceRunAgg]:
"""The run under its row lock, in its own transaction; it is saved when the block exits without an error."""
with self.repository.change(self.run.id, tenant_id=self.run.tenant_id) as locked:
assert locked is not None
yield locked.run
def invocation(self, name: str = "first") -> TraceInvocation:
return self.open_invocation(
agent_id=_application_trace_collection_identifier("agent"),
node_execution_id=None,
attempt=0,
call_key=name,
)
def open_invocation(
self, *, agent_id: str, node_execution_id: str | None, attempt: int, call_key: str
) -> TraceInvocation:
with self.change() as run:
return run.open_invocation(
agent_id=agent_id, node_execution_id=node_execution_id, attempt=attempt, call_key=call_key
)
class ScriptedProvider(TraceProvider[LangfuseConfig]):
"""Sends each trace as one JSON request through the egress, whose script decides the outcome."""
id = "langfuse"
config_model = LangfuseConfig
capabilities = ProviderCapabilities(native_cost=True, trace_url=True, span_url=True)
@override
def verify(self, config: LangfuseConfig) -> LangfuseConfig:
return config
@override
def project_url(self, config: LangfuseConfig) -> str | None:
return f"{config.host}/project/{config.project_key}"
@override
def send(self, config: LangfuseConfig, trace: Trace) -> SendResult:
reply = send_export_request(
self.egress,
"POST",
f"{config.host}/traces",
destination="Scripted",
headers={"Authorization": f"{config.public_key}:{config.secret_key}"},
content=trace.model_dump_json().encode(),
sensitive_url=False,
)
body = reply.json()
assert isinstance(body, dict)
return SendResult(
trace_id=str(body["trace_id"]), trace_url=body.get("trace_url"), span_urls=body.get("span_urls", {})
)
SCRIPTED_PROVIDERS = InstalledTraceProviders(ScriptedProvider)
"""Agent traces are delivered through ``ScriptedProvider``; no other provider is installed."""
class ForbiddenTraceProviders(TraceProviderCatalog):
"""Fails the test when the service resolves any provider."""
def __init__(self, reason: str) -> None:
self._reason = reason
@override
def resolve(self, provider_id: str) -> NoReturn:
pytest.fail(self._reason)
class StoredSecretCipher(TracingSecretCipher):
"""A stored ``encrypted-x`` secret decrypts to ``plaintext-x``; ``decrypted`` lists the ciphertexts read."""
def __init__(self) -> None:
self.decrypted: list[str] = []
@override
def encrypt(self, tenant_id: str, value: str) -> str:
return f"encrypted-{value.removeprefix('plaintext-')}"
@override
def decrypt(self, tenant_id: str, tokens: Sequence[str]) -> list[str]:
self.decrypted.extend(tokens)
return [token.replace("encrypted-", "plaintext-", 1) for token in tokens]
@override
def mask(self, value: str) -> str:
return obfuscated_token(value)
class ForbiddenSecretCipher(StoredSecretCipher):
"""Fails the test when the service reads any credential."""
def __init__(self, reason: str) -> None:
super().__init__()
self._reason = reason
@override
def decrypt(self, tenant_id: str, tokens: Sequence[str]) -> NoReturn:
pytest.fail(self._reason)
class UnreadableSecretCipher(StoredSecretCipher):
"""No stored secret can be decrypted, as when the workspace key is lost."""
@override
def decrypt(self, tenant_id: str, tokens: Sequence[str]) -> NoReturn:
raise ValueError("private-ciphertext")
def trace_records(sessions: sessionmaker[Session]) -> TraceRecordRepository:
return TraceRecordRepository(
sessions,
workflow_runs=DifyAPIRepositoryFactory.create_api_workflow_run_repository(sessions),
node_executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(sessions),
files_url="https://files.test",
enterprise_details=False,
)
@dataclass
class ServicePorts:
"""The service's collaborators besides its repository; a test replaces those it constrains.
``scheduled`` lists the runs the service scheduled for export.
"""
egress: Egress
providers: TraceProviderCatalog = SCRIPTED_PROVIDERS
cipher: TracingSecretCipher = field(default_factory=StoredSecretCipher)
settings: AgentTraceSettings = AGENT_TRACE_SETTINGS
scheduled: list[str] = field(default_factory=list[str])
def service(self, case: _ApplicationTraceCollectionCollection) -> ApplicationTracingService:
return self.service_on(case.repository, case.sessions)
def service_on(
self, repository: ApplicationTraceRepository, sessions: sessionmaker[Session]
) -> ApplicationTracingService:
"""The service whose runs, run queries, and source queries all go through ``repository``."""
return ApplicationTracingService(
runs=repository,
run_query=repository,
sources=repository,
records=trace_records(sessions),
builder=CanonicalTraceBuilder(),
providers=self.providers,
codec=ProviderConfigCodec(self.cipher),
egress=self.egress,
schedule_export=self.scheduled.append,
settings=self.settings,
)
def _application_trace_collection_accepted(trace_id: str) -> RecordingEgress:
"""An egress whose destination acknowledges exactly one trace."""
return RecordingEgress([response(200, {"trace_id": trace_id})], grpc_target=None)
def _application_trace_collection_no_egress() -> RecordingEgress:
"""An egress that fails any request the test did not script."""
return RecordingEgress([], grpc_target=None)
def sent_traces(egress: RecordingEgress) -> list[Trace]:
return [Trace.model_validate_json(request.content or b"") for request in egress.requests]
def _application_trace_collection_seed(
sessions: sessionmaker[Session],
name: str,
*,
capture_content: bool = False,
enabled: bool = True,
capture_agent: bool = True,
) -> _ApplicationTraceCollectionCollection:
app_id, tenant_id, message_id = (
_application_trace_collection_identifier(f"app-{name}"),
_application_trace_collection_identifier("tenant"),
_application_trace_collection_identifier(f"message-{name}"),
)
with sessions.begin() as session:
session.add(
make_app(
app_id=app_id,
tenant_id=tenant_id,
mode=AppMode.AGENT,
tracing=json.dumps(
{
"enabled": enabled,
"tracing_provider": "langfuse",
"capture_content": capture_content,
"capture_agent": capture_agent,
}
),
)
)
session.add(
make_message(
message_id=message_id,
app_id=app_id,
conversation_id=_application_trace_collection_identifier(f"conversation-{name}"),
inputs={},
query="hello",
message={},
answer="done",
message_unit_price=Decimal(0),
answer_unit_price=Decimal(0),
currency="USD",
from_source=ConversationFromSource.CONSOLE,
)
)
session.add(
TraceAppConfig(
app_id=app_id,
tracing_provider="langfuse",
tracing_config={
"public_key": "encrypted-public",
"secret_key": "encrypted-original",
"host": "https://langfuse.test",
"project_key": "original",
},
)
)
repository = ApplicationTraceRepository(sessions)
ServicePorts(_application_trace_collection_no_egress()).service_on(repository, sessions).start(
tenant_id=tenant_id,
app_id=app_id,
source_kind="message",
source_id=message_id,
session_id="session-original",
workflow_id=None,
)
# ``start`` returns no run for a source frozen as not collected, so read the frozen run itself.
run = repository.source_run(tenant_id=tenant_id, app_id=app_id, source_kind="message", source_id=message_id)
assert run is not None
return _ApplicationTraceCollectionCollection(repository, sessions, run)
def attach_conversation(case: _ApplicationTraceCollectionCollection, mode: AppMode) -> None:
"""Message records are assembled only for a message whose conversation belongs to its app."""
with case.sessions.begin() as session:
message = session.get(Message, case.run.source_id)
assert message is not None
session.add(
make_conversation(
conversation_id=message.conversation_id,
app_id=case.run.app_id,
mode=mode,
inputs={},
from_source=ConversationFromSource.CONSOLE,
)
)
def add_app_owner(case: _ApplicationTraceCollectionCollection, name: str) -> str:
"""Workflow records are read as the app creator, who must be a member of the app's workspace."""
owner_id = _application_trace_collection_identifier(f"owner-{name}")
with case.sessions.begin() as session:
if session.get(Tenant, case.run.tenant_id) is None:
session.add(make_tenant(tenant_id=case.run.tenant_id))
session.add(make_account(account_id=owner_id, email=f"{name}@example.com"))
session.add(TenantAccountJoin(tenant_id=case.run.tenant_id, account_id=owner_id, role=TenantAccountRole.OWNER))
app = session.get(App, case.run.app_id)
assert app is not None
app.created_by = owner_id
return owner_id
def trace_event(case: _ApplicationTraceCollectionCollection, payload: TracePayload) -> TraceEvent:
return TraceEvent(
event_id=uuid4().hex,
tenant_id=case.run.tenant_id,
app_id=case.run.app_id,
user_id=None,
occurred_at=datetime.now(UTC),
correlation=TraceCorrelation(
external_trace_id=None, session_id=None, parent_workflow_run_id=None, parent_node_execution_id=None
),
payload=payload,
)
def message_record(case: _ApplicationTraceCollectionCollection) -> MessageTraceInfo:
"""The record the delivery worker assembles when the message reports its end."""
with case.sessions() as session:
message = session.get(Message, case.run.source_id)
assert message is not None
conversation_id = message.conversation_id
record = trace_records(case.sessions).assemble(
trace_event(case, MessageCompleted(message_id=case.run.source_id, conversation_id=conversation_id))
)
assert isinstance(record, MessageTraceInfo)
return record
def _application_trace_collection_batch_for(
run: TraceRun,
invocation: TraceInvocation,
*,
final: bool = True,
batch_id: UUID | None = None,
span_id: str = "a" * 16,
) -> AgentTraceBatch:
return AgentTraceBatch(
run_id=UUID(run.id),
invocation_id=UUID(invocation.id),
batch_id=batch_id or uuid4(),
final=final,
spans=[
AgentTraceSpan(
trace_id=run.trace_id,
span_id=span_id,
parent_span_id=invocation.span_id,
name="Agent execution",
start_time_ns=1_000_000_000,
end_time_ns=2_000_000_000,
status="ok",
attributes={"gen_ai.operation.name": "invoke_agent"},
)
],
)
def _application_trace_collection_ingest(
collection: _ApplicationTraceCollectionCollection, batch: AgentTraceBatch
) -> tuple[int, list[str]]:
"""Ingest through the service; returns the spans it accepted and the runs it then scheduled for export."""
ports = ServicePorts(_application_trace_collection_no_egress())
return ports.service(collection).ingest(batch), ports.scheduled
def _application_trace_collection_terminal(collection: _ApplicationTraceCollectionCollection) -> None:
run = collection.run
root = Span(
span_id=run.root_span_id,
parent_span_id=None,
name="Agent turn",
kind=SpanKind.CHAIN,
start_time=datetime.fromtimestamp(0, UTC),
end_time=datetime.fromtimestamp(3, UTC),
status=SpanStatus.OK,
)
trace = Trace(
trace_id=run.trace_id,
external_trace_id=None,
session_id=run.session_id,
user_id=None,
parent=None,
spans=(root,),
)
with collection.change() as current:
current.capture_terminal(trace, now=naive_utc_now())
def _claim_at(
collection: _ApplicationTraceCollectionCollection, *, lease_seconds: int, now: datetime
) -> tuple[TraceRunAgg, str] | None:
"""Lease the run for one export attempt; returns the claimed run and the token that fences the attempt."""
token = str(uuid4())
with collection.repository.change(collection.run.id, tenant_id=collection.run.tenant_id) as locked:
assert locked is not None
claimed = locked.run.claim_export(
token=token, owner_exists=locked.owner_exists, collection_timeout=60, lease_seconds=lease_seconds, now=now
)
return (locked.run, token) if claimed else None
def _application_trace_collection_claim(
collection: _ApplicationTraceCollectionCollection,
) -> tuple[TraceRunAgg, str] | None:
return _claim_at(collection, lease_seconds=30, now=naive_utc_now())
def _accept_export(collection: _ApplicationTraceCollectionCollection, token: str, accepted: SendResult) -> None:
"""Record the destination's acceptance of the export attempt ``token``."""
with collection.change() as run:
run.export_accepted(
token,
provider_trace_id=accepted.trace_id,
trace_url=accepted.trace_url,
span_urls=accepted.span_urls,
now=naive_utc_now(),
)
class TestApplicationTraceCollectionCases:
@staticmethod
@pytest.fixture
def collection(sqlite_session_factory: sessionmaker[Session]) -> _ApplicationTraceCollectionCollection:
return _application_trace_collection_seed(sqlite_session_factory, "first")
@staticmethod
def test_batch_duplicate_is_idempotent_after_final_and_acceptance(
collection: _ApplicationTraceCollectionCollection,
) -> None:
invocation = collection.invocation()
batch = _application_trace_collection_batch_for(collection.run, invocation)
assert _application_trace_collection_ingest(collection, batch) == (1, [collection.run.id])
assert _application_trace_collection_ingest(collection, batch) == (1, [collection.run.id])
_application_trace_collection_terminal(collection)
leased = _application_trace_collection_claim(collection)
assert leased is not None
_accept_export(collection, leased[1], SendResult(trace_id="provider-id", trace_url=None, span_urls={}))
assert _application_trace_collection_ingest(collection, batch) == (1, [collection.run.id])
stored = collection.repository.get(collection.run.id)
assert stored is not None
assert stored.span_count == 1
with collection.sessions() as session:
assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 1
@staticmethod
def test_conflicting_batch_id_rolls_back_without_replacing_payload(
collection: _ApplicationTraceCollectionCollection,
) -> None:
batch = _application_trace_collection_batch_for(collection.run, collection.invocation(), final=False)
_application_trace_collection_ingest(collection, batch)
changed = batch.model_copy(update={"final": True})
with pytest.raises(TraceCollectionConflictError, match="tracing_batch_conflict"):
_application_trace_collection_ingest(collection, changed)
invocations = collection.repository.invocations(collection.run.id)
assert invocations[0].finalized is False
with collection.sessions() as session:
assert session.scalar(select(ApplicationTraceBatch.final)) is False
@staticmethod
def test_run_cannot_ingest_another_runs_invocation(collection: _ApplicationTraceCollectionCollection) -> None:
other = _application_trace_collection_seed(collection.sessions, "other")
foreign = other.invocation()
forged = _application_trace_collection_batch_for(collection.run, foreign)
with pytest.raises(TraceCollectionNotFoundError, match="tracing_invocation_not_found"):
_application_trace_collection_ingest(collection, forged)
with collection.sessions() as session:
assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 0
@staticmethod
def test_payload_limits_reject_whole_transaction(collection: _ApplicationTraceCollectionCollection) -> None:
batch = _application_trace_collection_batch_for(collection.run, collection.invocation())
limits = replace(AGENT_TRACE_SETTINGS, max_spans=100, max_bytes=1)
limited = ServicePorts(_application_trace_collection_no_egress(), settings=limits).service(collection)
with pytest.raises(TraceCollectionLimitError, match="tracing_payload_limit"):
limited.ingest(batch)
assert not collection.repository.batches(collection.run.id)
assert collection.repository.invocations(collection.run.id)[0].finalized is False
stored = collection.repository.get(collection.run.id)
assert stored is not None
assert stored.span_count == 0
@staticmethod
def test_wrong_trace_id_is_rejected_before_persistence(collection: _ApplicationTraceCollectionCollection) -> None:
batch = _application_trace_collection_batch_for(collection.run, collection.invocation())
altered = batch.model_copy(update={"spans": [batch.spans[0].model_copy(update={"trace_id": "f" * 32})]})
with pytest.raises(TraceCollectionConflictError, match="tracing_trace_id_mismatch"):
_application_trace_collection_ingest(collection, altered)
assert not collection.repository.batches(collection.run.id)
@staticmethod
def test_content_off_cleans_the_durable_batch_not_only_the_export(
collection: _ApplicationTraceCollectionCollection,
) -> None:
original = _application_trace_collection_batch_for(collection.run, collection.invocation(), final=False)
observation = original.spans[0].model_copy(
update={
"status": "error",
"status_message": "private error",
"attributes": {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "model-one",
"gen_ai.input.messages": "private prompt",
"gen_ai.tool.call.arguments": "private arguments",
"dify.payload": "private metadata",
"authorization": "private credential",
},
"events": [
AgentTraceEvent(
name="exception", timestamp_ns=1_500_000_000, attributes={"exception.message": "private event"}
)
],
}
)
batch = original.model_copy(update={"spans": [observation]})
_application_trace_collection_ingest(collection, batch)
batches = collection.repository.batches(collection.run.id)
assert "private" not in json.dumps(batches[0].spans)
assert batches[0].spans[0]["events"] == []
assert batches[0].spans[0]["status"] == "error"
assert batches[0].spans[0]["attributes"]["gen_ai.request.model"] == "model-one"
# A changed original payload is a conflict even if content-off sanitization
# would produce exactly the same persisted span.
changed = batch.model_copy(
update={"spans": [observation.model_copy(update={"status_message": "different private error"})]}
)
with pytest.raises(TraceCollectionConflictError, match="tracing_batch_conflict"):
_application_trace_collection_ingest(collection, changed)
@staticmethod
def test_invocation_key_replay_must_not_change_agent_binding(
collection: _ApplicationTraceCollectionCollection,
) -> None:
original = collection.invocation()
repeated = collection.invocation()
assert original.id == repeated.id
with pytest.raises(TraceRunConflictError, match="tracing_invocation_conflict"):
collection.open_invocation(
agent_id=_application_trace_collection_identifier("different-agent"),
node_execution_id=_application_trace_collection_identifier("different-node"),
attempt=2,
call_key="first",
)
@staticmethod
def test_claim_waits_for_all_invocations_final_and_then_only_one_lease(
collection: _ApplicationTraceCollectionCollection,
) -> None:
first, second = collection.invocation("first"), collection.invocation("second")
_application_trace_collection_terminal(collection)
_application_trace_collection_ingest(collection, _application_trace_collection_batch_for(collection.run, first))
assert _application_trace_collection_claim(collection) is None
_application_trace_collection_ingest(
collection, _application_trace_collection_batch_for(collection.run, second, span_id="b" * 16)
)
leased = _application_trace_collection_claim(collection)
assert leased is not None
assert _application_trace_collection_claim(collection) is None
assert leased[0].attempts == 1
@staticmethod
def test_expired_delivery_lease_accepts_confirmed_late_ack_for_same_attempt(
collection: _ApplicationTraceCollectionCollection,
) -> None:
_application_trace_collection_ingest(
collection, _application_trace_collection_batch_for(collection.run, collection.invocation())
)
_application_trace_collection_terminal(collection)
leased = _application_trace_collection_claim(collection)
assert leased is not None
with collection.sessions.begin() as session:
stored = session.get(ApplicationTraceRun, collection.run.id)
assert stored is not None
stored.export_started_at = naive_utc_now() - timedelta(seconds=60)
stored.export_heartbeat_at = stored.export_started_at
assert _application_trace_collection_claim(collection) is None
current = collection.repository.get(collection.run.id)
assert current is not None
assert current.state is TraceRunState.UNCERTAIN
_accept_export(collection, leased[1], SendResult(trace_id="late", trace_url=None, span_urls={}))
accepted = collection.repository.get(collection.run.id)
assert accepted is not None
assert accepted.state is TraceRunState.ACCEPTED
assert accepted.error_code is None
assert accepted.accepted_at is not None
assert accepted.provider_trace_id == "late"
assert _application_trace_collection_claim(collection) is None
@staticmethod
def test_incomplete_final_times_out_without_export(collection: _ApplicationTraceCollectionCollection) -> None:
collection.invocation()
_application_trace_collection_terminal(collection)
with collection.sessions.begin() as session:
row = session.get(ApplicationTraceRun, collection.run.id)
assert row is not None
row.terminal_at = naive_utc_now() - timedelta(seconds=120)
assert _application_trace_collection_claim(collection) is None
stored = collection.repository.get(collection.run.id)
assert stored is not None
assert stored.state is TraceRunState.FAILED
assert stored.error_code == "agent_trace_collection_incomplete"
@staticmethod
def test_running_trace_freezes_credentials_content_policy_and_session(
collection: _ApplicationTraceCollectionCollection,
) -> None:
original = collection.run
with collection.sessions.begin() as session:
app = session.get(App, original.app_id)
assert app is not None
app.tracing = json.dumps({"enabled": False, "tracing_provider": "opik", "capture_content": True})
config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == original.app_id))
assert config is not None
config.tracing_config = {"secret_key": "encrypted-new"}
service = ServicePorts(_application_trace_collection_no_egress()).service(collection)
repeated = service.start(
tenant_id=original.tenant_id,
app_id=original.app_id,
source_kind="message",
source_id=original.source_id,
session_id="changed",
workflow_id=None,
)
assert repeated is not None
assert repeated.provider == "langfuse"
assert repeated.provider_config["secret_key"] == "encrypted-original"
assert repeated.capture_content is False
assert repeated.config_revision == original.config_revision
assert repeated.session_id == "session-original"
@staticmethod
@pytest.mark.parametrize(
("outcome", "reply"),
[
(
TraceRunState.ACCEPTED,
response(200, {"trace_id": "vendor-trace", "trace_url": "https://trace.test/example"}),
),
(TraceRunState.UNCERTAIN, EgressError("acknowledgement lost", request_sent=True)),
(TraceRunState.RETRYING, response(503, {})),
],
)
def test_delivery_checkpoint_controls_replays(
collection: _ApplicationTraceCollectionCollection,
outcome: TraceRunState,
reply: EgressResponse | Exception,
) -> None:
_application_trace_collection_ingest(
collection, _application_trace_collection_batch_for(collection.run, collection.invocation())
)
_application_trace_collection_terminal(collection)
cipher = StoredSecretCipher()
egress = RecordingEgress([reply], grpc_target=None)
service = ServicePorts(egress, cipher=cipher).service(collection)
service.deliver(collection.run.id)
service.deliver(collection.run.id)
assert len(egress.requests) == 1
request = egress.requests[0]
assert request.url == "https://langfuse.test/traces"
assert request.headers["Authorization"] == "plaintext-public:plaintext-original"
assert cipher.decrypted == ["encrypted-public", "encrypted-original"]
stored = collection.repository.get(collection.run.id)
assert stored is not None
assert stored.state is outcome
assert stored.provider_config["secret_key"] == "encrypted-original"
assert "plaintext" not in json.dumps(stored.outer_trace)
@staticmethod
def test_source_owner_checks_app_and_tenant(collection: _ApplicationTraceCollectionCollection) -> None:
run = collection.run
service = ServicePorts(_application_trace_collection_no_egress()).service(collection)
foreign_tenant = _application_trace_collection_identifier("foreign-tenant")
assert (
service.start(
tenant_id=foreign_tenant,
app_id=run.app_id,
source_kind="message",
source_id=run.source_id,
session_id="x",
workflow_id=None,
)
is None
)
# ``start`` also returns no run for a source frozen as not collected; no run may be frozen at all.
assert (
collection.repository.source_run(
tenant_id=foreign_tenant, app_id=run.app_id, source_kind="message", source_id=run.source_id
)
is None
)
other = _application_trace_collection_seed(collection.sessions, "second-app")
assert (
service.start(
tenant_id=run.tenant_id,
app_id=other.run.app_id,
source_kind="message",
source_id=run.source_id,
session_id="x",
workflow_id=None,
)
is None
)
assert (
collection.repository.source_run(
tenant_id=run.tenant_id, app_id=other.run.app_id, source_kind="message", source_id=run.source_id
)
is None
)
@staticmethod
def test_disabled_source_decision_stays_disabled_after_config_enable(
sqlite_session_factory: sessionmaker[Session],
) -> None:
case = _application_trace_collection_seed(sqlite_session_factory, "initially-disabled", enabled=False)
attach_conversation(case, AppMode.AGENT)
service = ServicePorts(_application_trace_collection_no_egress()).service(case)
assert case.run.state is TraceRunState.NOT_COLLECTED
with case.sessions.begin() as session:
app = session.get(App, case.run.app_id)
assert app is not None
app.tracing = json.dumps({"enabled": True, "tracing_provider": "langfuse", "capture_content": True})
assert (
service.start(
tenant_id=case.run.tenant_id,
app_id=case.run.app_id,
source_kind="message",
source_id=case.run.source_id,
session_id="later-session",
workflow_id=None,
)
is None
)
assert service.capture_terminal(tenant_id=case.run.tenant_id, info=message_record(case)) is True
stored = case.repository.get(case.run.id)
assert stored is not None
assert stored.state is TraceRunState.NOT_COLLECTED
assert stored.provider_config == {}
assert stored.outer_trace is None
@staticmethod
def test_capture_agent_disabled_preserves_outer_only_content_policy(
sqlite_session_factory: sessionmaker[Session],
) -> None:
case = _application_trace_collection_seed(sqlite_session_factory, "outer-only", capture_agent=False)
attach_conversation(case, AppMode.AGENT)
ports = ServicePorts(_application_trace_collection_no_egress())
service = ports.service(case)
assert (
service.prepare_invocation(
tenant_id=case.run.tenant_id,
app_id=case.run.app_id,
agent_id=_application_trace_collection_identifier("agent"),
source_kind="message",
source_id=case.run.source_id,
session_id="session",
node_execution_id=None,
attempt=0,
call_key="call",
workflow_id=None,
)
is None
)
info = message_record(case).model_copy(update={"inputs": "private input", "outputs": "private output"})
assert service.capture_terminal(tenant_id=case.run.tenant_id, info=info) is True
assert ports.scheduled == [case.run.id]
stored = case.repository.get(case.run.id)
assert stored is not None
assert stored.outer_trace is not None
assert "private" not in json.dumps(stored.outer_trace)
# The message's synthetic model call is left to the Agent runtime, which reports none here.
assert len(stored.outer_trace["spans"]) == 1
assert case.repository.invocations(case.run.id) == []
@staticmethod
def test_deleting_selected_configuration_keeps_historical_destination(
collection: _ApplicationTraceCollectionCollection,
) -> None:
original = collection.run
service = AppTracingConfigService(
configs=SQLAlchemyAppTracingConfigRepository(session_factory=collection.sessions),
provider=ProviderConfigGateway(
providers=SCRIPTED_PROVIDERS,
codec=ProviderConfigCodec(StoredSecretCipher()),
egress=_application_trace_collection_no_egress(),
),
)
service.delete(RequestContext("request", None, "account", original.tenant_id), original.app_id, "langfuse")
stored = collection.repository.get(original.id)
assert stored is not None
assert stored.provider_config == original.provider_config
assert stored.provider_config["project_key"] == "original"
assert stored.config_revision == original.config_revision
with collection.sessions() as session:
assert session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == original.app_id)) is None
app = session.get(App, original.app_id)
assert app is not None
assert app.tracing is not None
assert json.loads(app.tracing)["enabled"] is False
assert json.loads(app.tracing)["tracing_provider"] is None
"""Exercise public orchestration, privacy, and delivery decisions against durable rows."""
_ApplicationTraceServiceBoundariesCollection = _ApplicationTraceCollectionCollection
_application_trace_service_boundaries_accepted = _application_trace_collection_accepted
_application_trace_service_boundaries_batch_for = _application_trace_collection_batch_for
_application_trace_service_boundaries_claim = _application_trace_collection_claim
_application_trace_service_boundaries_identifier = _application_trace_collection_identifier
_application_trace_service_boundaries_ingest = _application_trace_collection_ingest
_application_trace_service_boundaries_no_egress = _application_trace_collection_no_egress
_application_trace_service_boundaries_seed = _application_trace_collection_seed
_application_trace_service_boundaries_terminal = _application_trace_collection_terminal
FROZEN_LANGFUSE: dict[str, object] = {
"public_key": "encrypted-public",
"secret_key": "encrypted-original",
"host": "https://langfuse.test",
"project_key": "original",
}
class RunSource(TypedDict, closed=True):
"""The source a caller names to read or retry the status of a run."""
tenant_id: str
app_id: str
source_kind: TraceSourceKind
source_id: str
def source(run: TraceRun) -> RunSource:
return RunSource(tenant_id=run.tenant_id, app_id=run.app_id, source_kind=run.source_kind, source_id=run.source_id)
def not_collected(source_kind: TraceSourceKind, source_id: str) -> TraceRunStatus:
"""The status of a source nothing is collected for: it names no destination or delivery detail."""
return TraceRunStatus(
status=TraceRunState.NOT_COLLECTED,
source_kind=source_kind,
source_id=source_id,
provider=None,
trace_id=None,
trace_url=None,
span_url=None,
project_url=None,
config_revision=None,
capture_content=None,
capture_agent=None,
span_count=None,
attempts=None,
error_code=None,
accepted_at=None,
capabilities=None,
)
def freeze_config(case: _ApplicationTraceServiceBoundariesCollection, provider: str, config: dict[str, object]) -> None:
with case.sessions.begin() as session:
run = session.get(ApplicationTraceRun, case.run.id)
assert run is not None
run.provider = provider
run.provider_config = config
class TestApplicationTraceServiceBoundariesCases:
@staticmethod
def test_prepare_invocation_returns_only_opaque_context_and_respects_capture_gate(
sqlite_session_factory: sessionmaker[Session],
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "prepare", capture_content=True)
service = ServicePorts(_application_trace_service_boundaries_no_egress()).service(case)
context = service.prepare_invocation(
tenant_id=case.run.tenant_id,
app_id=case.run.app_id,
agent_id=_application_trace_service_boundaries_identifier("agent"),
source_kind="message",
source_id=case.run.source_id,
session_id="ignored-new-session",
node_execution_id=None,
attempt=0,
call_key="call",
workflow_id=None,
)
assert context is not None
assert context.run_id == UUID(case.run.id)
assert context.capture_content is True
assert context.session_id == "session-original"
assert "encrypted-original" not in context.model_dump_json()
disabled = ServicePorts(
_application_trace_service_boundaries_no_egress(), settings=replace(AGENT_TRACE_SETTINGS, enabled=False)
).service(case)
assert (
disabled.start(
tenant_id=case.run.tenant_id,
app_id=case.run.app_id,
source_kind="message",
source_id=case.run.source_id,
session_id="new",
workflow_id=None,
)
is None
)
@staticmethod
@pytest.mark.parametrize("missing", ["tenant", "app", "message"])
def test_status_denies_foreign_or_absent_source_without_destination_details(
sqlite_session_factory: sessionmaker[Session], missing: str
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "scope")
foreign = _application_trace_service_boundaries_identifier("foreign")
service = ServicePorts(_application_trace_service_boundaries_no_egress()).service(case)
with pytest.raises(TraceCollectionNotFoundError) as caught:
service.status(
tenant_id=foreign if missing == "tenant" else case.run.tenant_id,
app_id=foreign if missing == "app" else case.run.app_id,
source_kind="message",
source_id=foreign if missing == "message" else case.run.source_id,
node_execution_id=None,
)
assert caught.value.code == "tracing_source_not_found"
assert "encrypted-original" not in str(caught.value)
@staticmethod
def test_status_uses_frozen_destination_and_selects_latest_node_attempt(
sqlite_session_factory: sessionmaker[Session],
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "status")
with case.sessions.begin() as session:
config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == case.run.app_id))
assert config is not None
config.tracing_config = {**FROZEN_LANGFUSE, "host": "https://new.example", "project_key": "new"}
node_id = _application_trace_service_boundaries_identifier("selected-node")
invocation = case.open_invocation(
agent_id=_application_trace_service_boundaries_identifier("agent"),
node_execution_id=node_id,
attempt=1,
call_key="node:1",
)
_application_trace_service_boundaries_ingest(
case, _application_trace_service_boundaries_batch_for(case.run, invocation)
)
_application_trace_service_boundaries_terminal(case)
lease = _application_trace_service_boundaries_claim(case)
assert lease is not None
_accept_export(
case,
lease[1],
SendResult(
trace_id="provider-trace",
trace_url="https://frozen.example/trace",
span_urls={invocation.span_id: "https://frozen.example/span"},
),
)
service = ServicePorts(
_application_trace_service_boundaries_no_egress(),
cipher=ForbiddenSecretCipher("status must not decrypt credentials"),
).service(case)
result = service.status(**source(case.run), node_execution_id=node_id)
assert result.status is TraceRunState.ACCEPTED
assert result.provider == "langfuse"
assert result.trace_id == "provider-trace"
assert result.span_url == "https://frozen.example/span"
assert result.project_url == "https://langfuse.test/project/original"
assert result.config_revision == case.run.config_revision
assert result.capture_content is False
assert result.capabilities is not None
assert result.capabilities.provider == "langfuse"
assert "encrypted-original" not in str(result)
with pytest.raises(TraceCollectionNotFoundError, match="tracing_node_not_found"):
service.status(
**source(case.run), node_execution_id=_application_trace_service_boundaries_identifier("another-node")
)
@staticmethod
def test_new_run_freezes_updated_project_without_changing_previous_run(
sqlite_session_factory: sessionmaker[Session],
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "updated-project")
new_id = _application_trace_service_boundaries_identifier("new-message")
with case.sessions.begin() as session:
config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == case.run.app_id))
assert config is not None
config.tracing_config = {**FROZEN_LANGFUSE, "host": "https://new.example", "project_key": "new"}
message = session.get(Message, case.run.source_id)
assert message is not None
session.add(
make_message(
message_id=new_id,
app_id=case.run.app_id,
conversation_id=message.conversation_id,
inputs={},
query="new run",
message={},
answer="done",
message_unit_price=message.message_unit_price,
answer_unit_price=message.answer_unit_price,
currency=message.currency,
from_source=message.from_source,
)
)
service = ServicePorts(_application_trace_service_boundaries_no_egress()).service(case)
new_run = service.start(
tenant_id=case.run.tenant_id,
app_id=case.run.app_id,
source_kind="message",
source_id=new_id,
session_id="new-session",
workflow_id=None,
)
assert new_run is not None
previous = service.status(**source(case.run), node_execution_id=None)
assert previous.project_url == "https://langfuse.test/project/original"
assert (
service.status(**source(new_run), node_execution_id=None).project_url == "https://new.example/project/new"
)
@staticmethod
@pytest.mark.parametrize(
"frozen",
[
{key: value for key, value in FROZEN_LANGFUSE.items() if key != "public_key"},
{**FROZEN_LANGFUSE, "host": 1},
{**FROZEN_LANGFUSE, "host": "javascript:alert(1)"},
{**FROZEN_LANGFUSE, "host": "//example.com/project"},
{**FROZEN_LANGFUSE, "host": "https://user:[email protected]/project"},
{**FROZEN_LANGFUSE, "host": "https://@example.com"},
{**FROZEN_LANGFUSE, "host": "https://example.com\\@other.test"},
{**FROZEN_LANGFUSE, "host": "https://[invalid/project"},
],
)
def test_status_hides_unavailable_or_unsafe_frozen_project_urls(
sqlite_session_factory: sessionmaker[Session], frozen: dict[str, object]
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "invalid-project")
freeze_config(case, "langfuse", frozen)
result = (
ServicePorts(_application_trace_service_boundaries_no_egress())
.service(case)
.status(**source(case.run), node_execution_id=None)
)
assert result.project_url is None
assert "encrypted-original" not in str(result)
@staticmethod
@pytest.mark.parametrize(
("provider", "frozen"),
[
("langfuse", FROZEN_LANGFUSE),
("aliyun", {"license_key": "encrypted-license", "endpoint": "https://tracing.aliyun.example"}),
(
"tencent",
{"token": "encrypted-token", "endpoint": "https://apm.tencent.example", "service_name": "dify-app"},
),
],
)
def test_status_derives_project_link_from_frozen_provider_not_stored_urls(
sqlite_session_factory: sessionmaker[Session], provider: str, frozen: dict[str, object]
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "platform-console")
freeze_config(case, provider, {**frozen, "project_url": "https://stored.example/project"})
installed = EntryPointTraceProviderCatalog()
destination = installed.resolve(provider)
expected = destination(_application_trace_service_boundaries_no_egress()).project_url(
destination.config_model.model_validate(frozen)
)
result = (
ServicePorts(_application_trace_service_boundaries_no_egress(), providers=installed)
.service(case)
.status(**source(case.run), node_execution_id=None)
)
assert expected is not None
assert result.project_url == expected
assert result.capabilities is not None
assert result.capabilities.provider == provider
@staticmethod
@pytest.mark.parametrize("disabled", [True, False])
def test_uncollected_status_and_missing_run_retry_do_not_construct_provider(
sqlite_session_factory: sessionmaker[Session], disabled: bool
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "missing", enabled=not disabled)
if not disabled:
with case.sessions.begin() as session:
stored = session.get(ApplicationTraceRun, case.run.id)
assert stored is not None
session.delete(stored)
service = ServicePorts(
_application_trace_service_boundaries_no_egress(),
providers=ForbiddenTraceProviders("an uncollected source has no destination to construct"),
).service(case)
assert service.status(**source(case.run), node_execution_id=None) == not_collected(
"message", case.run.source_id
)
if not disabled:
with pytest.raises(TraceCollectionNotFoundError, match="tracing_run_not_found"):
service.retry(**source(case.run), acknowledge_possible_duplicates=True, node_execution_id=None)
@staticmethod
def test_retry_acknowledgement_guards_schedule_and_keeps_frozen_snapshot(
sqlite_session_factory: sessionmaker[Session],
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "retry")
_application_trace_service_boundaries_ingest(
case, _application_trace_service_boundaries_batch_for(case.run, case.invocation())
)
_application_trace_service_boundaries_terminal(case)
lease = _application_trace_service_boundaries_claim(case)
assert lease is not None
with case.change() as run:
run.export_uncertain(lease[1], now=naive_utc_now())
ports = ServicePorts(_application_trace_service_boundaries_no_egress())
service = ports.service(case)
with pytest.raises(TraceCollectionConflictError, match="tracing_duplicate_acknowledgement_required"):
service.retry(**source(case.run), acknowledge_possible_duplicates=False, node_execution_id=None)
assert ports.scheduled == []
result = service.retry(**source(case.run), acknowledge_possible_duplicates=True, node_execution_id=None)
assert ports.scheduled == [case.run.id]
assert result.status is TraceRunState.READY
assert result.config_revision == case.run.config_revision
current = case.repository.get(case.run.id)
assert current is not None
assert current.provider_config == case.run.provider_config
@staticmethod
@pytest.mark.parametrize(
("failure", "max_attempts", "state", "code"),
[
(ExportRetryableError("private-transport-error"), 1, TraceRunState.FAILED, "provider_retry_exhausted"),
(
ExportRetryableError("private-transport-error"),
2,
TraceRunState.RETRYING,
"provider_temporarily_unavailable",
),
(ExportRejectedError("private-rejection"), 2, TraceRunState.FAILED, "provider_export_rejected"),
(
ExportOutcomeUnknownError("private-acknowledgement"),
2,
TraceRunState.UNCERTAIN,
"provider_acceptance_unknown",
),
(ValueError("private-credential-error"), 2, TraceRunState.FAILED, "provider_export_rejected"),
],
)
def test_delivery_maps_provider_failures_to_stable_codes_without_vendor_details(
sqlite_session_factory: sessionmaker[Session],
caplog: pytest.LogCaptureFixture,
failure: Exception,
max_attempts: int,
state: TraceRunState,
code: str,
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "delivery")
_application_trace_service_boundaries_ingest(
case, _application_trace_service_boundaries_batch_for(case.run, case.invocation())
)
_application_trace_service_boundaries_terminal(case)
destination = RecordingEgress([failure], grpc_target=None)
service = ServicePorts(
destination, settings=replace(AGENT_TRACE_SETTINGS, max_export_attempts=max_attempts)
).service(case)
service.deliver(case.run.id)
current = case.repository.get(case.run.id)
assert current is not None
assert (current.state, current.error_code) == (state, code)
assert current.attempts == 1
with case.sessions() as session:
next_attempt_at = session.scalar(
select(ApplicationTraceRun.next_attempt_at).where(ApplicationTraceRun.id == case.run.id)
)
assert (next_attempt_at is not None) is (state is TraceRunState.RETRYING)
assert len(destination.requests) == 1
assert "private-" not in caplog.text
service.deliver(_application_trace_service_boundaries_identifier("missing-run"))
@staticmethod
@pytest.mark.parametrize("unreadable", ["decryption", "validation"])
def test_unreadable_frozen_configuration_fails_before_provider_io(
sqlite_session_factory: sessionmaker[Session],
caplog: pytest.LogCaptureFixture,
unreadable: str,
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "unreadable")
_application_trace_service_boundaries_ingest(
case, _application_trace_service_boundaries_batch_for(case.run, case.invocation())
)
_application_trace_service_boundaries_terminal(case)
if unreadable == "validation":
freeze_config(case, "langfuse", {**FROZEN_LANGFUSE, "host": "private-host-without-scheme"})
cipher = UnreadableSecretCipher() if unreadable == "decryption" else StoredSecretCipher()
destination = RecordingEgress([], grpc_target=None)
service = ServicePorts(destination, cipher=cipher).service(case)
service.deliver(case.run.id)
current = case.repository.get(case.run.id)
assert current is not None
assert (current.state, current.error_code) == (TraceRunState.FAILED, "tracing_config_unreadable")
assert destination.requests == []
assert "private-" not in caplog.text
with pytest.raises(TraceCollectionConflictError, match="tracing_retry_not_allowed"):
service.retry(**source(case.run), acknowledge_possible_duplicates=True, node_execution_id=None)
@staticmethod
def test_unavailable_frozen_provider_fails_before_decryption(
sqlite_session_factory: sessionmaker[Session],
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "retired-provider")
_application_trace_service_boundaries_ingest(
case, _application_trace_service_boundaries_batch_for(case.run, case.invocation())
)
_application_trace_service_boundaries_terminal(case)
freeze_config(case, "retired-provider", FROZEN_LANGFUSE)
ServicePorts(
_application_trace_service_boundaries_no_egress(),
cipher=ForbiddenSecretCipher("an unavailable provider must not decrypt credentials"),
).service(case).deliver(case.run.id)
current = case.repository.get(case.run.id)
assert current is not None
assert (current.state, current.error_code) == (TraceRunState.FAILED, "tracing_provider_unavailable")
@staticmethod
@pytest.mark.parametrize("capture_content", [False, True])
@pytest.mark.parametrize(
"status",
[
WorkflowNodeExecutionStatus.RUNNING,
WorkflowNodeExecutionStatus.SUCCEEDED,
WorkflowNodeExecutionStatus.FAILED,
],
)
def test_preview_terminal_uses_persisted_node_state_and_frozen_privacy(
sqlite_session_factory: sessionmaker[Session], status: WorkflowNodeExecutionStatus, capture_content: bool
) -> None:
case = _application_trace_service_boundaries_seed(
sqlite_session_factory, "preview", capture_content=capture_content
)
node_id = _application_trace_service_boundaries_identifier("preview-node")
workflow_id = _application_trace_service_boundaries_identifier("preview-definition")
with case.sessions.begin() as session:
session.add(make_workflow(workflow_id=workflow_id, tenant_id=case.run.tenant_id, app_id=case.run.app_id))
node_record = WorkflowNodeExecutionModel(
id=node_id,
node_execution_id=node_id,
tenant_id=case.run.tenant_id,
app_id=case.run.app_id,
workflow_id=workflow_id,
triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
index=0,
node_id="agent",
node_type="agent",
title="Agent preview",
inputs='{"secret":"private-input"}',
outputs='{"answer":"private-output"}',
process_data="{}",
status=status,
elapsed_time=2,
error="private-model-error" if status == WorkflowNodeExecutionStatus.FAILED else None,
created_by=_application_trace_service_boundaries_identifier("account"),
created_by_role=CreatorUserRole.ACCOUNT,
)
node_record.created_at = datetime(2026, 1, 1)
session.add(node_record)
destination = _application_trace_service_boundaries_accepted("preview-trace")
service = ServicePorts(destination).service(case)
run = service.start(
tenant_id=case.run.tenant_id,
app_id=case.run.app_id,
source_kind="node",
source_id=node_id,
workflow_id=workflow_id,
session_id="preview-session",
)
assert run is not None
service.deliver(run.id)
current = case.repository.get(run.id)
assert current is not None
if status == WorkflowNodeExecutionStatus.RUNNING:
assert current.outer_trace is None
assert current.state is TraceRunState.COLLECTING
assert destination.requests == []
else:
assert current.outer_trace is not None
assert current.state is TraceRunState.ACCEPTED
trace = Trace.model_validate(current.outer_trace)
root = trace.root
assert root.span_id == run.root_span_id
assert root.status.value == ("error" if status == WorkflowNodeExecutionStatus.FAILED else "ok")
assert ("private-" in trace.model_dump_json()) is capture_content
assert root.attributes["node_execution_id"] == node_id
assert (root.end_time - root.start_time).total_seconds() == 2
if not capture_content and status == WorkflowNodeExecutionStatus.FAILED:
assert root.error == "Node execution failed"
@staticmethod
@pytest.mark.parametrize(
("invalid", "code"),
[
("empty", "tracing_invocation_empty"),
("foreign_parent", "tracing_invocation_parent_invalid"),
("no_invocation_root", "tracing_invocation_root_missing"),
("self_parent", "tracing_payload_invalid"),
("conflicting_span", "tracing_span_conflict"),
],
)
def test_invalid_runtime_tree_is_rejected_before_provider_credentials_or_io(
sqlite_session_factory: sessionmaker[Session], invalid: str, code: str
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "invalid-runtime")
batch = _application_trace_service_boundaries_batch_for(case.run, case.invocation())
if invalid == "empty":
batch.spans = []
elif invalid == "foreign_parent":
batch.spans[0] = batch.spans[0].model_copy(update={"parent_span_id": "f" * 16})
elif invalid == "no_invocation_root":
# Every parent is a collected span, but none of them hangs from the invocation.
batch.spans = [
batch.spans[0].model_copy(update={"parent_span_id": "b" * 16}),
batch.spans[0].model_copy(update={"span_id": "b" * 16, "parent_span_id": "a" * 16}),
]
elif invalid == "self_parent":
batch.spans[0] = batch.spans[0].model_copy(update={"parent_span_id": batch.spans[0].span_id})
else:
_application_trace_service_boundaries_ingest(case, batch.model_copy(update={"final": False}))
batch = batch.model_copy(
update={
"batch_id": uuid4(),
"spans": [batch.spans[0].model_copy(update={"name": "conflicting-version"})],
}
)
_application_trace_service_boundaries_ingest(case, batch)
_application_trace_service_boundaries_terminal(case)
reason = "invalid trees must fail before credentials or provider requests"
ServicePorts(
_application_trace_service_boundaries_no_egress(),
providers=ForbiddenTraceProviders(reason),
cipher=ForbiddenSecretCipher(reason),
).service(case).deliver(case.run.id)
current = case.repository.get(case.run.id)
assert current is not None
assert current.state is TraceRunState.FAILED
assert current.error_code == code
@staticmethod
def test_delivery_requires_a_persisted_business_terminal(sqlite_session_factory: sessionmaker[Session]) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "no-terminal")
attach_conversation(case, AppMode.AGENT)
destination = _application_trace_service_boundaries_no_egress()
ServicePorts(destination).service(case).deliver(case.run.id)
current = case.repository.get(case.run.id)
assert current is not None
assert current.state is TraceRunState.COLLECTING
assert current.outer_trace is None
assert destination.requests == []
@staticmethod
def test_assembled_trace_nests_runtime_spans_under_their_invocation_with_trusted_identity(
sqlite_session_factory: sessionmaker[Session],
) -> None:
case = _application_trace_service_boundaries_seed(sqlite_session_factory, "assembled")
invocation = case.invocation()
_application_trace_service_boundaries_ingest(
case, _application_trace_service_boundaries_batch_for(case.run, invocation)
)
_application_trace_service_boundaries_terminal(case)
destination = _application_trace_service_boundaries_accepted("assembled-trace")
ServicePorts(destination).service(case).deliver(case.run.id)
[trace] = sent_traces(destination)
assert trace.trace_id == case.run.trace_id
assert trace.session_id == "session-original"
assert [(span.span_id, span.parent_span_id) for span in trace.spans] == [
(case.run.root_span_id, None),
(invocation.span_id, case.run.root_span_id),
("a" * 16, invocation.span_id),
]
attempt = trace.spans[1]
assert attempt.name == "Agent attempt 1"
assert attempt.attributes["invocation_id"] == invocation.id
assert (attempt.start_time, attempt.end_time) == (trace.spans[2].start_time, trace.spans[2].end_time)
for span in trace.spans:
assert span.attributes["dify.tracing.run_id"] == case.run.id
assert span.attributes["dify.source.id"] == case.run.source_id
assert span.attributes["tenant_id"] == case.run.tenant_id
"""Malformed durable traces must never be sent or offered for provider retry."""
_invalid_trace_delivery_batch_for = _application_trace_collection_batch_for
_invalid_trace_delivery_ingest = _application_trace_collection_ingest
_invalid_trace_delivery_no_egress = _application_trace_collection_no_egress
_invalid_trace_delivery_seed = _application_trace_collection_seed
_invalid_trace_delivery_terminal = _application_trace_collection_terminal
class TestInvalidTraceDeliveryCases:
@staticmethod
@pytest.mark.parametrize("invalid", ["cycle", "invalid_envelope"])
def test_invalid_tree_is_permanent_before_decryption_or_provider_io(
sqlite_session_factory: sessionmaker[Session], invalid: str
) -> None:
case = _invalid_trace_delivery_seed(sqlite_session_factory, str(uuid4()))
batch = _invalid_trace_delivery_batch_for(case.run, case.invocation())
if invalid == "cycle":
for span_id, parent_id in [("b" * 16, "c" * 16), ("c" * 16, "b" * 16)]:
batch.spans.append(
AgentTraceSpan(
trace_id=case.run.trace_id,
span_id=span_id,
parent_span_id=parent_id,
name="cyclic tool",
start_time_ns=1_000_000_000,
end_time_ns=2_000_000_000,
status="ok",
)
)
_invalid_trace_delivery_ingest(case, batch)
_invalid_trace_delivery_terminal(case)
if invalid == "invalid_envelope":
with case.sessions.begin() as session:
row = session.get(ApplicationTraceRun, case.run.id)
assert row is not None
row.outer_trace = {"corrupt": True}
reason = "invalid traces must fail before credentials or provider I/O"
ServicePorts(
_invalid_trace_delivery_no_egress(),
providers=ForbiddenTraceProviders(reason),
cipher=ForbiddenSecretCipher(reason),
).service(case).deliver(case.run.id)
current = case.repository.get(case.run.id)
assert current is not None
assert current.state is TraceRunState.FAILED
assert current.error_code == "tracing_payload_invalid"
assert current.attempts == 1
# A retry reports the run's status, which resolves its frozen provider; the forbidden ports guard only delivery.
with pytest.raises(TraceCollectionConflictError, match="tracing_retry_not_allowed"):
ServicePorts(_invalid_trace_delivery_no_egress()).service(case).retry(
**source(case.run), acknowledge_possible_duplicates=True, node_execution_id=None
)
"""Node logs resolve the caller's trace using SQL or configured LogStore ownership."""
_ApplicationTraceNodeAliasCollection = _ApplicationTraceCollectionCollection
_application_trace_node_alias_batch_for = _application_trace_collection_batch_for
_application_trace_node_alias_claim = _application_trace_collection_claim
_application_trace_node_alias_identifier = _application_trace_collection_identifier
_application_trace_node_alias_ingest = _application_trace_collection_ingest
_application_trace_node_alias_no_egress = _application_trace_collection_no_egress
_application_trace_node_alias_seed = _application_trace_collection_seed
_application_trace_node_alias_terminal = _application_trace_collection_terminal
@dataclass
class NodeAlias:
case: _ApplicationTraceNodeAliasCollection
node: dict[str, object]
workflow: dict[str, object]
requests: list[str]
logstore: bool
ports: ServicePorts
def service(self) -> ApplicationTracingService:
return self.ports.service(self.case)
def status(self, node_execution_id: str | None = None) -> TraceRunStatus:
return self.service().status(
tenant_id=self.case.run.tenant_id,
app_id=self.case.run.app_id,
source_kind="node",
source_id=str(self.node["id"]),
node_execution_id=node_execution_id,
)
def accept(self, *, spans: bool = True) -> tuple[str, str]:
span_urls: dict[str, str] = {}
for attempt in (0, 1):
invocation = self.case.open_invocation(
agent_id=_application_trace_node_alias_identifier("referenced-agent"),
node_execution_id=str(self.node["id"]),
attempt=attempt,
call_key=f"attempt-{attempt}",
)
_application_trace_node_alias_ingest(
self.case,
_application_trace_node_alias_batch_for(self.case.run, invocation, span_id=str(attempt + 1) * 16),
)
span_urls[invocation.span_id] = f"https://provider.test/span/{attempt}"
_application_trace_node_alias_terminal(self.case)
lease = _application_trace_node_alias_claim(self.case)
assert lease is not None
_accept_export(
self.case,
lease[1],
SendResult(
trace_id="provider-trace",
trace_url="https://provider.test/trace",
span_urls=span_urls if spans else {},
),
)
return "https://provider.test/trace", "https://provider.test/span/1"
class TestApplicationTraceNodeAliasCases:
@staticmethod
@pytest.fixture(params=["sql", "logstore"])
def alias(
request: pytest.FixtureRequest,
sqlite_engine: Engine,
sqlite_session_factory: sessionmaker[Session],
monkeypatch: pytest.MonkeyPatch,
) -> Iterator[NodeAlias]:
case = _application_trace_node_alias_seed(sqlite_session_factory, "node-alias")
with case.sessions.begin() as session:
app = session.get(App, case.run.app_id)
assert app is not None
app.mode = AppMode.WORKFLOW
common: dict[str, object] = {
"tenant_id": case.run.tenant_id,
"app_id": case.run.app_id,
"workflow_id": _application_trace_node_alias_identifier("definition"),
"created_by": _application_trace_node_alias_identifier("actor"),
"created_at": naive_utc_now().isoformat(),
"status": "succeeded",
}
workflow = {
**common,
"id": _application_trace_node_alias_identifier("workflow-source"),
"graph": json.dumps({"nodes": []}),
}
node = {
**common,
"id": _application_trace_node_alias_identifier("node-source"),
"workflow_run_id": workflow["id"],
"node_id": "reference-agent",
"node_type": "agent",
"title": "Reference Agent",
}
requests: list[str] = []
transactions: set[Connection] = set()
def begun(connection: Connection) -> None:
transactions.add(connection)
def ended(connection: Connection) -> None:
transactions.discard(connection)
if request.param == "sql":
with case.sessions.begin() as session:
session.add(_dict_to_workflow_run(workflow))
session.add(_dict_to_workflow_node_execution_model(node))
else:
apply_config_overrides(
monkeypatch,
LOGSTORE_DUAL_WRITE_ENABLED=False,
API_WORKFLOW_RUN_REPOSITORY=(
"extensions.logstore.repositories.logstore_api_workflow_run_repository.LogstoreAPIWorkflowRunRepository"
),
API_WORKFLOW_NODE_EXECUTION_REPOSITORY=(
"extensions.logstore.repositories.logstore_api_workflow_node_execution_repository."
"LogstoreAPIWorkflowNodeExecutionRepository"
),
)
for key, value in {
"LOGSTORE_DUAL_READ_ENABLED": "false",
"LOGSTORE_PG_MODE_ENABLED": "false",
"ALIYUN_SLS_ACCESS_KEY_ID": "test-access-key",
"ALIYUN_SLS_ACCESS_KEY_SECRET": "test-secret",
"ALIYUN_SLS_ENDPOINT": "https://sls.example.test",
"ALIYUN_SLS_REGION": "test-region",
}.items():
monkeypatch.setenv(key, value)
monkeypatch.setattr(AliyunLogStore, "_instance", None)
monkeypatch.setattr(AliyunLogStore, "_initialized", False)
monkeypatch.setattr(
AliyunLogStore, "_check_endpoint_connectivity", staticmethod(lambda _endpoint, _timeout: None)
)
def reject_http_transport(*_args: object, **_kwargs: object) -> None:
pytest.fail("Configured LogStore test escaped its isolated transport boundary")
monkeypatch.setattr(LogClient, "_getHttpResponse", reject_http_transport)
def get_logs(
_client: AliyunLogStore, *, query: str, logstore: str, **_kwargs: object
) -> list[dict[str, object]]:
assert not transactions, "Configured source reads must not hold a SQL transaction"
assert f'tenant_id:"{case.run.tenant_id}"' in query.replace(": ", ":")
assert logstore in {
AliyunLogStore.workflow_execution_logstore,
AliyunLogStore.workflow_node_execution_logstore,
}
is_workflow = logstore == AliyunLogStore.workflow_execution_logstore
if is_workflow:
assert f'app_id:"{case.run.app_id}"' in query
requests.append("workflow" if is_workflow else "node")
row = workflow if is_workflow else node
return [dict(row)] if row else []
monkeypatch.setattr(AliyunLogStore, "get_logs", get_logs)
def execute_sql(
client: AliyunLogStore, *, sql: str, query: str, logstore: str, **_kwargs: object
) -> list[dict[str, object]]:
# Node lookups use the SQL transport in SDK mode too; the real
# service still validates the returned node-to-workflow owner chain.
assert logstore == AliyunLogStore.workflow_node_execution_logstore
assert f'id:"{_application_trace_node_alias_identifier("node-source")}"' in query.replace(": ", ":")
assert f"tenant_id = '{case.run.tenant_id}'" in sql
assert f"id = '{_application_trace_node_alias_identifier('node-source')}'" in sql
assert f'FROM "{logstore}"' in sql
assert "ORDER BY log_version DESC LIMIT 1" in sql
return get_logs(client, query=query, logstore=logstore)
monkeypatch.setattr(AliyunLogStore, "execute_sql", execute_sql)
with case.sessions() as session:
assert session.scalar(select(WorkflowRun)) is None
assert session.scalar(select(WorkflowNodeExecutionModel)) is None
event.listen(sqlite_engine, "begin", begun)
event.listen(sqlite_engine, "commit", ended)
event.listen(sqlite_engine, "rollback", ended)
try:
service = ServicePorts(_application_trace_node_alias_no_egress()).service(case)
run = service.start(
tenant_id=case.run.tenant_id,
app_id=case.run.app_id,
source_kind="workflow",
source_id=str(workflow["id"]),
session_id="caller-session",
workflow_id=None,
)
assert run is not None
requests.clear()
yield NodeAlias(
_ApplicationTraceNodeAliasCollection(case.repository, case.sessions, run),
node,
workflow,
requests,
request.param == "logstore",
ServicePorts(_application_trace_node_alias_no_egress()),
)
finally:
event.remove(sqlite_engine, "begin", begun)
event.remove(sqlite_engine, "commit", ended)
event.remove(sqlite_engine, "rollback", ended)
@staticmethod
@pytest.mark.parametrize("spans", [True, False])
def test_alias_returns_parent_trace_and_latest_owned_invocation(alias: NodeAlias, spans: bool) -> None:
trace_url, span_url = alias.accept(spans=spans)
status = alias.status()
assert status.status is TraceRunState.ACCEPTED
assert status.source_kind == "node"
assert status.source_id == alias.node["id"]
assert status.trace_id == "provider-trace"
assert status.trace_url == trace_url
assert status.span_url == (span_url if spans else None)
assert status.config_revision == alias.case.run.config_revision
if alias.logstore:
assert alias.requests == ["node", "workflow"]
@staticmethod
@pytest.mark.parametrize("capture_agent", [True, False])
@pytest.mark.parametrize("accepted", [True, False])
def test_alias_without_internal_invocation_returns_parent_status(
alias: NodeAlias, capture_agent: bool, accepted: bool
) -> None:
with alias.case.sessions.begin() as session:
run = session.get(ApplicationTraceRun, alias.case.run.id)
assert run is not None
run.capture_agent = capture_agent
if accepted:
_application_trace_node_alias_terminal(alias.case)
lease = _application_trace_node_alias_claim(alias.case)
assert lease is not None
_accept_export(
alias.case,
lease[1],
SendResult(trace_id="outer-only", trace_url="https://provider.test/outer", span_urls={}),
)
status = alias.status()
assert status.status is (TraceRunState.ACCEPTED if accepted else TraceRunState.COLLECTING)
assert status.span_url is None
assert status.capture_agent is capture_agent
assert status.trace_url == ("https://provider.test/outer" if accepted else None)
@staticmethod
@pytest.mark.parametrize("state", [TraceRunState.COLLECTING, TraceRunState.NOT_COLLECTED])
def test_standalone_node_snapshot_wins_over_parent(alias: NodeAlias, state: TraceRunState) -> None:
alias.accept()
node_run = alias.service().start(
tenant_id=alias.case.run.tenant_id,
app_id=alias.case.run.app_id,
source_kind="node",
source_id=str(alias.node["id"]),
session_id="preview-session",
workflow_id=None,
)
assert node_run is not None
with alias.case.sessions.begin() as session:
run = session.get(ApplicationTraceRun, node_run.id)
assert run is not None
run.state = state
alias.requests.clear()
status = alias.status()
assert status.status is state
assert status.trace_url is None
if alias.logstore:
assert alias.requests == ["node"]
@staticmethod
def test_explicit_other_node_filter_cannot_select_sibling_span(alias: NodeAlias) -> None:
sibling = alias.case.open_invocation(
agent_id=_application_trace_node_alias_identifier("referenced-agent"),
node_execution_id=_application_trace_node_alias_identifier("sibling-node"),
attempt=0,
call_key="sibling",
)
_application_trace_node_alias_ingest(
alias.case, _application_trace_node_alias_batch_for(alias.case.run, sibling, span_id="f" * 16)
)
alias.accept()
with pytest.raises(TraceCollectionNotFoundError, match="tracing_node_not_found"):
alias.status(_application_trace_node_alias_identifier("sibling-node"))
with pytest.raises(TraceCollectionNotFoundError, match="tracing_node_not_found"):
alias.service().retry(
tenant_id=alias.case.run.tenant_id,
app_id=alias.case.run.app_id,
source_kind="node",
source_id=str(alias.node["id"]),
node_execution_id=_application_trace_node_alias_identifier("sibling-node"),
acknowledge_possible_duplicates=True,
)
assert not alias.ports.scheduled
@staticmethod
@pytest.mark.parametrize("state", [TraceRunState.FAILED, TraceRunState.UNCERTAIN, TraceRunState.ACCEPTED])
def test_alias_retry_uses_parent_guards_and_resolves_sources_once(alias: NodeAlias, state: TraceRunState) -> None:
alias.accept()
with alias.case.sessions.begin() as session:
run = session.get(ApplicationTraceRun, alias.case.run.id)
assert run is not None
run.state = state
service = alias.service()
def retry(acknowledge: bool) -> TraceRunStatus:
alias.requests.clear()
return service.retry(
tenant_id=alias.case.run.tenant_id,
app_id=alias.case.run.app_id,
source_kind="node",
source_id=str(alias.node["id"]),
acknowledge_possible_duplicates=acknowledge,
node_execution_id=None,
)
if state is TraceRunState.ACCEPTED:
with pytest.raises(TraceCollectionConflictError, match="tracing_already_accepted"):
retry(True)
assert not alias.ports.scheduled
else:
if state is TraceRunState.UNCERTAIN:
with pytest.raises(TraceCollectionConflictError, match="tracing_duplicate_acknowledgement_required"):
retry(False)
assert not alias.ports.scheduled
assert retry(state is TraceRunState.UNCERTAIN).status is TraceRunState.READY
assert alias.ports.scheduled == [alias.case.run.id]
if alias.logstore:
assert alias.requests == ["node", "workflow"]
@staticmethod
@pytest.mark.parametrize("source", ["node", "workflow"])
@pytest.mark.parametrize("field", ["tenant_id", "app_id", "workflow_id"])
def test_alias_rejects_broken_owner_chain(alias: NodeAlias, source: str, field: str) -> None:
alias.accept()
row = alias.node if source == "node" else alias.workflow
if alias.logstore:
row[field] = _application_trace_node_alias_identifier("foreign")
else:
with alias.case.sessions.begin() as session:
model = session.get(WorkflowNodeExecutionModel if source == "node" else WorkflowRun, str(row["id"]))
assert model is not None
# Keep field assignments explicit so ownership failures are visible.
if field == "tenant_id":
model.tenant_id = _application_trace_node_alias_identifier("foreign")
elif field == "app_id":
model.app_id = _application_trace_node_alias_identifier("foreign")
else:
model.workflow_id = _application_trace_node_alias_identifier("foreign")
with pytest.raises(TraceCollectionNotFoundError, match="tracing_source_not_found"):
alias.status()
@staticmethod
def test_alias_missing_parent_business_source_is_not_a_durable_owner_bypass(alias: NodeAlias) -> None:
if alias.logstore:
alias.workflow.clear()
else:
with alias.case.sessions.begin() as session:
parent = session.get(WorkflowRun, str(alias.workflow["id"]))
assert parent is not None
session.delete(parent)
with pytest.raises(TraceCollectionNotFoundError, match="tracing_source_not_found"):
alias.status()
@staticmethod
def test_owned_alias_without_parent_trace_is_not_collected(alias: NodeAlias) -> None:
with alias.case.sessions.begin() as session:
run = session.get(ApplicationTraceRun, alias.case.run.id)
assert run is not None
session.delete(run)
assert alias.status() == not_collected("node", str(alias.node["id"]))
"""Real SQLite state transitions and background renewal for delivery leases.
PostgreSQL row-lock scheduling and the deployed Celery pool still need dev
validation; these tests establish token and acknowledgement semantics.
"""
_ExportLeaseCollection = _ApplicationTraceCollectionCollection
_export_lease_accepted = _application_trace_collection_accepted
_export_lease_batch_for = _application_trace_collection_batch_for
_export_lease_claim = _application_trace_collection_claim
_export_lease_ingest = _application_trace_collection_ingest
_export_lease_seed = _application_trace_collection_seed
_export_lease_terminal = _application_trace_collection_terminal
class _SlowDestination(RecordingEgress):
"""Runs ``during_request`` while the export request is in flight."""
def __init__(self, replies: list[EgressResponse | Exception], during_request: Callable[[], None]) -> None:
super().__init__(replies, grpc_target=None)
self._during_request = during_request
@override
def request(
self,
method: str,
url: str,
*,
headers: Mapping[str, str],
content: bytes | None,
timeout: float,
sensitive_url: bool,
) -> EgressResponse:
self._during_request()
return super().request(
method, url, headers=headers, content=content, timeout=timeout, sensitive_url=sensitive_url
)
def _export_lease(locked: LockedTraceRun | None) -> str | None:
"""The token of the export attempt that holds the run; ``None`` when no attempt holds it."""
if locked is None or locked.run.state is not TraceRunState.EXPORTING:
return None
return locked.run.export_token
class _RenewalRecordingRepository(ApplicationTraceRepository):
"""Records the thread of every lease renewal; the first one can fail like a dropped connection.
A renewal is a change after which the export attempt that held the run still holds it.
"""
def __init__(self, sessions: sessionmaker[Session], *, transient_failure: bool) -> None:
super().__init__(sessions)
self._transient_failure = transient_failure
self.renewal_threads: list[int] = []
self.renewed = threading.Event()
@override
@contextmanager
def change(self, run_id: str, *, tenant_id: str) -> Generator[LockedTraceRun | None]:
renewal = False
with super().change(run_id, tenant_id=tenant_id) as locked:
lease = _export_lease(locked)
yield locked
renewal = lease is not None and _export_lease(locked) == lease
if renewal:
self.renewal_threads.append(threading.get_ident())
if self._transient_failure and len(self.renewal_threads) == 1:
raise RuntimeError("private-provider-value")
if renewal:
self.renewed.set()
class _UnavailableCompletionRepository(ApplicationTraceRepository):
"""Loses the database when the export outcome is recorded."""
@override
@contextmanager
def change(self, run_id: str, *, tenant_id: str) -> Generator[LockedTraceRun | None]:
with super().change(run_id, tenant_id=tenant_id) as locked:
lease = _export_lease(locked)
yield locked
if lease is not None and _export_lease(locked) is None:
raise TracingStoreUnavailableError("database unavailable")
def result(trace_id: str) -> SendResult:
return SendResult(trace_id=trace_id, trace_url=None, span_urls={})
def _renew_export_lease(case: _ExportLeaseCollection, token: str, now: datetime) -> bool:
with case.change() as run:
return run.renew_export_lease(token, now=now)
def _release_for_retry(case: _ExportLeaseCollection) -> None:
"""An operator's retry that acknowledges possible duplicates."""
with case.change() as run:
run.release_for_retry(acknowledge_possible_duplicates=True, now=naive_utc_now())
def _stored(case: _ExportLeaseCollection) -> ApplicationTraceRun:
"""The run's row: its record does not carry the export lease."""
with case.sessions() as session:
row = session.get(ApplicationTraceRun, case.run.id)
assert row is not None
return row
def _expire(case: _ExportLeaseCollection) -> str:
leased = _export_lease_claim(case)
assert leased is not None
with case.sessions.begin() as session:
row = session.get(ApplicationTraceRun, case.run.id)
assert row is not None
row.export_started_at = naive_utc_now() - timedelta(seconds=60)
row.export_heartbeat_at = row.export_started_at
assert _export_lease_claim(case) is None
assert _stored(case).state == TraceRunState.UNCERTAIN
return leased[1]
class TestExportLeaseCases:
@staticmethod
@pytest.fixture
def case(sqlite_session_factory: sessionmaker[Session]) -> _ExportLeaseCollection:
collection = _export_lease_seed(sqlite_session_factory, "lease-test")
_export_lease_ingest(collection, _export_lease_batch_for(collection.run, collection.invocation()))
_export_lease_terminal(collection)
return collection
@staticmethod
def test_renewal_keeps_long_export_alive_then_missing_heartbeat_expires(case: _ExportLeaseCollection) -> None:
now = naive_utc_now()
leased = _claim_at(case, lease_seconds=30, now=now)
assert leased is not None
token = leased[1]
now += timedelta(seconds=20)
assert _renew_export_lease(case, token, now)
now += timedelta(seconds=20)
assert _claim_at(case, lease_seconds=30, now=now) is None
current = _stored(case)
assert current.state == TraceRunState.EXPORTING
assert current.attempts == 1
assert current.export_started_at is not None
assert current.export_started_at + timedelta(seconds=30) < now
now += timedelta(seconds=11)
assert _claim_at(case, lease_seconds=30, now=now) is None
expired = _stored(case)
assert expired.state == TraceRunState.UNCERTAIN
assert expired.error_code == "export_lease_expired"
assert expired.export_token == token
assert not _renew_export_lease(case, token, now)
@staticmethod
def test_old_rows_without_heartbeat_use_started_at(case: _ExportLeaseCollection) -> None:
leased = _export_lease_claim(case)
assert leased is not None
with case.sessions.begin() as session:
row = session.get(ApplicationTraceRun, case.run.id)
assert row is not None
row.export_started_at = naive_utc_now() - timedelta(seconds=60)
row.export_heartbeat_at = None
assert _export_lease_claim(case) is None
current = _stored(case)
assert current.state == TraceRunState.UNCERTAIN
assert current.export_token == leased[1]
@staticmethod
def test_wrong_token_cannot_renew_or_complete(case: _ExportLeaseCollection) -> None:
leased = _export_lease_claim(case)
assert leased is not None
before = _stored(case)
assert not _renew_export_lease(case, "other-token", naive_utc_now())
_accept_export(case, "other-token", result("wrong"))
current = _stored(case)
assert current.export_token == leased[1]
assert current.export_heartbeat_at == before.export_heartbeat_at
assert current.state == TraceRunState.EXPORTING
assert current.provider_trace_id is None
@staticmethod
def test_late_success_wins_before_manual_retry_and_is_idempotent(case: _ExportLeaseCollection) -> None:
token = _expire(case)
confirmed = SendResult(
trace_id="confirmed",
trace_url="https://example.com/trace",
span_urls={"a" * 16: "https://example.com/span"},
)
_accept_export(case, token, confirmed)
current = _stored(case)
assert current.state == TraceRunState.ACCEPTED
assert current.error_code is None
assert current.export_token is None
assert current.accepted_at is not None
assert current.trace_url == confirmed.trace_url
assert current.span_urls == {"a" * 16: "https://example.com/span"}
_accept_export(case, token, result("duplicate-ack"))
assert _stored(case).provider_trace_id == "confirmed"
with pytest.raises(TraceRunConflictError, match="tracing_already_accepted"):
_release_for_retry(case)
@staticmethod
def test_manual_retry_revokes_old_heartbeat_and_ack_before_and_after_new_claim(
case: _ExportLeaseCollection,
) -> None:
old_token = _expire(case)
_release_for_retry(case)
assert not _renew_export_lease(case, old_token, naive_utc_now())
_accept_export(case, old_token, result("old"))
assert _stored(case).state == TraceRunState.READY
second = _export_lease_claim(case)
assert second is not None
assert second[1] != old_token
_accept_export(case, old_token, result("old"))
current = _stored(case)
assert current.state == TraceRunState.EXPORTING
assert current.export_token == second[1]
assert current.provider_trace_id is None
_accept_export(case, second[1], result("new"))
assert _stored(case).provider_trace_id == "new"
@staticmethod
@pytest.mark.parametrize("late_state", [TraceRunState.FAILED, TraceRunState.RETRYING, TraceRunState.UNCERTAIN])
def test_late_non_success_cannot_overwrite_lease_expiry(
case: _ExportLeaseCollection, late_state: TraceRunState
) -> None:
token = _expire(case)
now = naive_utc_now()
with case.change() as run:
if late_state is TraceRunState.FAILED:
run.export_failed(token, "late-failure", now=now)
elif late_state is TraceRunState.RETRYING:
run.export_deferred(token, max_attempts=AGENT_TRACE_SETTINGS.max_export_attempts, now=now)
else:
run.export_uncertain(token, now=now)
current = _stored(case)
assert current.state == TraceRunState.UNCERTAIN
assert current.error_code == "export_lease_expired"
@staticmethod
def test_other_uncertain_reason_cannot_be_promoted_by_late_ack(case: _ExportLeaseCollection) -> None:
token = _expire(case)
with case.sessions.begin() as session:
row = session.get(ApplicationTraceRun, case.run.id)
assert row is not None
row.error_code = "provider_acceptance_unknown"
_accept_export(case, token, result("late"))
assert _stored(case).state == TraceRunState.UNCERTAIN
@staticmethod
@pytest.mark.parametrize("transient_failure", [False, True])
@pytest.mark.parametrize(
("outcome", "reply"),
[
(TraceRunState.ACCEPTED, response(200, {"trace_id": "ack"})),
(TraceRunState.UNCERTAIN, EgressError("lost acknowledgement", request_sent=True)),
],
)
def test_service_heartbeats_during_export_and_stops_on_every_outcome(
case: _ExportLeaseCollection,
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
transient_failure: bool,
outcome: TraceRunState,
reply: EgressResponse | Exception,
) -> None:
clock = [naive_utc_now()]
# The service dates its claim, renewals, and outcome; the second claimant below passes the same clock.
monkeypatch.setattr(service_module, "naive_utc_now", lambda: clock[0])
renewals = _RenewalRecordingRepository(case.sessions, transient_failure=transient_failure)
observed: list[tuple[bool, bool, str]] = []
def while_exporting() -> None:
# The original start time is now beyond the lease; only a real background
# transaction can prevent a second claimant from marking it uncertain.
clock[0] += timedelta(seconds=3)
heartbeat_committed = renewals.renewed.wait(3)
second_claim = _claim_at(case, lease_seconds=1, now=clock[0])
observed.append((heartbeat_committed, second_claim is None, _stored(case).state))
ports = ServicePorts(
_SlowDestination([reply], while_exporting), settings=replace(AGENT_TRACE_SETTINGS, export_lease_seconds=1)
)
ports.service(replace(case, repository=renewals)).deliver(case.run.id)
assert observed == [(True, True, TraceRunState.EXPORTING)]
assert _stored(case).state == outcome
assert renewals.renewal_threads
assert all(thread_id != threading.get_ident() for thread_id in renewals.renewal_threads)
assert not any(thread.name == f"application-trace-lease-{case.run.id}" for thread in threading.enumerate())
assert "private-provider-value" not in caplog.text
@staticmethod
def test_heartbeat_stops_even_when_completion_database_write_raises(case: _ExportLeaseCollection) -> None:
unavailable = replace(case, repository=_UnavailableCompletionRepository(case.sessions))
destination = _export_lease_accepted("ack")
with pytest.raises(TracingStoreUnavailableError, match="database unavailable"):
ServicePorts(destination).service(unavailable).deliver(case.run.id)
# The database is lost only once the destination answered, so the export ran under its heartbeat.
assert len(destination.requests) == 1
assert not any(thread.name == f"application-trace-lease-{case.run.id}" for thread in threading.enumerate())
_application_trace_ownership_no_egress = _application_trace_collection_no_egress
_application_trace_ownership_seed = _application_trace_collection_seed
class TestApplicationTraceOwnershipCases:
@staticmethod
def test_naming_without_a_message_stays_with_its_agent_app(sqlite_session_factory: sessionmaker[Session]) -> None:
case = _application_trace_ownership_seed(sqlite_session_factory, "naming")
service = ServicePorts(_application_trace_ownership_no_egress()).service(case)
record = GenerateNameTraceInfo(
message_id=None,
message_data=None,
inputs="hello",
outputs="Greeting",
start_time=datetime(2026, 1, 1),
end_time=datetime(2026, 1, 1),
metadata={"app_id": case.run.app_id, "conversation_id": "conversation-1", "trace_event_id": "event-1"},
conversation_id="conversation-1",
tenant_id=case.run.tenant_id,
)
assert service.capture_terminal(tenant_id=case.run.tenant_id, info=record) is True
@staticmethod
def test_runs_frozen_for_an_unavailable_provider_report_no_capabilities(
sqlite_session_factory: sessionmaker[Session],
) -> None:
case = _application_trace_ownership_seed(sqlite_session_factory, "retired-capabilities")
freeze_config(case, "not-a-provider", FROZEN_LANGFUSE)
result = (
ServicePorts(_application_trace_ownership_no_egress())
.service(case)
.status(**source(case.run), node_execution_id=None)
)
assert result.provider == "not-a-provider"
assert result.capabilities is None