Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
"""Recover lost terminal notifications from real business rows through the trace record repository."""
import json
from collections.abc import Callable, Generator, Iterator
from dataclasses import dataclass, replace
from datetime import UTC, datetime, timedelta
from decimal import Decimal
from typing import Literal, cast, override
from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
import pytest
from cachetools import TTLCache
from dify_agent.protocol import RunEvent
from dify_agent.protocol.tracing import AgentTraceBatch, AgentTraceSpan
from dify_trace_langfuse.config import LangfuseConfig
from dify_trace_sdk.emitter import TraceScope
from dify_trace_sdk.errors import EgressError
from dify_trace_sdk.events import MessageCompleted, TraceCorrelation, TraceEvent, TracePayload
from dify_trace_sdk.exporting import send_export_request
from dify_trace_sdk.ids import span_id_for
from dify_trace_sdk.model import Trace
from dify_trace_sdk.provider import ProviderCapabilities, SendResult, TraceProvider
from dify_trace_sdk.testing import RecordingEgress, response
from flask import Flask
from sqlalchemy import Engine, delete, inspect, update
from sqlalchemy.orm import Session, scoped_session, sessionmaker
from clients.agent_backend import FakeAgentBackendRunClient, FakeAgentBackendScenario
from core.app.app_config.entities import (
EasyUIBasedAppConfig,
EasyUIBasedAppModelConfigFrom,
ModelConfigEntity,
PromptTemplateEntity,
)
from core.app.apps import base_app_queue_manager
from core.app.apps.agent_app.app_runner import AgentAppRunner
from core.app.apps.agent_app.session_store import AgentAppSessionScope
from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
from core.app.apps.exc import GenerateTaskStoppedError
from core.app.apps.execution_coordinator import set_app_task_stop_flag
from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager
from core.app.entities.app_invoke_entities import ChatAppGenerateEntity, DifyRunContext, InvokeFrom, UserFrom
from core.app.entities.queue_entities import (
AppQueueEvent,
MessageQueueMessage,
QueueLLMChunkEvent,
QueueMessageEndEvent,
QueueStopEvent,
)
from core.app.task_pipeline.easy_ui_based_generate_task_pipeline import EasyUIBasedGenerateTaskPipeline
from core.entities.provider_entities import ProviderQuotaType, QuotaConfiguration, QuotaUnit, SystemConfiguration
from domain.tracing.trace_run_agg import TraceInvocation, TraceRunState
from events.event_handlers import update_provider_when_message_created as receiver
from extensions.ext_database import db
from extensions.tracing import application_tracing_hooks, bind_tracing
from graphon.enums import WorkflowExecutionStatus
from graphon.model_runtime.entities.llm_entities import LLMUsage
from libs.datetime_utils import naive_utc_now
from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole
from models.application_trace import ApplicationTraceInvocation, ApplicationTraceRun
from models.enums import ConversationFromSource, CreatorUserRole, MessageStatus, WorkflowRunTriggeredFrom
from models.model import App, AppMode, Conversation, Message, TraceAppConfig
from models.workflow import WorkflowRun, WorkflowType
from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
from repositories.application_trace_repository import ApplicationTraceRepository
from repositories.factory import DifyAPIRepositoryFactory
from repositories.trace_record_repository import TraceRecordRepository
from services.tracing.adapters import StoredPolicyTraceScopes
from services.tracing.canonical import CanonicalTraceBuilder
from services.tracing.collection import AgentTraceSettings, TraceRun
from services.tracing.delivery import TraceRecordSource
from services.tracing.errors import SourceNotReadyError, TraceCollectionNotFoundError
from services.tracing.providers import ProviderConfigCodec
from services.tracing.records import BaseTraceInfo, MessageTraceInfo
from services.tracing.recovery import recover_terminal
from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
from services.tracing.scopes import TraceScopes
from services.tracing.service import ApplicationTracingService
from tests.helpers.unit.core.model_fixtures import make_model_config
from tests.helpers.unit.model_factories import make_account, make_app, make_conversation, make_message, make_tenant
from tests.helpers.unit.services.tracing_ports import (
AGENT_TRACE_SETTINGS,
InstalledTraceProviders,
WorkspaceBoundCipher,
)
from tests.unit_tests.core.app.apps.agent_app.test_app_runner import (
_CancelAndWaitFailingClient,
_ExplodingSessionStore,
_FakeQueueManager,
_FakeSessionStore,
_runner,
_soul,
_StoppedQueueManager,
_UsageCancellationClient,
)
def identifier(value: str) -> str:
return str(uuid5(NAMESPACE_URL, value))
@dataclass(frozen=True)
class Collection:
repository: ApplicationTraceRepository
sessions: sessionmaker[Session]
run: TraceRun
def invocation(self, name: str) -> TraceInvocation:
with self.repository.change(self.run.id, tenant_id=self.run.tenant_id) as locked:
assert locked is not None
return locked.run.open_invocation(
agent_id=identifier("agent"), node_execution_id=None, attempt=0, call_key=name
)
class ScriptedProvider(TraceProvider[LangfuseConfig]):
"""Sends each trace as one JSON request through the egress, whose script decides the outcome."""
id = "langfuse"
config_model = LangfuseConfig
capabilities = ProviderCapabilities(native_cost=True, trace_url=True, span_url=True)
@override
def verify(self, config: LangfuseConfig) -> LangfuseConfig:
return config
@override
def project_url(self, config: LangfuseConfig) -> str | None:
return f"{config.host}/project/{config.project_key}"
@override
def send(self, config: LangfuseConfig, trace: Trace) -> SendResult:
reply = send_export_request(
self.egress,
"POST",
f"{config.host}/traces",
destination="Scripted",
headers={"Authorization": f"{config.public_key}:{config.secret_key}"},
content=trace.model_dump_json().encode(),
sensitive_url=False,
)
body = reply.json()
assert isinstance(body, dict)
return SendResult(
trace_id=str(body["trace_id"]), trace_url=body.get("trace_url"), span_urls=body.get("span_urls", {})
)
def record_source(sessions: sessionmaker[Session]) -> TraceRecordRepository:
return TraceRecordRepository(
sessions,
workflow_runs=DifyAPIRepositoryFactory.create_api_workflow_run_repository(sessions),
node_executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(sessions),
files_url="https://files.test",
enterprise_details=False,
)
def tracing_service(
repository: ApplicationTraceRepository,
records: TraceRecordSource,
destination: RecordingEgress,
scheduled: list[str],
settings: AgentTraceSettings,
) -> ApplicationTracingService:
"""Agent trace collection over ``repository``; scheduled exports are recorded instead of enqueued."""
return ApplicationTracingService(
runs=repository,
run_query=repository,
sources=repository,
records=records,
builder=CanonicalTraceBuilder(),
providers=InstalledTraceProviders(ScriptedProvider),
codec=ProviderConfigCodec(WorkspaceBoundCipher()),
egress=destination,
schedule_export=scheduled.append,
settings=settings,
)
def seed(sessions: sessionmaker[Session], name: str, *, capture_content: bool, capture_agent: bool) -> Collection:
"""An Agent app tracing to Langfuse, one of its messages, and the run frozen when the message started."""
app_id, tenant_id, message_id = identifier(f"app-{name}"), identifier("tenant"), identifier(f"message-{name}")
cipher = WorkspaceBoundCipher()
with sessions.begin() as session:
session.add(
make_app(
app_id=app_id,
tenant_id=tenant_id,
mode=AppMode.AGENT,
tracing=json.dumps(
{
"enabled": True,
"tracing_provider": "langfuse",
"capture_content": capture_content,
"capture_agent": capture_agent,
}
),
)
)
session.add(
make_message(
message_id=message_id,
app_id=app_id,
conversation_id=identifier(f"conversation-{name}"),
inputs={},
query="hello",
message={},
answer="done",
message_unit_price=Decimal(0),
answer_unit_price=Decimal(0),
currency="USD",
from_source=ConversationFromSource.CONSOLE,
)
)
session.add(
TraceAppConfig(
app_id=app_id,
tracing_provider="langfuse",
tracing_config={
"public_key": cipher.encrypt(tenant_id, "public"),
"secret_key": cipher.encrypt(tenant_id, "original"),
"host": "https://langfuse.test",
"project_key": "original",
},
)
)
repository = ApplicationTraceRepository(sessions)
run = tracing_service(
repository, record_source(sessions), RecordingEgress([], grpc_target=None), [], AGENT_TRACE_SETTINGS
).start(
tenant_id=tenant_id,
app_id=app_id,
source_kind="message",
source_id=message_id,
session_id="session-original",
workflow_id=None,
)
assert run is not None
return Collection(repository, sessions, run)
def attach_conversation(collection: Collection, mode: AppMode) -> None:
"""Message records are assembled only for a message whose conversation belongs to its app."""
with collection.sessions.begin() as session:
message = session.get(Message, collection.run.source_id)
assert message is not None
session.add(
make_conversation(
conversation_id=message.conversation_id,
app_id=collection.run.app_id,
mode=mode,
inputs={},
from_source=ConversationFromSource.CONSOLE,
)
)
def add_app_owner(collection: Collection, name: str) -> str:
"""An account of the app's workspace that creates the app and, here, its workflow runs."""
owner_id = identifier(f"owner-{name}")
with collection.sessions.begin() as session:
if session.get(Tenant, collection.run.tenant_id) is None:
session.add(make_tenant(tenant_id=collection.run.tenant_id))
session.add(make_account(account_id=owner_id, email=f"{name}@example.com"))
session.add(
TenantAccountJoin(tenant_id=collection.run.tenant_id, account_id=owner_id, role=TenantAccountRole.OWNER)
)
app = session.get(App, collection.run.app_id)
assert app is not None
app.created_by = owner_id
return owner_id
def trace_event(run: TraceRun, payload: TracePayload) -> TraceEvent:
return TraceEvent(
event_id=uuid4().hex,
tenant_id=run.tenant_id,
app_id=run.app_id,
user_id=None,
occurred_at=datetime.now(UTC),
correlation=TraceCorrelation(
external_trace_id=None, session_id=None, parent_workflow_run_id=None, parent_node_execution_id=None
),
payload=payload,
)
def batch_for(run: TraceRun, invocation: TraceInvocation, *, final: bool, span_id: str) -> AgentTraceBatch:
return AgentTraceBatch(
run_id=UUID(run.id),
invocation_id=UUID(invocation.id),
batch_id=uuid4(),
final=final,
spans=[
AgentTraceSpan(
trace_id=run.trace_id,
span_id=span_id,
parent_span_id=invocation.span_id,
name="Agent execution",
start_time_ns=1_000_000_000,
end_time_ns=2_000_000_000,
status="ok",
attributes={"gen_ai.operation.name": "invoke_agent"},
)
],
)
def ingest(collection: Collection, batch: AgentTraceBatch) -> None:
"""Store a runtime batch in its run without scheduling the run's export."""
with collection.repository.change(str(batch.run_id), tenant_id=collection.run.tenant_id) as locked:
assert locked is not None
locked.run.ingest(batch, max_spans=100, max_bytes=1_000_000, now=naive_utc_now())
def claim(collection: Collection, run_id: str) -> str | None:
"""Claim the run for an export attempt; ``None`` when it may not export now."""
token = str(uuid4())
with collection.repository.change(run_id, tenant_id=collection.run.tenant_id) as locked:
assert locked is not None
claimed = locked.run.claim_export(
token=token, owner_exists=locked.owner_exists, collection_timeout=60, lease_seconds=30, now=naive_utc_now()
)
return token if claimed else None
def recover(
collection: Collection,
run: TraceRun,
*,
records: TraceRecordSource,
collector: ApplicationTracingService,
settings: AgentTraceSettings,
now: datetime,
) -> bool:
"""One reconciler recovery attempt of ``run`` at ``now``."""
current = collection.repository.load(run.id)
assert current is not None
return recover_terminal(
current,
runs=collection.repository,
sources=collection.repository,
records=records,
collector=collector,
collection_timeout=settings.collection_timeout_seconds,
now=now,
)
def sent_traces(egress: RecordingEgress) -> list[Trace]:
return [Trace.model_validate_json(request.content or b"") for request in egress.requests]
@dataclass(frozen=True)
class RecoveryCase:
collection: Collection
records: TraceRecordRepository
service: ApplicationTracingService
settings: AgentTraceSettings
now: datetime
scheduled: list[str]
destination: RecordingEgress
def recover(self, run: TraceRun, *, now: datetime) -> bool:
"""One reconciler recovery attempt of ``run`` at ``now``."""
return recover(
self.collection, run, records=self.records, collector=self.service, settings=self.settings, now=now
)
def recovery_case(
collection: Collection, destination: RecordingEgress, settings: AgentTraceSettings, now: datetime
) -> RecoveryCase:
"""The service the runtime hooks reach; scheduled exports are recorded instead of enqueued."""
records = record_source(collection.sessions)
scheduled: list[str] = []
service = tracing_service(collection.repository, records, destination, scheduled, settings)
return RecoveryCase(collection, records, service, settings, now, scheduled, destination)
class _Recorder:
def __init__(self) -> None:
self.events: list[TraceEvent] = []
def emit(self, event: TraceEvent) -> None:
self.events.append(event)
class _QueueManager(AppQueueManager):
"""Finite in-memory event source for the consumer's stream dispatch."""
def __init__(self, messages: list[MessageQueueMessage]) -> None:
self.messages = messages
@override
def listen(self) -> Generator[MessageQueueMessage]:
yield from self.messages
@override
def _publish(self, event: AppQueueEvent, pub_from: PublishFrom) -> None:
raise AssertionError("Stream dispatch must not publish queue events")
class _FailingRecords(TraceRecordSource):
"""Remembers every event recovery asks it to assemble, then fails with ``failure``."""
def __init__(self, failure: Exception) -> None:
self._failure = failure
self.events: list[TraceEvent] = []
@override
def assemble(self, event: TraceEvent) -> BaseTraceInfo:
self.events.append(event)
raise self._failure
def tracing_application(case: RecoveryCase) -> Flask:
"""An application bound like the composition root of a collecting deployment, with events kept in memory."""
application = Flask(__name__)
bind_tracing(
application,
scopes=StoredPolicyTraceScopes(
TraceScopes(
emitter=_Recorder(),
policies=SQLAlchemyAppTracingConfigRepository(session_factory=case.collection.sessions),
enterprise_enabled=lambda: False,
)
),
hooks=RuntimeApplicationTracingHooks(case.service),
)
return application
@pytest.fixture
def case(
sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
) -> RecoveryCase:
collection = seed(sqlite_session_factory, "recover", capture_content=True, capture_agent=True)
attach_conversation(collection, AppMode.AGENT)
invocation = collection.invocation("first")
ingest(collection, batch_for(collection.run, invocation, final=True, span_id="a" * 16))
now = naive_utc_now()
with sqlite_session_factory.begin() as session:
message = session.get(Message, collection.run.source_id)
stored_invocation = session.get(ApplicationTraceInvocation, invocation.id)
assert message is not None
assert stored_invocation is not None
message.created_at = now - timedelta(seconds=30)
message.updated_at = now - timedelta(seconds=10)
message.answer = "persisted final answer"
message.provider_response_latency = 1.0
message.message_tokens = 2
message.answer_tokens = 1
stored_invocation.finalized_at = now - timedelta(seconds=10)
monkeypatch.setattr(type(db), "engine", property(lambda _db: sqlite_engine))
monkeypatch.setattr(db, "session", scoped_session(sqlite_session_factory))
destination = RecordingEgress([response(200, {"trace_id": "provider-id"})], grpc_target=None)
return recovery_case(collection, destination, AGENT_TRACE_SETTINGS, now)
def collecting_within(case: RecoveryCase, collection_timeout_seconds: int) -> RecoveryCase:
"""The same case on a deployment that waits ``collection_timeout_seconds`` for a business terminal."""
return recovery_case(
case.collection,
case.destination,
replace(case.settings, collection_timeout_seconds=collection_timeout_seconds),
case.now,
)
def stop_requested(collection: Collection) -> bool:
"""The consumer's persisted stop intent, as both ledger readers of the message report it."""
run = collection.run
message = collection.repository.message(tenant_id=run.tenant_id, app_id=run.app_id, message_id=run.source_id)
progress = collection.repository.message_progress(
tenant_id=run.tenant_id, app_id=run.app_id, message_id=run.source_id
)
assert message is not None
assert progress is not None
assert message.agent_stop_requested is progress.agent_stop_requested
return progress.agent_stop_requested
def message_record(case: RecoveryCase) -> MessageTraceInfo:
"""The record the delivery worker assembles when the message reports its end."""
run = case.collection.run
message = case.collection.repository.message(tenant_id=run.tenant_id, app_id=run.app_id, message_id=run.source_id)
assert message is not None
record = case.records.assemble(
trace_event(run, MessageCompleted(message_id=run.source_id, conversation_id=message.conversation_id))
)
assert isinstance(record, MessageTraceInfo)
return record
def complete_metadata() -> str:
return json.dumps(
{
"annotation_reply": None,
"retriever_resources": [],
"reasoning": {},
"usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
}
)
def save_agent_stop(
case: RecoveryCase,
monkeypatch: pytest.MonkeyPatch,
answer: str,
*,
mode: AppMode = AppMode.AGENT,
consumer_session: Session | None = None,
process_stream: bool = False,
notify: Callable[[Message, ChatAppGenerateEntity], None] | None = None,
) -> list[TraceEvent]:
"""Run the real consumer save against the owned SQL rows; returns the trace events it emitted."""
def send(message: Message, *, application_generate_entity: ChatAppGenerateEntity) -> None:
if notify is not None:
notify(message, application_generate_entity)
monkeypatch.setattr("core.app.task_pipeline.easy_ui_based_generate_task_pipeline.message_was_created.send", send)
monkeypatch.setattr(
"core.app.task_pipeline.easy_ui_based_generate_task_pipeline.naive_utc_now",
lambda: case.now - timedelta(seconds=10),
)
with case.collection.sessions.begin() as session:
persisted = session.get(Message, case.collection.run.source_id)
assert persisted is not None
if persisted.message_metadata is None:
persisted.message_metadata = complete_metadata()
with case.collection.sessions() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
conversation = session.get(Conversation, message.conversation_id)
assert conversation is not None
recorder = _Recorder()
entity = ChatAppGenerateEntity(
task_id=identifier("stop-race-task"),
app_config=EasyUIBasedAppConfig(
tenant_id=case.collection.run.tenant_id,
app_id=case.collection.run.app_id,
app_mode=mode,
app_model_config_from=EasyUIBasedAppModelConfigFrom.APP_LATEST_CONFIG,
app_model_config_dict={},
model=ModelConfigEntity(provider="test-provider", model="test-model"),
prompt_template=PromptTemplateEntity(prompt_type=PromptTemplateEntity.PromptType.SIMPLE),
),
model_conf=make_model_config(provider="test-provider", model="test-model", mode="chat"),
inputs={},
files=[],
user_id=identifier("stop-race-user"),
stream=True,
invoke_from=InvokeFrom.DEBUGGER,
trace_scope=TraceScope(
emitter=recorder,
tenant_id=case.collection.run.tenant_id,
app_id=case.collection.run.app_id,
user_id=identifier("stop-race-user"),
correlation=TraceCorrelation(
external_trace_id=None, session_id=None, parent_workflow_run_id=None, parent_node_execution_id=None
),
),
)
stop = MessageQueueMessage(
task_id=entity.task_id,
app_mode=mode,
message_id=message.id,
conversation_id=conversation.id,
event=QueueStopEvent(stopped_by=QueueStopEvent.StopBy.USER_MANUAL),
)
pipeline = EasyUIBasedGenerateTaskPipeline(
entity, _QueueManager([stop] if process_stream else []), conversation, message, stream=True
)
pipeline._task_state.llm_result.message.content = answer
if process_stream:
# Token estimates and moderation are separate external boundaries;
# exercise real stream dispatch, SQL save, commit and notification.
monkeypatch.setattr(pipeline, "_handle_stop", lambda _event: None)
monkeypatch.setattr(pipeline, "handle_output_moderation_when_task_finished", lambda _answer: None)
list(pipeline._process_stream_response(publisher=None))
elif consumer_session is None:
with case.collection.sessions.begin() as session:
pipeline._save_message(session=session, preserve_existing_usage=True)
else:
pipeline._save_message(session=consumer_session, preserve_existing_usage=True)
consumer_session.commit()
return recorder.events
@pytest.mark.parametrize("receiver_fails", [False, True])
def test_stopped_agent_notifies_the_real_provider_receiver_once_after_commit(
case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, receiver_fails: bool
) -> None:
calls: list[str] = []
provider_updates: list[str] = []
monkeypatch.setattr(
receiver,
"_execute_provider_updates",
lambda updates: provider_updates.extend(operation.description for operation in updates),
)
def notify(message: Message, entity: ChatAppGenerateEntity) -> None:
# The production session factory does not expire on commit. Check real
# detached fields used by the actual receiver, not a mocked send order.
assert inspect(message).detached is True
assert not inspect(message).expired_attributes
assert message.answer == "partial answer"
with case.collection.sessions() as session:
saved = session.get(Message, message.id)
assert saved is not None
assert saved.answer == message.answer
assert stop_requested(case.collection) is True
system = SystemConfiguration(
enabled=True,
current_quota_type=ProviderQuotaType.TRIAL,
quota_configurations=[
QuotaConfiguration(
quota_type=ProviderQuotaType.TRIAL,
quota_unit=QuotaUnit.TOKENS,
quota_limit=100,
quota_used=0,
is_valid=True,
)
],
)
assert receiver._calculate_quota_usage(message=message, system_configuration=system, model_name="test") == 3
receiver.handle(message, application_generate_entity=entity)
calls.append(message.id)
if receiver_fails:
raise RuntimeError("receiver failed after commit")
if receiver_fails:
with pytest.raises(RuntimeError, match="receiver failed after commit"):
save_agent_stop(case, monkeypatch, "partial answer", process_stream=True, notify=notify)
else:
# The Agent worker reports a stopped Agent run's terminal itself.
assert save_agent_stop(case, monkeypatch, "partial answer", process_stream=True, notify=notify) == []
assert calls == [case.collection.run.source_id]
assert provider_updates == ["basic_last_used_update"]
assert stop_requested(case.collection) is True
@pytest.mark.parametrize("answer", ["", "partial answer before stop"])
@pytest.mark.parametrize("outcome", ["succeeded", "failed", "cancelled"])
@pytest.mark.parametrize("delivery", [TraceRunState.ACCEPTED, TraceRunState.UNCERTAIN])
def test_stopped_consumer_cannot_freeze_a_terminal_before_the_runtime_worker(
case: RecoveryCase,
monkeypatch: pytest.MonkeyPatch,
answer: str,
outcome: Literal["succeeded", "failed", "cancelled"],
delivery: TraceRunState,
) -> None:
destination = RecordingEgress(
[
response(200, {"trace_id": "provider-id"})
if delivery is TraceRunState.ACCEPTED
else EgressError("acknowledgement lost", request_sent=True)
],
grpc_target=None,
)
stopped = recovery_case(case.collection, destination, case.settings, case.now)
service = stopped.service
tenant_id = case.collection.run.tenant_id
# Simulate an old queued notification as well as the current consumer save:
# persisted state, not a detached event payload, must decide who owns the end.
notification = message_record(stopped)
queued = save_agent_stop(case, monkeypatch, answer)
assert service.capture_terminal(tenant_id=tenant_id, info=notification) is True
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.outer_trace is None, "a stopped SSE save is not the runtime terminal"
assert queued == []
assert stopped.recover(stored, now=case.now) is False
assert claim(case.collection, stored.id) is None
service.deliver(stored.id)
assert destination.requests == []
output = "real runtime answer" if outcome == "succeeded" else None
service.capture_agent_app_terminal(
tenant_id=stored.tenant_id,
app_id=stored.app_id,
message_id=stored.source_id,
outcome=outcome,
output=output,
)
# A duplicate worker result and late consumer notification cannot rewrite
# the frozen end or create a second ingestion attempt after acceptance.
service.deliver(stored.id)
service.deliver(stored.id)
exported = sent_traces(destination)
assert len(exported) == 1
trace = exported[0]
assert trace.root.attributes["message_execution_state"] == outcome
assert trace.root.status.value == ("ok" if outcome == "succeeded" else "error")
assert len(trace.spans) == 3
frozen = case.collection.repository.get(stored.id)
assert frozen is not None
assert frozen.state is delivery
service.capture_agent_app_terminal(
tenant_id=stored.tenant_id,
app_id=stored.app_id,
message_id=stored.source_id,
outcome="succeeded" if outcome != "succeeded" else "cancelled",
output="late result must not replace the accepted envelope",
)
assert service.capture_terminal(tenant_id=tenant_id, info=notification) is True
unchanged = case.collection.repository.get(stored.id)
assert unchanged is not None
assert unchanged.outer_trace == frozen.outer_trace
assert unchanged.attempts == frozen.attempts == 1
service.deliver(stored.id)
assert len(destination.requests) == 1
with case.collection.sessions() as session:
message = session.get(Message, stored.source_id)
assert message is not None
assert message.status is MessageStatus.NORMAL
assert message.answer == answer
assert message.message_tokens == 2
assert message.answer_tokens == 1
@pytest.mark.parametrize("mode", [AppMode.CHAT, AppMode.AGENT_CHAT])
def test_classic_stopped_consumer_keeps_its_existing_trace_notification(
case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, mode: AppMode
) -> None:
queued = save_agent_stop(case, monkeypatch, "classic partial answer", mode=mode, process_stream=True)
assert [event.payload for event in queued] == [
MessageCompleted(message_id=case.collection.run.source_id, conversation_id=identifier("conversation-recover"))
]
with case.collection.sessions() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
assert "agent_stop_requested" not in json.loads(message.message_metadata or "{}")
def test_worker_usage_flush_preserves_stop_intent_from_a_newer_consumer_save(
case: RecoveryCase, monkeypatch: pytest.MonkeyPatch
) -> None:
# The worker's scoped session may already have loaded this Message before
# the consumer commits its independent stop transaction.
cached_message = db.session.get(Message, case.collection.run.source_id)
assert cached_message is not None
assert cached_message.message_metadata is None
save_agent_stop(case, monkeypatch, "partial answer")
AgentAppRunner._persist_message_usage(
message_id=cached_message.id,
usage=LLMUsage.from_metadata({"prompt_tokens": 13, "completion_tokens": 8, "total_price": "0.21"}),
)
assert stop_requested(case.collection) is True
with case.collection.sessions() as session:
message = session.get(Message, cached_message.id)
assert message is not None
assert message.answer == "partial answer"
assert message.message_tokens == 13
assert message.answer_tokens == 8
assert json.loads(message.message_metadata or "{}")["usage"]["total_tokens"] == 21
def test_stop_save_preserves_worker_usage_committed_after_its_cached_message_read(
case: RecoveryCase, monkeypatch: pytest.MonkeyPatch
) -> None:
with case.collection.sessions.begin() as session:
initial = session.get(Message, case.collection.run.source_id)
assert initial is not None
initial.message_tokens = 0
initial.answer_tokens = 0
with case.collection.sessions() as consumer:
cached = consumer.get(Message, case.collection.run.source_id)
assert cached is not None
assert cached.message_tokens == 0
AgentAppRunner._persist_message_usage(
message_id=cached.id,
usage=LLMUsage.from_metadata({"prompt_tokens": 13, "completion_tokens": 8, "total_price": "0.21"}),
)
save_agent_stop(case, monkeypatch, "partial answer", consumer_session=consumer)
assert stop_requested(case.collection) is True
with case.collection.sessions() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
assert message.message_tokens == 13
assert message.answer_tokens == 8
assert message.total_price == Decimal("0.21")
assert json.loads(message.message_metadata or "{}")["usage"]["total_tokens"] == 21
@pytest.mark.parametrize("runtime_outcome", ["succeeded", "failed", "cancelled", "cancel_wait_failed"])
@pytest.mark.parametrize("final_batch_arrived", [False, True])
@pytest.mark.parametrize("snapshot_save_fails", [False, True])
def test_stopped_runner_hands_off_only_an_observed_runtime_terminal(
case: RecoveryCase,
monkeypatch: pytest.MonkeyPatch,
runtime_outcome: str,
final_batch_arrived: bool,
snapshot_save_fails: bool,
) -> None:
save_agent_stop(case, monkeypatch, "partial answer")
case = collecting_within(case, 60)
pending_invocation = None
if not final_batch_arrived:
pending_invocation = case.collection.invocation("awaiting-stop-final-batch")
ingest(case.collection, batch_for(case.collection.run, pending_invocation, final=False, span_id="b" * 16))
client = (
_CancelAndWaitFailingClient()
if runtime_outcome == "cancel_wait_failed"
else FakeAgentBackendRunClient(
scenario=FakeAgentBackendScenario.FAILED
if runtime_outcome == "failed"
else FakeAgentBackendScenario.SUCCESS
)
)
if runtime_outcome in {"succeeded", "failed"}:
# The terminal frame has arrived, but stop is already visible at the
# next check. Do not discard this known result and ask SDK cancellation
# to reclassify it (the SDK rejects non-cancelled terminal events).
arrived_terminal = client._events(client.run_id)[-1]
monkeypatch.setattr(client, "stream_events", lambda _run_id, **_kwargs: iter([arrived_terminal]))
def unexpected_cancel(*_args: object, **_kwargs: object) -> None:
raise AssertionError("a known runtime terminal must not be cancelled again")
monkeypatch.setattr(client, "cancel_run_and_wait", unexpected_cancel)
store = _ExplodingSessionStore() if snapshot_save_fails else _FakeSessionStore()
runner = _runner(client, store)
run = case.collection.run
context = DifyRunContext(
tenant_id=run.tenant_id,
app_id=run.app_id,
user_id=identifier("stop-race-user"),
user_from=UserFrom.ACCOUNT,
invoke_from=InvokeFrom.DEBUGGER,
)
scope = AgentAppSessionScope(
tenant_id=run.tenant_id,
app_id=run.app_id,
conversation_id=identifier("conversation-recover"),
agent_id=identifier("stop-agent"),
agent_config_snapshot_id=identifier("stop-snapshot"),
home_snapshot_id=None,
)
with tracing_application(case).app_context(), pytest.raises(GenerateTaskStoppedError):
runner._consume_stream(
client.run_id,
dify_context=context,
message_id=run.source_id,
queue_manager=cast(AppQueueManager, _StoppedQueueManager()),
model_name="test-model",
query="test query",
session_scope=scope,
binding_id=identifier("stop-binding"),
)
frozen = case.collection.repository.get(run.id)
assert frozen is not None
if runtime_outcome == "cancel_wait_failed":
assert frozen.outer_trace is None
assert store.saved == []
assert case.recover(frozen, now=case.now + timedelta(seconds=120)) is False
expired = case.collection.repository.get(run.id)
assert expired is not None
assert expired.state is TraceRunState.FAILED
assert expired.error_code == (
"business_terminal_not_persisted" if final_batch_arrived else "agent_trace_collection_incomplete"
)
return
assert frozen.outer_trace is not None
root = Trace.model_validate(frozen.outer_trace).root
assert root.attributes["message_execution_state"] == runtime_outcome
assert root.status.value == ("ok" if runtime_outcome == "succeeded" else "error")
assert len(store.saved) == (0 if snapshot_save_fails else 1)
if pending_invocation is not None:
assert claim(case.collection, run.id) is None
case.service.deliver(run.id)
assert case.destination.requests == []
ingest(case.collection, batch_for(run, pending_invocation, final=True, span_id="b" * 16))
case.service.deliver(run.id)
case.service.deliver(run.id)
exported = sent_traces(case.destination)
assert len(exported) == 1
assert exported[0].root.attributes["message_execution_state"] == runtime_outcome
assert len(exported[0].spans) == (3 if final_batch_arrived else 5)
accepted_run = case.collection.repository.get(run.id)
assert accepted_run is not None
assert accepted_run.state is TraceRunState.ACCEPTED
assert accepted_run.attempts == 1
assert stop_requested(case.collection) is True
with case.collection.sessions() as session:
message = session.get(Message, run.source_id)
assert message is not None
assert message.answer == "partial answer"
@pytest.mark.parametrize("incomplete", [False, True])
def test_missing_runtime_stop_terminal_converges_without_inventing_success(
case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, incomplete: bool
) -> None:
case = collecting_within(case, 60)
save_agent_stop(case, monkeypatch, "partial answer")
if incomplete:
invocation = case.collection.invocation("still-collecting")
ingest(case.collection, batch_for(case.collection.run, invocation, final=False, span_id="b" * 16))
assert case.recover(case.collection.run, now=case.now + timedelta(seconds=120)) is False
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.state is TraceRunState.FAILED
assert stored.error_code == (
"agent_trace_collection_incomplete" if incomplete else "business_terminal_not_persisted"
)
assert stored.outer_trace is None
assert claim(case.collection, stored.id) is None
@pytest.mark.parametrize("stop_on_end", [False, True])
def test_stop_during_final_queue_publication_preserves_the_already_observed_success(
case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, stop_on_end: bool
) -> None:
save_agent_stop(case, monkeypatch, "partial answer")
# Only Redis storage is in memory. Use the real queue's publish stop check
# rather than an already-stopped fake consumer or an exception-only stub.
flags: dict[str, object] = {}
monkeypatch.setattr(base_app_queue_manager.redis_client, "get", flags.get)
monkeypatch.setattr(
base_app_queue_manager.redis_client, "setex", lambda key, _ttl, value: flags.__setitem__(key, value)
)
monkeypatch.setattr(
"core.app.apps.agent_app.runtime_request_builder.resolve_model_context_window", lambda **_kwargs: None
)
task_id = identifier("stop-at-final-publish")
queue_manager = MessageBasedAppQueueManager(
task_id=task_id,
user_id=identifier("stop-race-user"),
invoke_from=InvokeFrom.DEBUGGER,
conversation_id=identifier("conversation-recover"),
app_mode=AppMode.AGENT,
message_id=case.collection.run.source_id,
)
clock = [0.0]
queue_manager._stopped_cache = TTLCache(maxsize=1, ttl=1, timer=lambda: clock[0])
store = _FakeSessionStore()
runner = _runner(FakeAgentBackendRunClient(), store)
original_publish = queue_manager._publish
def publish_after_stop(event: AppQueueEvent, pub_from: PublishFrom) -> None:
if isinstance(event, QueueMessageEndEvent if stop_on_end else QueueLLMChunkEvent):
set_app_task_stop_flag(task_id)
clock[0] = 2.0 # Expire the real queue's earlier cached false result.
original_publish(event, pub_from)
monkeypatch.setattr(queue_manager, "_publish", publish_after_stop)
run = case.collection.run
context = DifyRunContext(
tenant_id=run.tenant_id,
app_id=run.app_id,
user_id=identifier("stop-race-user"),
user_from=UserFrom.ACCOUNT,
invoke_from=InvokeFrom.DEBUGGER,
)
with tracing_application(case).app_context(), pytest.raises(GenerateTaskStoppedError):
runner.run(
dify_context=context,
agent_id=identifier("stop-agent"),
agent_config_snapshot_id=identifier("stop-snapshot"),
agent_soul=_soul(),
home_snapshot_id=None,
conversation_id=identifier("conversation-recover"),
query="test query",
message_id=run.source_id,
model_name="test-model",
queue_manager=queue_manager,
)
published = queue_manager._q.get_nowait()
assert isinstance(published, MessageQueueMessage)
assert isinstance(published.event, QueueLLMChunkEvent)
if stop_on_end:
published_end = queue_manager._q.get_nowait()
assert isinstance(published_end, MessageQueueMessage)
assert isinstance(published_end.event, QueueMessageEndEvent)
frozen = case.collection.repository.get(run.id)
assert frozen is not None
assert frozen.outer_trace is not None, "success arrived before the final queue publication saw Stop"
root = Trace.model_validate(frozen.outer_trace).root
assert root.attributes["message_execution_state"] == "succeeded"
assert root.status.value == "ok"
assert len(store.saved) == 1
@pytest.mark.parametrize("stop_during_cleanup", [False, True])
def test_runtime_cleanup_cancellation_does_not_reclassify_a_business_stream_failure(
case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, stop_during_cleanup: bool
) -> None:
client = _UsageCancellationClient()
queue_manager = _FakeQueueManager()
def disconnected_stream(*_args: object, **_kwargs: object) -> Iterator[RunEvent]:
raise RuntimeError("stream transport failed")
monkeypatch.setattr(client, "stream_events", disconnected_stream)
if stop_during_cleanup:
original_cancel = client.cancel_run_and_wait
def cancel_then_stop(*_args: object, **_kwargs: object) -> RunEvent:
terminal = original_cancel(client.run_id)
queue_manager.request_stop()
return terminal
monkeypatch.setattr(client, "cancel_run_and_wait", cancel_then_stop)
store = _FakeSessionStore()
runner = _runner(client, store)
run = case.collection.run
context = DifyRunContext(
tenant_id=run.tenant_id,
app_id=run.app_id,
user_id=identifier("cleanup-user"),
user_from=UserFrom.ACCOUNT,
invoke_from=InvokeFrom.DEBUGGER,
)
scope = AgentAppSessionScope(
tenant_id=run.tenant_id,
app_id=run.app_id,
conversation_id=identifier("conversation-recover"),
agent_id=identifier("cleanup-agent"),
agent_config_snapshot_id=identifier("cleanup-snapshot"),
home_snapshot_id=None,
)
with tracing_application(case).app_context():
with pytest.raises(GenerateTaskStoppedError if stop_during_cleanup else RuntimeError):
runner._consume_stream(
client.run_id,
dify_context=context,
message_id=run.source_id,
queue_manager=cast(AppQueueManager, queue_manager),
model_name="test-model",
query="test query",
session_scope=scope,
binding_id=identifier("cleanup-binding"),
)
if not stop_during_cleanup:
# This is the generator's existing generic exception handoff. A
# cleanup child-run cancellation must not win the application root.
hooks = application_tracing_hooks()
assert hooks is not None
hooks.capture_agent_app_terminal(
tenant_id=run.tenant_id,
app_id=run.app_id,
message_id=run.source_id,
outcome="failed",
output=None,
)
frozen = case.collection.repository.get(run.id)
assert frozen is not None
assert frozen.outer_trace is not None
root = Trace.model_validate(frozen.outer_trace).root
assert root.attributes["message_execution_state"] == ("cancelled" if stop_during_cleanup else "failed")
assert root.status.value == "error"
assert len(store.saved) == 1
with case.collection.sessions() as session:
message = session.get(Message, run.source_id)
assert message is not None
assert message.message_tokens == 13
assert message.answer_tokens == 8
assert message.total_price == Decimal("0.21")
case.service.deliver(run.id)
case.service.deliver(run.id)
delivered = case.collection.repository.get(run.id)
assert delivered is not None
assert delivered.state is TraceRunState.ACCEPTED
assert delivered.attempts == 1
assert len(case.destination.requests) == 1
@pytest.mark.parametrize("capture_agent", [False, True])
def test_missing_stop_worker_without_runtime_spans_also_reaches_a_delivery_failure(
case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, capture_agent: bool
) -> None:
collection = seed(
case.collection.sessions,
f"stop-without-spans-{capture_agent}",
capture_content=False,
capture_agent=capture_agent,
)
attach_conversation(collection, AppMode.AGENT)
stopped = recovery_case(
collection,
RecordingEgress([], grpc_target=None),
replace(AGENT_TRACE_SETTINGS, collection_timeout_seconds=60),
case.now,
)
save_agent_stop(stopped, monkeypatch, "")
assert stopped.recover(collection.run, now=case.now + timedelta(seconds=120)) is False
stored = collection.repository.get(collection.run.id)
assert stored is not None
assert stored.state is TraceRunState.FAILED
assert stored.error_code == "business_terminal_not_persisted"
assert stored.outer_trace is None
@pytest.mark.parametrize("outcome", ["succeeded", "cancelled", "failed"])
def test_worker_terminal_survives_closed_sse_without_inventing_a_message_write(
case: RecoveryCase, outcome: Literal["succeeded", "failed", "cancelled"]
) -> None:
with case.collection.sessions.begin() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
message.message_metadata = json.dumps({"usage": {"prompt_tokens": 2, "completion_tokens": 1}})
message.answer = ""
# A runtime final/usage flush still does not imply that an API message ended.
assert case.recover(case.collection.run, now=case.now) is False
output = "actual worker result" if outcome == "succeeded" else None
case.service.capture_agent_app_terminal(
tenant_id=case.collection.run.tenant_id,
app_id=case.collection.run.app_id,
message_id=case.collection.run.source_id,
outcome=outcome,
output=output,
)
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
root = Trace.model_validate(stored.outer_trace).root
assert root.attributes["message_execution_state"] == outcome
assert root.status.value == ("ok" if outcome == "succeeded" else "error")
assert root.outputs == (output or "")
assert case.scheduled == [stored.id]
with case.collection.sessions() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
assert message.answer == ""
assert message.message_metadata is not None
assert set(json.loads(message.message_metadata)) == {"usage"}
def test_worker_terminal_revalidates_message_tenant(case: RecoveryCase) -> None:
with pytest.raises(TraceCollectionNotFoundError, match="tracing_source_not_found"):
case.service.capture_agent_app_terminal(
tenant_id=identifier("foreign-tenant"),
app_id=case.collection.run.app_id,
message_id=case.collection.run.source_id,
outcome="succeeded",
output="must not export",
)
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.outer_trace is None
def test_recovers_real_message_terminal_once_through_the_record_repository(case: RecoveryCase) -> None:
with case.collection.sessions.begin() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
message.message_metadata = complete_metadata()
message.updated_at = case.now - timedelta(seconds=11)
assert case.recover(case.collection.run, now=case.now) is True
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.outer_trace is not None
assert stored.outer_trace["session_id"] == "session-original"
assert stored.outer_trace["spans"][0]["outputs"] == "persisted final answer"
assert case.recover(stored, now=case.now) is False
assert case.scheduled == [stored.id]
with case.collection.sessions() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
assert message.message_tokens == 2
assert message.answer_tokens == 1
assert message.answer == "persisted final answer"
def test_deliver_recovers_and_exports_once_after_terminal_queue_loss(case: RecoveryCase) -> None:
with case.collection.sessions.begin() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
message.message_metadata = complete_metadata()
message.updated_at = case.now - timedelta(seconds=11)
case.service.deliver(case.collection.run.id)
case.service.deliver(case.collection.run.id)
exported = sent_traces(case.destination)
assert len(exported) == 1
assert exported[0].spans[0].outputs == "persisted final answer"
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.state is TraceRunState.ACCEPTED
assert stored.provider_trace_id == "provider-id"
assert stored.attempts == 1
def test_early_usage_commit_is_not_treated_as_a_final_answer(case: RecoveryCase) -> None:
with case.collection.sessions.begin() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
message.message_metadata = json.dumps({"usage": {"prompt_tokens": 2, "completion_tokens": 1}})
message.answer = ""
message.updated_at = case.now - timedelta(seconds=11)
assert case.recover(case.collection.run, now=case.now) is False
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.outer_trace is None
assert stored.state is TraceRunState.COLLECTING
assert not case.scheduled
def test_final_without_business_commit_times_out_as_failed_not_success(case: RecoveryCase) -> None:
invocations = case.collection.repository.invocations(case.collection.run.id)
with case.collection.sessions.begin() as session:
invocation = session.get(ApplicationTraceInvocation, invocations[0].id)
assert invocation is not None
invocation.finalized_at = case.now - timedelta(hours=1)
assert case.recover(case.collection.run, now=case.now) is False
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.state is TraceRunState.FAILED
assert stored.error_code == "business_terminal_not_persisted"
assert stored.outer_trace is None
def test_final_settle_window_and_all_invocations_must_finish(case: RecoveryCase) -> None:
with case.collection.sessions.begin() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
message.message_metadata = complete_metadata()
message.updated_at = case.now - timedelta(seconds=1)
assert case.recover(case.collection.run, now=case.now) is False
second = case.collection.invocation("pending-second")
with case.collection.sessions.begin() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
message.updated_at = case.now - timedelta(seconds=10)
assert case.recover(case.collection.run, now=case.now) is False
with case.collection.sessions() as session:
invocation = session.get(ApplicationTraceInvocation, second.id)
assert invocation is not None
assert invocation.finalized is False
def test_explicit_message_failure_can_recover_without_fabricating_usage(case: RecoveryCase) -> None:
with case.collection.sessions.begin() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
message.status = MessageStatus.ERROR
message.error = "model initialization failed"
message.answer = ""
message.message_metadata = None
message.message_tokens = 0
message.answer_tokens = 0
message.updated_at = case.now - timedelta(seconds=11)
assert case.recover(case.collection.run, now=case.now) is True
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.outer_trace is not None
root = stored.outer_trace["spans"][0]
assert root["status"] == "error"
assert root["error"] == "model initialization failed"
assert root["outputs"] == ""
# The message's own model call is left to the Agent runtime; the root carries no usage of its own.
assert [(span["usage"], span["cost"]) for span in stored.outer_trace["spans"]] == [(None, None)]
def test_cross_tenant_source_is_rejected_before_the_record_repository_runs(case: RecoveryCase) -> None:
with case.collection.sessions.begin() as session:
app = session.get(App, case.collection.run.app_id)
assert app is not None
app.tenant_id = identifier("other-tenant")
records = _FailingRecords(AssertionError("a foreign source must never be assembled"))
service = tracing_service(case.collection.repository, records, case.destination, case.scheduled, case.settings)
recovered = recover(
case.collection, case.collection.run, records=records, collector=service, settings=case.settings, now=case.now
)
assert recovered is False
assert records.events == []
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.error_code == "tracing_source_not_found"
def test_source_not_ready_for_assembly_is_retried_later(case: RecoveryCase) -> None:
with case.collection.sessions.begin() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
message.message_metadata = complete_metadata()
message.updated_at = case.now - timedelta(seconds=11)
records = _FailingRecords(SourceNotReadyError())
service = tracing_service(case.collection.repository, records, case.destination, case.scheduled, case.settings)
recovered = recover(
case.collection, case.collection.run, records=records, collector=service, settings=case.settings, now=case.now
)
assert recovered is False
assert [event.payload for event in records.events] == [
MessageCompleted(message_id=case.collection.run.source_id, conversation_id=identifier("conversation-recover"))
]
assert records.events[0].correlation.session_id == "session-original"
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert (stored.state, stored.error_code, stored.outer_trace) == (TraceRunState.COLLECTING, None, None)
assert case.scheduled == []
def test_unresolved_recovery_rotates_behind_unchecked_sources(case: RecoveryCase) -> None:
other = seed(case.collection.sessions, "other-recovery", capture_content=False, capture_agent=True)
assert case.collection.repository.pending_ids(limit=1) == [case.collection.run.id]
assert case.recover(case.collection.run, now=case.now) is False
assert case.collection.repository.pending_ids(limit=1) == [other.run.id]
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.state is TraceRunState.COLLECTING
@pytest.mark.parametrize("status", [MessageStatus.PAUSED, MessageStatus.NORMAL])
def test_no_final_marker_or_paused_message_is_not_recovered(case: RecoveryCase, status: MessageStatus) -> None:
with case.collection.sessions.begin() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
message.status = status
message.updated_at = case.now - timedelta(seconds=10)
assert case.recover(case.collection.run, now=case.now) is False
assert not case.scheduled
def add_workflow_run(case: RecoveryCase, status: WorkflowExecutionStatus, *, finished: bool, created_by: str) -> str:
workflow_run_id = identifier("recover-workflow-run")
workflow_run = WorkflowRun(
id=workflow_run_id,
tenant_id=case.collection.run.tenant_id,
app_id=case.collection.run.app_id,
workflow_id=identifier("workflow"),
type=WorkflowType.WORKFLOW,
triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
version="1",
graph="{}",
inputs="{}",
outputs=json.dumps({"result": "persisted workflow output"}),
status=status,
error="actual workflow failure"
if status in {WorkflowExecutionStatus.FAILED, WorkflowExecutionStatus.STOPPED}
else None,
created_by=created_by,
created_by_role=CreatorUserRole.ACCOUNT,
finished_at=case.now - timedelta(seconds=10) if finished else None,
elapsed_time=20,
total_tokens=0,
total_steps=0,
)
workflow_run.created_at = case.now - timedelta(seconds=30)
with case.collection.sessions.begin() as session:
session.add(workflow_run)
return workflow_run_id
def start_workflow(case: RecoveryCase, workflow_run_id: str) -> TraceRun:
run = case.service.start(
tenant_id=case.collection.run.tenant_id,
app_id=case.collection.run.app_id,
source_kind="workflow",
source_id=workflow_run_id,
session_id="workflow-session",
workflow_id=None,
)
assert run is not None
return run
@pytest.mark.parametrize(
("status", "finished"),
[
(WorkflowExecutionStatus.RUNNING, False),
(WorkflowExecutionStatus.PAUSED, True),
(WorkflowExecutionStatus.SUCCEEDED, False),
(WorkflowExecutionStatus.SUCCEEDED, True),
(WorkflowExecutionStatus.PARTIAL_SUCCEEDED, True),
(WorkflowExecutionStatus.FAILED, True),
(WorkflowExecutionStatus.STOPPED, True),
],
)
def test_workflow_recovery_requires_committed_terminal_state(
case: RecoveryCase, status: WorkflowExecutionStatus, finished: bool
) -> None:
owner_id = add_app_owner(case.collection, "workflow")
workflow_run_id = add_workflow_run(case, status, finished=finished, created_by=owner_id)
run = start_workflow(case, workflow_run_id)
expected = finished and status in {
WorkflowExecutionStatus.SUCCEEDED,
WorkflowExecutionStatus.PARTIAL_SUCCEEDED,
WorkflowExecutionStatus.FAILED,
WorkflowExecutionStatus.STOPPED,
}
assert case.recover(run, now=case.now) is expected
stored = case.collection.repository.get(run.id)
assert stored is not None
if expected:
assert stored.outer_trace is not None
assert stored.root_span_id == span_id_for(workflow_run_id)
root = stored.outer_trace["spans"][0]
assert root["span_id"] == span_id_for(workflow_run_id)
assert root["outputs"] == {"result": "persisted workflow output"}
succeeded = status in {WorkflowExecutionStatus.SUCCEEDED, WorkflowExecutionStatus.PARTIAL_SUCCEEDED}
assert root["status"] == ("ok" if succeeded else "error")
assert stored.outer_trace["session_id"] == "workflow-session"
assert stored.outer_trace["user_id"] == owner_id
else:
assert stored.outer_trace is None
assert (stored.state, stored.error_code) == (TraceRunState.COLLECTING, None)
@pytest.mark.parametrize("owner_state", ["no_creator", "missing_account", "not_a_member"])
def test_workflow_recovery_does_not_depend_on_the_app_creator(case: RecoveryCase, owner_state: str) -> None:
owner_id = add_app_owner(case.collection, "workflow")
with case.collection.sessions.begin() as session:
app = session.get(App, case.collection.run.app_id)
assert app is not None
if owner_state == "no_creator":
app.created_by = None
elif owner_state == "missing_account":
account = session.get(Account, owner_id)
assert account is not None
session.delete(account)
else:
session.execute(delete(TenantAccountJoin).where(TenantAccountJoin.account_id == owner_id))
workflow_run_id = add_workflow_run(case, WorkflowExecutionStatus.SUCCEEDED, finished=True, created_by=owner_id)
run = start_workflow(case, workflow_run_id)
assert case.recover(run, now=case.now) is True
stored = case.collection.repository.get(run.id)
assert stored is not None
assert stored.outer_trace is not None
assert stored.root_span_id == span_id_for(workflow_run_id)
@pytest.mark.parametrize("formatted", [None, "", " \n\t", "{}", "[]", "null", " { } ", "[\n]", {}, []])
@pytest.mark.parametrize("capture_content", [False, True])
def test_early_error_root_preserves_known_query_only_when_content_is_enabled(
case: RecoveryCase, formatted: object, capture_content: bool
) -> None:
query = "KNOWN_USER_QUERY_BEFORE_FORMATTED_MESSAGE"
with case.collection.sessions.begin() as session:
# Historical rows can contain JSON strings/lists despite the ORM's dict annotation.
session.execute(
update(Message.metadata.tables["messages"])
.where(Message.id == case.collection.run.source_id)
.values(message=formatted, query=query, answer="")
)
run = session.get(ApplicationTraceRun, case.collection.run.id)
assert run is not None
run.capture_content = capture_content
case.service.capture_agent_app_terminal(
tenant_id=case.collection.run.tenant_id,
app_id=case.collection.run.app_id,
message_id=case.collection.run.source_id,
outcome="failed",
output=None,
)
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
root = Trace.model_validate(stored.outer_trace).root
assert root.status.value == "error"
assert root.attributes["message_execution_state"] == "failed"
assert root.inputs == ({"query": query} if capture_content else None)
assert root.outputs == ("" if capture_content else None)
case.service.deliver(stored.id)
[assembled] = sent_traces(case.destination)
assert (query in assembled.model_dump_json()) is capture_content
assert all(span.inputs is None for span in assembled.spans if span.span_id != assembled.root.span_id)
with case.collection.sessions() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
assert message.message == formatted
assert message.query == query
assert message.answer == ""
@pytest.mark.parametrize(
("formatted", "expected"),
[
(
{"messages": [{"role": "user", "content": "actual formatted input"}]},
{"messages": [{"role": "user", "content": "actual formatted input"}]},
),
(
'{"messages":[{"role":"user","content":"actual formatted input"}]}',
{"messages": [{"role": "user", "content": "actual formatted input"}]},
),
(
[{"role": "user", "content": "actual formatted input"}],
[{"role": "user", "content": "actual formatted input"}],
),
(
'[{"role":"user","content":"actual formatted input"}]',
[{"role": "user", "content": "actual formatted input"}],
),
("opaque non-JSON formatted input", "opaque non-JSON formatted input"),
("0", "0"),
("false", "false"),
],
)
def test_worker_handoff_never_replaces_existing_formatted_inputs(
case: RecoveryCase, formatted: object, expected: object
) -> None:
query = "QUERY_MUST_NOT_REPLACE_FORMATTED_INPUTS"
with case.collection.sessions.begin() as session:
session.execute(
update(Message.metadata.tables["messages"])
.where(Message.id == case.collection.run.source_id)
.values(message=formatted, query=query, answer="")
)
case.service.capture_agent_app_terminal(
tenant_id=case.collection.run.tenant_id,
app_id=case.collection.run.app_id,
message_id=case.collection.run.source_id,
outcome="failed",
output=None,
)
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
trace = Trace.model_validate(stored.outer_trace)
assert trace.root.inputs == expected
assert query not in trace.model_dump_json()
with case.collection.sessions() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
assert message.message == formatted
def test_worker_handoff_does_not_invent_input_when_query_is_empty(case: RecoveryCase) -> None:
with case.collection.sessions.begin() as session:
message = session.get(Message, case.collection.run.source_id)
assert message is not None
message.message = {}
message.query = ""
message.answer = ""
case.service.capture_agent_app_terminal(
tenant_id=case.collection.run.tenant_id,
app_id=case.collection.run.app_id,
message_id=case.collection.run.source_id,
outcome="failed",
output=None,
)
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert Trace.model_validate(stored.outer_trace).root.inputs == {}
def test_worker_handoff_leaves_a_not_collected_run_untouched(case: RecoveryCase) -> None:
with case.collection.sessions.begin() as session:
run = session.get(ApplicationTraceRun, case.collection.run.id)
assert run is not None
run.state = TraceRunState.NOT_COLLECTED
case.service.capture_agent_app_terminal(
tenant_id=case.collection.run.tenant_id,
app_id=case.collection.run.app_id,
message_id=case.collection.run.source_id,
outcome="succeeded",
output="answer",
)
stored = case.collection.repository.get(case.collection.run.id)
assert stored is not None
assert stored.outer_trace is None
assert case.scheduled == []
assert claim(case.collection, case.collection.run.id) is None