Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

39.9 KB
Raw Blame
from __future__ import annotations

import json
from unittest.mock import AsyncMock

import pytest
import sqlalchemy as sa
from sqlalchemy.orm import sessionmaker

from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from tests.web.services.test_task_execution_event_store import engine as engine_fixture
from tests.web.services.test_task_execution_event_store import (
    task_id as task_id_fixture,
)
from xagent.core.agent.checkpoint import (
    ExecutionEventPersistenceError,
    TraceCheckpointStore,
)
from xagent.core.agent.runtime import PatternRuntime
from xagent.core.agent.trace import (
    TraceAction,
    TraceCategory,
    TraceEventType,
    TraceScope,
)
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.task import Task, TaskStatus, TraceEvent
from xagent.web.models.task_execution_event import TaskExecutionEvent
from xagent.web.services.chat_history_service import persist_user_message_no_commit
from xagent.web.services.managed_task_lease import finalize_managed_task_lease_result
from xagent.web.services.task_execution_controller import (
    TaskControlState,
    apply_task_control_transition,
)
from xagent.web.services.task_lease_service import acquire_task_lease
from xagent.web.tracing import ExecutionEventTraceAdapter, create_task_tracer

engine = engine_fixture
task_id = task_id_fixture


@pytest.fixture
def canonical(engine, task_id, monkeypatch):
    # Use this fixture's session factory, not a prior test's global async engine.
    # The runtime compatibility test below supplies both backends explicitly.
    monkeypatch.setenv("XAGENT_ASYNC_TRACE_DB_ENABLED", "false")
    factory = sessionmaker(engine)
    with factory() as db:
        task = db.get(Task, task_id)
        task.conversation_storage_version = 2
        db.commit()
    monkeypatch.setattr("xagent.web.models.database._SessionLocal", factory)
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_db", lambda: iter([factory()])
    )
    return factory, task_id


def facts(db, task_id):
    return list(
        db.scalars(
            sa.select(TaskExecutionEvent)
            .where(
                TaskExecutionEvent.task_id == task_id,
            )
            .order_by(TaskExecutionEvent.sequence)
        )
    )


def test_acceptance_and_compatibility_row_share_transaction(canonical):
    factory, task_id = canonical
    with factory() as db:
        task = db.get(Task, task_id)
        user_id = task.user_id
        persist_user_message_no_commit(
            db, task_id, user_id, "hello", turn_id="t1", attachments=[]
        )
        assert [e.kind for e in facts(db, task_id)] == ["input_accepted"]
        db.rollback()
        assert facts(db, task_id) == []
        assert db.query(TaskChatMessage).count() == 0
        first = persist_user_message_no_commit(
            db, task_id, user_id, "hello", turn_id="t1", attachments=[]
        )
        db.commit()
        second = persist_user_message_no_commit(
            db, task_id, user_id, "hello", turn_id="t1", attachments=[]
        )
        db.commit()
        assert first.id == second.id
        assert len(facts(db, task_id)) == 1
        assert second.attachments == []


@pytest.mark.asyncio
async def test_factory_commits_recoverable_state_before_observers(canonical):
    factory, task_id = canonical
    tracer = create_task_tracer(task_id)
    assert tracer.records_execution_events
    assert any(isinstance(h, ExecutionEventTraceAdapter) for h in tracer.handlers)
    observer = AsyncMock()
    tracer.handlers.append(observer)
    payload = {
        "execution_id": "run-root",
        "pattern": "ReActPattern",
        "label": "after_llm",
        "context": {
            "messages": [
                {"role": "user", "content": "hello", "metadata": {"turn_id": "t1"}}
            ]
        },
        "pattern_state": {"adopted_plan": "完整计划" * 10000, "pending_tool_calls": []},
    }
    await TraceCheckpointStore(tracer).save(payload)
    with factory() as db:
        rows = facts(db, task_id)
        assert [e.kind for e in rows] == ["recovery_state", "input_applied"]
        assert rows[0].payload["data"]["snapshot"] == payload
        assert rows[1].payload["recovery_event_id"] == rows[0].event_id
        assert db.query(TraceEvent).count() == 1
    observer.handle_event.assert_awaited_once()


@pytest.mark.asyncio
async def test_failed_fact_commit_stops_runtime_before_broadcast(
    canonical, monkeypatch
):
    _, task_id = canonical
    tracer = create_task_tracer(task_id)
    observer = AsyncMock()
    tracer.handlers = [observer]

    def fail(*args, **kwargs):
        raise OSError("database unavailable")

    monkeypatch.setattr(
        "xagent.web.services.task_execution_event_writer.append_task_execution_event_no_commit",
        fail,
    )
    runtime = PatternRuntime(tracer=TraceCheckpointStore(tracer))
    with pytest.raises(ExecutionEventPersistenceError):
        await runtime.on_tool_start(
            tool_call={
                "id": "call1",
                "name": "write",
                "args": {},
                "tool_attempt_id": "attempt1",
                "assistant_message_id": "batch1",
            }
        )
    observer.handle_event.assert_not_awaited()


@pytest.mark.asyncio
async def test_observer_failure_does_not_invalidate_fact(canonical):
    factory, task_id = canonical
    tracer = create_task_tracer(task_id)
    tracer.handlers = [
        AsyncMock(handle_event=AsyncMock(side_effect=OSError("socket closed")))
    ]
    await tracer.trace_event(
        TraceEventType(TraceScope.TASK, TraceAction.START, TraceCategory.GENERAL),
        task_id=str(task_id),
        data={"input": "hello"},
        require_persisted=True,
    )
    with factory() as db:
        assert len(facts(db, task_id)) == 1


@pytest.mark.asyncio
async def test_attempt_result_keeps_batch_identity_and_blocks_blind_replay(canonical):
    factory, task_id = canonical
    tracer = create_task_tracer(task_id)
    tracer.handlers = [
        h for h in tracer.handlers if isinstance(h, ExecutionEventTraceAdapter)
    ]
    runtime = PatternRuntime(tracer=TraceCheckpointStore(tracer))
    call = {
        "id": "provider-duplicate-id",
        "name": "write",
        "args": {"value": 1},
        "tool_attempt_id": "attempt1",
        "assistant_message_id": "batch1",
    }
    await runtime.on_tool_start(tool_call=call)
    await runtime.on_tool_end(
        tool_call=call, result={"success": True, "output": "长结果" * 10000}
    )
    with pytest.raises(ExecutionEventPersistenceError):
        await runtime.on_tool_start(tool_call=call)
    with factory() as db:
        rows = facts(db, task_id)
        assert len(rows) == 2
        assert {r.assistant_message_id for r in rows} == {"batch1"}
        assert {r.tool_attempt_id for r in rows} == {"attempt1"}
        assert rows[1].payload["data"]["result"]["output"] == "长结果" * 10000


@pytest.mark.parametrize(
    "status",
    [
        TaskStatus.COMPLETED,
        TaskStatus.FAILED,
        TaskStatus.PAUSED,
        TaskStatus.WAITING_FOR_USER,
    ],
)
def test_channel_outcome_and_transcript_commit_with_lease(canonical, status):
    factory, task_id = canonical
    with factory() as db:
        lease = acquire_task_lease(db, task_id, new_run=True)
        assert lease is not None
        assert finalize_managed_task_lease_result(
            db,
            lease,
            status=status,
            assistant_content="result",
            execution_result={"output": "result"},
        )
        rows = facts(db, task_id)
        assert rows[-1].kind == "execution_settled"
        assert rows[-1].payload["status"] == status.value
        assert any(row.kind == "assistant_message" for row in rows)
        assert db.get(Task, task_id).runner_id is None


def test_control_and_event_rollback_together(canonical):
    factory, task_id = canonical
    with factory() as db:
        task = db.get(Task, task_id)
        apply_task_control_transition(
            task, TaskControlState.PAUSED, status=TaskStatus.PAUSED
        )
        assert facts(db, task_id)[-1].kind == "control_state_changed"
        db.rollback()
        assert facts(db, task_id) == []
        assert db.get(Task, task_id).status == TaskStatus.PENDING


@pytest.mark.asyncio
async def test_real_react_loop_records_batch_before_tool_and_retains_identity(
    canonical,
):
    from tests.core.agent.test_react import FakeLLM, FakeTool
    from xagent.core.agent import ExecutionContext, ReActPattern

    factory, task_id = canonical
    tracer = create_task_tracer(task_id)
    tracer.handlers = [
        h for h in tracer.handlers if isinstance(h, ExecutionEventTraceAdapter)
    ]
    runtime = PatternRuntime(tracer=TraceCheckpointStore(tracer))
    context = ExecutionContext(system_prompt="Calculate")
    context.add_user_message("2+2", metadata={"turn_id": "real-turn"})
    tool = FakeTool()
    pattern = ReActPattern(max_iterations=3)
    result = await pattern.run(
        context=context,
        tools=[tool],
        runtime=runtime,
        llm=FakeLLM(
            responses=[
                {
                    "content": "calculate",
                    "tool_calls": [
                        {
                            "id": "call1",
                            "function": {
                                "name": "calculator",
                                "arguments": '{"expression":"2+2"}',
                            },
                        }
                    ],
                },
                {"content": "4", "done": True},
            ]
        ),
    )
    assert result["success"]
    assert len(tool.calls) == 1
    with factory() as db:
        rows = facts(db, task_id)
        tools = [row for row in rows if row.kind.startswith("tool_execution_")]
        state = next(
            row
            for row in rows
            if row.kind == "recovery_state"
            and row.payload["data"]["snapshot"]["label"] == "after_llm"
        )
        snapshot = state.payload["data"]["snapshot"]
        saved_call = snapshot["pattern_state"]["pending_tool_calls"][0]
        assert state.sequence < tools[0].sequence < tools[1].sequence
        assert {row.tool_attempt_id for row in tools} == {saved_call["tool_attempt_id"]}
        assert {row.assistant_message_id for row in tools} == {
            saved_call["assistant_message_id"]
        }
        restored = ReActPattern()
        restored.load_state(snapshot["pattern_state"])
        assert (
            restored.pending_tool_calls[0]["tool_attempt_id"]
            == saved_call["tool_attempt_id"]
        )


@pytest.mark.asyncio
async def test_outbound_stream_is_committed_before_websocket_and_failure_is_strict(
    canonical, monkeypatch
):
    from xagent.web.services import task_execution as websocket

    factory, task_id = canonical
    monkeypatch.setattr(websocket, "get_db", lambda: iter([factory()]))
    broadcasts = []

    async def broadcast(event, task_id):
        with factory() as db:
            assert facts(db, task_id)[-1].payload["data"]["content"] == "hello"
        broadcasts.append(event)

    monkeypatch.setattr(websocket, "publish_task_event", broadcast)
    handler = websocket.make_agent_outbound_handler(task_id, authoritative=True)
    payload = {
        "type": "final_answer_end",
        "content": "hello",
        "stream_id": "stream1",
        "message_id": "final_answer_1",
    }
    await handler(payload)
    assert len(broadcasts) == 1

    def fail(*args, **kwargs):
        raise OSError("write failure")

    monkeypatch.setattr(
        "xagent.web.services.task_execution_event_writer.append_task_execution_event_no_commit",
        fail,
    )
    with pytest.raises(ExecutionEventPersistenceError):
        await handler(payload)
    assert len(broadcasts) == 1


def test_compatibility_failure_rolls_back_outcome_and_retains_lease(
    canonical, monkeypatch
):
    factory, task_id = canonical
    with factory() as db:
        lease = acquire_task_lease(db, task_id, new_run=True)

        def fail(*args, **kwargs):
            raise OSError("event storage unavailable")

        monkeypatch.setattr(
            "xagent.web.services.task_execution_event_writer.append_task_execution_event_no_commit",
            fail,
        )
        with pytest.raises(OSError):
            finalize_managed_task_lease_result(
                db, lease, status=TaskStatus.COMPLETED, assistant_content="result"
            )
        db.expire_all()
        task = db.get(Task, task_id)
        assert task.status == TaskStatus.RUNNING
        assert task.runner_id == lease.runner_id
        assert db.query(TaskChatMessage).count() == 0
        assert facts(db, task_id) == []


@pytest.mark.asyncio
async def test_replaced_lease_cannot_append_or_broadcast(canonical):
    from xagent.web.services.task_lease_service import bind_task_lease_context

    factory, task_id = canonical
    with factory() as db:
        lease = acquire_task_lease(db, task_id, new_run=True)
        db.get(Task, task_id).run_id = "replacement-run"
        db.commit()
    tracer = create_task_tracer(task_id)
    observer = AsyncMock()
    tracer.handlers = [observer]
    with bind_task_lease_context(lease):
        with pytest.raises(ExecutionEventPersistenceError):
            await tracer.trace_event(
                TraceEventType(
                    TraceScope.TASK, TraceAction.START, TraceCategory.GENERAL
                ),
                task_id=str(task_id),
            )
    observer.handle_event.assert_not_awaited()
    with factory() as db:
        assert facts(db, task_id) == []


def test_assistant_projection_replay_has_one_fact_and_one_row(canonical):
    from xagent.web.services.chat_history_service import (
        persist_assistant_message_no_commit,
    )

    factory, task_id = canonical
    with factory() as db:
        task = db.get(Task, task_id)
        task.run_id = "run1"
        task.status = TaskStatus.COMPLETED
        db.commit()
        for _ in range(2):
            persist_assistant_message_no_commit(
                db,
                task_id,
                task.user_id,
                "done",
                message_type="assistant_response",
                content_is_reconciled=True,
            )
            db.commit()
        rows = facts(db, task_id)
        assert len(rows) == 1
        assert db.query(TaskChatMessage).one().execution_event_id == rows[0].event_id


def test_command_fact_and_inbox_are_one_transaction(canonical):
    from xagent.web.models.task_command import TaskExecutionCommand
    from xagent.web.services.task_command_transport import (
        TaskCommandKind,
        stage_task_command,
    )

    factory, task_id = canonical
    with factory() as db:
        task = db.get(Task, task_id)
        stage_task_command(
            db,
            task_id=task_id,
            actor_user_id=task.user_id,
            command_id="answer1",
            kind=TaskCommandKind.RESUME,
            payload={"response": "yes"},
        )
        assert facts(db, task_id)[-1].payload["payload"] == {"response": "yes"}
        db.rollback()
        assert facts(db, task_id) == []
        assert db.query(TaskExecutionCommand).count() == 0


def test_pre_runner_failure_is_a_fact_and_failed_commit_is_not_broadcastable(
    canonical, monkeypatch
):
    from xagent.web.services.task_execution import _terminal_task_error_payload

    factory, task_id = canonical
    monkeypatch.setattr(
        "xagent.web.services.task_execution.get_session_local", lambda: factory
    )
    _terminal_task_error_payload(task_id, "sandbox unavailable")
    with factory() as db:
        rows = facts(db, task_id)
        assert rows[-1].kind == "execution_settled"
        assert rows[-1].payload["result"]["error"] == "sandbox unavailable"
        assert rows[-1].payload["status"] == TaskStatus.FAILED.value

    def fail(*args, **kwargs):
        raise OSError("commit failure")

    monkeypatch.setattr(
        "xagent.web.services.task_execution_event_writer.append_task_execution_event_no_commit",
        fail,
    )
    with pytest.raises(ExecutionEventPersistenceError):
        _terminal_task_error_payload(task_id, "sandbox unavailable")


def test_settlement_serializes_execution_context_without_losing_state(canonical):
    from xagent.core.agent import ExecutionContext
    from xagent.web.services.task_execution_event_writer import (
        stage_result_fact_no_commit,
    )

    factory, task_id = canonical
    context = ExecutionContext(execution_id="run-context")
    context.add_assistant_message("完整回复" * 10000)
    with factory() as db:
        task = db.get(Task, task_id)
        task.status = TaskStatus.COMPLETED
        stage_result_fact_no_commit(db, task, {"agent_result": {"context": context}})
        db.commit()
        assert (
            facts(db, task_id)[0].payload["result"]["agent_result"]["context"]
            == context.to_dict()
        )
        with pytest.raises(TypeError, match="Unsupported execution fact"):
            stage_result_fact_no_commit(db, task, {"unknown": object()})


@pytest.mark.asyncio
@pytest.mark.parametrize("native_async", [False, True], ids=["sync", "async"])
async def test_fact_and_checkpoint_projection_use_current_trace_database_runtime(
    canonical, engine, monkeypatch, native_async
):
    from xagent.web.services import trace_handlers
    from xagent.web.services.task_lease_service import bind_task_lease_context
    from xagent.web.services.trace_database import TraceDatabaseRuntime

    factory, task_id = canonical
    source = engine
    if native_async and engine.dialect.name == "postgresql":
        # This fixture uses creator= for disposable databases; recover its URL
        # so the async driver connects to the same isolated database.
        connection = engine.raw_connection()
        try:
            parameters = connection.driver_connection.info.dsn_parameters
            source = sa.create_engine(
                sa.URL.create(
                    "postgresql",
                    username=parameters["user"],
                    host=parameters["host"],
                    port=int(parameters["port"]),
                    database=parameters["dbname"],
                )
            )
        finally:
            connection.close()
    database_runtime = TraceDatabaseRuntime(source, use_async=native_async, limit=1)
    monkeypatch.setattr(
        trace_handlers, "get_trace_database_runtime", lambda: database_runtime
    )
    with factory() as db:
        lease = acquire_task_lease(db, task_id, new_run=True)
    tracer = create_task_tracer(task_id)
    store = TraceCheckpointStore(tracer)
    payload = {
        "execution_id": "root",
        "context": {},
        "label": "after_llm",
        "pattern_state": {"pending_tool_calls": []},
    }
    try:
        with bind_task_lease_context(lease):
            await store.save(payload)
            assert await tracer.load_latest_checkpoint("root") == payload
        with factory() as db:
            task = db.get(Task, task_id)
            assert task.last_checkpoint_trace_event_id is not None
            row = db.get(TraceEvent, task.last_checkpoint_trace_event_id)
            event = next(e for e in facts(db, task_id) if e.kind == "recovery_state")
            assert row.event_id == event.payload["protocol_event_id"]
            assert event.run_id == lease.run_id
            assert event.payload["data"]["snapshot"] == payload
    finally:
        await database_runtime.close()
        if source is not engine:
            source.dispose()


@pytest.mark.asyncio
@pytest.mark.parametrize("producer", ["root", "child", "outbound"])
async def test_replaced_attempt_in_same_run_cannot_write_facts(
    canonical, producer, monkeypatch
):
    from xagent.web.services import task_execution
    from xagent.web.services.task_lease_service import bind_task_lease_context

    factory, task_id = canonical
    with factory() as db:
        lease = acquire_task_lease(db, task_id, new_run=True)
        task = db.get(Task, task_id)
        task.lease_attempt_id = "replacement-attempt"
        db.commit()
    monkeypatch.setattr(task_execution, "get_db", lambda: iter([factory()]))
    broadcast = AsyncMock()
    monkeypatch.setattr(task_execution, "publish_task_event", broadcast)
    with bind_task_lease_context(lease):
        with pytest.raises(ExecutionEventPersistenceError):
            if producer == "outbound":
                await task_execution.make_agent_outbound_handler(
                    task_id, authoritative=True
                )({"type": "final_answer_delta", "delta": "stale"})
            else:
                adapter = ExecutionEventTraceAdapter(
                    task_id, build_id="child" if producer == "child" else None
                )
                from xagent.core.agent.trace import TraceEvent as CoreTraceEvent

                await adapter.commit_event(
                    CoreTraceEvent(
                        TraceEventType(
                            TraceScope.TASK, TraceAction.START, TraceCategory.GENERAL
                        ),
                        task_id=str(task_id),
                    )
                )
    broadcast.assert_not_awaited()
    with factory() as db:
        assert facts(db, task_id) == []
        assert db.query(TraceEvent).count() == 0


@pytest.mark.asyncio
async def test_outbound_question_preserves_source_identity_in_atomic_projection(
    canonical, monkeypatch
):
    from xagent.web.services import task_execution

    factory, task_id = canonical
    monkeypatch.setattr(task_execution, "get_db", lambda: iter([factory()]))
    monkeypatch.setattr(task_execution, "publish_task_event", AsyncMock())
    await task_execution.make_agent_outbound_handler(task_id, authoritative=True)(
        {
            "event_id": "question-event",
            "message": "Which file?",
            "expect_response": True,
        }
    )
    with factory() as db:
        assert len(facts(db, task_id)) == 1
        row = db.query(TaskChatMessage).one()
        assert row.source_event_id == "question-event"
        event = next(e for e in facts(db, task_id) if e.kind == "agent_message")
        assert row.execution_event_id == event.event_id
        assert event.payload["protocol_event_id"] == "question-event"
        assert db.query(TraceEvent).one().event_id == "question-event"


@pytest.mark.parametrize("status", [TaskStatus.PAUSED, TaskStatus.WAITING_FOR_USER])
def test_repeated_resting_settlements_keep_distinct_facts_in_same_run(
    canonical, status
):
    from xagent.web.services.task_execution_event_writer import (
        stage_result_fact_no_commit,
    )

    factory, task_id = canonical
    with factory() as db:
        lease = acquire_task_lease(db, task_id, new_run=True)
        run_id = lease.run_id
        for index in range(2):
            if index:
                lease = acquire_task_lease(db, task_id, expected_run_id=run_id)
            assert lease.run_id == run_id
            result = {"output": f"rest {index}"}
            assert finalize_managed_task_lease_result(
                db,
                lease,
                status=status,
                assistant_content=result["output"],
                execution_result=result,
            )
            # Replaying the same settlement keeps its original identity.
            stage_result_fact_no_commit(db, db.get(Task, task_id), result)
            db.commit()
        settled = [e for e in facts(db, task_id) if e.kind == "execution_settled"]
        assert len(settled) == 2
        assert [e.payload["result"]["output"] for e in settled] == ["rest 0", "rest 1"]
        assert db.query(TaskChatMessage).count() == 2
        assert db.get(Task, task_id).runner_id is None


def test_racing_message_claim_has_only_one_delivery_owner(canonical, engine):
    from concurrent.futures import ThreadPoolExecutor
    from threading import Event, current_thread

    from xagent.web.services.chat_history_service import (
        claim_user_message_delivery_no_commit,
    )

    factory, task_id = canonical
    waiting = Event()

    def before_execute(connection, cursor, statement, parameters, context, many):
        if (
            current_thread().name.startswith("second-claim")
            and "UPDATE tasks" in statement
            and "conversation_event_sequence" in statement
        ):
            waiting.set()

    sa.event.listen(engine, "before_cursor_execute", before_execute)
    try:
        with (
            factory() as winner,
            ThreadPoolExecutor(
                max_workers=1, thread_name_prefix="second-claim"
            ) as pool,
        ):
            user_id = winner.get(Task, task_id).user_id
            first = claim_user_message_delivery_no_commit(
                winner, task_id, user_id, "one input", turn_id="same-turn"
            )
            assert first.claimed

            def second_claim():
                with factory() as db:
                    claim = claim_user_message_delivery_no_commit(
                        db, task_id, user_id, "one input", turn_id="same-turn"
                    )
                    result = claim.claimed, claim.payload_matches
                    db.commit()
                    return result

            future = pool.submit(second_claim)
            try:
                assert waiting.wait(DB_PROGRESS_TIMEOUT), (
                    "second claimant did not reach the task lock"
                )
            finally:
                winner.commit()
            assert future.result(timeout=DB_PROGRESS_TIMEOUT) == (False, True)
        with factory() as db:
            assert db.query(TaskChatMessage).count() == 1
            assert (
                len([e for e in facts(db, task_id) if e.kind == "input_accepted"]) == 1
            )
    finally:
        sa.event.remove(engine, "before_cursor_execute", before_execute)


@pytest.mark.asyncio
@pytest.mark.parametrize("failed_action", [TraceAction.START, TraceAction.END])
async def test_compaction_fact_failure_does_not_fall_back_or_mutate_context(
    failed_action,
):
    from types import SimpleNamespace
    from unittest.mock import Mock

    from xagent.core.agent.trace import Tracer

    failure = ExecutionEventPersistenceError("one uncertain compaction fact")
    written = []

    async def writer(event):
        written.append(event)
        if (
            event.event_type.category == TraceCategory.LLM
            and event.event_type.action == failed_action
        ):
            raise failure

    tracer = Tracer()
    tracer.event_writer = writer
    runtime = PatternRuntime(tracer=tracer)
    context = SimpleNamespace(
        execution_id="compaction",
        messages=["preserve"],
        metadata={},
        build_llm_compact_request_if_needed=Mock(
            return_value={"messages": [], "max_tokens": 100, "metadata": {}}
        ),
        compact_with_llm_response=Mock(),
        compact_if_needed=Mock(),
    )
    llm = SimpleNamespace(chat=AsyncMock(return_value={"content": "summary"}))
    with pytest.raises(ExecutionEventPersistenceError) as caught:
        await runtime.compact_context_if_needed(context=context, llm=llm)
    assert caught.value is failure
    context.compact_if_needed.assert_not_called()
    context.compact_with_llm_response.assert_not_called()
    assert context.messages == ["preserve"]
    assert len(written) == (1 if failed_action == TraceAction.START else 2)


@pytest.mark.postgresql
def test_trace_fact_and_command_acceptance_do_not_deadlock(canonical, engine):
    from concurrent.futures import ThreadPoolExecutor
    from threading import Event, current_thread

    from xagent.core.agent.trace import TraceEvent as CoreTraceEvent
    from xagent.web.services.task_command_transport import (
        TaskCommandKind,
        enqueue_task_command,
    )
    from xagent.web.services.task_lease_service import bind_task_lease_context

    if engine.dialect.name != "postgresql":
        pytest.skip("PostgreSQL foreign-key row locks")
    factory, task_id = canonical
    with factory() as db:
        lease = acquire_task_lease(db, task_id, new_run=True)
        user_id = db.get(Task, task_id).user_id
    trace_locked = Event()
    command_waiting = Event()

    def after_execute(connection, cursor, statement, parameters, context, many):
        if (
            current_thread().name.startswith("trace-fact")
            and "UPDATE tasks" in statement
            and not trace_locked.is_set()
        ):
            trace_locked.set()
            assert command_waiting.wait(DB_PROGRESS_TIMEOUT)

    def before_execute(connection, cursor, statement, parameters, context, many):
        if (
            current_thread().name.startswith("accept-command")
            and "UPDATE tasks" in statement
            and "conversation_event_sequence" in statement
        ):
            # The command INSERT has already acquired the task FK KEY SHARE.
            command_waiting.set()

    def write_trace():
        with factory() as db, bind_task_lease_context(lease):
            db.execute(sa.text("SET LOCAL lock_timeout = '5s'"))
            ExecutionEventTraceAdapter(task_id)._save_trace_event(
                db,
                CoreTraceEvent(
                    TraceEventType(
                        TraceScope.TASK, TraceAction.START, TraceCategory.GENERAL
                    ),
                    task_id=str(task_id),
                ),
            )
            db.commit()

    def accept_command():
        assert trace_locked.wait(DB_PROGRESS_TIMEOUT)
        with factory() as db:
            db.execute(sa.text("SET LOCAL lock_timeout = '5s'"))
            enqueue_task_command(
                db,
                task_id=task_id,
                actor_user_id=user_id,
                command_id="concurrent-command",
                kind=TaskCommandKind.RESUME,
                payload={},
            )

    sa.event.listen(engine, "after_cursor_execute", after_execute)
    sa.event.listen(engine, "before_cursor_execute", before_execute)
    try:
        with (
            ThreadPoolExecutor(
                max_workers=1, thread_name_prefix="trace-fact"
            ) as writers,
            ThreadPoolExecutor(
                max_workers=1, thread_name_prefix="accept-command"
            ) as commands,
        ):
            writer = writers.submit(write_trace)
            command = commands.submit(accept_command)
            writer.result(timeout=10)
            command.result(timeout=10)
        with factory() as db:
            assert len(facts(db, task_id)) == 2
            assert db.query(TraceEvent).count() == 1
    finally:
        sa.event.remove(engine, "after_cursor_execute", after_execute)
        sa.event.remove(engine, "before_cursor_execute", before_execute)


def test_withdrawn_input_reuses_acceptance_when_handed_to_new_run(canonical):
    from xagent.web.services.chat_history_service import (
        claim_user_message_delivery_no_commit,
        withdraw_pending_user_message_delivery_sync,
    )

    factory, tid = canonical
    with factory() as db:
        task = db.get(Task, tid)
        task.run_id = "ended-run"
        claim = claim_user_message_delivery_no_commit(
            db, tid, task.user_id, "new input", turn_id="turn"
        )
        assert claim.claimed
        db.commit()
        event_id = claim.message.execution_event_id
        user_id = task.user_id
    assert withdraw_pending_user_message_delivery_sync(tid, "turn")
    with factory() as db:
        db.get(Task, tid).run_id = "new-run"
        row = persist_user_message_no_commit(
            db, tid, user_id, "new input", turn_id="turn", delivery_status="pending"
        )
        db.commit()
        assert row.execution_event_id == event_id
        assert len(facts(db, tid)) == 1
        assert facts(db, tid)[0].run_id == "ended-run"
        assert db.query(TaskChatMessage).count() == 1


def test_orphan_delivery_fact_and_projection_are_atomic(canonical, monkeypatch):
    from xagent.web.services import task_execution_event_writer as writer
    from xagent.web.services.task_lease_recovery import (
        reconcile_orphaned_pending_deliveries_no_commit,
    )
    from xagent.web.services.task_lease_service import utc_now

    factory, tid = canonical
    with factory() as db:
        task = db.get(Task, tid)
        task.status = TaskStatus.COMPLETED
        persist_user_message_no_commit(
            db,
            tid,
            task.user_id,
            "pending input",
            turn_id="turn",
            delivery_status="pending",
        )
        db.commit()
        original = writer.stage_delivery_fact_no_commit

        def fail_after_fact(*args, **kwargs):
            original(*args, **kwargs)
            raise OSError("failed transaction")

        with monkeypatch.context() as failure:
            failure.setattr(writer, "stage_delivery_fact_no_commit", fail_after_fact)
            with pytest.raises(OSError):
                reconcile_orphaned_pending_deliveries_no_commit(
                    db, now=utc_now(), task_id=tid
                )
            db.rollback()
        assert db.query(TaskChatMessage).one().delivery_status == "pending"
        assert len(facts(db, tid)) == 1
        assert (
            reconcile_orphaned_pending_deliveries_no_commit(
                db, now=utc_now(), task_id=tid
            )
            == 1
        )
        db.commit()
        db.expire_all()
        assert db.query(TaskChatMessage).one().delivery_status == "dispatched"
        assert facts(db, tid)[-1].payload == {"status": "dispatched"}
        assert (
            reconcile_orphaned_pending_deliveries_no_commit(
                db, now=utc_now(), task_id=tid
            )
            == 0
        )
        assert len(facts(db, tid)) == 2


@pytest.mark.parametrize("with_checkpoint", [False, True])
def test_expired_lease_outcome_fact_and_task_are_atomic(
    canonical, monkeypatch, with_checkpoint
):
    from datetime import timedelta

    from xagent.core.agent.checkpoint import CHECKPOINT_TYPE
    from xagent.web.services import task_execution_event_writer as writer
    from xagent.web.services.task_lease_recovery import (
        TASK_LEASE_EXPIRED_ERROR,
        recover_task_lease_candidate_no_commit,
    )
    from xagent.web.services.task_lease_service import (
        get_expired_task_lease_candidates,
        utc_now,
    )

    factory, tid = canonical
    with factory() as db:
        task = db.get(Task, tid)
        task.status = TaskStatus.RUNNING
        task.control_state = "running"
        task.run_id = "expired-run"
        task.runner_id = "dead-runner"
        task.lease_expires_at = utc_now() - timedelta(seconds=5)
        if with_checkpoint:
            writer.append_fact_no_commit(
                db,
                task_id=tid,
                kind="recovery_state",
                key="expired-checkpoint",
                run_id="expired-run",
                payload={
                    "protocol_event_id": "expired-checkpoint",
                    "data": {
                        "checkpoint_type": CHECKPOINT_TYPE,
                        "snapshot_schema_version": 1,
                        "execution_id": str(tid),
                        "snapshot": {
                            "execution_id": str(tid),
                            "context": {"messages": []},
                        },
                    },
                },
            )
            task.last_checkpoint_event_id = "expired-checkpoint"
        db.commit()
        candidate = get_expired_task_lease_candidates(db, cutoff=utc_now(), limit=1)[0]
        original = writer.stage_result_fact_no_commit

        def fail_after_fact(*args, **kwargs):
            original(*args, **kwargs)
            raise OSError("failed recovery transaction")

        with monkeypatch.context() as failure:
            failure.setattr(writer, "stage_result_fact_no_commit", fail_after_fact)
            with pytest.raises(OSError, match="failed recovery transaction"):
                recover_task_lease_candidate_no_commit(
                    db, candidate, recovered_at=utc_now()
                )
            db.rollback()
        assert db.get(Task, tid).status == TaskStatus.RUNNING
        assert db.get(Task, tid).runner_id == "dead-runner"
        assert db.get(Task, tid).state_version == candidate.state_version
        assert len(facts(db, tid)) == int(with_checkpoint)

        expected = TaskStatus.PAUSED if with_checkpoint else TaskStatus.FAILED
        assert (
            recover_task_lease_candidate_no_commit(
                db, candidate, recovered_at=utc_now()
            )
            == expected
        )
        db.commit()
        assert db.get(Task, tid).status == expected
        assert db.get(Task, tid).runner_id is None
        assert len(facts(db, tid)) == 1 + int(with_checkpoint)
        outcome = facts(db, tid)[-1]
        assert outcome.kind == "execution_settled"
        assert outcome.run_id == "expired-run"
        assert outcome.payload == {
            "status": expected.value,
            "result": {"error": None if with_checkpoint else TASK_LEASE_EXPIRED_ERROR},
        }
        assert (
            recover_task_lease_candidate_no_commit(
                db, candidate, recovered_at=utc_now()
            )
            is None
        )
        assert len(facts(db, tid)) == 1 + int(with_checkpoint)


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "action,field", [(TraceAction.START, "messages"), (TraceAction.END, "response")]
)
async def test_llm_projection_is_capped_while_fact_stays_complete(
    canonical, monkeypatch, action, field
):
    factory, task_id = canonical
    monkeypatch.setenv("XAGENT_MAX_TRACE_PAYLOAD_BYTES", "50000")
    content = "x" * 200_000
    value = [{"role": "user", "content": content}] if field == "messages" else content
    runtime = PatternRuntime(tracer=create_task_tracer(task_id))
    await runtime._emit_trace_event(
        TraceEventType(TraceScope.ACTION, action, TraceCategory.LLM),
        task_id=str(task_id),
        step_id="llm-step",
        data={field: value, "model": "test-model"},
    )
    with factory() as db:
        fact = facts(db, task_id)[0]
        projection = db.query(TraceEvent).filter_by(task_id=task_id).one()
        assert fact.payload["data"][field] == value
        assert len(json.dumps(projection.data).encode()) <= 50000
        assert projection.data["model"] == "test-model"


@pytest.mark.asyncio
async def test_stream_deltas_publish_without_durable_writes(
    canonical, monkeypatch, engine
):
    from xagent.web.services import task_execution as websocket
    from xagent.web.services.task_lease_service import bind_task_lease_context

    factory, task_id = canonical
    monkeypatch.setattr(websocket, "get_db", lambda: iter([factory()]))
    publish = AsyncMock()
    monkeypatch.setattr(websocket, "publish_task_event", publish)
    handler = websocket.make_agent_outbound_handler(task_id, authoritative=True)
    with factory() as db:
        lease = acquire_task_lease(db, task_id, new_run=True)
    statements = []

    def record(_conn, _cursor, statement, _parameters, _context, _executemany):
        statements.append(statement)

    sa.event.listen(engine, "before_cursor_execute", record)
    try:
        with bind_task_lease_context(lease):
            for chunk in ("hello", " ", "world"):
                await handler(
                    {
                        "type": "final_answer_delta",
                        "delta": chunk,
                        "stream_id": "s",
                        "message_id": "final_answer_1",
                    }
                )
    finally:
        sa.event.remove(engine, "before_cursor_execute", record)
    assert [call.args[0]["delta"] for call in publish.await_args_list] == [
        "hello",
        " ",
        "world",
    ]
    assert statements and all(
        statement.lstrip().upper().startswith("SELECT") for statement in statements
    )
    with factory() as db:
        assert facts(db, task_id) == []
    with bind_task_lease_context(lease):
        await handler(
            {
                "type": "final_answer_end",
                "content": "hello world",
                "stream_id": "s",
                "message_id": "final_answer_1",
            }
        )
    with factory() as db:
        assert [row.kind for row in facts(db, task_id)] == ["final_answer_end"]
        assert facts(db, task_id)[0].payload["data"]["content"] == "hello world"