Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

23.3 KB
Raw Blame
"""Same-identity retries of V2 trace-fact commits.

Faults are raised at ``Session.commit`` so a "lost acknowledgement" really
commits first: the retry must then replay the committed fact, never append a
second one or run a tool twice.
"""

from __future__ import annotations

import asyncio
import logging
import sqlite3
import uuid
from collections.abc import Callable
from copy import deepcopy

import pytest
import sqlalchemy as sa
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.orm import Session

from tests.web.services.test_task_execution_event_writer import (
    canonical as canonical_fixture,
)
from tests.web.services.test_task_execution_event_writer import engine as engine_fixture
from tests.web.services.test_task_execution_event_writer import (
    facts,
)
from tests.web.services.test_task_execution_event_writer import (
    task_id as task_id_fixture,
)
from xagent.core.agent.checkpoint import (
    ExecutionEventPersistenceError,
    TraceCheckpointStore,
)
from xagent.core.agent.runtime import PatternRuntime
from xagent.core.agent.trace import (
    TraceAction,
    TraceCategory,
    TraceEvent,
    TraceEventType,
    TraceScope,
)
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.task import Task
from xagent.web.models.task import TraceEvent as StoredTraceEvent
from xagent.web.models.task_execution_event import TaskExecutionEvent
from xagent.web.services.chat_history_service import persist_user_message_no_commit
from xagent.web.services.db_write_retry import DbWriteRetryPolicy
from xagent.web.services.task_execution_event_store import ExecutionEventConflict
from xagent.web.services.task_lease_service import (
    acquire_task_lease,
    bind_task_lease_context,
)
from xagent.web.services.trace_handlers import CommitState, DatabaseTraceHandler
from xagent.web.tracing import ExecutionEventTraceAdapter, create_task_tracer

canonical = canonical_fixture
engine = engine_fixture
task_id = task_id_fixture


@pytest.fixture(autouse=True)
def immediate_retries(monkeypatch):
    monkeypatch.setenv("XAGENT_DB_WRITE_RETRY_ATTEMPTS", "3")
    monkeypatch.setenv("XAGENT_DB_WRITE_RETRY_BASE_DELAY_MS", "0")
    monkeypatch.setenv("XAGENT_DB_WRITE_RETRY_MAX_DELAY_MS", "0")


@pytest.fixture
def attempts(monkeypatch):
    """Count transactions the trace writer runs, across all handlers."""
    calls: list[str] = []
    original = DatabaseTraceHandler._save_trace_event

    def save(self, db, event, **kwargs):
        calls.append(str(event.id))
        return original(self, db, event, **kwargs)

    monkeypatch.setattr(DatabaseTraceHandler, "_save_trace_event", save)
    return calls


@pytest.fixture(params=[False, True], ids=["sync", "async"])
def trace_runtime(request, canonical, engine, monkeypatch):
    """Run writes through the bounded sync worker or the async prepared path."""
    from xagent.web.services import trace_handlers
    from xagent.web.services.trace_database import TraceDatabaseRuntime

    native_async = request.param
    source = engine
    if native_async and engine.dialect.name == "postgresql":
        # This fixture uses creator= for disposable databases; recover its URL
        # so the async driver connects to the same isolated database.
        connection = engine.raw_connection()
        try:
            parameters = connection.driver_connection.info.dsn_parameters
            source = sa.create_engine(
                sa.URL.create(
                    "postgresql",
                    username=parameters["user"],
                    host=parameters["host"],
                    port=int(parameters["port"]),
                    database=parameters["dbname"],
                )
            )
        finally:
            connection.close()
    database_runtime = TraceDatabaseRuntime(source, use_async=native_async, limit=1)
    monkeypatch.setattr(
        trace_handlers, "get_trace_database_runtime", lambda: database_runtime
    )
    yield database_runtime
    if source is not engine:
        source.dispose()


def invalidated() -> OperationalError:
    # psycopg2 reports a lost COMMIT acknowledgement exactly like this.
    return OperationalError(
        "COMMIT",
        {},
        Exception("server closed the connection unexpectedly"),
        connection_invalidated=True,
    )


def holds_fact(kind: str | None = None) -> Callable[[Session], bool]:
    """Match a commit whose transaction staged or read a fact of ``kind``."""

    def matches(db: Session) -> bool:
        return any(
            isinstance(row, TaskExecutionEvent) and kind in (None, row.kind)
            for row in db.identity_map.values()
        )

    return matches


def fail_commits(
    monkeypatch,
    target: Callable[[Session], bool],
    *,
    committed: bool | list[bool],
    times: int = 1,
    error: Callable[[], Exception] = invalidated,
    between: Callable[[], None] | None = None,
) -> list[Exception]:
    """Fail the first ``times`` matching commits, after or before COMMIT.

    ``committed`` may give each failure its own outcome, in order.
    ``between`` runs after each failed transaction has committed or rolled
    back, i.e. between two attempts, outside the writer's locks.
    """
    raised: list[Exception] = []
    original = Session.commit
    outcomes = committed if isinstance(committed, list) else [committed] * times

    def commit(self):
        if len(raised) >= len(outcomes) or not target(self):
            return original(self)
        if outcomes[len(raised)]:
            original(self)
        else:
            self.rollback()
        if between is not None:
            between()
        raised.append(error())
        raise raised[-1]

    monkeypatch.setattr(Session, "commit", commit)
    return raised


def task_start(task_id: int, **data) -> TraceEvent:
    return TraceEvent(
        TraceEventType(TraceScope.TASK, TraceAction.START, TraceCategory.GENERAL),
        task_id=str(task_id),
        data=data,
    )


def projections(db, event_id: str) -> int:
    return db.query(StoredTraceEvent).filter_by(event_id=event_id).count()


@pytest.mark.asyncio
@pytest.mark.parametrize("committed", [False, True], ids=["rolled-back", "lost-ack"])
async def test_retryable_commit_failure_reruns_into_one_fact(
    canonical, trace_runtime, attempts, monkeypatch, committed
):
    factory, task_id = canonical
    raised = fail_commits(monkeypatch, holds_fact(), committed=committed)
    event = task_start(task_id, input="hello")
    original_data = deepcopy(event.data)
    try:
        await ExecutionEventTraceAdapter(task_id).commit_event(event)
    finally:
        await trace_runtime.close()
    assert len(raised) == 1
    assert len(attempts) == 2
    assert event.data == original_data
    with factory() as db:
        rows = facts(db, task_id)
        assert [row.idempotency_key for row in rows] == [f"runtime:{event.id}"]
        assert projections(db, str(event.id)) == 1
        assert db.get(Task, task_id).conversation_event_sequence == 1


def react_call(task_id: int, tool):
    """Run one real ReAct tool call through the V2 fact writer."""
    from tests.core.agent.test_react import FakeLLM
    from xagent.core.agent import ExecutionContext, ReActPattern

    tracer = create_task_tracer(task_id)
    tracer.handlers = [
        h for h in tracer.handlers if isinstance(h, ExecutionEventTraceAdapter)
    ]
    context = ExecutionContext(system_prompt="Calculate")
    context.add_user_message("2+2", metadata={"turn_id": "retry-turn"})
    return ReActPattern(max_iterations=3).run(
        context=context,
        tools=[tool],
        runtime=PatternRuntime(tracer=TraceCheckpointStore(tracer)),
        llm=FakeLLM(
            responses=[
                {
                    "content": "calculate",
                    "tool_calls": [
                        {
                            "id": "call1",
                            "function": {
                                "name": "calculator",
                                "arguments": '{"expression":"2+2"}',
                            },
                        }
                    ],
                },
                {"content": "4", "done": True},
            ]
        ),
    )


def tool_facts(db, task_id):
    return [row for row in facts(db, task_id) if row.kind.startswith("tool_")]


@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["tool_execution_start", "tool_execution_end"])
async def test_lost_tool_ack_runs_the_tool_once(
    canonical, trace_runtime, monkeypatch, kind
):
    from tests.core.agent.test_react import FakeTool

    factory, task_id = canonical
    raised = fail_commits(monkeypatch, holds_fact(kind), committed=True)
    tool = FakeTool()
    try:
        result = await react_call(task_id, tool)
    finally:
        await trace_runtime.close()
    assert result["success"]
    assert len(raised) == 1
    assert len(tool.calls) == 1
    with factory() as db:
        rows = tool_facts(db, task_id)
        assert [row.kind for row in rows] == [
            "tool_execution_start",
            "tool_execution_end",
        ]
        assert len({row.tool_attempt_id for row in rows}) == 1
        for row in rows:
            assert projections(db, row.payload["protocol_event_id"]) == 1


@pytest.mark.asyncio
async def test_start_replaced_between_attempts_is_still_refused(
    canonical, monkeypatch, attempts
):
    from tests.core.agent.test_react import FakeTool

    factory, task_id = canonical

    def replace_start():
        # Another writer's start for the same attempt, not this call's own.
        with factory() as db:
            db.execute(
                sa.update(TaskExecutionEvent)
                .where(TaskExecutionEvent.kind == "tool_execution_start")
                .values(event_id=str(uuid.uuid4()))
            )
            db.commit()

    fail_commits(
        monkeypatch,
        holds_fact("tool_execution_start"),
        committed=True,
        between=replace_start,
    )
    tool = FakeTool()
    with pytest.raises(ExecutionEventPersistenceError) as caught:
        await react_call(task_id, tool)
    assert isinstance(caught.value.__cause__, RuntimeError)
    assert "already started" in str(caught.value.__cause__)
    assert tool.calls == []
    with factory() as db:
        assert [row.kind for row in tool_facts(db, task_id)] == ["tool_execution_start"]


def tool_start(task_id: int, attempt: str) -> TraceEvent:
    return TraceEvent(
        TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.TOOL),
        task_id=str(task_id),
        step_id="tool-step",
        data={
            "tool_name": "write",
            "tool_call_id": "call1",
            "tool_attempt_id": attempt,
            "assistant_message_id": "batch1",
        },
    )


def tool_end(task_id: int, attempt: str) -> TraceEvent:
    return TraceEvent(
        TraceEventType(TraceScope.ACTION, TraceAction.END, TraceCategory.TOOL),
        task_id=str(task_id),
        step_id="tool-step",
        data={
            "tool_name": "write",
            "tool_call_id": "call1",
            "tool_attempt_id": attempt,
            "assistant_message_id": "batch1",
            "result": {"success": True},
        },
    )


@pytest.mark.asyncio
@pytest.mark.parametrize("claim", ["none", "other", "own-with-end"])
async def test_start_guard_accepts_only_this_calls_own_start(
    canonical, trace_runtime, claim
):
    try:
        await _check_start_guard(canonical, claim)
    finally:
        await trace_runtime.close()


async def _check_start_guard(canonical, claim):
    factory, task_id = canonical
    adapter = ExecutionEventTraceAdapter(task_id)
    first = tool_start(task_id, "attempt1")
    first.require_persisted = True
    state = CommitState()
    await adapter._save_to_database(first, commit_state=state)
    assert state.start_event_id is not None
    if claim == "none":
        retry = CommitState()
    elif claim == "other":
        retry = CommitState(start_event_id=str(uuid.uuid4()))
    else:
        retry = CommitState(start_event_id=state.start_event_id)
        await adapter.commit_event(tool_end(task_id, "attempt1"))
    second = tool_start(task_id, "attempt1")
    second.require_persisted = True
    with pytest.raises(RuntimeError, match="already started"):
        await adapter._save_to_database(second, commit_state=retry)
    # The own start alone is the idempotent rerun this guard admits.
    if claim != "own-with-end":
        rerun = tool_start(task_id, "attempt1")
        rerun.require_persisted = True
        await adapter._save_to_database(
            rerun, commit_state=CommitState(start_event_id=state.start_event_id)
        )
        assert rerun.id == first.id
    with factory() as db:
        starts = [r for r in facts(db, task_id) if r.kind == "tool_execution_start"]
        assert [r.event_id for r in starts] == [state.start_event_id]


@pytest.mark.asyncio
async def test_lease_replaced_between_attempts_is_not_retried(
    canonical, monkeypatch, attempts
):
    factory, task_id = canonical
    with factory() as db:
        lease = acquire_task_lease(db, task_id, new_run=True)

    def replace_lease():
        with factory() as db:
            db.get(Task, task_id).run_id = "replacement-run"
            db.commit()

    raised = fail_commits(
        monkeypatch,
        holds_fact(),
        committed=False,
        between=replace_lease,
    )
    with bind_task_lease_context(lease):
        with pytest.raises(ExecutionEventPersistenceError) as caught:
            await ExecutionEventTraceAdapter(task_id).commit_event(task_start(task_id))
    assert len(raised) == 1
    assert len(attempts) == 2
    assert "lost its task lease" in str(caught.value.__cause__)
    with factory() as db:
        assert facts(db, task_id) == []
        assert db.query(StoredTraceEvent).count() == 0


def integrity_error() -> IntegrityError:
    return IntegrityError("INSERT", {}, Exception("duplicate key value"))


@pytest.mark.asyncio
@pytest.mark.parametrize("error", [integrity_error, lambda: ValueError("bad")])
async def test_non_retryable_error_is_attempted_once(
    canonical, monkeypatch, attempts, error
):
    factory, task_id = canonical
    raised = fail_commits(monkeypatch, holds_fact(), committed=False, error=error)
    with pytest.raises(ExecutionEventPersistenceError) as caught:
        await ExecutionEventTraceAdapter(task_id).commit_event(task_start(task_id))
    assert caught.value.__cause__ is raised[0]
    assert len(attempts) == 1
    with factory() as db:
        assert facts(db, task_id) == []


@pytest.mark.asyncio
async def test_payload_drift_under_same_identity_conflicts_without_retry(
    canonical, attempts
):
    factory, task_id = canonical
    adapter = ExecutionEventTraceAdapter(task_id)
    event = task_start(task_id, input="hello")
    await adapter.commit_event(event)
    event.data = {"input": "changed"}
    with pytest.raises(ExecutionEventPersistenceError) as caught:
        await adapter.commit_event(event)
    assert isinstance(caught.value.__cause__, ExecutionEventConflict)
    assert len(attempts) == 2
    with factory() as db:
        assert [row.payload["data"] for row in facts(db, task_id)] == [
            {"input": "hello"}
        ]


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("orig", "expected"),
    [
        (Exception("server closed the connection unexpectedly"), 4),
        # SQLite reports BUSY only after its busy timeout: one retry at most.
        (sqlite3.OperationalError("database is locked"), 2),
    ],
    ids=["connection", "sqlite-busy"],
)
async def test_exhausted_retries_keep_the_database_error_as_cause(
    canonical, monkeypatch, attempts, orig, expected
):
    factory, task_id = canonical
    monkeypatch.setenv("XAGENT_DB_WRITE_RETRY_ATTEMPTS", "4")
    raised = fail_commits(
        monkeypatch,
        holds_fact(),
        committed=False,
        times=10,
        error=lambda: OperationalError("COMMIT", {}, orig),
    )
    with pytest.raises(ExecutionEventPersistenceError) as caught:
        await ExecutionEventTraceAdapter(task_id).commit_event(task_start(task_id))
    assert len(attempts) == expected
    assert len(raised) == expected
    assert caught.value.__cause__ is raised[-1]
    with factory() as db:
        assert facts(db, task_id) == []


@pytest.mark.asyncio
async def test_cancellation_during_backoff_stops_retrying(
    canonical, monkeypatch, attempts
):
    factory, task_id = canonical
    backing_off = asyncio.Event()

    def delay(self, retry, *, rng=None):
        backing_off.set()
        return 60.0

    monkeypatch.setattr(DbWriteRetryPolicy, "delay", delay)
    fail_commits(monkeypatch, holds_fact(), committed=False)
    write = asyncio.create_task(
        ExecutionEventTraceAdapter(task_id).commit_event(task_start(task_id))
    )
    await asyncio.wait_for(backing_off.wait(), timeout=10)
    write.cancel()
    with pytest.raises(asyncio.CancelledError):
        await write
    assert len(attempts) == 1
    with factory() as db:
        assert facts(db, task_id) == []


@pytest.mark.asyncio
async def test_v1_task_writes_are_not_retried(canonical, monkeypatch, attempts):
    factory, task_id = canonical
    with factory() as db:
        db.get(Task, task_id).conversation_storage_version = 1
        db.commit()
    tracer = create_task_tracer(task_id)
    # No fact writer: the V2 commit path, and its retries, never runs.
    assert tracer.event_writer is None
    raised = fail_commits(
        monkeypatch,
        lambda db: any(
            isinstance(row, StoredTraceEvent)
            for row in [*db.new, *db.identity_map.values()]
        ),
        committed=False,
    )
    with pytest.raises(RuntimeError):
        await tracer.trace_event(
            TraceEventType(TraceScope.TASK, TraceAction.START, TraceCategory.GENERAL),
            task_id=str(task_id),
            require_persisted=True,
        )
    assert len(raised) == 1
    assert len(attempts) == 1
    with factory() as db:
        assert db.query(StoredTraceEvent).count() == 0


@pytest.mark.asyncio
async def test_compaction_lost_ack_keeps_the_first_watermark(canonical, monkeypatch):
    factory, task_id = canonical
    with factory() as db:
        message = persist_user_message_no_commit(
            db, task_id, db.get(Task, task_id).user_id, "earlier", turn_id="turn-1"
        )
        db.commit()
        anchor = facts(db, task_id)[0]
        expected = {
            "scope_id": "root",
            "event_id": anchor.event_id,
            "sequence": anchor.sequence,
        }
        data = {"summary": "Earlier conversation", "watermark_message_id": message.id}

    def drop_transcript():
        # A recomputed watermark would now fail; the rerun must reuse the first.
        with factory() as db:
            db.execute(
                sa.delete(TaskChatMessage).where(TaskChatMessage.task_id == task_id)
            )
            db.commit()

    raised = fail_commits(
        monkeypatch,
        holds_fact("action_end_compact"),
        committed=True,
        between=drop_transcript,
    )
    event = TraceEvent(
        TraceEventType(TraceScope.ACTION, TraceAction.END, TraceCategory.COMPACT),
        task_id=str(task_id),
        step_id="compact-step",
        data=deepcopy(data),
    )
    await ExecutionEventTraceAdapter(task_id).commit_event(event)
    assert len(raised) == 1
    assert event.data == data
    with factory() as db:
        summaries = [r for r in facts(db, task_id) if r.kind == "action_end_compact"]
        assert len(summaries) == 1
        assert summaries[0].payload["transcript_watermark"] == expected
        assert summaries[0].payload["data"] == data
        assert projections(db, str(event.id)) == 1


@pytest.mark.asyncio
async def test_start_rolled_back_then_lost_ack_admits_only_the_second_insert(
    canonical, monkeypatch, attempts
):
    factory, task_id = canonical
    # Attempt 1 rolls back, attempt 2 commits but loses its ack, attempt 3
    # must recognise attempt 2's row, not attempt 1's, as this call's own.
    raised = fail_commits(
        monkeypatch,
        holds_fact("tool_execution_start"),
        committed=[False, True],
    )
    await ExecutionEventTraceAdapter(task_id).commit_event(
        tool_start(task_id, "attempt1")
    )
    assert len(raised) == 2
    assert len(attempts) == 3
    with factory() as db:
        assert [row.kind for row in tool_facts(db, task_id)] == ["tool_execution_start"]


def settlement_start(task_id: int, attempt: str) -> TraceEvent:
    event = tool_start(task_id, attempt)
    event.data = {**event.data, "settlement_delivery": True}
    return event


@pytest.mark.asyncio
async def test_lost_settlement_ack_replays_the_settlement_fact(
    canonical, monkeypatch, attempts
):
    factory, task_id = canonical
    adapter = ExecutionEventTraceAdapter(task_id)
    await adapter.commit_event(tool_start(task_id, "attempt1"))
    await adapter.commit_event(tool_end(task_id, "attempt1"))
    raised = fail_commits(monkeypatch, holds_fact(), committed=True)
    event = settlement_start(task_id, "attempt1")
    await adapter.commit_event(event)
    assert len(raised) == 1
    with factory() as db:
        rows = tool_facts(db, task_id)
        assert [row.idempotency_key for row in rows] == [
            "tool:attempt1:tool_execution_start",
            "tool:attempt1:tool_execution_end",
            "tool:attempt1:settlement:tool_execution_start",
        ]
        assert projections(db, str(event.id)) == 1


@pytest.mark.asyncio
async def test_rerun_waits_a_bounded_time_for_the_task_lock(
    canonical, engine, monkeypatch, attempts
):
    if engine.dialect.name != "postgresql":
        pytest.skip("lock_timeout is a PostgreSQL setting")
    from xagent.web.services import db_write_retry
    from xagent.web.services.task_execution_event_store import (
        lock_task_execution_events_no_commit,
    )

    factory, task_id = canonical
    monkeypatch.setattr(db_write_retry, "_RERUN_LOCK_TIMEOUT", "200ms")
    holder = factory()

    def abandon_lock():
        # A dropped connection's transaction can keep the task row locked.
        lock_task_execution_events_no_commit(holder, task_id)

    fail_commits(monkeypatch, holds_fact(), committed=False, between=abandon_lock)
    try:
        with pytest.raises(ExecutionEventPersistenceError) as caught:
            await asyncio.wait_for(
                ExecutionEventTraceAdapter(task_id).commit_event(task_start(task_id)),
                timeout=10,
            )
    finally:
        holder.rollback()
        holder.close()
    # Both reruns gave up on the lock instead of waiting for it indefinitely.
    assert len(attempts) == 3
    assert getattr(caught.value.__cause__.orig, "pgcode", None) == "55P03"


@pytest.mark.asyncio
@pytest.mark.parametrize("times", [1, 10], ids=["recovered", "exhausted"])
async def test_retried_attempts_warn_and_only_a_failed_write_errors(
    canonical, monkeypatch, caplog, times
):
    factory, task_id = canonical
    fail_commits(monkeypatch, holds_fact(), committed=False, times=times)
    with caplog.at_level(logging.WARNING):
        try:
            await ExecutionEventTraceAdapter(task_id).commit_event(task_start(task_id))
        except ExecutionEventPersistenceError:
            assert times == 10
    errors = [r.getMessage() for r in caplog.records if r.levelno >= logging.ERROR]
    warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
    assert warnings
    if times == 1:
        assert errors == []
    else:
        assert len(errors) == 1
        assert errors[0].startswith("execution_event_commit_failed ")
        assert "site=trace_fact" in errors[0]