Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

22.6 KB
Raw Blame
from __future__ import annotations

import asyncio
import sqlite3
from types import SimpleNamespace
from typing import Any

import pytest

from xagent.core.agent import Agent, AgentRunner, ExecutionContext, PatternRuntime
from xagent.core.agent.checkpoint import (
    CHECKPOINT_EVENT_TYPE,
    CHECKPOINT_TYPE,
    LEGACY_CHECKPOINT_TYPES,
    CheckpointAccessRefusedError,
    CheckpointCorruptError,
    CheckpointPersistenceError,
    CheckpointUnavailableError,
    TraceCheckpointStore,
    read_latest_checkpoint_payload,
)
from xagent.core.agent.trace import (
    ConsoleTraceHandler,
    TraceEvent,
    TraceHandler,
    Tracer,
)


class PersistentTraceBackend:
    def __init__(self) -> None:
        self.events: list[dict[str, Any]] = []

    async def trace_event(
        self,
        event_type: Any,
        *,
        task_id: str | None = None,
        data: dict[str, Any] | None = None,
        require_persisted: bool = False,
        **_: Any,
    ) -> str:
        event_id = f"event-{len(self.events) + 1}"
        self.events.append(
            {
                "event_id": event_id,
                "event_type": str(event_type),
                "task_id": task_id,
                "data": dict(data or {}),
                "require_persisted": require_persisted,
            }
        )
        return event_id

    async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any] | None:
        for event in reversed(self.events):
            data = event["data"]
            if (
                data.get("checkpoint_type") == CHECKPOINT_TYPE
                and data.get("root_execution_id") == execution_id
            ):
                return dict(data)
        return None


class BestEffortTraceBackend:
    async def trace_event(
        self,
        event_type: Any,
        *,
        task_id: str | None = None,
        data: dict[str, Any] | None = None,
    ) -> str:
        del event_type, task_id, data
        return "best-effort"


class NoneReturningCheckpointBackend:
    async def checkpoint(self, **_: Any) -> None:
        return None


class NoneReturningTraceEventBackend:
    async def trace_event(
        self,
        event_type: Any,
        *,
        task_id: str | None = None,
        data: dict[str, Any] | None = None,
        require_persisted: bool = False,
    ) -> None:
        del event_type, task_id, data, require_persisted
        return None


class RaisingCheckpointBackend:
    def __init__(self, error: BaseException) -> None:
        self.error = error

    async def checkpoint(self, **_: Any) -> None:
        raise self.error


class LegacyCheckpointBackend:
    def __init__(self, payload: dict[str, Any]) -> None:
        self.payload = payload

    async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any]:
        legacy_type = next(iter(LEGACY_CHECKPOINT_TYPES))
        return {
            "checkpoint_type": legacy_type,
            "root_execution_id": execution_id,
            "snapshot": dict(self.payload),
        }


class LegacyNamedCheckpointBackend:
    def __init__(self, payload: dict[str, Any]) -> None:
        self.payload = payload

    def get_latest_checkpoint(self, execution_id: str) -> dict[str, Any]:
        return {
            "checkpoint_type": CHECKPOINT_TYPE,
            "root_execution_id": execution_id,
            "snapshot": dict(self.payload),
        }


class FailingTraceHandler(TraceHandler):
    async def handle_event(self, _: TraceEvent) -> None:
        raise ValueError("checkpoint write failed: secret=supersecret")


class FakeLLM:
    async def chat(self, **_: Any) -> str:
        return "done"


class CheckpointingPattern:
    async def run(self, *, context: Any, runtime: Any, **_: Any) -> dict[str, Any]:
        context.add_assistant_message("done")
        await runtime.checkpoint(
            "final",
            context=context,
            pattern=self,
            status="completed",
        )
        return {"success": True, "output": "done"}


@pytest.mark.asyncio
async def test_trace_checkpoint_store_persists_full_snapshot_event() -> None:
    backend = PersistentTraceBackend()
    store = TraceCheckpointStore(backend)
    payload = {
        "type": "checkpoint",
        "label": "before_llm",
        "execution_id": "exec-checkpoint",
        "context": {"messages": []},
        "pattern": "ReActPattern",
        "pattern_state": {"current_iteration": 0},
        "status": "thinking",
    }

    event_id = await store.checkpoint(**payload)
    loaded = await store.load_latest_checkpoint("exec-checkpoint")

    assert event_id == "event-1"
    assert backend.events[0]["require_persisted"] is True
    assert backend.events[0]["data"]["checkpoint_type"] == CHECKPOINT_TYPE
    assert backend.events[0]["data"]["snapshot"] == payload
    assert loaded == payload


@pytest.mark.asyncio
async def test_required_trace_error_preserves_redacted_handler_cause() -> None:
    tracer = Tracer()
    tracer.add_handler(FailingTraceHandler())

    with pytest.raises(RuntimeError) as exc_info:
        await tracer.trace_event(
            CHECKPOINT_EVENT_TYPE,
            task_id="trace-cause",
            require_persisted=True,
        )

    message = str(exc_info.value)
    assert "First failure: ValueError: checkpoint write failed" in message
    assert "supersecret" not in message
    assert isinstance(exc_info.value.__cause__, ValueError)


@pytest.mark.asyncio
async def test_trace_checkpoint_store_reads_legacy_checkpoint_marker() -> None:
    payload = {
        "type": "checkpoint",
        "label": "before_llm",
        "execution_id": "legacy-exec",
        "context": {"messages": []},
    }
    store = TraceCheckpointStore(LegacyCheckpointBackend(payload))

    loaded = await store.load_latest_checkpoint("legacy-exec")

    assert loaded == payload


@pytest.mark.asyncio
async def test_trace_checkpoint_store_supports_legacy_reader_name() -> None:
    payload = {
        "type": "checkpoint",
        "execution_id": "legacy-reader-name",
        "context": {"messages": []},
    }
    store = TraceCheckpointStore(LegacyNamedCheckpointBackend(payload))

    loaded = await store.load_latest_checkpoint("legacy-reader-name")

    assert loaded == payload


@pytest.mark.asyncio
async def test_trace_checkpoint_store_rejects_best_effort_trace_event() -> None:
    store = TraceCheckpointStore(BestEffortTraceBackend())

    with pytest.raises(CheckpointPersistenceError):
        await store.checkpoint(
            type="checkpoint",
            label="before_llm",
            execution_id="exec-best-effort",
        )


@pytest.mark.asyncio
async def test_trace_checkpoint_store_rejects_none_checkpoint_writer() -> None:
    store = TraceCheckpointStore(NoneReturningCheckpointBackend())

    with pytest.raises(CheckpointPersistenceError):
        await store.checkpoint(
            type="checkpoint",
            label="before_llm",
            execution_id="exec-none-writer",
        )


@pytest.mark.asyncio
async def test_trace_checkpoint_store_rejects_none_trace_event_writer() -> None:
    store = TraceCheckpointStore(NoneReturningTraceEventBackend())

    with pytest.raises(CheckpointPersistenceError):
        await store.checkpoint(
            type="checkpoint",
            label="before_llm",
            execution_id="exec-none-trace-event",
        )


@pytest.mark.asyncio
@pytest.mark.parametrize("error_type", [RuntimeError, sqlite3.OperationalError])
async def test_trace_checkpoint_store_normalizes_writer_failures(
    error_type: type[Exception],
) -> None:
    failure = error_type("checkpoint unavailable")
    store = TraceCheckpointStore(RaisingCheckpointBackend(failure))

    with pytest.raises(CheckpointPersistenceError) as exc_info:
        await store.checkpoint(
            type="checkpoint",
            label="before_llm",
            execution_id="exec-failed-writer",
        )

    assert exc_info.value.__cause__ is failure


@pytest.mark.asyncio
@pytest.mark.parametrize("failure", [asyncio.CancelledError(), SystemExit()])
async def test_trace_checkpoint_store_preserves_base_exception_semantics(
    failure: BaseException,
) -> None:
    store = TraceCheckpointStore(RaisingCheckpointBackend(failure))

    with pytest.raises(type(failure)) as exc_info:
        await store.checkpoint(
            type="checkpoint",
            label="before_llm",
            execution_id="exec-interrupted-writer",
        )

    assert exc_info.value is failure


@pytest.mark.asyncio
@pytest.mark.parametrize("error_type", [RuntimeError, sqlite3.OperationalError])
async def test_pattern_runtime_normalizes_unwrapped_writer_failures(
    error_type: type[Exception],
) -> None:
    """``PatternRuntime`` normalizes even without ``TraceCheckpointStore``.

    Only ``xagent/service.py`` wraps the tracer; the fallback runtimes in
    ``auto.py``, ``react.py`` and ``dag.py`` pass a bare tracer straight
    through, so the boundary has to normalize centrally.
    """

    failure = error_type("checkpoint unavailable")
    runtime = PatternRuntime(
        tracer=RaisingCheckpointBackend(failure),
        execution_id="exec-unwrapped",
    )

    with pytest.raises(CheckpointPersistenceError) as exc_info:
        await runtime.checkpoint(
            "before_llm",
            context=ExecutionContext(execution_id="exec-unwrapped"),
            pattern=SimpleNamespace(status="running"),
        )

    assert exc_info.value.__cause__ is failure


@pytest.mark.asyncio
@pytest.mark.parametrize("failure", [asyncio.CancelledError(), SystemExit()])
async def test_pattern_runtime_checkpoint_preserves_base_exception_semantics(
    failure: BaseException,
) -> None:
    runtime = PatternRuntime(
        tracer=RaisingCheckpointBackend(failure),
        execution_id="exec-unwrapped-interrupted",
    )

    with pytest.raises(type(failure)) as exc_info:
        await runtime.checkpoint(
            "before_llm",
            context=ExecutionContext(execution_id="exec-unwrapped-interrupted"),
            pattern=SimpleNamespace(status="running"),
        )

    assert exc_info.value is failure


@pytest.mark.asyncio
async def test_pattern_runtime_does_not_double_wrap_normalized_failures() -> None:
    """An error the store already normalized must pass through unchanged."""

    failure = RuntimeError("database unavailable")
    runtime = PatternRuntime(
        tracer=TraceCheckpointStore(RaisingCheckpointBackend(failure)),
        execution_id="exec-wrapped",
    )

    with pytest.raises(CheckpointPersistenceError) as exc_info:
        await runtime.checkpoint(
            "before_llm",
            context=ExecutionContext(execution_id="exec-wrapped"),
            pattern=SimpleNamespace(status="running"),
        )

    # Exactly one level of normalization: the cause is the writer's own
    # error, not another CheckpointPersistenceError.
    assert exc_info.value.__cause__ is failure


class PersistedAwareTraceEventBackend:
    """A plain event tracer that can be asked for persisted delivery."""

    def __init__(self) -> None:
        self.calls: list[dict[str, Any]] = []

    async def trace_event(
        self,
        event_type: Any,
        *,
        task_id: str | None = None,
        data: dict[str, Any] | None = None,
        require_persisted: bool = False,
    ) -> str:
        self.calls.append({"task_id": task_id, "require_persisted": require_persisted})
        return "evt-1"


class BestEffortTraceEventBackend:
    """A plain event tracer with no way to guarantee persistence."""

    def __init__(self) -> None:
        self.calls = 0

    async def trace_event(
        self,
        event_type: Any,
        *,
        task_id: str | None = None,
        data: dict[str, Any] | None = None,
    ) -> str:
        self.calls += 1
        return "evt-1"


class NoWriterBackend:
    """A tracer with no checkpoint capability at all."""


@pytest.mark.asyncio
async def test_pattern_runtime_requires_persisted_delivery_from_event_tracer() -> None:
    """The bare-tracer event fallback must ask for persisted delivery."""

    backend = PersistedAwareTraceEventBackend()
    runtime = PatternRuntime(tracer=backend, execution_id="exec-persisted")

    await runtime.checkpoint(
        "before_llm",
        context=ExecutionContext(execution_id="exec-persisted"),
        pattern=SimpleNamespace(status="running"),
    )

    assert backend.calls == [{"task_id": "exec-persisted", "require_persisted": True}]


@pytest.mark.asyncio
async def test_pattern_runtime_rejects_a_best_effort_event_tracer() -> None:
    """A tracer that cannot promise persistence is not a checkpoint writer.

    Returning normally here would tell the caller the transition is durable
    when the write was only best-effort.
    """

    backend = BestEffortTraceEventBackend()
    runtime = PatternRuntime(tracer=backend, execution_id="exec-best-effort")

    with pytest.raises(CheckpointPersistenceError) as exc_info:
        await runtime.checkpoint(
            "before_llm",
            context=ExecutionContext(execution_id="exec-best-effort"),
            pattern=SimpleNamespace(status="running"),
        )

    # The capability check must reject it up front. Without that check the
    # call would still fail -- passing an unsupported ``require_persisted``
    # raises TypeError, which the boundary normalizes -- so assert on the
    # specific reason, not merely on the error type.
    assert "cannot guarantee checkpoint persistence" in str(exc_info.value)
    assert exc_info.value.__cause__ is None
    assert backend.calls == 0


@pytest.mark.asyncio
async def test_pattern_runtime_tolerates_a_tracer_without_any_writer() -> None:
    """A span-only observability tracer is the "not configured" mode.

    It carries no event writer at all, so it is treated like ``tracer=None``
    rather than failing the run. Raising here would break every execution
    that passes a spans-only tracer, which is a supported shape. The
    best-effort *event* tracer above is the case that does raise: it claims
    to record the checkpoint without being able to promise it survives.
    """

    runtime = PatternRuntime(tracer=NoWriterBackend(), execution_id="exec-no-writer")

    payload = await runtime.checkpoint(
        "before_llm",
        context=ExecutionContext(execution_id="exec-no-writer"),
        pattern=SimpleNamespace(status="running"),
    )

    assert payload["label"] == "before_llm"


@pytest.mark.asyncio
async def test_pattern_runtime_without_a_tracer_stays_a_no_op() -> None:
    """``tracer=None`` is the documented no-tracing mode, not a failure.

    The fallback ``PatternRuntime`` constructions in auto.py / react.py /
    dag.py build one of these, and they must keep working.
    """

    runtime = PatternRuntime(execution_id="exec-no-tracer")

    payload = await runtime.checkpoint(
        "before_llm",
        context=ExecutionContext(execution_id="exec-no-tracer"),
        pattern=SimpleNamespace(status="running"),
    )

    assert payload["label"] == "before_llm"


@pytest.mark.asyncio
async def test_bare_tracer_checkpoint_round_trips_through_the_real_reader() -> None:
    """A bare tracer's checkpoint must be findable on a cold resume.

    Goes through the real ``EphemeralCheckpointTraceHandler`` and its real
    ``load_latest_checkpoint``. Before the fallback wrote the canonical
    envelope, this call reported success while the handler dropped the event,
    so the lookup returned ``None`` and a resume could replay work that had
    already run.

    The ``DatabaseTraceHandler`` path is PostgreSQL-only and is not exercised
    here; it selects on the same ``checkpoint_type`` /
    ``READABLE_CHECKPOINT_TYPES`` filter that the wire-format assertions
    below pin.
    """

    from xagent.web.tracing import EphemeralCheckpointTraceHandler

    store: dict[str, dict[str, Any]] = {}
    handler = EphemeralCheckpointTraceHandler(store)
    tracer = Tracer()
    tracer.add_handler(handler)
    runtime = PatternRuntime(tracer=tracer, execution_id="exec-bare-roundtrip")

    await runtime.checkpoint(
        "after_tool",
        context=ExecutionContext(execution_id="exec-bare-roundtrip"),
        pattern=SimpleNamespace(status="running"),
    )

    restored = await handler.load_latest_checkpoint("exec-bare-roundtrip")

    assert restored is not None
    assert restored["label"] == "after_tool"
    assert restored["execution_id"] == "exec-bare-roundtrip"


@pytest.mark.asyncio
async def test_bare_tracer_rejects_a_console_only_handler() -> None:
    """A ``Tracer`` with only observational handlers is not durable.

    ``Tracer.trace_event(require_persisted=True)`` only proves every
    currently attached handler dispatched without raising; it does not prove
    any of them can answer a checkpoint read. A ``Tracer`` wired with only
    ``ConsoleTraceHandler`` would previously "succeed" here (return a
    non-``None`` event id) and still lose the checkpoint, since
    ``ConsoleTraceHandler`` has no ``load_latest_checkpoint``. This must be
    rejected instead of silently reported as durable.
    """

    tracer = Tracer()
    tracer.add_handler(ConsoleTraceHandler())
    runtime = PatternRuntime(tracer=tracer, execution_id="exec-console-only")

    with pytest.raises(CheckpointPersistenceError) as exc_info:
        await runtime.checkpoint(
            "before_llm",
            context=ExecutionContext(execution_id="exec-console-only"),
            pattern=SimpleNamespace(status="running"),
        )

    assert "no checkpoint-reading handler" in str(exc_info.value)


@pytest.mark.asyncio
async def test_runner_can_use_trace_checkpoint_store_for_resume() -> None:
    backend = PersistentTraceBackend()
    store = TraceCheckpointStore(backend)
    agent = Agent(
        name="checkpointed",
        patterns=[CheckpointingPattern()],
        llm=FakeLLM(),
    )
    runner = AgentRunner(agent=agent, tracer=store)

    result = await runner.run(task="Say done", execution_id="exec-runner-store")
    loaded = await store.load_latest_checkpoint("exec-runner-store")

    assert result["success"] is True
    assert loaded is not None
    assert loaded["label"] == "final"
    assert loaded["context"]["messages"][-1]["content"] == "done"


class RaisingCheckpointReader:
    """Sole checkpoint-capable handler; a second handler must never run."""

    def __init__(self, error: Exception) -> None:
        self.error = error

    async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any]:
        del execution_id
        raise self.error


class UnreachableCheckpointReader:
    def __init__(self) -> None:
        self.calls = 0

    async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any] | None:
        del execution_id
        self.calls += 1
        raise AssertionError("a prior capable reader already answered")


@pytest.mark.asyncio
async def test_tracer_returns_first_reader_result_verbatim_including_none() -> None:
    """The first capable handler's result is authoritative -- Tracer must not
    probe a second handler after a valid ``None`` (absent) result."""
    tracer = Tracer()

    class NoneReturningHandler:
        async def load_latest_checkpoint(
            self, execution_id: str
        ) -> dict[str, Any] | None:
            del execution_id
            return None

    unreachable = UnreachableCheckpointReader()
    tracer.add_handler(NoneReturningHandler())
    tracer.add_handler(unreachable)

    result = await tracer.load_latest_checkpoint("exec-first-reader-wins")

    assert result is None
    assert unreachable.calls == 0


@pytest.mark.asyncio
async def test_tracer_propagates_first_reader_exception() -> None:
    """A raising first reader must abort the lookup, not fall through to a
    second handler -- today's production stacks all run a single reader, so
    treating a raise as "try the next one" would hide the failure."""
    tracer = Tracer()
    unreachable = UnreachableCheckpointReader()
    tracer.add_handler(RaisingCheckpointReader(CheckpointUnavailableError("down")))
    tracer.add_handler(unreachable)

    with pytest.raises(CheckpointUnavailableError):
        await tracer.load_latest_checkpoint("exec-raising-reader")

    assert unreachable.calls == 0


@pytest.mark.asyncio
async def test_read_latest_checkpoint_payload_propagates_reader_exception() -> None:
    reader = RaisingCheckpointReader(CheckpointUnavailableError("down"))

    with pytest.raises(CheckpointUnavailableError):
        await read_latest_checkpoint_payload(reader, "exec-raising-probe")


@pytest.mark.asyncio
async def test_trace_checkpoint_store_propagates_reader_exception() -> None:
    store = TraceCheckpointStore(
        RaisingCheckpointReader(CheckpointUnavailableError("down"))
    )

    with pytest.raises(CheckpointUnavailableError):
        await store.load_latest_checkpoint("exec-raising-store")


@pytest.mark.asyncio
async def test_trace_checkpoint_store_returns_none_without_raising() -> None:
    """A wrapped reader's authoritative ``None`` must pass through as
    absence, not be misread as an unrecognized (corrupt) shape."""

    class NoneReturningReader:
        async def load_latest_checkpoint(
            self, execution_id: str
        ) -> dict[str, Any] | None:
            del execution_id
            return None

    store = TraceCheckpointStore(NoneReturningReader())

    assert await store.load_latest_checkpoint("exec-none-passthrough") is None


@pytest.mark.asyncio
async def test_trace_checkpoint_store_unrecognized_shape_raises_corrupt() -> None:
    class UnrecognizedShapeReader:
        async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any]:
            del execution_id
            return {"unexpected": "shape"}

    store = TraceCheckpointStore(UnrecognizedShapeReader())

    with pytest.raises(CheckpointCorruptError):
        await store.load_latest_checkpoint("exec-unrecognized-shape")


@pytest.mark.asyncio
async def test_trace_checkpoint_store_readable_type_without_snapshot_raises_corrupt() -> (
    None
):
    class NoSnapshotReader:
        async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any]:
            return {
                "checkpoint_type": CHECKPOINT_TYPE,
                "root_execution_id": execution_id,
            }

    store = TraceCheckpointStore(NoSnapshotReader())

    with pytest.raises(CheckpointCorruptError):
        await store.load_latest_checkpoint("exec-no-snapshot")


def test_checkpoint_access_refused_error_reason_defaults_to_active_run() -> None:
    """Existing call sites that construct this error without ``reason`` keep
    working, and get the most common refusal classification for free."""
    error = CheckpointAccessRefusedError("refused")
    assert error.reason == "active_run"


def test_checkpoint_access_refused_error_carries_an_explicit_reason() -> None:
    error = CheckpointAccessRefusedError("refused", reason="lease_mismatch")
    assert error.reason == "lease_mismatch"