Enterprise agent platform for building, deploying, and operating AI teammates.
from __future__ import annotations
import asyncio
import copy
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from xagent.core.agent.context import ContextManager
from xagent.core.agent.runner import (
AgentRunner,
UserMessageInjectionConflictError,
UserMessageInjectionOutcome,
UserMessageInjectionRejectedError,
)
from xagent.core.agent.runtime import ExecutionInterrupted, PatternRuntime
@pytest.fixture
def live():
manager = ContextManager()
context = manager.create_context("atomic")
context.add_user_message("original")
tracer = SimpleNamespace(
load_latest_checkpoint=AsyncMock(return_value=None), checkpoint=AsyncMock()
)
runner = AgentRunner(
SimpleNamespace(llm=None), tracer=tracer, context_manager=manager
)
yield runner, context, tracer
@pytest.mark.asyncio
async def test_candidate_is_invisible_until_persisted(live):
runner, context, tracer = live
entered, release = asyncio.Event(), asyncio.Event()
async def write(**payload):
assert [m["content"] for m in payload["context"]["messages"]] == [
"original",
"new",
]
entered.set()
await release.wait()
tracer.checkpoint.side_effect = write
task = asyncio.create_task(
runner.inject_user_message("atomic", "new", turn_id="turn")
)
await entered.wait()
try:
assert [m.content for m in context.messages] == ["original"]
assert "_pending_user_message_trace_turn_id" not in context.metadata
finally:
release.set()
await task
assert [m.content for m in context.messages] == ["original", "new"]
@pytest.mark.asyncio
async def test_confirmed_failed_write_leaves_no_ghost(live):
runner, context, tracer = live
tracer.checkpoint.side_effect = RuntimeError("write failed")
with pytest.raises(UserMessageInjectionRejectedError, match="write failed"):
await runner.inject_user_message("atomic", "new", turn_id="turn")
assert [m.content for m in context.messages] == ["original"]
tracer.checkpoint.side_effect = None
result = await runner.inject_user_message("atomic", "new", turn_id="turn")
assert result.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert tracer.checkpoint.await_count == 2
@pytest.mark.asyncio
async def test_lost_ack_readback_confirms_message(live):
runner, context, tracer = live
async def write(**payload):
tracer.load_latest_checkpoint.return_value = payload
raise RuntimeError("lost acknowledgement")
tracer.checkpoint.side_effect = write
result = await runner.inject_user_message("atomic", "new", turn_id="turn")
assert result.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert [m.content for m in context.messages] == ["original", "new"]
@pytest.mark.asyncio
async def test_uncertain_write_blocks_stale_checkpoint(live):
runner, context, tracer = live
async def write(**payload):
tracer.load_latest_checkpoint.side_effect = RuntimeError("read unavailable")
raise RuntimeError("write outcome unknown")
tracer.checkpoint.side_effect = write
result = await runner.inject_user_message("atomic", "new", turn_id="turn")
assert result.outcome is UserMessageInjectionOutcome.OUTCOME_UNKNOWN
assert [m.content for m in context.messages] == ["original"]
runtime = PatternRuntime(execution_id="atomic", tracer=tracer)
with pytest.raises(ExecutionInterrupted):
await runtime.checkpoint("late", context=context, pattern=SimpleNamespace())
assert tracer.checkpoint.await_count == 1
@pytest.mark.asyncio
async def test_ordinary_checkpoint_waits_for_injection_publication(live):
runner, context, tracer = live
entered, release = asyncio.Event(), asyncio.Event()
snapshots = []
async def write(**payload):
if payload["label"] == "user_message_injected":
entered.set()
await release.wait()
snapshots.append(payload)
tracer.checkpoint.side_effect = write
injection = asyncio.create_task(
runner.inject_user_message("atomic", "new", turn_id="turn")
)
await entered.wait()
runtime = PatternRuntime(execution_id="atomic", tracer=tracer)
ordinary = asyncio.create_task(
runtime.checkpoint("ordinary", context=context, pattern=SimpleNamespace())
)
await asyncio.sleep(0)
assert not ordinary.done()
# The executing pattern can still append output while persistence awaits.
context.add_assistant_message("concurrent output")
release.set()
await asyncio.gather(injection, ordinary)
assert [m.content for m in context.messages] == [
"original",
"concurrent output",
"new",
]
assert snapshots[-1]["label"] == "ordinary"
assert snapshots[-1]["context"]["messages"][-1]["content"] == "new"
@pytest.mark.asyncio
async def test_concurrent_duplicate_injection_writes_once(live):
runner, context, tracer = live
first, second = await asyncio.gather(
runner.inject_user_message("atomic", "new", turn_id="turn"),
runner.inject_user_message("atomic", "new", turn_id="turn"),
)
assert {first.outcome, second.outcome} == {
UserMessageInjectionOutcome.POSTED_FRESH,
UserMessageInjectionOutcome.POSTED_REPLAY,
}
assert tracer.checkpoint.await_count == 1
assert len(context.messages) == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("cancel_at", ["write", "readback"])
async def test_cancelled_uncertain_injection_blocks_stale_writes(live, cancel_at):
runner, context, tracer = live
entered = asyncio.Event()
async def block():
entered.set()
await asyncio.Event().wait()
async def write(**payload):
await block()
# AsyncMock's side_effect must itself await the suspended read.
async def read(_):
await block()
async def fail_write(**payload):
tracer.load_latest_checkpoint.side_effect = read
raise RuntimeError("uncertain")
tracer.checkpoint.side_effect = write if cancel_at == "write" else fail_write
task = asyncio.create_task(
runner.inject_user_message("atomic", "new", turn_id="turn")
)
await entered.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert [m.content for m in context.messages] == ["original"]
runtime = PatternRuntime(execution_id="atomic", tracer=tracer)
with pytest.raises(ExecutionInterrupted):
await runtime.checkpoint("late", context=context, pattern=SimpleNamespace())
@pytest.mark.asyncio
async def test_concurrent_cold_injections_share_context(live):
runner, context, tracer = live
payload = {"context": context.to_dict()}
runner.context_manager.remove_context("atomic")
reads, both = 0, asyncio.Event()
async def read(_):
nonlocal reads
reads += 1
if reads == 2:
both.set()
await both.wait()
return payload
async def write(**written):
nonlocal payload
# A real store suspends, letting the other reader publish meanwhile.
await asyncio.sleep(0)
payload = written
tracer.load_latest_checkpoint.side_effect = read
tracer.checkpoint.side_effect = write
a, b = await asyncio.gather(
runner.inject_user_message(
"atomic", "one", turn_id="one", request_interrupt=False
),
runner.inject_user_message(
"atomic", "two", turn_id="two", request_interrupt=False
),
)
assert a.context is b.context
# No run holds the restored context, so the last injection evicts it.
assert runner.context_manager.get_context("atomic") is None
assert {m.content for m in a.context.messages} == {"original", "one", "two"}
assert {
m["content"] for m in tracer.checkpoint.call_args.kwargs["context"]["messages"]
} == {"original", "one", "two"}
@pytest.mark.asyncio
async def test_cancel_before_write_does_not_poison_context(live):
from xagent.core.agent.context.execution import context_checkpoint_gate
runner, context, tracer = live
gate = context_checkpoint_gate(context)
async with gate.shared():
task = asyncio.create_task(
runner.inject_user_message("atomic", "new", turn_id="turn")
)
await asyncio.sleep(0)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert not gate.injection_uncertain
tracer.checkpoint.assert_not_awaited()
result = await runner.inject_user_message("atomic", "new", turn_id="turn")
assert result.outcome is UserMessageInjectionOutcome.POSTED_FRESH
@pytest.mark.asyncio
@pytest.mark.parametrize(
"payload", ["malformed", {"context": {}}, {"context": {"messages": ["bad"]}}]
)
async def test_malformed_readback_cannot_prove_absence(live, payload):
runner, context, tracer = live
async def write(**_):
tracer.load_latest_checkpoint.return_value = payload
raise RuntimeError("unknown write")
tracer.checkpoint.side_effect = write
result = await runner.inject_user_message("atomic", "new", turn_id="turn")
assert result.outcome is UserMessageInjectionOutcome.OUTCOME_UNKNOWN
assert [m.content for m in context.messages] == ["original"]
@pytest.mark.asyncio
async def test_live_unknown_stops_run_and_explicit_resume_reloads_checkpoint(live):
runner, context, tracer = live
entered, release = asyncio.Event(), asyncio.Event()
durable = {"context": context.to_dict()}
class Pattern:
async def run(self, *, context, runtime, **kwargs):
entered.set()
await release.wait()
await runtime.checkpoint("after", context=context, pattern=self)
return {"success": True, "output": "done"}
runner.agent.patterns = [Pattern()]
operation = asyncio.create_task(
runner.run("original", execution_id="atomic", checkpoint=durable)
)
await entered.wait()
async def ambiguous_write(**payload):
nonlocal durable
durable = payload
tracer.load_latest_checkpoint.side_effect = RuntimeError("read unavailable")
raise RuntimeError("ack lost")
tracer.checkpoint.side_effect = ambiguous_write
try:
outcome = await runner.inject_user_message("atomic", "new", turn_id="turn")
assert outcome.outcome is UserMessageInjectionOutcome.OUTCOME_UNKNOWN
release.set()
result = await operation
assert result["status"] == "interrupted"
assert tracer.checkpoint.await_count == 1
tracer.load_latest_checkpoint.side_effect = None
tracer.load_latest_checkpoint.return_value = durable
tracer.checkpoint.side_effect = None
result = await runner.resume("atomic")
assert result["success"]
restored = result["context"]
assert [m.content for m in restored.messages].count("new") == 1
finally:
release.set()
operation.cancel()
await asyncio.gather(operation, return_exceptions=True)
@pytest.mark.asyncio
@pytest.mark.parametrize("cancel_at", ["callback", "watermark"])
async def test_post_commit_cancellation_still_interrupts_live_run(live, cancel_at):
from xagent.core.agent.runner import ExecutionControl
runner, context, tracer = live
entered = asyncio.Event()
async def callback(**kwargs):
if cancel_at == "callback":
entered.set()
await asyncio.Event().wait()
else:
context.metadata["_user_message_trace_watermark"] = "turn"
async def write(**payload):
if payload["label"] == "user_message_trace_watermark":
entered.set()
await asyncio.Event().wait()
runner.callbacks = [SimpleNamespace(on_user_message_posted=callback)]
tracer.checkpoint.side_effect = write
runtime = PatternRuntime(execution_id="atomic", tracer=tracer)
runner._active_controls["atomic"] = ExecutionControl(runtime=runtime, task=None)
operation = asyncio.create_task(
runner.inject_user_message("atomic", "new", turn_id="turn")
)
try:
await asyncio.wait_for(entered.wait(), 5)
operation.cancel()
with pytest.raises(asyncio.CancelledError):
await operation
assert await runtime.should_interrupt()
assert context.messages[-1].content == "new"
finally:
operation.cancel()
await asyncio.gather(operation, return_exceptions=True)
@pytest.mark.asyncio
async def test_run_completion_waits_for_in_flight_injection(live):
runner, context, tracer = live
running, finish, writing, drain = (asyncio.Event() for _ in range(4))
class Pattern:
async def run(self, **kwargs):
running.set()
await finish.wait()
return {"success": True, "output": "prior output"}
runner.agent.patterns = [Pattern()]
operation = asyncio.create_task(
runner.run(
"original", execution_id="atomic", checkpoint={"context": context.to_dict()}
)
)
await running.wait()
async def write(**payload):
writing.set()
await drain.wait()
tracer.load_latest_checkpoint.side_effect = RuntimeError("read unavailable")
raise RuntimeError("lost acknowledgement")
tracer.checkpoint.side_effect = write
injection = asyncio.create_task(
runner.inject_user_message("atomic", "new", turn_id="turn")
)
try:
await writing.wait()
finish.set()
await asyncio.sleep(0)
assert not operation.done()
drain.set()
assert (await injection).outcome is UserMessageInjectionOutcome.OUTCOME_UNKNOWN
result = await operation
assert result.get("status") != "interrupted"
assert result["injection_outcome_unknown"] is True
assert result["success"]
assert result["output"] == "prior output"
assert [
m.content for m in runner.context_manager.get_context("atomic").messages
].count("new") == 0
finally:
finish.set()
drain.set()
await asyncio.gather(operation, injection, return_exceptions=True)
@pytest.mark.asyncio
async def test_live_input_after_result_publication_defers_until_old_run_finishes(live):
runner, context, tracer = live
publishing, finish = asyncio.Event(), asyncio.Event()
async def on_run_end(**kwargs):
publishing.set()
await finish.wait()
runner.callbacks = [SimpleNamespace(on_run_end=on_run_end)]
runner.agent.patterns = [
SimpleNamespace(run=AsyncMock(return_value={"success": True, "output": "done"}))
]
operation = asyncio.create_task(
runner.run(
"original", execution_id="atomic", checkpoint={"context": context.to_dict()}
)
)
try:
await publishing.wait()
result = await runner.inject_user_message("atomic", "new", turn_id="turn")
assert result.outcome is UserMessageInjectionOutcome.NOT_POSTED
tracer.checkpoint.assert_not_awaited()
finish.set()
finished = await operation
assert finished["success"]
# The finished run's context was evicted; the deferred input restores
# from the checkpoint.
assert runner.context_manager.get_context("atomic") is None
tracer.load_latest_checkpoint.return_value = {
"context": finished["context"].to_dict()
}
# The existing deferred path posts without interrupting the finished run.
result = await runner.inject_user_message(
"atomic", "new", turn_id="turn", request_interrupt=False
)
assert result.outcome is UserMessageInjectionOutcome.POSTED_FRESH
finally:
finish.set()
await asyncio.gather(operation, return_exceptions=True)
@pytest.mark.asyncio
async def test_explicit_new_input_reloads_uncertain_idle_context(live):
from xagent.core.agent.context.execution import context_checkpoint_gate
runner, old_context, tracer = live
durable = None
async def write(**payload):
nonlocal durable
durable = payload
tracer.load_latest_checkpoint.side_effect = RuntimeError("unavailable")
raise RuntimeError("lost ack")
tracer.checkpoint.side_effect = write
first = await runner.inject_user_message(
"atomic", "first", turn_id="first", request_interrupt=False
)
assert first.outcome is UserMessageInjectionOutcome.OUTCOME_UNKNOWN
tracer.load_latest_checkpoint.side_effect = None
tracer.load_latest_checkpoint.return_value = durable
tracer.checkpoint.side_effect = None
second = await runner.inject_user_message(
"atomic", "second", turn_id="second", request_interrupt=False
)
assert second.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert second.context is not old_context
assert [m.content for m in second.context.messages] == [
"original",
"first",
"second",
]
assert context_checkpoint_gate(old_context).injection_uncertain
assert tracer.checkpoint.await_count == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("active_run", [False, True])
async def test_fenced_context_rejects_later_input_without_writing(live, active_run):
from xagent.core.agent.context.execution import context_checkpoint_gate
from xagent.core.agent.runner import track_user_message_injection
runner, context, tracer = live
async def write(**payload):
tracer.load_latest_checkpoint.side_effect = RuntimeError("unavailable")
raise RuntimeError("lost ack")
tracer.checkpoint.side_effect = write
first = await runner.inject_user_message("atomic", "first", turn_id="first")
assert first.outcome is UserMessageInjectionOutcome.OUTCOME_UNKNOWN
assert context_checkpoint_gate(context).injection_uncertain
tracer.load_latest_checkpoint.side_effect = None
tracer.checkpoint.side_effect = None
if active_run:
runner._active_controls["atomic"] = SimpleNamespace(
runtime=SimpleNamespace(last_checkpoint=None)
)
with track_user_message_injection() as attempt:
second = await runner.inject_user_message(
"atomic",
"second",
turn_id="second",
# A live interrupt is fenced even when no run is active; a deferred
# input is fenced only while the old execution still runs.
request_interrupt=not active_run,
)
assert second.outcome is UserMessageInjectionOutcome.REJECTED_RETRYABLE
assert attempt.outcome is UserMessageInjectionOutcome.REJECTED_RETRYABLE
# Truthy so that no caller mistakes it for a deferrable NOT_POSTED.
assert second.outcome
assert second.context is context
assert runner.context_manager.get_context("atomic") is context
assert [m.content for m in context.messages] == ["original"]
assert context_checkpoint_gate(context).injection_uncertain
assert tracer.checkpoint.await_count == 1
@pytest.mark.asyncio
async def test_confirmed_readback_does_not_leave_transient_unknown_result(live):
from xagent.core.agent.context.execution import context_checkpoint_gate
runner, context, tracer = live
started, interrupt, reading, release = (asyncio.Event() for _ in range(4))
payload = None
class Pattern:
async def run(self, **kwargs):
started.set()
await interrupt.wait()
raise ExecutionInterrupted("user pause")
runner.agent.patterns = [Pattern()]
operation = asyncio.create_task(
runner.run(
"original", execution_id="atomic", checkpoint={"context": context.to_dict()}
)
)
await started.wait()
async def read(_):
reading.set()
await release.wait()
return payload
async def write(**candidate):
nonlocal payload
payload = candidate
tracer.load_latest_checkpoint.side_effect = read
raise RuntimeError("lost ack")
tracer.checkpoint.side_effect = write
injection = asyncio.create_task(
runner.inject_user_message("atomic", "new", turn_id="new")
)
try:
await reading.wait()
interrupt.set()
gate = context_checkpoint_gate(runner.context_manager.get_context("atomic"))
async def finalizer_waiting():
while not gate._waiters:
await asyncio.sleep(0)
await asyncio.wait_for(finalizer_waiting(), 2)
release.set()
assert (await injection).outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert (await operation)["injection_outcome_unknown"] is False
finally:
release.set()
interrupt.set()
await asyncio.gather(operation, injection, return_exceptions=True)
@pytest.mark.asyncio
async def test_exceptional_run_exit_closes_live_admission(live):
from xagent.core.agent.checkpoint import CheckpointPersistenceError
runner, context, tracer = live
class Pattern:
async def run(self, **kwargs):
raise CheckpointPersistenceError("failed checkpoint")
runner.agent.patterns = [Pattern()]
with pytest.raises(CheckpointPersistenceError):
await runner.run(
"original", execution_id="atomic", checkpoint={"context": context.to_dict()}
)
posted = await runner.inject_user_message("atomic", "late", turn_id="late")
assert posted.outcome is UserMessageInjectionOutcome.NOT_POSTED
tracer.checkpoint.assert_not_awaited()
@pytest.mark.asyncio
async def test_acceptance_evidence_survives_postwrite_registry_error(live):
from xagent.core.agent.registry import ExecutionRegistry
from xagent.core.agent.runner import track_user_message_injection
runner, context, tracer = live
registry = ExecutionRegistry()
registry.register("atomic", runner)
token = registry.subscribe(lambda _: registry.unsubscribe(token))
with track_user_message_injection() as attempt:
with pytest.raises(RuntimeError, match="dictionary changed"):
await registry.post_user_message("atomic", "new", turn_id="new")
assert attempt.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert context.messages[-1].content == "new"
@pytest.mark.asyncio
@pytest.mark.parametrize("after_write", [False, True])
async def test_cancellation_evidence_comes_from_write_boundary(live, after_write):
from xagent.core.agent.runner import track_user_message_injection
runner, context, tracer = live
entered = asyncio.Event()
async def blocked(*args, **kwargs):
entered.set()
await asyncio.Event().wait()
if after_write:
tracer.checkpoint.side_effect = blocked
else:
tracer.load_latest_checkpoint.side_effect = blocked
with track_user_message_injection() as attempt:
operation = asyncio.create_task(
runner.inject_user_message("atomic", "new", turn_id="new")
)
await entered.wait()
operation.cancel()
with pytest.raises(asyncio.CancelledError):
await operation
assert attempt.outcome is (
UserMessageInjectionOutcome.OUTCOME_UNKNOWN
if after_write
else UserMessageInjectionOutcome.NOT_POSTED
)
@pytest.mark.asyncio
async def test_write_only_checkpoint_wrapper_cannot_claim_authoritative_absence(live):
from xagent.core.agent.checkpoint import (
CheckpointUnavailableError,
TraceCheckpointStore,
)
runner, context, tracer = live
backend = SimpleNamespace(checkpoint=AsyncMock())
runner.tracer = TraceCheckpointStore(backend)
with pytest.raises(CheckpointUnavailableError, match="no readable backend"):
await runner.inject_user_message("atomic", "new", turn_id="new")
backend.checkpoint.assert_not_awaited()
_O = UserMessageInjectionOutcome
@pytest.mark.parametrize(
"attempt,posted,error,expected",
[
# Returned outcomes.
(_O.POSTED_FRESH, _O.POSTED_FRESH, None, "ACCEPTED"),
(_O.POSTED_REPLAY, _O.POSTED_REPLAY, None, "ACCEPTED"),
(_O.OUTCOME_UNKNOWN, _O.OUTCOME_UNKNOWN, None, "UNKNOWN"),
(_O.REJECTED_RETRYABLE, _O.REJECTED_RETRYABLE, None, "NOT_ACCEPTED_RETRYABLE"),
(_O.NOT_POSTED, _O.NOT_POSTED, None, "DEFER"),
# A service layer that forwards the value without recording evidence.
(_O.NOT_POSTED, _O.POSTED_FRESH, None, "ACCEPTED"),
(_O.NOT_POSTED, None, None, "DEFER"),
# Recorded acceptance wins over any later error, including cancellation.
(_O.POSTED_FRESH, None, asyncio.CancelledError(), "ACCEPTED"),
(_O.POSTED_REPLAY, None, RuntimeError("registry event"), "ACCEPTED"),
(_O.OUTCOME_UNKNOWN, None, asyncio.CancelledError(), "UNKNOWN"),
(_O.OUTCOME_UNKNOWN, None, RuntimeError("late"), "UNKNOWN"),
(
_O.NOT_POSTED,
None,
UserMessageInjectionRejectedError("absent"),
"NOT_ACCEPTED_RETRYABLE",
),
(_O.REJECTED_RETRYABLE, None, RuntimeError("lease"), "NOT_ACCEPTED_RETRYABLE"),
(_O.NOT_POSTED, None, RuntimeError("read failed"), "FAILED_BEFORE_WRITE"),
(_O.NOT_POSTED, None, asyncio.CancelledError(), "FAILED_BEFORE_WRITE"),
(
_O.NOT_POSTED,
None,
UserMessageInjectionConflictError("conflict"),
"FAILED_BEFORE_WRITE",
),
],
)
def test_classify_injection_rules(attempt, posted, error, expected):
from xagent.core.agent.runner import InjectionDisposition, classify_injection
assert classify_injection(attempt, posted=posted, error=error) is getattr(
InjectionDisposition, expected
)
def test_classify_injection_rejects_untyped_outcome():
from xagent.core.agent.runner import classify_injection
with pytest.raises(TypeError):
classify_injection(_O.NOT_POSTED, posted=False)
@pytest.mark.asyncio
async def test_real_runner_outcomes_classify_at_the_write_boundary(live):
"""The classifier agrees with what the runner actually records."""
from xagent.core.agent.runner import (
InjectionDisposition,
classify_injection,
track_user_message_injection,
)
runner, context, tracer = live
# Confirmed absence: the write failed and the read-back shows no turn.
tracer.checkpoint.side_effect = RuntimeError("write failed")
with track_user_message_injection() as attempt:
with pytest.raises(UserMessageInjectionRejectedError) as rejected:
await runner.inject_user_message("atomic", "new", turn_id="absent")
assert (
classify_injection(attempt.outcome, error=rejected.value)
is InjectionDisposition.NOT_ACCEPTED_RETRYABLE
)
# Accepted, then cancelled while tracing the accepted turn.
tracer.checkpoint.side_effect = None
async def cancelled_callback(**kwargs):
raise asyncio.CancelledError()
runner.callbacks = [SimpleNamespace(on_user_message_posted=cancelled_callback)]
with track_user_message_injection() as attempt:
with pytest.raises(asyncio.CancelledError) as cancelled:
await runner.inject_user_message("atomic", "new", turn_id="accepted")
assert context.messages[-1].content == "new"
assert (
classify_injection(attempt.outcome, error=cancelled.value)
is InjectionDisposition.ACCEPTED
)
def _cold_checkpoint(execution_id: str | None) -> dict:
from xagent.core.agent.context import ExecutionContext
context = ExecutionContext(execution_id=execution_id or "unused")
context.add_user_message("original")
data = context.to_dict()
if execution_id is None:
data.pop("execution_id")
return {"context": data}
def _yielding_reader(checkpoint: dict) -> AsyncMock:
async def read(_execution_id):
# Yield so a regression back into the lookup loop times out cleanly.
await asyncio.sleep(0)
return checkpoint
return AsyncMock(side_effect=read)
@pytest.mark.asyncio
async def test_cold_start_adopts_the_lookup_id_when_the_checkpoint_has_none():
manager = ContextManager()
tracer = SimpleNamespace(
load_latest_checkpoint=_yielding_reader(_cold_checkpoint(None)),
checkpoint=AsyncMock(),
)
runner = AgentRunner(
SimpleNamespace(llm=None), tracer=tracer, context_manager=manager
)
result = await asyncio.wait_for(
runner.inject_user_message(
"cold", "new", turn_id="turn", request_interrupt=False
),
5,
)
assert result.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert result.context.execution_id == "cold"
assert manager.get_context("cold") is None
@pytest.mark.asyncio
async def test_cold_start_without_an_id_still_drops_a_derived_output_language():
from xagent.core.agent.language import (
OUTPUT_LANGUAGE_METADATA_KEY,
OUTPUT_LANGUAGE_SOURCE_METADATA_KEY,
)
checkpoint = _cold_checkpoint(None)
checkpoint["context"]["metadata"] = {
OUTPUT_LANGUAGE_METADATA_KEY: "fr",
OUTPUT_LANGUAGE_SOURCE_METADATA_KEY: "detected",
}
manager = ContextManager()
tracer = SimpleNamespace(
load_latest_checkpoint=_yielding_reader(checkpoint),
checkpoint=AsyncMock(),
)
runner = AgentRunner(
SimpleNamespace(llm=None), tracer=tracer, context_manager=manager
)
result = await asyncio.wait_for(
runner.inject_user_message(
"cold", "new", turn_id="turn", request_interrupt=False
),
5,
)
# The id is filled in on a copy; the migration must still reach it.
assert result.context.execution_id == "cold"
assert OUTPUT_LANGUAGE_METADATA_KEY not in result.context.metadata
assert OUTPUT_LANGUAGE_SOURCE_METADATA_KEY not in result.context.metadata
@pytest.mark.asyncio
async def test_cold_start_rejects_a_checkpoint_of_another_execution():
from xagent.core.agent.checkpoint import CheckpointCorruptError
manager = ContextManager()
tracer = SimpleNamespace(
load_latest_checkpoint=_yielding_reader(_cold_checkpoint("other")),
checkpoint=AsyncMock(),
)
runner = AgentRunner(
SimpleNamespace(llm=None), tracer=tracer, context_manager=manager
)
with pytest.raises(CheckpointCorruptError, match="different execution"):
await asyncio.wait_for(
runner.inject_user_message("cold", "new", turn_id="turn"), 5
)
tracer.checkpoint.assert_not_awaited()
@pytest.mark.asyncio
async def test_write_only_tracer_cannot_prove_a_failed_write_absent():
from xagent.core.agent.checkpoint import TraceCheckpointStore
from xagent.core.agent.trace import Tracer
manager = ContextManager()
context = manager.create_context("write-only")
context.add_user_message("original")
class FailingWriter:
async def handle_event(self, event):
raise RuntimeError("write outcome unknown")
tracer = Tracer()
tracer.add_handler(FailingWriter())
runner = AgentRunner(
SimpleNamespace(llm=None),
tracer=TraceCheckpointStore(tracer),
context_manager=manager,
)
# Tracer exposes load_latest_checkpoint but no handler can read, so its
# empty answer must not become "confirmed absent".
result = await runner.inject_user_message("write-only", "new", turn_id="t")
assert result.outcome is UserMessageInjectionOutcome.OUTCOME_UNKNOWN
assert [m.content for m in context.messages] == ["original"]
@pytest.mark.asyncio
async def test_lost_baseline_on_readback_is_uncertain_not_absent(live):
runner, context, tracer = live
tracer.load_latest_checkpoint.return_value = {"context": context.to_dict()}
async def write(**payload):
# The store answers, but the checkpoint read before the write is gone.
tracer.load_latest_checkpoint.return_value = None
raise RuntimeError("write outcome unknown")
tracer.checkpoint.side_effect = write
result = await runner.inject_user_message("atomic", "new", turn_id="turn")
assert result.outcome is UserMessageInjectionOutcome.OUTCOME_UNKNOWN
assert [m.content for m in context.messages] == ["original"]
# --- Context cache lifetime -------------------------------------------------
class _DurableStore:
"""Checkpoint store whose reads return copies of the latest write."""
def __init__(self) -> None:
self.latest: dict | None = None
self.writes: list[str] = []
self.reads = 0
self.read_hook = None
self.fail_labels: set[str] = set()
self.ambiguous_labels: set[str] = set()
self.reads_fail = False
async def load_latest_checkpoint(self, execution_id):
self.reads += 1
if self.reads_fail:
raise RuntimeError("read unavailable")
snapshot = copy.deepcopy(self.latest)
if self.read_hook is not None:
hook, self.read_hook = self.read_hook, None
return await hook(snapshot)
return snapshot
async def checkpoint(self, **payload):
label = payload["label"]
if label in self.fail_labels:
raise RuntimeError(f"{label} write failed")
if label in self.ambiguous_labels:
# The write lands but its acknowledgement and read-back are lost.
self.latest = copy.deepcopy(payload)
self.reads_fail = True
raise RuntimeError("ack lost")
self.writes.append(label)
self.latest = copy.deepcopy(payload)
def seed(self, execution_id: str, *messages: str) -> None:
from xagent.core.agent.context import ExecutionContext
context = ExecutionContext(execution_id=execution_id)
for message in messages:
context.add_user_message(message)
self.latest = {"type": "checkpoint", "context": context.to_dict()}
def messages(self) -> list[str]:
assert self.latest is not None
return [m["content"] for m in self.latest["context"]["messages"]]
class _AnsweringPattern:
"""Checkpoints like a real pattern, then returns an answer the runner appends."""
def __init__(self, answer: str = "A answer", *, before_return=None) -> None:
self.answer = answer
self.before_return = before_return
def get_state(self):
return {"marker": "state"}
async def run(self, *, context, runtime, **kwargs):
await runtime.checkpoint("step", context=context, pattern=self)
if self.before_return is not None:
await self.before_return()
return {"success": True, "output": self.answer}
@pytest.fixture
def durable():
store = _DurableStore()
runner = AgentRunner(
SimpleNamespace(llm=None),
tracer=store,
context_manager=ContextManager(),
workspace_enabled=False,
)
yield runner, store
@pytest.mark.asyncio
async def test_follow_up_after_another_process_extended_the_checkpoint(durable):
from xagent.core.agent.context import ExecutionContext
runner, store = durable
runner.agent.patterns = [_AnsweringPattern()]
assert (await runner.run("m1", execution_id="aba"))["success"]
# Process B takes the task over, answers another turn and checkpoints.
other = ExecutionContext.from_dict(store.latest["context"])
other.add_user_message("m2 from B")
other.add_assistant_message("B answer")
store.latest = {**store.latest, "context": other.to_dict()}
posted = await runner.inject_user_message(
"aba", "m3", turn_id="m3", request_interrupt=False
)
assert posted.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert store.messages() == ["m1", "A answer", "m2 from B", "B answer", "m3"]
assert runner.context_manager.get_context("aba") is None
@pytest.mark.asyncio
@pytest.mark.parametrize("stage", ["memory_session", "resume_workspace"])
async def test_setup_failure_before_the_run_leaves_no_cached_context(durable, stage):
from xagent.core.agent.context import ExecutionContext
runner, store = durable
runner.agent.patterns = [_AnsweringPattern()]
kwargs = {}
if stage == "memory_session":
runner.memory_manager = SimpleNamespace(
get_or_create_session=AsyncMock(side_effect=RuntimeError("setup failed"))
)
else:
runner.workspace_enabled = True
runner.workspace_manager = SimpleNamespace(
get_or_create_workspace=AsyncMock(side_effect=RuntimeError("setup failed"))
)
seed = ExecutionContext(execution_id="setup")
seed.add_user_message("m1")
kwargs["checkpoint"] = {"context": seed.to_dict()}
with pytest.raises(RuntimeError, match="setup failed"):
await runner.run("m1", execution_id="setup", **kwargs)
assert runner.context_manager.get_context("setup") is None
@pytest.mark.asyncio
async def test_live_input_without_a_local_run_defers_and_writes_nothing(durable):
runner, store = durable
store.seed("cold", "m1")
posted = await runner.inject_user_message("cold", "live", turn_id="live")
assert posted.outcome is UserMessageInjectionOutcome.NOT_POSTED
assert store.writes == []
assert runner.context_manager.get_context("cold") is None
@pytest.mark.asyncio
async def test_deferred_cold_start_leaves_the_cache_empty(durable):
runner, store = durable
store.seed("cold", "m1")
posted = await runner.inject_user_message(
"cold", "later", turn_id="later", request_interrupt=False
)
assert posted.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert store.messages() == ["m1", "later"]
assert runner.context_manager.get_context("cold") is None
@pytest.mark.asyncio
async def test_fenced_run_keeps_its_context_until_deferred_input_reloads(durable):
runner, store = durable
started, release = asyncio.Event(), asyncio.Event()
async def wait():
started.set()
await release.wait()
runner.agent.patterns = [_AnsweringPattern(before_return=wait)]
operation = asyncio.create_task(runner.run("m1", execution_id="fence"))
try:
await started.wait()
store.ambiguous_labels.add("user_message_injected")
first = await runner.inject_user_message("fence", "uncertain", turn_id="u")
assert first.outcome is UserMessageInjectionOutcome.OUTCOME_UNKNOWN
store.ambiguous_labels.clear()
store.reads_fail = False
release.set()
result = await operation
finally:
release.set()
await asyncio.gather(operation, return_exceptions=True)
assert result["injection_outcome_unknown"] is True
# The fenced context never writes again, not even the tail checkpoint.
assert "run_end_tail" not in store.writes
fenced = runner.context_manager.get_context("fence")
assert fenced is result["context"]
live = await runner.inject_user_message("fence", "live", turn_id="live")
assert live.outcome is UserMessageInjectionOutcome.REJECTED_RETRYABLE
assert runner.context_manager.get_context("fence") is fenced
deferred = await runner.inject_user_message(
"fence", "deferred", turn_id="deferred", request_interrupt=False
)
assert deferred.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert deferred.context is not fenced
assert store.messages() == ["m1", "uncertain", "deferred"]
assert runner.context_manager.get_context("fence") is None
@pytest.mark.asyncio
async def test_context_stays_cached_while_an_injection_outlives_the_run(durable):
runner, store = durable
started, release_run = asyncio.Event(), asyncio.Event()
in_callback, release_callback = asyncio.Event(), asyncio.Event()
async def wait():
started.set()
await release_run.wait()
async def on_user_message_posted(*, message, **_):
if message.content == "I":
in_callback.set()
await release_callback.wait()
runner.agent.patterns = [_AnsweringPattern(before_return=wait)]
runner.callbacks = [SimpleNamespace(on_user_message_posted=on_user_message_posted)]
operation = asyncio.create_task(runner.run("m1", execution_id="held"))
injection = None
try:
await started.wait()
injection = asyncio.create_task(
runner.inject_user_message("held", "I", turn_id="I")
)
await in_callback.wait()
release_run.set()
result = await operation
held = runner.context_manager.get_context("held")
assert held is result["context"]
later = await runner.inject_user_message(
"held", "J", turn_id="J", request_interrupt=False
)
assert later.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert later.context is held
release_callback.set()
assert (await injection).outcome is UserMessageInjectionOutcome.POSTED_FRESH
finally:
release_run.set()
release_callback.set()
await asyncio.gather(
operation, *([injection] if injection else []), return_exceptions=True
)
assert store.messages() == ["m1", "I", "A answer", "J"]
assert runner.context_manager.get_context("held") is None
@pytest.mark.asyncio
async def test_cold_start_reader_racing_an_eviction_reads_again(durable):
runner, store = durable
store.seed("race", "m1")
reading, release = asyncio.Event(), asyncio.Event()
async def stale_read(snapshot):
reading.set()
await release.wait()
return snapshot
store.read_hook = stale_read
reader = asyncio.create_task(
runner.inject_user_message("race", "R", turn_id="R", request_interrupt=False)
)
try:
await reading.wait()
other = await runner.inject_user_message(
"race", "J", turn_id="J", request_interrupt=False
)
assert other.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert runner.context_manager.get_context("race") is None
release.set()
assert (await reader).outcome is UserMessageInjectionOutcome.POSTED_FRESH
finally:
release.set()
await asyncio.gather(reader, return_exceptions=True)
# R discarded the snapshot it read before J's write and read again.
assert store.reads == 3
assert store.messages() == ["m1", "J", "R"]
assert runner.context_manager.get_context("race") is None
assert runner.context_manager._cold_starts == {}
@pytest.mark.asyncio
async def test_completed_run_persists_the_answer_appended_after_its_checkpoint(
durable,
):
runner, store = durable
runner.agent.patterns = [_AnsweringPattern()]
result = await runner.run("m1", execution_id="tail")
assert result["success"]
assert store.writes == ["step", "run_end_tail"]
assert store.messages() == ["m1", "A answer"]
# The pattern state of its last checkpoint is carried over unchanged.
assert store.latest["pattern"] == "_AnsweringPattern"
assert store.latest["pattern_state"] == {"marker": "state"}
posted = await runner.inject_user_message(
"tail", "next", turn_id="next", request_interrupt=False
)
assert posted.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert store.messages() == ["m1", "A answer", "next"]
class _SelfRecordingPattern:
def __init__(self, *, checkpoint: bool) -> None:
self.checkpoint = checkpoint
async def run(self, *, context, runtime, **kwargs):
if self.checkpoint:
context.add_assistant_message("recorded")
await runtime.checkpoint("final", context=context, pattern=self)
return {"success": True, "output": "recorded"}
@pytest.mark.asyncio
@pytest.mark.parametrize("checkpointed", [True, False])
async def test_tail_checkpoint_is_skipped_without_new_state(durable, checkpointed):
runner, store = durable
runner.agent.patterns = [_SelfRecordingPattern(checkpoint=checkpointed)]
assert (await runner.run("m1", execution_id="same"))["success"]
# Unchanged after the pattern's checkpoint, or no checkpoint in this run.
assert store.writes == (["final"] if checkpointed else [])
@pytest.mark.asyncio
async def test_tail_checkpoint_failure_keeps_the_result(durable, caplog):
runner, store = durable
runner.agent.patterns = [_AnsweringPattern()]
store.fail_labels.add("run_end_tail")
with caplog.at_level("WARNING", logger="xagent.core.agent.runner"):
result = await runner.run("m1", execution_id="tail-fails")
assert result["success"] is True
assert result["output"] == "A answer"
assert store.writes == ["step"]
assert "Final context checkpoint failed for tail-fails" in caplog.text
assert runner.context_manager.get_context("tail-fails") is None
@pytest.mark.asyncio
async def test_tail_fact_failure_aborts_completion(durable, monkeypatch):
from xagent.core.agent.checkpoint import ExecutionEventPersistenceError
runner, store = durable
runner.agent.patterns = [_AnsweringPattern()]
on_run_end = AsyncMock()
runner.callbacks = [SimpleNamespace(on_run_end=on_run_end)]
checkpoint = store.checkpoint
async def fail_tail(**payload):
if payload["label"] == "run_end_tail":
raise ExecutionEventPersistenceError("fact write failed")
await checkpoint(**payload)
monkeypatch.setattr(store, "checkpoint", fail_tail)
with pytest.raises(ExecutionEventPersistenceError, match="fact write failed"):
await runner.run("m1", execution_id="tail-fact-fails")
on_run_end.assert_not_awaited()
assert store.writes == ["step"]
assert runner.context_manager.get_context("tail-fact-fails") is None
@pytest.mark.asyncio
async def test_waiting_result_is_not_tail_persisted_so_resume_keeps_waiting(durable):
from xagent.core.agent import ReActPattern
runner, store = durable
class FakeLLM:
def __init__(self, responses):
self.responses = responses
self.calls = []
async def chat(self, **kwargs):
self.calls.append(kwargs)
return self.responses.pop(0)
llm = FakeLLM(
[
{
"tool_calls": [
{
"id": "call-send",
"function": {
"name": "send_message",
"arguments": (
'{"message":"Choose A or B",'
'"message_type":"question","expect_response":true}'
),
},
}
]
}
]
)
runner.agent = SimpleNamespace(llm=llm, patterns=[ReActPattern(max_iterations=2)])
first = await runner.run("Ask", execution_id="waiting")
assert first["status"] == "waiting_for_user"
# The runner appended the question in memory only; ReAct's checkpoint
# records the message count a reply must exceed.
assert "run_end_tail" not in store.writes
resumed = await runner.resume("waiting")
assert resumed["status"] == "waiting_for_user"
assert len(llm.calls) == 1
@pytest.mark.asyncio
async def test_watermark_repersist_skips_a_context_replaced_meanwhile(durable):
from xagent.core.agent.context import ExecutionContext
runner, store = durable
store.seed("replaced", "m1")
in_callback, release = asyncio.Event(), asyncio.Event()
async def on_user_message_posted(*, context, **_):
in_callback.set()
await release.wait()
context.metadata["_user_message_trace_watermark"] = "traced"
runner.callbacks = [SimpleNamespace(on_user_message_posted=on_user_message_posted)]
injection = asyncio.create_task(
runner.inject_user_message(
"replaced", "I", turn_id="I", request_interrupt=False
)
)
try:
await in_callback.wait()
# A resumed run caches a newer context and checkpoints past I.
newer = ExecutionContext.from_dict(store.latest["context"])
newer.add_assistant_message("resumed answer")
runner.context_manager.set_context(newer)
store.latest = {**store.latest, "context": newer.to_dict()}
release.set()
result = await injection
finally:
release.set()
await asyncio.gather(injection, return_exceptions=True)
assert result.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert store.writes == ["user_message_injected"]
assert store.messages() == ["m1", "I", "resumed answer"]
assert runner.context_manager.get_context("replaced") is newer
runner.context_manager.remove_context("replaced")