Enterprise agent platform for building, deploying, and operating AI teammates.
from __future__ import annotations
import asyncio
import logging
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import httpx
import openai
import pytest
from xagent.core.agent import (
Agent,
ContextManager,
DAGPattern,
ExecutionContext,
ExecutionPlan,
PatternRuntime,
PlanStep,
TraceEventCallback,
)
from xagent.core.agent import runtime as runtime_module
from xagent.core.agent.attachments import build_image_context_references
from xagent.core.agent.checkpoint import (
CheckpointCorruptError,
CheckpointPersistenceError,
CheckpointUnavailableError,
)
from xagent.core.agent.context.execution import (
COMPACT_THRESHOLD_SOURCE_CONTEXT_WINDOW,
COMPACT_THRESHOLD_SOURCE_DEFAULT,
TOOL_EVIDENCE_REMOVED_METADATA_KEY,
TURN_CONTEXT_HEADER,
context_checkpoint_gate,
tool_evidence_state,
)
from xagent.core.agent.context.memory_retrieval import (
MEMORY_CONTEXT_METADATA_KEY,
MEMORY_RETRIEVAL_SCOPE_FINGERPRINT_METADATA_KEY,
RETRIEVED_MEMORIES_METADATA_KEY,
)
from xagent.core.agent.interruption import InterruptionReason
from xagent.core.agent.language import (
OUTPUT_LANGUAGE_METADATA_KEY,
OUTPUT_LANGUAGE_SOURCE_METADATA_KEY,
reset_output_language_to_request_context,
)
from xagent.core.agent.pattern.dag import dag as dag_module
from xagent.core.agent.runner import AgentRunner, UserMessageInjectionOutcome
from xagent.core.agent.runtime import LLMCallInterrupted
from xagent.core.execution_scope import ExecutionScope, ExecutionScopeContext
from xagent.core.guardrails import input as input_review
from xagent.core.guardrails.output import bind_output_review_agent
from xagent.core.model.chat.basic.xinference import XinferenceLLM
from xagent.core.model.chat.exceptions import (
LLMEmptyContentError,
LLMTimeoutError,
ModelProviderError,
ModelProviderRetryableError,
)
from xagent.core.model.chat.routing_context import xinference_routing_key
from xagent.core.task_runtime import PREFERRED_INPUT_MODALITIES_METADATA_KEY
from xagent.core.user_context import current_user_id
def _without_turn_context(content: object) -> str:
"""The message text without the per-turn context appended to the request."""
return str(content).split(f"\n\n{TURN_CONTEXT_HEADER}", 1)[0]
def _all_text(messages: list[dict[str, Any]]) -> str:
return "\n\n".join(str(message.get("content") or "") for message in messages)
@pytest.fixture(autouse=True)
def reset_compact_warnings(monkeypatch: pytest.MonkeyPatch) -> None:
# The once-per-model warning set is process-global; isolate it per test.
monkeypatch.setattr(runtime_module, "_COMPACT_WINDOW_WARNED_MODELS", set())
@dataclass
class FakeWorkspace:
id: str
workspace_dir: Path
input_dir: Path
output_dir: Path
temp_dir: Path
allowed_external_dirs: list[Path]
class FakeWorkspaceManager:
def __init__(self, tmp_path: Path) -> None:
self.tmp_path = tmp_path
self.calls: list[dict[str, Any]] = []
self.thread_ids: list[int] = []
def get_or_create_workspace(
self,
base_dir: str,
task_id: str,
allowed_external_dirs: list[str] | None = None,
scope_segments: tuple[str, ...] = (),
) -> FakeWorkspace:
self.thread_ids.append(threading.get_ident())
self.calls.append(
{
"base_dir": base_dir,
"task_id": task_id,
"allowed_external_dirs": allowed_external_dirs,
}
)
workspace_dir = self.tmp_path / task_id
return FakeWorkspace(
id=task_id,
workspace_dir=workspace_dir,
input_dir=workspace_dir / "input",
output_dir=workspace_dir / "output",
temp_dir=workspace_dir / "temp",
allowed_external_dirs=[Path(path) for path in allowed_external_dirs or []],
)
class FakeMemoryManager:
def __init__(self) -> None:
self.calls: list[dict[str, Any]] = []
def get_or_create_session(
self,
*,
execution_id: str,
user_id: str | None = None,
session_id: str | None = None,
) -> dict[str, Any]:
self.calls.append(
{
"execution_id": execution_id,
"user_id": user_id,
"session_id": session_id,
}
)
return {
"session_id": session_id or f"memory-{execution_id}",
"snapshot": {"summary": f"resume {execution_id}"},
}
class AsyncMemoryManager(FakeMemoryManager):
async def get_or_create_session(
self,
*,
execution_id: str,
user_id: str | None = None,
session_id: str | None = None,
) -> dict[str, Any]:
return super().get_or_create_session(
execution_id=execution_id,
user_id=user_id,
session_id=session_id,
)
class FakePattern:
def __init__(self, result: dict[str, Any]) -> None:
self.result = result
self.calls: list[dict[str, Any]] = []
async def run(self, **kwargs: Any) -> dict[str, Any]:
self.calls.append(kwargs)
return dict(self.result)
class CheckpointingPattern(FakePattern):
"""Checkpoints once, as real patterns do, so later input can restore it."""
async def run(self, **kwargs: Any) -> dict[str, Any]:
await kwargs["runtime"].checkpoint(
"before_llm", context=kwargs["context"], pattern=self
)
return await super().run(**kwargs)
class FailingPattern:
def __init__(self, error: str) -> None:
self.error = error
async def run(self, **_: Any) -> dict[str, Any]:
return {"success": False, "error": self.error}
class LLMInterruptedPattern:
async def run(self, **_: Any) -> dict[str, Any]:
raise LLMCallInterrupted("paused during LLM call")
class StatefulPattern:
def __init__(self) -> None:
self.state: dict[str, Any] = {}
self.calls: list[dict[str, Any]] = []
def load_state(self, state: dict[str, Any]) -> None:
self.state = state
async def run(self, **kwargs: Any) -> dict[str, Any]:
self.calls.append(kwargs)
return {
"success": True,
"output": self.state["output"],
"message_count": len(kwargs["context"].messages),
}
class MemoryCheckpointPattern(StatefulPattern):
def __init__(self, *, nested_auto: bool = False) -> None:
super().__init__()
self.nested_auto = nested_auto
self.active_step_contexts: dict[str, dict[str, Any]] = {}
if nested_auto:
del self.active_step_contexts
self.dag_state: dict[str, Any] | None = None
self.child_system_context: str | None = None
def load_state(self, state: dict[str, Any]) -> None:
super().load_state(state)
if self.nested_auto:
self.dag_state = state.get("dag_state")
else:
self.active_step_contexts = dict(state.get("active_step_contexts", {}))
async def run(self, **kwargs: Any) -> dict[str, Any]:
contexts = (
(self.dag_state or {}).get("active_step_contexts", {})
if self.nested_auto
else self.active_step_contexts
)
child = ExecutionContext.from_dict(contexts["step_1"])
self.child_system_context = child._system_context()
self.calls.append(kwargs)
return {"success": True, "output": "ok"}
class InjectingPattern:
def __init__(self, runner: AgentRunner, execution_id: str) -> None:
self.runner = runner
self.execution_id = execution_id
async def run(self, *, context: ExecutionContext, **_: Any) -> dict[str, Any]:
injected = await self.runner.inject_user_message(
self.execution_id,
"Injected while resumed.",
request_interrupt=False,
)
return {
"success": True,
"same_context": injected.context is context,
"messages": [message.content for message in context.messages],
}
class TrackingCallback:
def __init__(self) -> None:
self.events: list[tuple[str, str]] = []
async def on_run_start(self, **payload: Any) -> None:
context = payload["context"]
self.events.append(("start", context.execution_id))
async def on_run_end(self, **payload: Any) -> None:
context = payload["context"]
self.events.append(("end", context.execution_id))
class FailingUserMessageCallback:
async def on_user_message_posted(self, **_: Any) -> None:
raise RuntimeError("trace callback failed")
class StatusAwareTeardownTool:
def __init__(self) -> None:
self.teardown_calls: list[tuple[str | None, str | None]] = []
async def setup(self, task_id: str | None = None) -> None:
return None
async def teardown(
self,
task_id: str | None = None,
execution_status: str | None = None,
) -> None:
self.teardown_calls.append((task_id, execution_status))
class RecordingTraceEventTracer:
def __init__(self) -> None:
self.events: list[dict[str, Any]] = []
async def trace_event(
self,
event_type: Any,
*,
task_id: str | None = None,
step_id: str | None = None,
data: dict[str, Any] | None = None,
) -> str:
self.events.append(
{
"event_type": getattr(event_type, "value", str(event_type)),
"task_id": task_id,
"step_id": step_id,
"data": data or {},
}
)
return str(len(self.events))
class InterruptingPattern:
def __init__(
self,
runner: AgentRunner,
execution_id: str,
*,
before_interrupt_check: Any | None = None,
) -> None:
self.runner = runner
self.execution_id = execution_id
self.before_interrupt_check = before_interrupt_check
async def run(
self,
*,
context: ExecutionContext,
runtime: PatternRuntime,
**_: Any,
) -> dict[str, Any]:
if callable(self.before_interrupt_check):
maybe_result = self.before_interrupt_check()
if maybe_result is not None:
await maybe_result
else:
self.runner.pause(self.execution_id, reason="pause before step")
if await runtime.should_interrupt():
await runtime.checkpoint(
"interrupted",
context=context,
pattern=self,
status="interrupted",
metadata={"safe_point": "during_pattern"},
)
return {
"success": False,
"status": "interrupted",
"error": runtime.interrupt_reason or "interrupted",
}
return {"success": True, "output": "continued"}
class TracerCheckpointStore:
def __init__(self) -> None:
self.by_execution_id: dict[str, dict[str, Any]] = {}
async def checkpoint(self, **payload: Any) -> None:
self.by_execution_id[str(payload["execution_id"])] = dict(payload)
async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any] | None:
payload = self.by_execution_id.get(execution_id)
return dict(payload) if payload is not None else None
class RecordingCheckpointTracer(RecordingTraceEventTracer, TracerCheckpointStore):
def __init__(self) -> None:
RecordingTraceEventTracer.__init__(self)
TracerCheckpointStore.__init__(self)
class EmptyCanonicalCheckpointStore:
def __init__(self) -> None:
self.legacy_reads = 0
async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any] | None:
del execution_id
return None
def get_latest_checkpoint(self, execution_id: str) -> dict[str, Any] | None:
del execution_id
self.legacy_reads += 1
raise AssertionError("canonical empty result must end checkpoint lookup")
def test_user_message_injection_outcome_truthiness_contract() -> None:
"""Preserve baseline truthiness; consumers handle unknown explicitly first."""
assert UserMessageInjectionOutcome.NOT_POSTED == ""
assert not UserMessageInjectionOutcome.NOT_POSTED
assert UserMessageInjectionOutcome.POSTED_FRESH
assert UserMessageInjectionOutcome.POSTED_REPLAY
assert UserMessageInjectionOutcome.OUTCOME_UNKNOWN
assert UserMessageInjectionOutcome.REJECTED_RETRYABLE
def test_user_message_injection_outcome_member_set_has_not_drifted() -> None:
"""Adding outcomes requires auditing every success and interaction-close path."""
assert {member.name for member in UserMessageInjectionOutcome} == {
"NOT_POSTED",
"POSTED_FRESH",
"POSTED_REPLAY",
"OUTCOME_UNKNOWN",
"REJECTED_RETRYABLE",
}
@pytest.mark.asyncio
async def test_runner_treats_canonical_empty_checkpoint_as_authoritative() -> None:
checkpoint_store = EmptyCanonicalCheckpointStore()
runner = AgentRunner(
agent=Agent(name="checkpoint-reader", patterns=[], llm=None),
tracer=checkpoint_store,
)
result = await runner.inject_user_message(
"missing-execution",
"Continue",
request_interrupt=False,
)
assert result.context is None
assert result.outcome is UserMessageInjectionOutcome.NOT_POSTED
assert checkpoint_store.legacy_reads == 0
class ContextlessCheckpointStore:
"""Returns a recognized checkpoint payload that carries no context.
Every production checkpoint writer persists a ``context`` dict; a
stored payload without one is malformed. The runner must classify it
as corrupt rather than silently building fresh state on resume."""
async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any]:
return {"type": "checkpoint", "execution_id": execution_id}
class UnavailableCheckpointStore:
"""Every read fails -- distinct from ``EmptyCanonicalCheckpointStore``,
whose ``None`` is an authoritative "no checkpoint" the runner may act
on by building fresh state."""
async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any]:
del execution_id
raise CheckpointUnavailableError("checkpoint store unavailable")
class FlakyOnceCheckpointStore:
"""Fails the first read, then behaves like a normal checkpoint store.
Models a transient failure during ``inject_user_message``'s baseline
read: the retry after the failure must actually persist the message,
not find a "ghost" confirmation left behind by the rejected attempt.
"""
def __init__(self) -> None:
self.by_execution_id: dict[str, dict[str, Any]] = {}
self.load_calls = 0
async def checkpoint(self, **payload: Any) -> None:
self.by_execution_id[str(payload["execution_id"])] = dict(payload)
async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any] | None:
self.load_calls += 1
if self.load_calls == 1:
raise CheckpointUnavailableError("transient")
payload = self.by_execution_id.get(execution_id)
return dict(payload) if payload is not None else None
@pytest.mark.asyncio
async def test_run_resume_does_not_build_fresh_context_on_unavailable() -> None:
runner = AgentRunner(
agent=Agent(name="checkpoint-reader", patterns=[], llm=None),
tracer=UnavailableCheckpointStore(),
)
build_context_calls: list[Any] = []
original_build_context = runner._build_context
async def spy_build_context(*args: Any, **kwargs: Any) -> Any:
build_context_calls.append((args, kwargs))
return await original_build_context(*args, **kwargs)
runner._build_context = spy_build_context # type: ignore[method-assign]
with pytest.raises(CheckpointUnavailableError):
await runner.run(
task=None,
execution_id="exec-resume-unavailable",
resume=True,
)
assert build_context_calls == []
assert runner.context_manager.get_context("exec-resume-unavailable") is None
@pytest.mark.asyncio
async def test_inject_user_message_propagates_unavailable() -> None:
"""Distinct from the canonical-empty-checkpoint case above: a read
failure must not be swallowed into the same ``None`` "no checkpoint"
result -- the caller cannot tell a real failure from genuine absence."""
runner = AgentRunner(
agent=Agent(name="checkpoint-reader", patterns=[], llm=None),
tracer=UnavailableCheckpointStore(),
)
with pytest.raises(CheckpointUnavailableError):
await runner.inject_user_message(
"missing-execution-unavailable",
"Continue",
request_interrupt=False,
)
@pytest.mark.asyncio
async def test_inject_rejection_leaves_no_dedupe_residue() -> None:
tracer = FlakyOnceCheckpointStore()
runner = AgentRunner(
agent=Agent(name="checkpoint-reader", patterns=[], llm=None),
tracer=tracer,
)
context = ExecutionContext(execution_id="exec-residue")
runner.context_manager.set_context(context)
with pytest.raises(CheckpointUnavailableError):
await runner.inject_user_message(
"exec-residue",
"Continue",
turn_id="turn-1",
request_interrupt=False,
)
# The rejected attempt must not have mutated the in-memory context.
assert context.messages == []
# A retry must actually persist the message -- it must not find a
# ghost confirmation left behind by the failed attempt above.
result = await runner.inject_user_message(
"exec-residue",
"Continue",
turn_id="turn-1",
request_interrupt=False,
)
assert result.context is context
assert result.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert len(context.messages) == 1
assert tracer.by_execution_id["exec-residue"]["context"]["messages"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"case", ["blocked", "unavailable", "delegated", "builtin", "workforce_builder"]
)
async def test_runner_reviews_input_before_building_context(monkeypatch, case) -> None:
unavailable = case == "unavailable"
hook = AsyncMock(
return_value=input_review.InputReviewResult(
refusal="Review refused.", unavailable=unavailable
)
)
monkeypatch.setattr(input_review, "_review", hook)
runner = AgentRunner(
agent=Agent(
name="reviewed", patterns=[], llm=None, metadata={"agent_type": case}
)
)
if case == "delegated":
with bind_output_review_agent(42):
result = await runner.run("Candidate", execution_id="reviewed")
else:
result = await runner.run("Candidate", execution_id="reviewed")
assert result["success"] is (case == "blocked")
assert result["output"] == "Review refused."
assert runner.context_manager.get_context("reviewed") is None
request = hook.await_args.args[0]
assert request.source == ("new_run" if case in {"blocked", "unavailable"} else case)
assert request.delegated_agent_id == (42 if case == "delegated" else None)
@pytest.mark.asyncio
async def test_rejected_reply_leaves_context_and_checkpoint_untouched(
monkeypatch,
) -> None:
tracer = FlakyOnceCheckpointStore()
runner = AgentRunner(
agent=Agent(name="reviewed", patterns=[], llm=None), tracer=tracer
)
context = ExecutionContext(execution_id="reviewed")
runner.context_manager.set_context(context)
hook = AsyncMock(
return_value=input_review.InputReviewResult(
refusal="Try later.", unavailable=True
)
)
monkeypatch.setattr(input_review, "_review", hook)
with pytest.raises(input_review.InputReviewRejected, match="Try later") as exc:
await runner.inject_user_message(
"reviewed",
display_message="Candidate",
execution_message="Candidate with context",
turn_id="reply",
)
assert exc.value.unavailable
assert context.messages == []
assert tracer.load_calls == 0 and tracer.by_execution_id == {}
request = hook.await_args.args[0]
assert (request.source, request.text, request.execution_text) == (
"reply",
"Candidate",
"Candidate with context",
)
@pytest.mark.asyncio
@pytest.mark.parametrize("timeout", [False, True])
async def test_input_review_contains_host_failure(monkeypatch, caplog, timeout) -> None:
async def hook(_request):
if timeout:
await asyncio.Event().wait()
raise RuntimeError("Private prompt content")
monkeypatch.setattr(input_review, "_review", hook)
monkeypatch.setattr(input_review, "get_input_review_timeout_seconds", lambda: 0)
result = await input_review.review_input(
"run", "Candidate", execution_text="Candidate"
)
assert result.unavailable and result.refusal == input_review.UNAVAILABLE_INPUT
assert "Private prompt content" not in caplog.text
@pytest.mark.asyncio
@pytest.mark.parametrize(
"verdict", [None, input_review.InputReviewResult(unavailable=True)]
)
async def test_input_review_rejects_invalid_or_incomplete_verdict(
monkeypatch, verdict
) -> None:
monkeypatch.setattr(input_review, "_review", AsyncMock(return_value=verdict))
result = await input_review.review_input(
"run", "Candidate", execution_text="Candidate"
)
assert result.unavailable and result.refusal == input_review.UNAVAILABLE_INPUT
@pytest.mark.asyncio
async def test_reply_review_refusal_survives_live_run_finishing(monkeypatch) -> None:
tracer = TracerCheckpointStore()
runner = AgentRunner(
agent=Agent(name="reviewed", patterns=[], llm=None), tracer=tracer
)
context = ExecutionContext(execution_id="reviewed")
runner.context_manager.set_context(context)
async def refuse(_request):
context_checkpoint_gate(context).run_finishing = True
return input_review.InputReviewResult(refusal="Reply refused.")
hook = AsyncMock(side_effect=refuse)
monkeypatch.setattr(input_review, "_review", hook)
with pytest.raises(input_review.InputReviewRejected, match="Reply refused"):
await runner.inject_user_message("reviewed", "Reply", turn_id="turn")
hook.assert_awaited_once()
assert context.messages == [] and tracer.by_execution_id == {}
def test_input_history_filters_both_prompt_representations() -> None:
messages = [
{"role": "user", "content": " Candidate "},
{"role": "user", "content": "Candidate with context"},
{"role": "assistant", "content": "Candidate"},
{"role": "user", "content": "Other question"},
]
excluded = frozenset(
map(input_review.input_fingerprint, ("Candidate", "Candidate with context"))
)
assert input_review.filter_input_history(messages, excluded) == messages[2:]
@pytest.mark.asyncio
async def test_allowed_turn_replays_only_approved_history(monkeypatch) -> None:
hook = AsyncMock(
return_value=input_review.InputReviewResult(
excluded_inputs=frozenset(
{input_review.input_fingerprint("Rejected earlier")}
)
)
)
monkeypatch.setattr(input_review, "_review", hook)
runner = AgentRunner(
agent=Agent(
name="reviewed",
patterns=[FakePattern({"success": True, "output": "Done"})],
llm=None,
)
)
result = await runner.run(
"New turn",
initial_messages=[
{"role": "user", "content": "Approved earlier"},
{"role": "user", "content": "Rejected earlier"},
],
)
assert [
message.content
for message in result["context"].messages
if message.role == "user"
] == ["Approved earlier", "New turn"]
@pytest.mark.asyncio
async def test_reply_review_does_not_hold_checkpoint_gate_and_replay_skips_review(
monkeypatch,
) -> None:
review_started, finish_review = asyncio.Event(), asyncio.Event()
async def review(_request):
review_started.set()
await finish_review.wait()
return input_review.InputReviewResult()
hook = AsyncMock(side_effect=review)
monkeypatch.setattr(input_review, "_review", hook)
runner = AgentRunner(
agent=Agent(name="reviewed", patterns=[], llm=None),
tracer=TracerCheckpointStore(),
)
context = ExecutionContext(execution_id="reviewed")
runner.context_manager.set_context(context)
injection = asyncio.create_task(
runner.inject_user_message(
"reviewed", "Reply", turn_id="turn", request_interrupt=False
)
)
try:
async with asyncio.timeout(5):
await review_started.wait()
async with context_checkpoint_gate(context).shared():
assert not injection.done()
finally:
finish_review.set()
result = await injection
assert result.outcome is UserMessageInjectionOutcome.POSTED_FRESH
replay = await runner.inject_user_message(
"reviewed", "Reply", turn_id="turn", request_interrupt=False
)
assert replay.outcome is UserMessageInjectionOutcome.POSTED_REPLAY
hook.assert_awaited_once()
@pytest.mark.asyncio
async def test_runner_builds_context_and_invokes_pattern(tmp_path: Path) -> None:
event_loop_thread = threading.get_ident()
workspace_manager = FakeWorkspaceManager(tmp_path)
memory_manager = FakeMemoryManager()
callback = TrackingCallback()
pattern = FakePattern({"success": True, "output": "done"})
agent = Agent(
name="writer",
patterns=[pattern],
tools=["local-tool"],
llm="fake-llm",
system_prompt="System prompt",
)
runner = AgentRunner(
agent=agent,
workspace_manager=workspace_manager,
memory_manager=memory_manager,
callbacks=[callback],
workspace_base_dir=str(tmp_path / "workspaces"),
)
result = await runner.run(
task="Write a summary",
execution_id="exec-1",
user_id="user-1",
session_id="session-1",
allowed_external_dirs=[str(tmp_path / "kb")],
extra_tools=["extra-tool"],
metadata={"source": "test"},
)
assert result["success"] is True
assert result["execution_id"] == "exec-1"
context = result["context"]
assert isinstance(context, ExecutionContext)
assert context.system_prompt == "System prompt"
assert context.user_id == "user-1"
assert context.session_id == "session-1"
assert context.workspace_id == "exec-1"
assert context.memory_session_id == "session-1"
assert context.memory_snapshot == {"summary": "resume exec-1"}
assert context.metadata["task"] == "Write a summary"
assert context.metadata["source"] == "test"
assert [message.role for message in context.messages] == ["user", "assistant"]
assert context.messages[0].content == "Write a summary"
assert context.messages[1].content == "done"
# No run is left, so the checkpoint is authoritative again.
assert ContextManager().get_context("exec-1") is None
pattern_call = pattern.calls[0]
assert pattern_call["task"] == "Write a summary"
assert pattern_call["context"] is context
assert pattern_call["tools"] == ["local-tool", "extra-tool"]
assert pattern_call["llm"] == "fake-llm"
assert isinstance(pattern_call["runtime"], PatternRuntime)
assert workspace_manager.calls[0]["task_id"] == "exec-1"
assert workspace_manager.thread_ids[0] != event_loop_thread
assert callback.events == [("start", "exec-1"), ("end", "exec-1")]
@pytest.mark.asyncio
async def test_runner_inserts_synthetic_user_turn_before_a_leading_assistant_initial_message(
tmp_path: Path,
) -> None:
# The marketplace Hire flow seeds a persona greeting as a task's very
# first persisted message (see seed_assistant_message in
# src/xagent/web/api/chat.py) - initial_messages then starts with role
# "assistant" and no prior user turn. Anthropic's Messages API (and
# every claude_compatible provider routed through it) rejects a request
# whose first message isn't role "user", so the runner must correct
# this before it's ever replayed into context.
workspace_manager = FakeWorkspaceManager(tmp_path)
memory_manager = FakeMemoryManager()
pattern = FakePattern({"success": True, "output": "done"})
agent = Agent(name="writer", patterns=[pattern], tools=[], llm="fake-llm")
runner = AgentRunner(
agent=agent,
workspace_manager=workspace_manager,
memory_manager=memory_manager,
workspace_base_dir=str(tmp_path / "workspaces"),
)
result = await runner.run(
task="Let's get started",
execution_id="exec-seed",
user_id="user-1",
initial_messages=[
{
"role": "assistant",
"content": "Hi - I'm Maya, your Social Media Content Manager.",
}
],
)
context = result["context"]
assert [message.role for message in context.messages] == [
"user",
"assistant",
"user",
"assistant",
]
assert context.messages[0].content == "(conversation start)"
assert context.messages[0].metadata.get("_xagent_synthetic") == "leading_user_turn"
assert (
context.messages[1].content
== "Hi - I'm Maya, your Social Media Content Manager."
)
assert context.messages[2].content == "Let's get started"
@pytest.mark.asyncio
async def test_runner_does_not_insert_synthetic_turn_for_user_first_initial_messages(
tmp_path: Path,
) -> None:
workspace_manager = FakeWorkspaceManager(tmp_path)
memory_manager = FakeMemoryManager()
pattern = FakePattern({"success": True, "output": "done"})
agent = Agent(name="writer", patterns=[pattern], tools=[], llm="fake-llm")
runner = AgentRunner(
agent=agent,
workspace_manager=workspace_manager,
memory_manager=memory_manager,
workspace_base_dir=str(tmp_path / "workspaces"),
)
result = await runner.run(
task="Follow up",
execution_id="exec-normal",
user_id="user-1",
initial_messages=[
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hello, how can I help?"},
],
)
context = result["context"]
assert [message.role for message in context.messages] == [
"user",
"assistant",
"user",
"assistant",
]
assert context.messages[0].content == "Hi"
@pytest.mark.asyncio
async def test_runner_passes_waiting_status_to_tool_teardown(tmp_path: Path) -> None:
tool = StatusAwareTeardownTool()
agent = Agent(
name="interactive",
patterns=[
FakePattern(
{
"success": False,
"status": "waiting_for_user",
"message": "Provide the missing value.",
}
)
],
)
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(
task="Run an interactive tool",
execution_id="interaction-task",
extra_tools=[tool],
)
assert result["status"] == "waiting_for_user"
assert tool.teardown_calls == [("interaction-task", "waiting_for_user")]
class LiveStepTasksPattern:
"""A pattern that reports a waiting_for_user exit while its
has_live_step_tasks() predicate is under test control, to pin the
runner-side guard independently of any real pattern implementation."""
def __init__(self, *, live_step_tasks: bool) -> None:
self._live_step_tasks = live_step_tasks
async def run(self, **kwargs: Any) -> dict[str, Any]:
return {
"success": False,
"status": "waiting_for_user",
"message": "Pick one.",
"clarification_draft": {"source": "test"},
}
def has_live_step_tasks(self) -> bool:
return self._live_step_tasks
@pytest.mark.asyncio
async def test_runner_raises_when_waiting_exit_still_has_live_step_tasks(
tmp_path: Path,
) -> None:
agent = Agent(
name="interactive",
patterns=[LiveStepTasksPattern(live_step_tasks=True)],
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
with pytest.raises(AssertionError, match="live step tasks"):
await runner.run(task="Run an interactive tool", execution_id="live-tasks-task")
@pytest.mark.asyncio
async def test_runner_allows_waiting_exit_once_step_tasks_are_clear(
tmp_path: Path,
) -> None:
agent = Agent(
name="interactive",
patterns=[LiveStepTasksPattern(live_step_tasks=False)],
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(
task="Run an interactive tool", execution_id="no-live-tasks-task"
)
assert result["status"] == "waiting_for_user"
@pytest.mark.asyncio
async def test_runner_passes_failed_status_when_tool_setup_raises(
tmp_path: Path,
) -> None:
class SetupFailingTool(StatusAwareTeardownTool):
async def setup(self, task_id: str | None = None) -> None:
raise RuntimeError("setup failed")
tool = SetupFailingTool()
runner = AgentRunner(
agent=Agent(
name="setup-failure",
patterns=[FakePattern({"success": True, "output": "unused"})],
),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
with pytest.raises(RuntimeError, match="setup failed"):
await runner.run(
task="Initialize tools",
execution_id="setup-failure-task",
extra_tools=[tool],
)
assert tool.teardown_calls == [("setup-failure-task", "failed")]
@pytest.mark.asyncio
async def test_runner_awaits_async_memory_manager(tmp_path: Path) -> None:
memory_manager = AsyncMemoryManager()
pattern = FakePattern({"success": True, "output": "done"})
agent = Agent(name="writer", patterns=[pattern])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
memory_manager=memory_manager,
)
result = await runner.run(
task="Write a summary",
execution_id="exec-async-memory",
session_id="session-async",
)
assert result["success"] is True
assert result["context"].memory_session_id == "session-async"
assert result["context"].memory_snapshot == {"summary": "resume exec-async-memory"}
@pytest.mark.asyncio
async def test_runner_tries_multiple_patterns_and_collects_failures(
tmp_path: Path,
) -> None:
first = FailingPattern("first failed")
second = FakePattern({"success": True, "message": "second worked"})
agent = Agent(name="writer", patterns=[first, second])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(task="Recover", execution_id="exec-2")
assert result["success"] is True
assert result["pattern"] == "FakePattern"
context = result["context"]
assert [message.content for message in context.messages] == [
"Recover",
"second worked",
]
class CheckpointFailingPattern:
"""Raises the typed durability error, as a DAG step does when its
checkpoint writer refuses the write."""
def __init__(self) -> None:
self.calls = 0
self.routing_key: str | None = None
async def run(self, **_: Any) -> dict[str, Any]:
self.calls += 1
self.routing_key = xinference_routing_key.get()
raise CheckpointPersistenceError("checkpoint writer unavailable")
class RaisingPattern:
"""Raises an ordinary exception, which stays a recoverable failure."""
def __init__(self) -> None:
self.calls = 0
async def run(self, **_: Any) -> dict[str, Any]:
self.calls += 1
raise RuntimeError("ordinary pattern failure")
@pytest.mark.asyncio
async def test_runner_aborts_the_run_on_a_checkpoint_durability_failure(
tmp_path: Path,
) -> None:
"""A durability failure must not fall through to the next pattern.
The state transition was never committed, so running the fallback could
repeat a non-idempotent side effect the first pattern already performed,
or report success while the checkpoint needed for recovery is missing.
"""
first = CheckpointFailingPattern()
second = FakePattern({"success": True, "message": "second worked"})
agent = Agent(name="writer", patterns=[first, second])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
with pytest.raises(CheckpointPersistenceError):
await runner.run(task="Durability", execution_id="exec-durability")
assert first.calls == 1
assert second.calls == []
assert first.routing_key is not None
assert xinference_routing_key.get() is None
@pytest.mark.asyncio
async def test_runner_still_falls_back_after_an_ordinary_pattern_exception(
tmp_path: Path,
) -> None:
"""The contrast: an ordinary raised exception stays recoverable."""
first = RaisingPattern()
second = FakePattern({"success": True, "message": "second worked"})
agent = Agent(name="writer", patterns=[first, second])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(task="Recover", execution_id="exec-recover-raise")
assert first.calls == 1
assert len(second.calls) == 1
assert result["success"] is True
@pytest.mark.asyncio
async def test_runner_reports_a_terminal_trace_for_a_custom_pattern_that_never_calls_on_pattern_error(
tmp_path: Path,
) -> None:
"""A custom pattern that only raises must still get a terminal trace.
The ``AgentPattern`` interface requires only ``run()``; unlike
``DAGPattern``/``ReActPattern``, a custom pattern is not required to call
``runtime.on_pattern_error()`` itself before letting
``CheckpointPersistenceError`` propagate. Without a fallback report here,
the abort would still correctly propagate to the caller, but no terminal
``trace_error`` would ever be recorded -- an audit/trace visibility gap.
"""
tracer = RecordingTraceEventTracer()
first = CheckpointFailingPattern()
agent = Agent(name="writer", patterns=[first])
runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
with pytest.raises(CheckpointPersistenceError):
await runner.run(task="Durability", execution_id="exec-custom-pattern-trace")
error_events = [
event for event in tracer.events if event["event_type"] == "task_error_general"
]
assert error_events
assert error_events[-1]["data"]["error_type"] == "agent_pattern_error"
@pytest.mark.asyncio
async def test_runner_returns_aggregate_error_when_all_patterns_fail(
tmp_path: Path,
) -> None:
agent = Agent(
name="writer",
patterns=[FailingPattern("first failed"), FailingPattern("second failed")],
)
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(task="Impossible", execution_id="exec-3")
assert result["success"] is False
assert result["patterns_attempted"] == 2
assert len(result["pattern_errors"]) == 2
assert result["context"].messages[0].content == "Impossible"
assert [entry["interruption_reason"] for entry in result["pattern_errors"]] == [
None,
None,
]
assert "interruption_reason" not in result
@pytest.mark.asyncio
async def test_runner_does_not_promote_code_from_only_one_failed_pattern(
tmp_path: Path,
) -> None:
agent = Agent(
name="writer",
patterns=[
FakePattern(
{
"success": False,
"status": "invalid_tool_protocol",
"failure_code": "excel_workbook_access_denied",
}
),
FailingPattern("unclassified failure"),
],
)
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(
task="Impossible", execution_id="exec-mixed-failure-codes"
)
assert result["success"] is False
assert "failure_code" not in result
class ErrorRaisingPattern:
def __init__(self, error: Exception) -> None:
self.error = error
async def run(self, **_: Any) -> dict[str, Any]:
raise self.error
@pytest.mark.asyncio
async def test_runner_attaches_agreed_interruption_reason(tmp_path: Path) -> None:
provider_down = RuntimeError("chat failed")
provider_down.__cause__ = LLMTimeoutError("first token timeout")
agent = Agent(
name="writer",
patterns=[
ErrorRaisingPattern(LLMTimeoutError("timed out")),
ErrorRaisingPattern(provider_down),
],
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Impossible", execution_id="exec-reason-agree")
assert result["success"] is False
assert [entry["interruption_reason"] for entry in result["pattern_errors"]] == [
"llm_unavailable",
"llm_unavailable",
]
assert result["interruption_reason"] == "llm_unavailable"
@pytest.mark.asyncio
async def test_runner_single_raising_pattern_reports_reason(tmp_path: Path) -> None:
agent = Agent(
name="writer", patterns=[ErrorRaisingPattern(LLMEmptyContentError("empty"))]
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Impossible", execution_id="exec-reason-single")
assert result["pattern_errors"][0]["interruption_reason"] == "model_output_invalid"
assert result["pattern_errors"][0]["exception_type"] == "LLMEmptyContentError"
assert result["interruption_reason"] == "model_output_invalid"
def _model_provider_error(
status_code: int = 403, *, cls: type[ModelProviderError] = ModelProviderError
) -> ModelProviderError:
return cls(
prefix="Provider API error",
kind="access_denied",
status_code=status_code,
provider_code="provider_code_4204",
provider_message="Model is decommissioned",
sdk_message=f"Error code: {status_code} - Model is decommissioned",
details=["request_id=req-1"],
)
def _rate_limited_provider_error() -> ModelProviderError:
error = _model_provider_error(429, cls=ModelProviderRetryableError)
request = httpx.Request("POST", "https://provider.example/v1/chat/completions")
error.__cause__ = openai.RateLimitError(
"slow down", response=httpx.Response(429, request=request), body=None
)
return error
@pytest.mark.asyncio
@pytest.mark.parametrize(
"build_error",
[_model_provider_error, _rate_limited_provider_error],
ids=["non_retryable", "retryable_rate_limit"],
)
async def test_runner_single_pattern_model_provider_error_carries_details(
tmp_path: Path, build_error: Any
) -> None:
error = build_error()
agent = Agent(name="writer", patterns=[ErrorRaisingPattern(error)])
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Impossible", execution_id="exec-model-error")
assert result["success"] is False
assert result["diagnostic_error"] == str(error)
assert result["model_error"] == error.structured_fields()
assert result["error"].startswith("All 1 patterns failed")
entry = result["pattern_errors"][0]
assert entry["exception_type"] == type(error).__name__
assert entry["error"] == str(error)
if isinstance(error, ModelProviderRetryableError):
assert result["interruption_reason"] == "llm_unavailable"
else:
assert result.get("interruption_reason") is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"build_patterns",
[
lambda: [ErrorRaisingPattern(RuntimeError("chat failed"))],
lambda: [
ErrorRaisingPattern(_model_provider_error()),
FailingPattern("structured failure"),
],
lambda: [
ErrorRaisingPattern(_model_provider_error()),
ErrorRaisingPattern(_model_provider_error()),
],
],
ids=["other_exception", "two_patterns_one_provider_error", "two_provider_errors"],
)
async def test_runner_omits_model_error_keys_unless_sole_pattern_raised_it(
tmp_path: Path, build_patterns: Any
) -> None:
patterns = build_patterns()
agent = Agent(name="writer", patterns=patterns)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Impossible", execution_id="exec-no-model-error")
assert result["success"] is False
assert result["error"].startswith(f"All {len(patterns)} patterns failed")
assert "diagnostic_error" not in result
assert "model_error" not in result
@pytest.mark.asyncio
@pytest.mark.parametrize(
"second",
[
ErrorRaisingPattern(ValueError("bad tool arguments")),
ErrorRaisingPattern(LLMEmptyContentError("empty")),
FailingPattern("structured failure"),
],
)
async def test_runner_omits_interruption_reason_when_patterns_disagree(
tmp_path: Path, second: Any
) -> None:
agent = Agent(
name="writer",
patterns=[ErrorRaisingPattern(LLMTimeoutError("timed out")), second],
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Impossible", execution_id="exec-reason-mixed")
assert result["success"] is False
assert result["pattern_errors"][0]["interruption_reason"] == "llm_unavailable"
assert result["pattern_errors"][1]["interruption_reason"] != "llm_unavailable"
assert "interruption_reason" not in result
@pytest.mark.asyncio
async def test_runner_classifies_unsuccessful_pattern_results(tmp_path: Path) -> None:
agent = Agent(
name="writer",
patterns=[
FakePattern({"success": False, "status": "invalid_tool_protocol"}),
FakePattern({"success": False, "status": "invalid_tool_protocol"}),
],
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Impossible", execution_id="exec-reason-result")
assert [entry["interruption_reason"] for entry in result["pattern_errors"]] == [
"model_output_invalid",
"model_output_invalid",
]
assert result["interruption_reason"] == "model_output_invalid"
@pytest.mark.asyncio
async def test_runner_single_unsuccessful_pattern_result_carries_reason(
tmp_path: Path,
) -> None:
"""Production runs one pattern; its returned result must carry the reason."""
agent = Agent(
name="writer",
patterns=[FakePattern({"success": False, "status": "invalid_tool_protocol"})],
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Impossible", execution_id="exec-reason-single")
assert result["status"] == "invalid_tool_protocol"
assert result["interruption_reason"] == "model_output_invalid"
@pytest.mark.asyncio
async def test_runner_single_pattern_keeps_its_own_interruption_reason(
tmp_path: Path,
) -> None:
# The enum member, not its value: only the classifier's own-reason branch
# turns it into a plain string. ``InterruptionReason`` subclasses ``str``,
# so equality alone would pass either way; the type is what is pinned.
agent = Agent(
name="writer",
patterns=[
FakePattern(
{
"success": False,
"interruption_reason": InterruptionReason.LLM_UNAVAILABLE,
}
)
],
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Impossible", execution_id="exec-reason-own")
assert result["interruption_reason"] == "llm_unavailable"
assert type(result["interruption_reason"]) is str
@pytest.mark.asyncio
@pytest.mark.parametrize("own_reason", ["not_a_real_reason", 42, None])
async def test_runner_single_pattern_drops_invalid_own_reason(
tmp_path: Path, own_reason: object
) -> None:
agent = Agent(
name="writer",
patterns=[
FakePattern(
{"success": False, "error": "x", "interruption_reason": own_reason}
)
],
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Impossible", execution_id="exec-reason-bad")
assert "interruption_reason" not in result
@pytest.mark.asyncio
async def test_runner_single_terminal_pattern_result_has_no_reason(
tmp_path: Path,
) -> None:
agent = Agent(
name="writer",
patterns=[FakePattern({"success": False, "error": "bad request"})],
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Impossible", execution_id="exec-reason-none")
assert "interruption_reason" not in result
@pytest.mark.asyncio
async def test_runner_success_after_failed_pattern_has_no_interruption_reason(
tmp_path: Path,
) -> None:
agent = Agent(
name="writer",
patterns=[
ErrorRaisingPattern(LLMTimeoutError("timed out")),
FakePattern({"success": True, "output": "done"}),
],
)
runner = AgentRunner(agent=agent, workspace_manager=FakeWorkspaceManager(tmp_path))
result = await runner.run(task="Answer", execution_id="exec-reason-success")
assert result["success"] is True
assert "interruption_reason" not in result
assert "pattern_errors" not in result
@pytest.mark.asyncio
async def test_runner_returns_single_pattern_failure_result(tmp_path: Path) -> None:
agent = Agent(
name="writer",
patterns=[
FakePattern(
{
"success": False,
"status": "failed",
"failure_reason": "structured_failure",
"error": "failed with details",
}
)
],
)
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(task="Impossible", execution_id="exec-single-fail")
assert result["success"] is False
assert result["status"] == "failed"
assert result["failure_reason"] == "structured_failure"
assert result["error"] == "failed with details"
assert "pattern_errors" not in result
@pytest.mark.asyncio
async def test_runner_does_not_add_empty_user_message_for_missing_task(
tmp_path: Path,
) -> None:
agent = Agent(name="writer", patterns=[FakePattern({"success": True})])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(task=None, execution_id="exec-empty-task")
assert result["success"] is True
assert result["context"].messages == []
@pytest.mark.asyncio
async def test_initial_messages_replay_tool_pairs(tmp_path: Path) -> None:
agent = Agent(name="writer", patterns=[FakePattern({"success": True})])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
initial_messages = [
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {"name": "read_file", "arguments": "{}"},
}
],
},
{
"role": "tool",
"tool_name": "read_file",
"tool_call_id": "call-1",
"raw_result": {"output": "file contents"},
},
]
result = await runner.run(
task=None,
execution_id="exec-replay",
initial_messages=initial_messages,
)
assert result["success"] is True
messages = result["context"].messages
# initial_messages[0] is role "assistant", so AgentRunner.run prepends a
# synthetic leading user turn (Anthropic's Messages API rejects a request
# whose first message isn't role "user") before replaying the
# assistant/tool pair.
assert [message.role for message in messages] == ["user", "assistant", "tool"]
synthetic_message = messages[0]
assert synthetic_message.content == "(conversation start)"
assert synthetic_message.metadata["_xagent_synthetic"] == "leading_user_turn"
assistant_message = messages[1]
assert assistant_message.content == ""
assert assistant_message.tool_calls == initial_messages[0]["tool_calls"]
tool_message = messages[2]
assert tool_message.content == "Tool read_file returned: file contents"
assert tool_message.tool_call_id == "call-1"
assert tool_message.metadata["raw_result"] == {"output": "file contents"}
assert tool_message.metadata["tool_name"] == "read_file"
# The synthetic turn is prepended, not interleaved, so tool-call pairing
# is undisturbed: every "tool" message is still immediately preceded by
# the assistant message declaring its tool_call_id.
for index, message in enumerate(messages):
if message.role == "tool":
assert messages[index - 1].role == "assistant"
assert message.tool_call_id in {
call["id"] for call in (messages[index - 1].tool_calls or [])
}
@pytest.mark.asyncio
async def test_initial_assistant_with_empty_content_and_tool_calls_survives(
tmp_path: Path,
) -> None:
agent = Agent(name="writer", patterns=[FakePattern({"success": True})])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
initial_messages = [
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call-2",
"type": "function",
"function": {"name": "list_files", "arguments": "{}"},
}
],
},
]
result = await runner.run(
task=None,
execution_id="exec-empty-content-tool-calls",
initial_messages=initial_messages,
)
assert result["success"] is True
messages = result["context"].messages
# The original defect this test guards: an assistant message with
# content="" and non-empty tool_calls must not be dropped by the replay
# loop's "is there anything to keep" check. The leading synthetic user
# turn (added because initial_messages[0] is role "assistant") must not
# mask that — the assistant message must still be present right after it.
assert [message.role for message in messages] == ["user", "assistant"]
synthetic_message = messages[0]
assert synthetic_message.content == "(conversation start)"
assert synthetic_message.metadata["_xagent_synthetic"] == "leading_user_turn"
assistant_message = messages[1]
assert assistant_message.content == ""
assert assistant_message.tool_calls == initial_messages[0]["tool_calls"]
@pytest.mark.asyncio
async def test_initial_messages_starting_with_user_no_synthetic_turn(
tmp_path: Path,
) -> None:
"""A realistically-shaped reconstruction that already starts with a user
transcript message, followed by an assistant/tool pair, must NOT trigger
the synthetic leading-user-turn correction: it is only needed when the
replay would otherwise start with role "assistant".
"""
agent = Agent(name="writer", patterns=[FakePattern({"success": True})])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
initial_messages = [
{"role": "user", "content": "Please read the file"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call-3",
"type": "function",
"function": {"name": "read_file", "arguments": "{}"},
}
],
},
{
"role": "tool",
"tool_name": "read_file",
"tool_call_id": "call-3",
"raw_result": {"output": "file contents"},
},
]
result = await runner.run(
task=None,
execution_id="exec-replay-user-first",
initial_messages=initial_messages,
)
assert result["success"] is True
messages = result["context"].messages
assert [message.role for message in messages] == ["user", "assistant", "tool"]
assert messages[0].content == "Please read the file"
assert not any(
message.metadata.get("_xagent_synthetic") == "leading_user_turn"
for message in messages
)
@pytest.mark.asyncio
async def test_initial_messages_plain_roles_unchanged(tmp_path: Path) -> None:
agent = Agent(name="writer", patterns=[FakePattern({"success": True})])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
initial_messages = [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "Hello there"},
{"role": "user", "content": ""}, # dropped: no content/context_refs
]
result = await runner.run(
task=None,
execution_id="exec-plain-initial",
initial_messages=initial_messages,
)
assert result["success"] is True
messages = result["context"].messages
assert [(message.role, message.content) for message in messages] == [
("system", "You are helpful."),
("user", "Hello there"),
]
@pytest.mark.asyncio
async def test_runner_stops_on_llm_call_interrupt(tmp_path: Path) -> None:
fallback = FakePattern({"success": True, "output": "should not run"})
agent = Agent(name="writer", patterns=[LLMInterruptedPattern(), fallback])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(task="Pause me", execution_id="exec-llm-interrupt")
assert result["success"] is False
assert result["status"] == "interrupted"
assert result["error"] == "paused during LLM call"
assert result["pattern"] == "LLMInterruptedPattern"
assert fallback.calls == []
@pytest.mark.asyncio
async def test_runner_restores_context_and_pattern_from_checkpoint(
tmp_path: Path,
) -> None:
event_loop_thread = threading.get_ident()
checkpoint_context = ExecutionContext(execution_id="exec-resume")
checkpoint_context.add_user_message("Original task")
checkpoint_context.metadata[PREFERRED_INPUT_MODALITIES_METADATA_KEY] = ["text"]
checkpoint = {
"context": checkpoint_context.to_dict(),
"pattern": "StatefulPattern",
"pattern_state": {"output": "restored"},
}
pattern = StatefulPattern()
agent = Agent(name="writer", patterns=[pattern])
workspace_manager = FakeWorkspaceManager(tmp_path)
runner = AgentRunner(
agent=agent,
workspace_manager=workspace_manager,
)
result = await runner.run(
task="Should not be appended",
execution_id="exec-resume",
checkpoint=checkpoint,
metadata={PREFERRED_INPUT_MODALITIES_METADATA_KEY: ["image"]},
)
assert result["success"] is True
assert result["output"] == "restored"
assert result["message_count"] == 1
assert pattern.state == {"output": "restored"}
assert result["context"].metadata[PREFERRED_INPUT_MODALITIES_METADATA_KEY] == [
"image"
]
assert [message.content for message in result["context"].messages] == [
"Original task",
"restored",
]
assert len(workspace_manager.thread_ids) == 1
assert workspace_manager.thread_ids[0] != event_loop_thread
@pytest.mark.asyncio
async def test_xinference_routing_key_survives_checkpoint_resume(
tmp_path: Path,
) -> None:
keys: list[str | None] = []
messages = [{"role": "user", "content": "Continue the task"}]
class ModelHandle:
async def chat(self, *, generate_config: dict[str, Any], **kwargs: Any) -> Any:
keys.append(generate_config.get("routing_key"))
if generate_config["stream"]:
async def chunks():
yield {
"choices": [
{"delta": {"content": "ok"}, "finish_reason": "stop"}
]
}
return chunks()
return {"content": "ok"}
class ModelPattern(FakePattern):
async def run(self, **kwargs: Any) -> dict[str, Any]:
llm = kwargs["llm"]
await llm.chat(messages=messages)
async for _ in llm.stream_chat(messages=messages):
pass
return await super().run(**kwargs)
llm = XinferenceLLM()
llm._client = MagicMock()
llm._model_handle = ModelHandle()
llm._supports_session_affinity = True
runner = AgentRunner(
agent=Agent(name="writer", llm=llm, patterns=[ModelPattern({"success": True})]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
scope_a = ExecutionScope(workspace_segments=("workspace-a",))
scope_b = ExecutionScope(workspace_segments=("workspace-b",))
def recorded_key() -> str:
nonstream_key, stream_key = keys
keys.clear()
assert nonstream_key is not None and nonstream_key == stream_key
return nonstream_key
token = current_user_id.set(42)
try:
with ExecutionScopeContext(scope_a):
first = await runner.run(task="Initial task", execution_id="task-1")
assert first["success"]
first_key = recorded_key()
restored = await runner.run(
task=None,
checkpoint={"context": first["context"].to_dict()},
resume=True,
)
assert restored["success"] and recorded_key() == first_key
other = await runner.run(task="Another task", execution_id="task-2")
assert other["success"]
other_task_key = recorded_key()
with ExecutionScopeContext(scope_b):
other_scope = await runner.run(
task="Different scope", execution_id="task-1"
)
assert other_scope["success"]
other_scope_key = recorded_key()
finally:
current_user_id.reset(token)
token = current_user_id.set(43)
try:
with ExecutionScopeContext(scope_a):
other_owner = await runner.run(
task="Different owner", execution_id="task-1"
)
assert other_owner["success"]
other_owner_key = recorded_key()
finally:
current_user_id.reset(token)
await llm.chat(messages=messages)
assert len({first_key, other_task_key, other_scope_key, other_owner_key}) == 4
assert keys == [None]
@pytest.mark.asyncio
async def test_runner_clears_checkpointed_modality_preference(
tmp_path: Path,
) -> None:
checkpoint_context = ExecutionContext(execution_id="exec-clear-modality")
checkpoint_context.add_user_message("Original task")
checkpoint_context.metadata[PREFERRED_INPUT_MODALITIES_METADATA_KEY] = ["image"]
checkpoint = {
"context": checkpoint_context.to_dict(),
"pattern": "StatefulPattern",
"pattern_state": {"output": "restored"},
}
pattern = StatefulPattern()
runner = AgentRunner(
agent=Agent(name="writer", patterns=[pattern]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(
task="Should not be appended",
execution_id="exec-clear-modality",
checkpoint=checkpoint,
metadata={PREFERRED_INPUT_MODALITIES_METADATA_KEY: []},
)
assert PREFERRED_INPUT_MODALITIES_METADATA_KEY not in result["context"].metadata
def test_merge_context_metadata_restored_clears_absent_modality_key(
tmp_path: Path,
) -> None:
"""Restored merges clear the modality key just like fresh-context merges."""
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
context = ExecutionContext(execution_id="exec-merge-absent")
context.metadata[PREFERRED_INPUT_MODALITIES_METADATA_KEY] = ["image"]
context.metadata["execution_type"] = "checkpointed"
runner._merge_context_metadata(context, {}, restored=True)
assert PREFERRED_INPUT_MODALITIES_METADATA_KEY not in context.metadata
assert context.metadata["execution_type"] == "checkpointed"
def test_merge_context_metadata_restored_overlays_execution_identity(
tmp_path: Path,
) -> None:
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
context = ExecutionContext(execution_id="exec-trusted-identity")
context.metadata.update(
{"task_source": "spoofed", "run_id": "stale", "other": "checkpointed"}
)
runner._merge_context_metadata(
context,
{"task_source": "slack", "run_id": "run-current"},
restored=True,
)
assert context.metadata["task_source"] == "slack"
assert context.metadata["run_id"] == "run-current"
assert context.metadata["other"] == "checkpointed"
@pytest.mark.parametrize("key", ["task_source", "run_id"])
def test_merge_context_metadata_restored_keeps_identity_against_none(
tmp_path: Path, key: str
) -> None:
"""A resume entry point with no trusted source must not erase the real one.
Several resume paths pass these keys unconditionally with a None value.
Overwriting the checkpointed identity with None would permanently deny a
pending approval that was gated under it.
"""
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
context = ExecutionContext(execution_id="exec-trusted-identity")
context.metadata.update({"task_source": "slack", "run_id": "run-original"})
runner._merge_context_metadata(context, {key: None}, restored=True)
assert context.metadata["task_source"] == "slack"
assert context.metadata["run_id"] == "run-original"
def test_merge_context_metadata_replaces_legacy_raw_memory_reason(
tmp_path: Path,
) -> None:
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
context = ExecutionContext(execution_id="exec-safe-memory-reason")
context.metadata.update(
{
"memory_available": False,
"memory_availability_reason": "host resolver secret shard eu-3",
"other": "checkpointed",
}
)
runner._merge_context_metadata(
context,
{
"memory_available": False,
"memory_availability_reason": "unavailable",
},
restored=True,
)
assert context.metadata["memory_available"] is False
assert context.metadata["memory_availability_reason"] == "unavailable"
assert context.metadata["other"] == "checkpointed"
@pytest.mark.asyncio
async def test_runner_empty_resume_metadata_preserves_non_modality_metadata(
tmp_path: Path,
) -> None:
"""Resume metadata is authoritative for the modality key only.
Every other checkpointed metadata entry survives an empty resume metadata
mapping; the modality preference is cleared because the current run did not
declare one.
"""
checkpoint_context = ExecutionContext(execution_id="exec-preserve-metadata")
checkpoint_context.add_user_message("Original task")
checkpoint_context.metadata.update(
{
PREFERRED_INPUT_MODALITIES_METADATA_KEY: ["image"],
"execution_type": "checkpointed",
}
)
checkpoint = {
"context": checkpoint_context.to_dict(),
"pattern": "StatefulPattern",
"pattern_state": {"output": "restored"},
}
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(
task="Should not be appended",
execution_id="exec-preserve-metadata",
checkpoint=checkpoint,
metadata={},
)
assert PREFERRED_INPUT_MODALITIES_METADATA_KEY not in result["context"].metadata
assert result["context"].metadata["execution_type"] == "checkpointed"
@pytest.mark.asyncio
async def test_runner_registers_restored_context_for_live_message_injection(
tmp_path: Path,
) -> None:
checkpoint_context = ExecutionContext(execution_id="exec-restore-inject")
checkpoint_context.add_user_message("Original task")
checkpoint = {
"context": checkpoint_context.to_dict(),
"pattern": "InjectingPattern",
"pattern_state": {},
}
agent = Agent(name="writer", patterns=[])
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
agent.patterns = [InjectingPattern(runner, "exec-restore-inject")]
result = await runner.run(
task=None,
execution_id="exec-restore-inject",
checkpoint=checkpoint,
)
assert result["success"] is True
assert result["same_context"] is True
assert result["messages"] == ["Original task", "Injected while resumed."]
@pytest.mark.asyncio
async def test_runner_pause_requests_interrupt_for_active_execution(
tmp_path: Path,
) -> None:
tracer = TracerCheckpointStore()
agent = Agent(name="writer", patterns=[])
runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
agent.patterns = [InterruptingPattern(runner, "exec-pause")]
result = await runner.run(task="Calculate 6*7", execution_id="exec-pause")
assert result["success"] is False
assert result["status"] == "interrupted"
assert tracer.by_execution_id["exec-pause"]["label"] == "interrupted"
assert (
tracer.by_execution_id["exec-pause"]["metadata"]["safe_point"]
== "during_pattern"
)
@pytest.mark.asyncio
async def test_runner_inject_user_message_updates_live_context_and_requests_interrupt(
tmp_path: Path,
) -> None:
tracer = TracerCheckpointStore()
agent = Agent(name="writer", patterns=[])
runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
agent.patterns = [
InterruptingPattern(
runner,
"exec-inject",
before_interrupt_check=lambda: runner.inject_user_message(
"exec-inject",
"Use metric units.",
reason="new user message",
),
)
]
result = await runner.run(task="Calculate 6*7", execution_id="exec-inject")
context = result["context"]
assert result["success"] is False
assert result["status"] == "interrupted"
user_messages = [msg.content for msg in context.messages if msg.role == "user"]
assert user_messages == ["Calculate 6*7", "Use metric units."]
checkpoint_messages = tracer.by_execution_id["exec-inject"]["context"]["messages"]
assert any(
message["role"] == "user" and message["content"] == "Use metric units."
for message in checkpoint_messages
)
@pytest.mark.asyncio
async def test_runner_resume_restores_from_latest_checkpoint_after_restart(
tmp_path: Path,
) -> None:
tracer = TracerCheckpointStore()
execution_id = "exec-restart"
first_agent = Agent(name="writer", patterns=[])
first_runner = AgentRunner(
agent=first_agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
first_agent.patterns = [InterruptingPattern(first_runner, execution_id)]
interrupted = await first_runner.run(
task="Calculate 6*7",
execution_id=execution_id,
)
assert interrupted["status"] == "interrupted"
await first_runner.inject_user_message(
execution_id,
"Reply with only the number.",
request_interrupt=False,
)
agent = Agent(
name="writer",
patterns=[FakePattern({"success": True, "response": "42"})],
)
resumed_runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
resumed = await resumed_runner.resume(execution_id)
assert resumed["success"] is True
assert resumed["response"] == "42"
resumed_contents = [message.content for message in resumed["context"].messages]
assert "Reply with only the number." in resumed_contents
assert resumed_contents.index(
"Reply with only the number."
) < resumed_contents.index("42")
@pytest.mark.asyncio
async def test_runner_inject_user_message_with_files_dispatches_trace_callback(
tmp_path: Path,
) -> None:
"""End-to-end coverage of the continuation chip path: a websocket-style
``post_user_message`` call with attachments must (a) attach the files to
the new Message so they survive checkpoints and (b) fire the trace
callback so the chip is broadcast live (instead of only appearing after
a page reload via historical replay)."""
tracer = RecordingCheckpointTracer()
agent = Agent(name="writer", patterns=[CheckpointingPattern({"success": True})])
runner = AgentRunner(
agent=agent,
tracer=tracer,
callbacks=[TraceEventCallback()],
workspace_manager=FakeWorkspaceManager(tmp_path),
)
await runner.run(task="Original task", execution_id="exec-cont-files")
files = [
{
"file_id": "fid-cont",
"name": "follow-up.pdf",
"size": 2048,
"type": "application/pdf",
}
]
result = await runner.post_user_message(
"exec-cont-files",
"Use the attached PDF.",
request_interrupt=False,
files=files,
)
assert result.context is not None
new_user_message = next(
msg for msg in reversed(result.context.messages) if msg.role == "user"
)
assert new_user_message.metadata.get("files") == files
turn_id = new_user_message.metadata.get("turn_id")
assert isinstance(turn_id, str) and turn_id
user_message_events = [
event
for event in tracer.events
if event["event_type"] == "task_start_message"
and event["data"].get("message") == "Use the attached PDF."
]
assert len(user_message_events) == 1
assert user_message_events[0]["data"]["turn_id"] == turn_id
assert user_message_events[0]["data"]["files"] == files
assert user_message_events[0]["data"]["attachments"] == files
@pytest.mark.asyncio
async def test_runner_post_user_message_alias_matches_inject_behavior(
tmp_path: Path,
) -> None:
tracer = TracerCheckpointStore()
agent = Agent(name="writer", patterns=[FakePattern({"success": True})])
runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
checkpoint_context = ExecutionContext(execution_id="exec-alias")
checkpoint_context.add_user_message("Original task")
await tracer.checkpoint(
type="checkpoint",
execution_id="exec-alias",
pattern="FakePattern",
label="before_llm",
status="interrupted",
context=checkpoint_context.to_dict(),
pattern_state={},
metadata={},
)
result = await runner.post_user_message(
"exec-alias",
"Follow-up from user.",
request_interrupt=False,
)
assert result.context is not None
user_messages = [
message.content for message in result.context.messages if message.role == "user"
]
assert user_messages == ["Original task", "Follow-up from user."]
@pytest.mark.asyncio
async def test_runner_post_user_message_deduplicates_explicit_turn_id_after_failure(
tmp_path: Path,
) -> None:
tracer = TracerCheckpointStore()
execution_id = "exec-idempotent-message"
checkpoint_context = ExecutionContext(execution_id=execution_id)
checkpoint_context.add_user_message("Original task")
await tracer.checkpoint(
type="checkpoint",
execution_id=execution_id,
pattern="FakePattern",
label="waiting_for_user",
status="waiting_for_user",
context=checkpoint_context.to_dict(),
pattern_state={},
metadata={},
)
failing_runner = AgentRunner(
agent=Agent(name="writer", patterns=[FakePattern({"success": True})]),
tracer=tracer,
callbacks=[FailingUserMessageCallback()],
workspace_manager=FakeWorkspaceManager(tmp_path),
)
failing_runner.pause = MagicMock(return_value=True)
# A live input needs a run of this execution active in this process.
failing_runner.context_manager.set_context(
ExecutionContext.from_dict(checkpoint_context.to_dict())
)
accepted = await failing_runner.post_user_message(
execution_id,
"Choose B",
turn_id="a2a:42:msg-1",
request_interrupt=True,
)
assert accepted.context is not None
assert accepted.outcome is UserMessageInjectionOutcome.POSTED_FRESH
failing_runner.pause.assert_called_once_with(
execution_id,
reason="new user message",
)
failing_runner.context_manager.remove_context(execution_id)
retry_runner = AgentRunner(
agent=Agent(name="writer", patterns=[FakePattern({"success": True})]),
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await retry_runner.post_user_message(
execution_id,
"Choose B",
turn_id="a2a:42:msg-1",
request_interrupt=False,
)
# The retry cold-starts from the checkpoint the first (pre-callback-
# failure) attempt already persisted, so this is the short-circuit
# replaying an already-seen turn id, not a second fresh write.
assert result.context is not None
assert result.outcome is UserMessageInjectionOutcome.POSTED_REPLAY
retried_messages = [
message
for message in result.context.messages
if message.role == "user" and message.metadata.get("turn_id") == "a2a:42:msg-1"
]
assert len(retried_messages) == 1
assert retried_messages[0].content == "Choose B"
@pytest.mark.asyncio
async def test_runner_rejects_reused_turn_id_with_different_content(
tmp_path: Path,
) -> None:
tracer = TracerCheckpointStore()
execution_id = "exec-conflicting-message"
checkpoint_context = ExecutionContext(execution_id=execution_id)
checkpoint_context.add_user_message(
"Choose A",
metadata={"turn_id": "a2a:42:msg-1"},
)
await tracer.checkpoint(
type="checkpoint",
execution_id=execution_id,
pattern="FakePattern",
label="waiting_for_user",
status="waiting_for_user",
context=checkpoint_context.to_dict(),
pattern_state={},
metadata={},
)
runner = AgentRunner(
agent=Agent(name="writer", patterns=[FakePattern({"success": True})]),
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
with pytest.raises(ValueError, match="different user message"):
await runner.post_user_message(
execution_id,
"Choose B",
turn_id="a2a:42:msg-1",
request_interrupt=False,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("scenario", ["fresh", "replay", "conflicting_content"])
async def test_runner_inject_user_message_reports_fresh_vs_replay(
tmp_path: Path, scenario: str
) -> None:
"""The three states this contract exists to name: a first write reports
POSTED_FRESH, a repeat of the same turn id with the same content
short-circuits and reports POSTED_REPLAY without persisting anything
new, and a repeat with different content still raises -- unchanged from
before this contract existed."""
tracer = TracerCheckpointStore()
execution_id = "exec-fresh-replay-grid"
agent = Agent(name="writer", patterns=[CheckpointingPattern({"success": True})])
runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
await runner.run(task="Original task", execution_id=execution_id)
first = await runner.inject_user_message(
execution_id,
"Choose B",
turn_id="turn-fresh-replay-grid",
request_interrupt=False,
)
assert first.outcome is UserMessageInjectionOutcome.POSTED_FRESH
assert first.context is not None
if scenario == "fresh":
return
if scenario == "replay":
second = await runner.inject_user_message(
execution_id,
"Choose B",
turn_id="turn-fresh-replay-grid",
request_interrupt=False,
)
assert second.outcome is UserMessageInjectionOutcome.POSTED_REPLAY
# The idle context was evicted, so the replay is read from the checkpoint.
assert second.context is not None
matching = [
message
for message in second.context.messages
if message.role == "user"
and message.metadata.get("turn_id") == "turn-fresh-replay-grid"
]
assert len(matching) == 1
return
assert scenario == "conflicting_content"
with pytest.raises(ValueError, match="different user message"):
await runner.inject_user_message(
execution_id,
"Choose C",
turn_id="turn-fresh-replay-grid",
request_interrupt=False,
)
@pytest.mark.asyncio
async def test_runner_post_user_message_preserves_display_and_execution_contract(
tmp_path: Path,
) -> None:
tracer = TracerCheckpointStore()
agent = Agent(name="writer", patterns=[FakePattern({"success": True})])
runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
checkpoint_context = ExecutionContext(execution_id="exec-display-contract")
checkpoint_context.add_user_message("Original task")
await tracer.checkpoint(
type="checkpoint",
execution_id="exec-display-contract",
pattern="FakePattern",
label="before_llm",
status="interrupted",
context=checkpoint_context.to_dict(),
pattern_state={},
metadata={},
)
execution_message = "Read file\n\n## UPLOADED FILES\nfile_id=file-123"
files = [{"file_id": "file-123", "name": "notes.txt"}]
result = await runner.post_user_message(
"exec-display-contract",
execution_message=execution_message,
display_message="Read file",
files=files,
turn_id="client-turn-123",
request_interrupt=False,
)
assert result.context is not None
latest_user = [
message for message in result.context.messages if message.role == "user"
][-1]
assert latest_user.content == execution_message
assert latest_user.metadata["display_message"] == "Read file"
assert latest_user.metadata["files"] == files
turn_id = latest_user.metadata.get("turn_id")
assert turn_id == "client-turn-123"
checkpoint_messages = tracer.by_execution_id["exec-display-contract"]["context"][
"messages"
]
latest_checkpoint_user = [
message for message in checkpoint_messages if message["role"] == "user"
][-1]
assert latest_checkpoint_user["content"] == execution_message
assert latest_checkpoint_user["metadata"]["display_message"] == "Read file"
assert latest_checkpoint_user["metadata"]["files"] == files
assert latest_checkpoint_user["metadata"]["turn_id"] == turn_id
@pytest.mark.asyncio
async def test_runner_initial_user_message_preserves_display_metadata(
tmp_path: Path,
) -> None:
tracer = RecordingTraceEventTracer()
agent = Agent(
name="writer",
patterns=[FakePattern({"success": True, "response": "Done"})],
)
runner = AgentRunner(
agent=agent,
tracer=tracer,
callbacks=[TraceEventCallback()],
workspace_manager=FakeWorkspaceManager(tmp_path),
)
execution_message = "Read file\n\n## UPLOADED FILES\nfile_id=file-123"
files = [{"file_id": "file-123", "name": "notes.txt"}]
result = await runner.run(
task=execution_message,
execution_id="exec-initial-display",
metadata={"request_context": {"display_message": "Read file", "files": files}},
)
first_user = next(
message for message in result["context"].messages if message.role == "user"
)
assert first_user.content == execution_message
assert first_user.metadata["display_message"] == "Read file"
assert first_user.metadata["files"] == files
assert result["context"].current_user_request_text(prefer_display=True) == (
"Read file"
)
turn_id = first_user.metadata.get("turn_id")
assert isinstance(turn_id, str) and turn_id
user_event = next(
event for event in tracer.events if event["event_type"] == "task_start_message"
)
assert user_event["data"]["message"] == "Read file"
assert user_event["data"]["turn_id"] == turn_id
@pytest.mark.parametrize(
("request_context", "expected"),
[
pytest.param({}, None, id="missing"),
pytest.param({"display_message": None}, "", id="null"),
pytest.param({"display_message": 17}, "", id="non-string"),
pytest.param({"display_message": ""}, "", id="blank"),
pytest.param({"display_message": " \n\t"}, " \n\t", id="whitespace"),
pytest.param({"display_message": "Read file"}, "Read file", id="text"),
],
)
def test_runner_normalizes_initial_display_message_state(
request_context: dict[str, Any], expected: str | None
) -> None:
runner = AgentRunner(agent=Agent(name="writer", patterns=[]))
context = ExecutionContext(metadata={"request_context": request_context})
metadata = runner._initial_user_message_metadata(context)
if expected is None:
assert "display_message" not in metadata
else:
assert metadata["display_message"] == expected
@pytest.mark.asyncio
async def test_runner_attaches_uploaded_image_refs_to_initial_user_message(
tmp_path: Path,
) -> None:
agent = Agent(
name="vision",
patterns=[FakePattern({"success": True, "response": "Done"})],
)
runner = AgentRunner(
agent=agent,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
references = build_image_context_references(
[{"file_id": "image-123", "name": "diagram.png", "type": "image/png"}]
)
result = await runner.run(
task="What is shown?",
execution_id="exec-initial-image",
task_context_refs=references,
)
first_user = next(
message for message in result["context"].messages if message.role == "user"
)
assert first_user.context_refs == references
@pytest.mark.asyncio
async def test_runner_attaches_uploaded_image_refs_to_injected_user_message(
tmp_path: Path,
) -> None:
tracer = TracerCheckpointStore()
agent = Agent(name="vision", patterns=[CheckpointingPattern({"success": True})])
runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
await runner.run(task="Start", execution_id="exec-injected-image")
result = await runner.inject_user_message(
"exec-injected-image",
"Inspect the new image",
files=[{"file_id": "image-456", "name": "screen.jpg", "type": "image/jpeg"}],
request_interrupt=False,
)
assert result.context is not None
assert result.context.messages[-1].context_refs[0].file_id == "image-456"
@pytest.mark.asyncio
async def test_runner_post_user_message_rejects_execution_without_display(
tmp_path: Path,
) -> None:
tracer = TracerCheckpointStore()
agent = Agent(name="writer", patterns=[FakePattern({"success": True})])
runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
checkpoint_context = ExecutionContext(execution_id="exec-display-required")
checkpoint_context.add_user_message("Original task")
await tracer.checkpoint(
type="checkpoint",
execution_id="exec-display-required",
pattern="FakePattern",
label="before_llm",
status="interrupted",
context=checkpoint_context.to_dict(),
pattern_state={},
metadata={},
)
with pytest.raises(ValueError, match="requires display_message"):
await runner.post_user_message(
"exec-display-required",
execution_message="Read file\n\n## UPLOADED FILES\nfile_id=file-123",
request_interrupt=False,
)
@pytest.mark.asyncio
async def test_trace_callback_does_not_emit_completion_for_interrupted_run(
tmp_path: Path,
) -> None:
tracer = RecordingTraceEventTracer()
agent = Agent(
name="paused",
patterns=[
FakePattern(
{
"success": False,
"status": "interrupted",
"error": "Paused by user.",
}
)
],
)
runner = AgentRunner(
agent=agent,
tracer=tracer,
callbacks=[TraceEventCallback()],
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(task="Pause this", execution_id="exec-paused")
assert result["status"] == "interrupted"
event_types = [event["event_type"] for event in tracer.events]
assert event_types == ["task_start_message"]
assert "task_end_general" not in event_types
@pytest.mark.asyncio
async def test_trace_callback_unwraps_final_answer_and_omits_success_context(
tmp_path: Path,
) -> None:
tracer = RecordingTraceEventTracer()
agent = Agent(
name="writer",
patterns=[
FakePattern(
{
"success": True,
"output": (
'```json\n{"action":"final_answer",'
'"action_input":"Done cleanly."}\n```'
),
"message": (
'```json\n{"action":"final_answer",'
'"action_input":"Done cleanly."}\n```'
),
}
)
],
)
runner = AgentRunner(
agent=agent,
tracer=tracer,
callbacks=[TraceEventCallback()],
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(task="Finish", execution_id="exec-success")
assert result["output"] == "Done cleanly."
assert result["message"] == "Done cleanly."
ai_event = next(
event for event in tracer.events if event["event_type"] == "task_end_message"
)
assert ai_event["data"]["content"] == "Done cleanly."
assert "context" not in ai_event["data"]
class _FakeLLM:
def __init__(self, context_window: Any, model_name: str = "fake-model") -> None:
self.context_window = context_window
self.model_name = model_name
def _threshold_runner(
context_window: Any, model_name: str = "fake-model"
) -> AgentRunner:
agent = Agent(
name="t",
patterns=[FakePattern({})],
llm=_FakeLLM(context_window, model_name),
)
return AgentRunner(agent=agent)
def test_resolve_compact_threshold_uses_window_ratio(monkeypatch) -> None:
monkeypatch.delenv("XAGENT_COMPACT_THRESHOLD_RATIO", raising=False)
# 128000 * 0.75
assert _threshold_runner(128000)._resolve_compact_threshold() == (
96000,
COMPACT_THRESHOLD_SOURCE_CONTEXT_WINDOW,
)
def test_resolve_compact_threshold_respects_ratio_env(monkeypatch) -> None:
monkeypatch.setenv("XAGENT_COMPACT_THRESHOLD_RATIO", "0.8")
assert _threshold_runner(200000)._resolve_compact_threshold() == (
160000,
COMPACT_THRESHOLD_SOURCE_CONTEXT_WINDOW,
)
@pytest.mark.parametrize("window", [None, 0, -1, "128000"])
def test_resolve_compact_threshold_falls_back_to_default(monkeypatch, window) -> None:
monkeypatch.delenv("XAGENT_COMPACT_THRESHOLD_DEFAULT", raising=False)
# None / non-positive / non-int all fall back to the global default.
assert _threshold_runner(window)._resolve_compact_threshold() == (
32000,
COMPACT_THRESHOLD_SOURCE_DEFAULT,
)
def test_resolve_compact_threshold_default_env_override(monkeypatch) -> None:
monkeypatch.setenv("XAGENT_COMPACT_THRESHOLD_DEFAULT", "50000")
assert _threshold_runner(None)._resolve_compact_threshold() == (
50000,
COMPACT_THRESHOLD_SOURCE_DEFAULT,
)
def test_resolve_compact_threshold_missing_llm() -> None:
agent = Agent(name="t", patterns=[FakePattern({})], llm=None)
assert AgentRunner(agent=agent)._resolve_compact_threshold() == (
32000,
COMPACT_THRESHOLD_SOURCE_DEFAULT,
)
def test_resolve_compact_threshold_warns_once_per_model_on_fallback(
monkeypatch, caplog
) -> None:
monkeypatch.delenv("XAGENT_COMPACT_THRESHOLD_DEFAULT", raising=False)
runner = _threshold_runner(None, model_name="moonshotai.kimi-k2.5")
with caplog.at_level(logging.WARNING, logger="xagent.core.agent.runtime"):
runner._resolve_compact_threshold()
runner._resolve_compact_threshold()
_threshold_runner(None, model_name="other-model")._resolve_compact_threshold()
_threshold_runner(128000, model_name="sized")._resolve_compact_threshold()
fallback_records = [
record
for record in caplog.records
if "context_window" in record.getMessage()
and "compaction threshold" in record.getMessage()
]
assert len(fallback_records) == 2
assert "moonshotai.kimi-k2.5" in fallback_records[0].getMessage()
assert "32000" in fallback_records[0].getMessage()
assert "other-model" in fallback_records[1].getMessage()
def test_resolve_compact_threshold_does_not_warn_for_virtual_models(
monkeypatch, caplog
) -> None:
monkeypatch.delenv("XAGENT_COMPACT_THRESHOLD_DEFAULT", raising=False)
class VirtualLLM:
model_name = "auto"
context_window = None
async def prepare_for_call(self, messages: Any, **_: Any) -> Any:
return self
agent = Agent(name="t", patterns=[FakePattern({})], llm=VirtualLLM())
with caplog.at_level(logging.WARNING, logger="xagent.core.agent.runtime"):
resolved = AgentRunner(agent=agent)._resolve_compact_threshold()
# The window is resolved per call and the threshold recomputed then.
assert resolved == (32000, COMPACT_THRESHOLD_SOURCE_DEFAULT)
assert not any("compaction threshold" in r.getMessage() for r in caplog.records)
@pytest.mark.asyncio
async def test_run_records_compact_threshold_source_on_context(monkeypatch) -> None:
monkeypatch.delenv("XAGENT_COMPACT_THRESHOLD_RATIO", raising=False)
captured: dict[str, Any] = {}
class CapturingPattern(FakePattern):
async def run(self, **kwargs: Any) -> dict[str, Any]:
context = kwargs["context"]
captured["threshold"] = context.compact_config.threshold
captured["source"] = context.compact_config.threshold_source
return await super().run(**kwargs)
agent = Agent(name="t", patterns=[CapturingPattern({})], llm=_FakeLLM(128000))
await AgentRunner(agent=agent).run("hello")
assert captured == {
"threshold": 96000,
"source": COMPACT_THRESHOLD_SOURCE_CONTEXT_WINDOW,
}
@pytest.mark.asyncio
async def test_run_resume_warns_when_restored_threshold_is_the_default(
tmp_path: Path, caplog
) -> None:
"""A task resumed in a fresh process has no in-memory record of why its
compaction threshold is what it is; ``AgentRunner.run`` re-issues the
fallback warning from the restored checkpoint so the missing
``context_window`` column is still visible in this process's log."""
tracer = TracerCheckpointStore()
execution_id = "exec-resume-warn"
checkpoint_context = ExecutionContext(execution_id=execution_id)
checkpoint_context.add_user_message("Original task")
assert (
checkpoint_context.compact_config.threshold_source
== COMPACT_THRESHOLD_SOURCE_DEFAULT
)
tracer.by_execution_id[execution_id] = {
"execution_id": execution_id,
"context": checkpoint_context.to_dict(),
}
agent = Agent(
name="writer",
patterns=[FakePattern({"success": True, "message": "ok"})],
llm=_FakeLLM(None, "moonshotai.kimi-k2.5"),
)
runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
with caplog.at_level(logging.WARNING, logger="xagent.core.agent.runtime"):
result = await runner.run(task=None, execution_id=execution_id, resume=True)
assert result["success"] is True
warnings = [
record.getMessage()
for record in caplog.records
if "resumed task" in record.getMessage()
]
assert len(warnings) == 1
assert "moonshotai.kimi-k2.5" in warnings[0]
@pytest.mark.asyncio
async def test_run_resume_stays_silent_when_model_now_has_a_window(
tmp_path: Path, caplog
) -> None:
"""The model row backing this resumed task now has a ``context_window``
(populated after the checkpoint was written, or simply since a fresh
process last saw it), so the restored default threshold is not running
blind and must not be re-warned about."""
tracer = TracerCheckpointStore()
execution_id = "exec-resume-silent"
checkpoint_context = ExecutionContext(execution_id=execution_id)
checkpoint_context.add_user_message("Original task")
tracer.by_execution_id[execution_id] = {
"execution_id": execution_id,
"context": checkpoint_context.to_dict(),
}
agent = Agent(
name="writer",
patterns=[FakePattern({"success": True, "message": "ok"})],
llm=_FakeLLM(256_000, "sized"),
)
runner = AgentRunner(
agent=agent,
tracer=tracer,
workspace_manager=FakeWorkspaceManager(tmp_path),
)
with caplog.at_level(logging.WARNING, logger="xagent.core.agent.runtime"):
result = await runner.run(task=None, execution_id=execution_id, resume=True)
assert result["success"] is True
assert not any("resumed task" in record.getMessage() for record in caplog.records)
def _memory_checkpoint(*, nested_auto: bool = False) -> dict[str, Any]:
root = ExecutionContext(
execution_id="memory-resume",
metadata={
MEMORY_RETRIEVAL_SCOPE_FINGERPRINT_METADATA_KEY: "scope-a",
MEMORY_CONTEXT_METADATA_KEY: "SCOPE_A_PRIVATE_NOTE",
RETRIEVED_MEMORIES_METADATA_KEY: {
"react": [{"content": "SCOPE_A_PRIVATE_NOTE"}]
},
},
)
root.add_user_message("Continue")
child = root.create_child_context(metadata={"dag_step_id": "step_1"})
state: dict[str, Any] = {
"output": "restored",
"active_step_contexts": {"step_1": child.to_dict()},
}
if nested_auto:
state = {"output": "restored", "dag_state": state}
return {
"context": root.to_dict(),
"pattern_state": state,
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("trusted_scope", "expected_scope"),
[
("scope-b", "scope-b"),
(None, None),
(12, None),
],
)
async def test_resume_reconciles_root_and_active_dag_memory_before_prompt(
tmp_path: Path,
trusted_scope: object,
expected_scope: str | None,
) -> None:
pattern = MemoryCheckpointPattern()
runner = AgentRunner(
agent=Agent(name="memory-resume", patterns=[pattern]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(
task=None,
execution_id="memory-resume",
checkpoint=_memory_checkpoint(),
metadata={MEMORY_RETRIEVAL_SCOPE_FINGERPRINT_METADATA_KEY: trusted_scope},
)
assert result["success"] is True
assert pattern.child_system_context is not None
assert "SCOPE_A_PRIVATE_NOTE" not in pattern.child_system_context
for metadata in (
result["context"].metadata,
pattern.active_step_contexts["step_1"]["metadata"],
):
assert MEMORY_CONTEXT_METADATA_KEY not in metadata
assert RETRIEVED_MEMORIES_METADATA_KEY not in metadata
assert metadata.get(MEMORY_RETRIEVAL_SCOPE_FINGERPRINT_METADATA_KEY) == (
expected_scope
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("trusted_scope", "expect_private_memory"),
[
("scope-b", False),
(None, False),
("scope-a", True),
],
)
async def test_resume_reconciles_real_dag_child_before_step_prompt(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
trusted_scope: str | None,
expect_private_memory: bool,
) -> None:
root = ExecutionContext(
execution_id="real-dag-memory-resume",
metadata={
MEMORY_RETRIEVAL_SCOPE_FINGERPRINT_METADATA_KEY: "scope-a",
MEMORY_CONTEXT_METADATA_KEY: "SCOPE_A_PRIVATE_NOTE",
RETRIEVED_MEMORIES_METADATA_KEY: {
"react": [{"content": "SCOPE_A_PRIVATE_NOTE"}]
},
},
)
root.add_user_message("Continue the waiting step")
step = PlanStep(id="step_1", task="Finish the private-memory task")
child = root.create_child_context(
metadata={"dag_step_id": step.id, "dag_step_name": step.task}
)
checkpoint_pattern = DAGPattern(lambda **_: None)
checkpoint_pattern.status = "waiting_for_user"
checkpoint_pattern.plan = ExecutionPlan(steps=[step])
step.status = "running"
checkpoint_pattern.active_step_ids = [step.id]
checkpoint_pattern.active_step_contexts[step.id] = child.to_dict()
checkpoint_pattern.active_step_pattern_states[step.id] = {}
checkpoint_pattern.planned_user_message_count = 1
captured_prompts: list[str] = []
class CapturingReAct:
def __init__(self, **_: Any) -> None:
self.state: dict[str, Any] = {}
def seed_memory_input(self, memory_text: str) -> None:
del memory_text
def load_state(self, state: dict[str, Any]) -> None:
self.state = dict(state)
def get_state(self) -> dict[str, Any]:
return dict(self.state)
async def run(self, *, context: ExecutionContext, **_: Any) -> dict[str, Any]:
captured_prompts.append(context._system_context())
return {"success": False, "status": "waiting_for_user"}
monkeypatch.setattr(dag_module, "ReActPattern", CapturingReAct)
resumed_pattern = DAGPattern(lambda **_: None)
runner = AgentRunner(
agent=Agent(
name="real-dag-memory-resume",
patterns=[resumed_pattern],
llm=MagicMock(),
),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(
task=None,
execution_id=root.execution_id,
checkpoint={
"context": root.to_dict(),
"pattern_state": checkpoint_pattern.get_state(),
},
metadata={MEMORY_RETRIEVAL_SCOPE_FINGERPRINT_METADATA_KEY: trusted_scope},
)
assert result["status"] == "waiting_for_user"
assert len(captured_prompts) == 1
assert ("SCOPE_A_PRIVATE_NOTE" in captured_prompts[0]) is expect_private_memory
@pytest.mark.asyncio
async def test_resume_preserves_same_scope_and_legacy_host_absence(
tmp_path: Path,
) -> None:
for metadata in (
{MEMORY_RETRIEVAL_SCOPE_FINGERPRINT_METADATA_KEY: "scope-a"},
{},
):
pattern = MemoryCheckpointPattern()
runner = AgentRunner(
agent=Agent(name="memory-resume", patterns=[pattern]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(
task=None,
execution_id="memory-resume",
checkpoint=_memory_checkpoint(),
metadata=metadata,
)
assert result["context"].metadata[MEMORY_CONTEXT_METADATA_KEY] == (
"SCOPE_A_PRIVATE_NOTE"
)
assert "SCOPE_A_PRIVATE_NOTE" in (pattern.child_system_context or "")
@pytest.mark.asyncio
async def test_resume_reconciles_nested_auto_dag_children_before_prompt(
tmp_path: Path,
) -> None:
pattern = MemoryCheckpointPattern(nested_auto=True)
runner = AgentRunner(
agent=Agent(name="memory-resume", patterns=[pattern]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
await runner.run(
task=None,
execution_id="memory-resume",
checkpoint=_memory_checkpoint(nested_auto=True),
metadata={MEMORY_RETRIEVAL_SCOPE_FINGERPRINT_METADATA_KEY: "scope-b"},
)
assert "SCOPE_A_PRIVATE_NOTE" not in (pattern.child_system_context or "")
child_metadata = (pattern.dag_state or {})["active_step_contexts"]["step_1"][
"metadata"
]
assert child_metadata[MEMORY_RETRIEVAL_SCOPE_FINGERPRINT_METADATA_KEY] == (
"scope-b"
)
@pytest.mark.asyncio
async def test_run_resume_raises_corrupt_on_contextless_checkpoint() -> None:
runner = AgentRunner(
agent=Agent(name="checkpoint-reader", patterns=[], llm=None),
tracer=ContextlessCheckpointStore(),
)
build_context_calls: list[Any] = []
original_build_context = runner._build_context
async def spy_build_context(*args: Any, **kwargs: Any) -> Any:
build_context_calls.append((args, kwargs))
return await original_build_context(*args, **kwargs)
runner._build_context = spy_build_context # type: ignore[method-assign]
with pytest.raises(CheckpointCorruptError):
await runner.run(
task=None,
execution_id="exec-resume-contextless",
resume=True,
)
assert build_context_calls == []
assert runner.context_manager.get_context("exec-resume-contextless") is None
@pytest.mark.asyncio
async def test_inject_user_message_raises_corrupt_on_contextless_checkpoint() -> None:
"""A found checkpoint without a context dict is malformed, not absent:
returning ``None`` here would be indistinguishable from "no checkpoint"
and the caller would defer forever against a row that can never resume."""
runner = AgentRunner(
agent=Agent(name="checkpoint-reader", patterns=[], llm=None),
tracer=ContextlessCheckpointStore(),
)
with pytest.raises(CheckpointCorruptError):
await runner.inject_user_message(
"exec-inject-contextless",
message="hello",
)
@pytest.mark.asyncio
async def test_resume_drops_legacy_router_output_language(tmp_path: Path) -> None:
checkpoint_context = ExecutionContext(execution_id="exec-legacy-router-language")
checkpoint_context.metadata["pattern"] = "auto"
checkpoint_context.metadata[OUTPUT_LANGUAGE_METADATA_KEY] = "Simplified Chinese"
checkpoint_context.metadata[OUTPUT_LANGUAGE_SOURCE_METADATA_KEY] = "auto_router"
checkpoint_context.add_user_message("Summarize the release notes in one paragraph.")
child_context = ExecutionContext(execution_id="exec-legacy-router-language_child")
child_context.metadata[OUTPUT_LANGUAGE_METADATA_KEY] = "Simplified Chinese"
child_context.metadata[OUTPUT_LANGUAGE_SOURCE_METADATA_KEY] = "auto_router"
checkpoint = {
"context": checkpoint_context.to_dict(),
"pattern": "StatefulPattern",
"pattern_state": {
"output": "restored",
"active_step_contexts": {"step_1": child_context.to_dict()},
},
}
pattern = StatefulPattern()
runner = AgentRunner(
agent=Agent(name="writer", patterns=[pattern]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(
task=None,
execution_id="exec-legacy-router-language",
checkpoint=checkpoint,
)
assert result["success"] is True
metadata = result["context"].metadata
assert OUTPUT_LANGUAGE_METADATA_KEY not in metadata
assert OUTPUT_LANGUAGE_SOURCE_METADATA_KEY not in metadata
restored_child = pattern.state["active_step_contexts"]["step_1"]["metadata"]
assert OUTPUT_LANGUAGE_METADATA_KEY not in restored_child
assert OUTPUT_LANGUAGE_SOURCE_METADATA_KEY not in restored_child
sent = result["context"].get_messages_for_llm()
system_content = sent[0]["content"]
assert "Output language: Simplified Chinese" not in system_content
# The request is sent once, as the latest user message.
assert "Summarize the release notes in one paragraph." not in system_content
assert [
_without_turn_context(m["content"]) for m in sent if m["role"] == "user"
] == ["Summarize the release notes in one paragraph."]
@pytest.mark.asyncio
async def test_resume_drops_legacy_plan_output_language(tmp_path: Path) -> None:
checkpoint_context = ExecutionContext(execution_id="exec-legacy-plan-language")
checkpoint_context.metadata["pattern"] = "dag_plan_execute"
checkpoint_context.metadata[OUTPUT_LANGUAGE_METADATA_KEY] = "Simplified Chinese"
checkpoint_context.metadata[OUTPUT_LANGUAGE_SOURCE_METADATA_KEY] = "dag_plan"
checkpoint_context.add_user_message("Summarize the release notes in one paragraph.")
child_context = ExecutionContext(execution_id="exec-legacy-plan-language_child")
child_context.metadata[OUTPUT_LANGUAGE_METADATA_KEY] = "Simplified Chinese"
child_context.metadata[OUTPUT_LANGUAGE_SOURCE_METADATA_KEY] = "dag_plan"
checkpoint = {
"context": checkpoint_context.to_dict(),
"pattern": "StatefulPattern",
"pattern_state": {
"output": "restored",
"active_step_contexts": {"step_1": child_context.to_dict()},
},
}
pattern = StatefulPattern()
runner = AgentRunner(
agent=Agent(name="writer", patterns=[pattern]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(
task=None,
execution_id="exec-legacy-plan-language",
checkpoint=checkpoint,
)
assert result["success"] is True
metadata = result["context"].metadata
assert OUTPUT_LANGUAGE_METADATA_KEY not in metadata
assert OUTPUT_LANGUAGE_SOURCE_METADATA_KEY not in metadata
restored_child = pattern.state["active_step_contexts"]["step_1"]["metadata"]
assert OUTPUT_LANGUAGE_METADATA_KEY not in restored_child
assert OUTPUT_LANGUAGE_SOURCE_METADATA_KEY not in restored_child
sent = result["context"].get_messages_for_llm()
system_content = sent[0]["content"]
assert "Output language: Simplified Chinese" not in system_content
# The request is sent once, as the latest user message.
assert "Summarize the release notes in one paragraph." not in system_content
assert [
_without_turn_context(m["content"]) for m in sent if m["role"] == "user"
] == ["Summarize the release notes in one paragraph."]
@pytest.mark.asyncio
async def test_resume_keeps_caller_supplied_output_language(tmp_path: Path) -> None:
checkpoint_context = ExecutionContext(execution_id="exec-caller-language")
checkpoint_context.metadata["request_context"] = {
OUTPUT_LANGUAGE_METADATA_KEY: "French"
}
checkpoint_context.metadata[OUTPUT_LANGUAGE_METADATA_KEY] = "French"
checkpoint_context.add_user_message("Summarize the release notes.")
checkpoint = {
"context": checkpoint_context.to_dict(),
"pattern": "StatefulPattern",
"pattern_state": {"output": "restored"},
}
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(
task=None,
execution_id="exec-caller-language",
checkpoint=checkpoint,
)
assert result["success"] is True
assert result["context"].metadata[OUTPUT_LANGUAGE_METADATA_KEY] == "French"
system_content = result["context"].get_messages_for_llm()[0]["content"]
assert "Output language: French" in system_content
class _StoredContextCheckpointStore:
def __init__(self, context: ExecutionContext) -> None:
self.payload = {"type": "checkpoint", "context": context.to_dict()}
async def load_latest_checkpoint(self, execution_id: str) -> dict[str, Any]:
del execution_id
return self.payload
def _cold_start_runner(context: ExecutionContext) -> AgentRunner:
return AgentRunner(
agent=Agent(name="writer", patterns=[], llm=None),
tracer=_StoredContextCheckpointStore(context),
)
@pytest.mark.asyncio
async def test_inject_user_message_cold_start_drops_legacy_output_language() -> None:
stored = ExecutionContext(execution_id="exec-inject-legacy-language")
stored.metadata["pattern"] = "auto"
stored.metadata[OUTPUT_LANGUAGE_METADATA_KEY] = "Simplified Chinese"
stored.metadata[OUTPUT_LANGUAGE_SOURCE_METADATA_KEY] = "auto_router"
stored.add_user_message("Summarize the release notes.")
result = await _cold_start_runner(stored).inject_user_message(
"exec-inject-legacy-language",
message="continue",
)
assert result.context is not None
assert OUTPUT_LANGUAGE_METADATA_KEY not in result.context.metadata
assert OUTPUT_LANGUAGE_SOURCE_METADATA_KEY not in result.context.metadata
@pytest.mark.asyncio
async def test_inject_user_message_cold_start_keeps_caller_output_language() -> None:
stored = ExecutionContext(execution_id="exec-inject-caller-language")
stored.metadata["request_context"] = {OUTPUT_LANGUAGE_METADATA_KEY: "French"}
stored.metadata[OUTPUT_LANGUAGE_METADATA_KEY] = "French"
stored.metadata[OUTPUT_LANGUAGE_SOURCE_METADATA_KEY] = "auto_router"
stored.add_user_message("Summarize the release notes.")
result = await _cold_start_runner(stored).inject_user_message(
"exec-inject-caller-language",
message="continue",
)
assert result.context is not None
assert result.context.metadata[OUTPUT_LANGUAGE_METADATA_KEY] == "French"
assert OUTPUT_LANGUAGE_SOURCE_METADATA_KEY not in result.context.metadata
def test_resume_migration_only_touches_execution_context_nodes() -> None:
"""The migration owns ExecutionContext metadata and nothing else: a
``metadata`` dict inside a message, a tool argument, or a step result is
someone else's payload, and a cold start would persist a silent edit."""
root = ExecutionContext(execution_id="exec-migration-ownership")
root.metadata["pattern"] = "dag_plan_execute"
root.metadata[OUTPUT_LANGUAGE_METADATA_KEY] = "Simplified Chinese"
root.metadata[OUTPUT_LANGUAGE_SOURCE_METADATA_KEY] = "dag_plan"
root.add_user_message(
"Translate the attached note.",
metadata={OUTPUT_LANGUAGE_METADATA_KEY: "French"},
)
child = ExecutionContext(execution_id="exec-migration-ownership_child")
child.metadata[OUTPUT_LANGUAGE_METADATA_KEY] = "Simplified Chinese"
child.metadata[OUTPUT_LANGUAGE_SOURCE_METADATA_KEY] = "dag_plan"
checkpoint = {
"context": root.to_dict(),
"pattern": "DAGPattern",
"metadata": {OUTPUT_LANGUAGE_METADATA_KEY: "French"},
"pattern_state": {
"active_step_contexts": {"step_1": child.to_dict()},
"step_results": {
"step_1": {"metadata": {OUTPUT_LANGUAGE_METADATA_KEY: "French"}}
},
"active_step_pattern_states": {
"step_1": {
"last_response": {
"tool_calls": [
{
"arguments": {
"metadata": {OUTPUT_LANGUAGE_METADATA_KEY: "French"}
}
}
]
}
}
},
},
}
reset_output_language_to_request_context(checkpoint)
assert OUTPUT_LANGUAGE_METADATA_KEY not in checkpoint["context"]["metadata"]
assert OUTPUT_LANGUAGE_SOURCE_METADATA_KEY not in checkpoint["context"]["metadata"]
restored_child = checkpoint["pattern_state"]["active_step_contexts"]["step_1"]
assert OUTPUT_LANGUAGE_METADATA_KEY not in restored_child["metadata"]
assert OUTPUT_LANGUAGE_SOURCE_METADATA_KEY not in restored_child["metadata"]
message_metadata = checkpoint["context"]["messages"][0]["metadata"]
assert message_metadata[OUTPUT_LANGUAGE_METADATA_KEY] == "French"
assert checkpoint["metadata"][OUTPUT_LANGUAGE_METADATA_KEY] == "French"
step_result = checkpoint["pattern_state"]["step_results"]["step_1"]
assert step_result["metadata"][OUTPUT_LANGUAGE_METADATA_KEY] == "French"
tool_arguments = checkpoint["pattern_state"]["active_step_pattern_states"][
"step_1"
]["last_response"]["tool_calls"][0]["arguments"]
assert tool_arguments["metadata"][OUTPUT_LANGUAGE_METADATA_KEY] == "French"
def test_resume_migration_reaches_a_nested_auto_pattern_child_context() -> None:
child = ExecutionContext(execution_id="exec-migration-nested_child")
child.metadata[OUTPUT_LANGUAGE_METADATA_KEY] = "Simplified Chinese"
child.metadata[OUTPUT_LANGUAGE_SOURCE_METADATA_KEY] = "auto_router"
checkpoint = {
"context": ExecutionContext(execution_id="exec-migration-nested").to_dict(),
"pattern_state": {
"dag_state": {"active_step_contexts": {"step_1": child.to_dict()}}
},
}
reset_output_language_to_request_context(checkpoint)
nested = checkpoint["pattern_state"]["dag_state"]["active_step_contexts"]["step_1"]
assert OUTPUT_LANGUAGE_METADATA_KEY not in nested["metadata"]
@pytest.mark.asyncio
async def test_runner_does_not_try_another_pattern_after_fact_commit_failure() -> None:
from xagent.core.agent.checkpoint import ExecutionEventPersistenceError
class UncertainPattern:
async def run(self, **_: Any) -> dict[str, Any]:
raise ExecutionEventPersistenceError("tool result commit failed")
fallback = FakePattern({"success": True, "output": "must not retry"})
runner = AgentRunner(
agent=Agent(
name="fact-failure", patterns=[UncertainPattern(), fallback], llm=None
),
workspace_enabled=False,
)
with pytest.raises(ExecutionEventPersistenceError):
await runner.run(task="write once", execution_id="uncertain-effect")
assert fallback.calls == []
@pytest.mark.parametrize(
"client_value, engine_value",
[(True, False), ("true", False), (1, False), (False, True)],
ids=["client_true", "client_string", "client_one", "client_false_on_latched_run"],
)
@pytest.mark.parametrize("surface", ["top_level", "request_context"])
def test_a_client_cannot_set_the_tool_evidence_marker(
tmp_path: Path, surface: str, client_value: object, engine_value: bool
) -> None:
"""Both surfaces that carry client input into metadata refuse this key.
Each cell sends the opposite of what the engine currently holds, so a cell
goes red if the client's value lands -- including the dangerous direction,
a client sending False to clear a run that really did lose observations.
"""
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
context = ContextManager().create_context(execution_id="exec-reserved-key")
context.metadata[TOOL_EVIDENCE_REMOVED_METADATA_KEY] = engine_value
client_keys = {
TOOL_EVIDENCE_REMOVED_METADATA_KEY: client_value,
"some_client_key": "kept",
}
metadata = (
dict(client_keys)
if surface == "top_level"
else {"request_context": dict(client_keys)}
)
runner._merge_context_metadata(context, metadata)
assert context.metadata[TOOL_EVIDENCE_REMOVED_METADATA_KEY] is engine_value
# Proves the merge actually ran; without it, the line above could pass
# simply because nothing was merged at all.
assert context.metadata["some_client_key"] == "kept"
def test_a_restored_context_takes_no_client_metadata_at_all(tmp_path: Path) -> None:
"""The restore branch returns before both filters because it merges nothing.
A run rebuilt from a checkpoint keeps what it latched: the current turn's
metadata contributes only the modality preference, so neither surface that
carries client input reaches it.
"""
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
context = ContextManager().create_context(execution_id="exec-restored-key")
context.metadata[TOOL_EVIDENCE_REMOVED_METADATA_KEY] = True
client_keys = {
TOOL_EVIDENCE_REMOVED_METADATA_KEY: False,
"some_client_key": "kept",
}
runner._merge_context_metadata(
context,
{**client_keys, "request_context": dict(client_keys)},
restored=True,
)
assert context.metadata[TOOL_EVIDENCE_REMOVED_METADATA_KEY] is True
assert "some_client_key" not in context.metadata
assert "request_context" not in context.metadata
def test_a_restored_context_with_no_marker_key_is_never_backfilled(
tmp_path: Path,
) -> None:
"""A checkpoint written before this key existed must stay keyless on resume.
Backfilling either value here would erase the distinction the third state
exists to carry: False would tell a run that really did lose observations
that nothing was removed, and True would tell a run that lost nothing
that something was. The restore branch returns before either client-input
filter runs, so nothing it does can write this key in either direction.
"""
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
context = ContextManager().create_context(execution_id="exec-restored-no-key")
context.metadata.pop(TOOL_EVIDENCE_REMOVED_METADATA_KEY, None)
client_keys = {
TOOL_EVIDENCE_REMOVED_METADATA_KEY: True,
"some_client_key": "kept",
}
runner._merge_context_metadata(
context,
{**client_keys, "request_context": dict(client_keys)},
restored=True,
)
assert TOOL_EVIDENCE_REMOVED_METADATA_KEY not in context.metadata
assert tool_evidence_state(context) == "unknown"
@pytest.mark.parametrize("stored", [True, False], ids=["removed", "intact"])
def test_a_restored_context_with_a_marker_key_keeps_its_stored_value(
tmp_path: Path, stored: bool
) -> None:
"""A checkpoint that does carry the key is never recomputed on restore."""
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
context = ContextManager().create_context(execution_id="exec-restored-has-key")
context.metadata[TOOL_EVIDENCE_REMOVED_METADATA_KEY] = stored
client_keys = {
TOOL_EVIDENCE_REMOVED_METADATA_KEY: not stored,
"some_client_key": "kept",
}
runner._merge_context_metadata(
context,
{**client_keys, "request_context": dict(client_keys)},
restored=True,
)
assert context.metadata[TOOL_EVIDENCE_REMOVED_METADATA_KEY] is stored
assert tool_evidence_state(context) == ("removed" if stored else "intact")
def test_the_marker_is_stamped_before_any_client_metadata_is_merged(
tmp_path: Path,
) -> None:
"""The refusal must not depend on the stamp happening to win a race.
``create_context`` writes False before the merge runs, so the ordering is
asserted here and a later refactor that moves the stamp cannot pass quietly.
"""
runner = AgentRunner(
agent=Agent(name="writer", patterns=[StatefulPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
context = ContextManager().create_context(execution_id="exec-reserved-order")
seen: list[object] = []
original = runner._apply_request_context
def record(ctx: ExecutionContext, request_context: dict[str, Any]) -> None:
seen.append(ctx.metadata.get(TOOL_EVIDENCE_REMOVED_METADATA_KEY))
original(ctx, request_context)
runner._apply_request_context = record # type: ignore[method-assign]
runner._merge_context_metadata(context, {"request_context": {"a": 1}})
assert seen == [False]
@pytest.mark.asyncio
async def test_runner_finishes_cache_when_pattern_exits_without_error_hook(
tmp_path: Path,
) -> None:
from xagent.core.agent.runtime import ExecutionInterrupted
events = []
runtimes = []
async def notify(payload):
events.append(payload)
class InterruptedPattern:
async def run(self, **kwargs):
runtime = kwargs["runtime"]
runtimes.append(runtime)
await runtime.on_pattern_start(context=kwargs["context"], pattern=self)
runtime._cache_session.register(("runner-interrupt", "model"), notify)
raise ExecutionInterrupted("stopped without pattern error hook")
runner = AgentRunner(
agent=Agent(name="interrupted", patterns=[InterruptedPattern()]),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.run(task="work", execution_id="cache-interrupted")
assert result["status"] == "interrupted"
runtime = runtimes[0]
if runtime._cache_session._publisher is not None:
await runtime._cache_session._publisher
assert not runtime.pattern_error_reported
assert runtime._cache_pattern_count == 0
assert [item["phase"] for item in events] == ["finished"]