Enterprise agent platform for building, deploying, and operating AI teammates.
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
import pytest
from pydantic import BaseModel
from xagent.core.agent import ExecutionContext, PatternRuntime, ReActPattern
from xagent.core.agent.clarification import (
ClarificationDraft,
draft_from_waiting_request,
)
from xagent.core.agent.trace import TraceAction
from xagent.web.api.websocket import SharedWebSocketTracer
from xagent.web.services.task_event_trace_handler import TaskEventTraceHandler
from xagent.web.services.trace_handlers import DatabaseTraceHandler
class CalculatorArgs(BaseModel):
expression: str
class FakeLLM:
def __init__(self, responses: list[Any]) -> None:
self.responses = responses
self.calls: list[dict[str, Any]] = []
async def chat(self, **kwargs: Any) -> Any:
self.calls.append(kwargs)
return self.responses.pop(0)
class ResumableTool:
"""A tool with its own suspended interaction state, resolved by a follow-up reply."""
def __init__(self) -> None:
self.metadata = SimpleNamespace(
name="approval_gate",
description="Run an action after the user responds.",
)
self.user_response: str | None = None
def args_type(self) -> type[BaseModel]:
return CalculatorArgs
def resume_user_interaction(self, *, interaction_id: str, response: str) -> None:
self.user_response = response
async def run_json_async(self, args: dict[str, Any]) -> Any:
if self.user_response is None:
return {
"success": False,
"status": "waiting_for_user",
"interaction_id": "interaction-1",
"message": "Should the action continue?",
"message_type": "confirmation",
"interactions": [
{
"type": "select_one",
"field": "decision",
"label": "Decision",
"options": [
{"label": "Continue", "value": "continue"},
{"label": "Stop", "value": "stop"},
],
}
],
}
return {"success": True, "expression": args["expression"]}
def _send_message_llm() -> FakeLLM:
return FakeLLM(
[
{
"tool_calls": [
{
"id": "call-send",
"function": {
"name": "send_message",
"arguments": (
'{"message":"Choose A or B",'
'"message_type":"question","expect_response":true}'
),
},
}
]
}
]
)
def _ask_user_question_llm() -> FakeLLM:
return FakeLLM(
[
{
"tool_calls": [
{
"id": "call-ask",
"function": {
"name": "ask_user_question",
"arguments": (
'{"message":"Pick one","interactions":'
'[{"type":"select_one","field":"choice","label":"Choice"}]}'
),
},
}
]
}
]
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("llm_factory", "tools_factory", "expected_source"),
[
(_send_message_llm, lambda: [], "send_message"),
(_ask_user_question_llm, lambda: [], "ask_user_question"),
],
)
async def test_waiting_return_carries_draft_matching_recomputed_value(
llm_factory: Any, tools_factory: Any, expected_source: str
) -> None:
"""The returned ``clarification_draft`` equals the draft recomputed
from the same ``waiting_for_user_request`` right after the run.
"""
pattern = ReActPattern(max_iterations=2)
runtime = PatternRuntime(execution_id="exec-tp1")
context = ExecutionContext(execution_id="exec-tp1")
context.add_user_message("Ask")
result = await pattern.run(
context=context, tools=tools_factory(), llm=llm_factory(), runtime=runtime
)
assert result["status"] == "waiting_for_user"
expected = draft_from_waiting_request(
pattern.waiting_for_user_request,
execution_id=context.execution_id,
step_id=None,
)
assert result["clarification_draft"] == expected
assert result["clarification_draft"].source == expected_source
assert (
result["clarification_draft"].event_id
== runtime.outbound_messages[-1]["event_id"]
)
assert (
pattern.waiting_for_user_request["event_id"]
== result["clarification_draft"].event_id
)
@pytest.mark.asyncio
async def test_waiting_return_via_tool_carries_tool_waiting_draft() -> None:
"""The multi-tool waiting point also carries
a draft equal to the one recomputed from the same request.
"""
tool = ResumableTool()
pattern = ReActPattern(max_iterations=2)
runtime = PatternRuntime(execution_id="exec-tp1-tool")
context = ExecutionContext(execution_id="exec-tp1-tool")
context.add_user_message("Run the gated action.")
llm = FakeLLM(
[
{
"tool_calls": [
{
"id": "wait-call",
"function": {
"name": "approval_gate",
"arguments": '{"expression":"2+2"}',
},
}
]
}
]
)
result = await pattern.run(context=context, tools=[tool], llm=llm, runtime=runtime)
assert result["status"] == "waiting_for_user"
expected = draft_from_waiting_request(
pattern.waiting_for_user_request,
execution_id=context.execution_id,
step_id=None,
)
assert result["clarification_draft"] == expected
assert result["clarification_draft"].source == "tool_waiting"
assert (
result["clarification_draft"].event_id
== runtime.outbound_messages[-1]["event_id"]
)
assert (
pattern.waiting_for_user_request["event_id"]
== result["clarification_draft"].event_id
)
@pytest.mark.asyncio
async def test_empty_message_send_message_still_yields_an_answerable_draft() -> None:
"""A ``send_message`` call with an empty ``message`` and
``expect_response=True`` is schema-valid (the tool only requires the
``message`` key to be present, not non-empty) and reaches
``waiting_for_user``.
It used to derive no draft at all -- no message, no ``"interactions"``
key, no ``"requests"`` list -- which is the worst version of the bug the
default free-text field exists for: nothing to read and nothing to answer
with. The appended field now supplies the ``"interactions"`` key, so a
draft is derivable. Only the draft: a blank message is still refused
downstream (``resolve_publishable_clarification`` returns
``NotApplicable("empty_question")``, and ``websocket.py`` persists no chat
row for an empty message), so this case ends the turn with an answerable
field published and nothing durable holding it.
"""
llm = FakeLLM(
[
{
"tool_calls": [
{
"id": "call-send-empty",
"function": {
"name": "send_message",
"arguments": (
'{"message":"","message_type":"question",'
'"expect_response":true}'
),
},
}
]
}
]
)
pattern = ReActPattern(max_iterations=2)
context = ExecutionContext(execution_id="exec-empty-message")
context.add_user_message("Ask")
result = await pattern.run(context=context, tools=[], llm=llm)
assert result["status"] == "waiting_for_user"
draft = result["clarification_draft"]
assert draft is not None
assert draft.source == "send_message"
assert draft.message == ""
assert [item["field"] for item in draft.interactions] == ["response"]
@pytest.mark.asyncio
async def test_completed_run_after_resume_carries_no_draft() -> None:
"""Once ``waiting_for_user_request`` is cleared by
a real user reply, the returned result no longer carries a draft.
"""
pattern = ReActPattern(max_iterations=2)
context = ExecutionContext(execution_id="exec-tp1-clear")
context.add_user_message("Ask")
first = await pattern.run(context=context, tools=[], llm=_send_message_llm())
assert first["status"] == "waiting_for_user"
context.add_user_message("B")
resumed_pattern = ReActPattern(max_iterations=2)
resumed_pattern.load_state(pattern.get_state())
resumed_llm = FakeLLM([{"content": "Continuing with B."}])
resumed = await resumed_pattern.run(context=context, tools=[], llm=resumed_llm)
assert resumed["success"] is True
assert resumed_pattern.waiting_for_user_request is None
assert "clarification_draft" not in resumed
@pytest.mark.asyncio
async def test_reentry_without_new_message_returns_stable_draft_and_sends_nothing() -> (
None
):
"""Re-entering with no new user message stays waiting, carries the
same draft as the first run, and sends zero new outbound messages.
"""
pattern = ReActPattern(max_iterations=2)
context = ExecutionContext(execution_id="exec-tp3")
context.add_user_message("Ask")
first = await pattern.run(context=context, tools=[], llm=_send_message_llm())
assert first["status"] == "waiting_for_user"
resumed_pattern = ReActPattern(max_iterations=2)
resumed_pattern.load_state(pattern.get_state())
resumed_runtime = PatternRuntime()
resumed_llm = FakeLLM([{"content": "Should not run"}])
resumed = await resumed_pattern.run(
context=context, tools=[], llm=resumed_llm, runtime=resumed_runtime
)
assert resumed["status"] == "waiting_for_user"
assert resumed_llm.calls == []
assert len(resumed_runtime.outbound_messages) == 0
assert resumed["clarification_draft"] == first["clarification_draft"]
assert resumed["clarification_draft"].event_id
assert (
resumed["clarification_draft"].event_id == first["clarification_draft"].event_id
)
@pytest.mark.asyncio
async def test_marker_survives_trace_serialization_of_a_dirty_interaction_id() -> None:
"""A marker computed before a control-character injection matches
the marker recomputed after the injected request round-trips through the
real trace serializer and a JSON encode/decode cycle -- ``_marker_clean``
and ``clean_string`` share a domain, so persistence cannot desync them.
The injected id carries both a byte the domain rejects (``\\x01``) and
one it keeps (``\\t``), so the before/after comparison actually depends
on the two filters agreeing on ``\\t``. That comparison alone only
catches a *web*-side narrowing, though: both markers finish with a call
to this module's own ``_marker_clean``, so a *core*-side narrowing of
``_MARKER_KEEP`` would apply identically on both sides of the comparison
and never show up as a mismatch. The direct equality assertion below
closes that gap by pinning ``_MARKER_KEEP`` against the actual serializer's
behavior for every C0 control character.
"""
import json
from xagent.core.agent.checkpoint import TraceCheckpointStore
from xagent.core.agent.clarification import _MARKER_KEEP
from xagent.web.services.trace_handlers import DatabaseTraceHandler
class RecordingTraceBackend:
def __init__(self) -> None:
self.events: list[dict[str, Any]] = []
async def trace_event(
self,
event_type: Any,
*,
task_id: str | None = None,
data: dict[str, Any] | None = None,
require_persisted: bool = False,
) -> str:
del event_type, require_persisted
self.events.append({"task_id": task_id, "data": dict(data or {})})
return f"event-{len(self.events)}"
tool = ResumableTool()
pattern = ReActPattern(max_iterations=2)
context = ExecutionContext(execution_id="exec-tp4")
context.add_user_message("Run the gated action.")
llm = FakeLLM(
[
{
"tool_calls": [
{
"id": "wait-call",
"function": {
"name": "approval_gate",
"arguments": '{"expression":"2+2"}',
},
}
]
}
]
)
result = await pattern.run(context=context, tools=[tool], llm=llm)
assert result["status"] == "waiting_for_user"
assert pattern.waiting_for_user_request is not None
pattern.waiting_for_user_request["requests"][0]["interaction_id"] = (
"wait\tcall\x01x"
)
marker_before = draft_from_waiting_request(
pattern.waiting_for_user_request,
execution_id=context.execution_id,
step_id=None,
).turn_marker
backend = RecordingTraceBackend()
store = TraceCheckpointStore(backend)
await store.checkpoint(
type="checkpoint",
label="waiting_for_user",
execution_id="exec-tp4",
pattern="ReActPattern",
pattern_state=pattern.get_state(),
status="waiting_for_user",
)
event_payload = backend.events[0]["data"]
handler = DatabaseTraceHandler(1)
serialized = handler._serialize_data_for_json(event_payload)
assert "_serialization_error" not in serialized
restored_event_payload = json.loads(json.dumps(serialized))
restored_state = restored_event_payload["snapshot"]["pattern_state"]
resumed_pattern = ReActPattern(max_iterations=2)
resumed_pattern.load_state(restored_state)
marker_after = draft_from_waiting_request(
resumed_pattern.waiting_for_user_request,
execution_id="exec-tp4",
step_id=None,
).turn_marker
assert marker_after == marker_before
# Pin the actual C0-control behavior rather than the implementation syntax.
# This independently detects either side narrowing or widening its keep set.
controls = "".join(chr(value) for value in range(32))
preserved = handler._serialize_data_for_json({"value": controls})["value"]
assert set(preserved) == set(_MARKER_KEEP)
assert preserved == "\t\n\r"
def test_marker_distinguishes_requests_with_ambiguous_raw_concatenation() -> None:
"""Two request sets whose raw values would collide under naive
``"|"``-joining (a ``tool_call_id`` containing ``|`` vs. an
``interaction_id`` containing ``|``) still produce distinct markers,
because every component carries its own length prefix.
"""
request_a = {
"kind": "tool_waiting_for_user",
"requests": [
{
"tool_call_id": "a|b",
"tool_name": "t",
"interaction_id": "c\nd",
"message": "m",
"message_type": "question",
"interactions": [],
}
],
"message": "m",
"message_type": "question",
"interactions": [],
"task_text": None,
"message_count": 1,
}
request_b = {
"kind": "tool_waiting_for_user",
"requests": [
{
"tool_call_id": "a",
"tool_name": "t",
"interaction_id": "b|c\nd",
"message": "m",
"message_type": "question",
"interactions": [],
}
],
"message": "m",
"message_type": "question",
"interactions": [],
"task_text": None,
"message_count": 1,
}
draft_a = draft_from_waiting_request(request_a, execution_id="e", step_id=None)
draft_b = draft_from_waiting_request(request_b, execution_id="e", step_id=None)
assert draft_a is not None and draft_b is not None
# Naive "|"-joining without a length prefix would collapse both raw
# concatenations to "a|b|c\nd" -- the markers must still differ.
assert draft_a.turn_marker != draft_b.turn_marker
class RecordingTracer:
"""Fake tracer that records every ``trace_event`` call verbatim.
Unlike ``RecordingTraceBackend`` in ``test_clarification_draft.py``
(which sits behind ``TraceCheckpointStore`` and only ever sees
checkpoint-shaped payloads), this attaches directly to
``PatternRuntime`` so it also captures the pattern-start/pattern-end
trace events that ``ReActPattern.run`` emits on its own -- including
the ``on_pattern_end`` event, whose ``data["result"]`` is the same
result dict ``pattern.run`` returns, with the live
``ClarificationDraft`` still attached.
"""
def __init__(self) -> None:
self.events: list[dict[str, Any]] = []
async def trace_event(
self,
event_type: Any,
*,
task_id: str | None = None,
step_id: str | None = None,
data: dict[str, Any] | None = None,
require_persisted: bool = False,
) -> str:
del require_persisted
self.events.append(
{
"event_type": event_type,
"task_id": task_id,
"step_id": step_id,
"data": dict(data or {}),
}
)
return f"event-{len(self.events)}"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("handler_factory", "serialize_method"),
[
(lambda: DatabaseTraceHandler(1), "_serialize_data_for_json"),
(lambda: TaskEventTraceHandler(1), "_serialize_data"),
(lambda: SharedWebSocketTracer(ws=None, task_id=1), "_serialize_data"),
],
ids=[
"database_trace_handler",
"websocket_trace_handler",
"shared_websocket_tracer",
],
)
async def test_pattern_end_trace_payload_with_draft_survives_real_serializer(
handler_factory: Any, serialize_method: str
) -> None:
"""The actual trace-event payload built by ``PatternRuntime.on_pattern_end``
(``{"pattern": ..., "result": result, "status": ...}``, see
``runtime.py``'s ``on_pattern_end`` / ``_emit_pattern_trace``) round-trips
through the real database and WebSocket trace-handler serializers without
falling back to the ``_serialization_error`` stub, and ``result``/
``status`` survive the round trip with their content intact.
The companion cell
``test_pattern_end_trace_payload_without_to_dict_collapses_to_stub``
guards the failure direction: it removes ``ClarificationDraft.to_dict``
and asserts the payload collapses to the ``_serialization_error`` stub.
"""
tracer = RecordingTracer()
runtime = PatternRuntime(execution_id="exec-trace-payload", tracer=tracer)
pattern = ReActPattern(max_iterations=2)
context = ExecutionContext(execution_id="exec-trace-payload")
context.add_user_message("Ask")
result = await pattern.run(
context=context, tools=[], llm=_send_message_llm(), runtime=runtime
)
assert result["status"] == "waiting_for_user"
assert result["clarification_draft"] is not None
end_events = [
event
for event in tracer.events
if event["event_type"].action == TraceAction.END and "result" in event["data"]
]
assert len(end_events) == 1
payload = end_events[0]["data"]
assert payload["result"] is result
handler = handler_factory()
serialized = getattr(handler, serialize_method)(payload)
assert "_serialization_error" not in serialized
assert serialized["status"] == "waiting_for_user"
assert serialized["result"]["status"] == "waiting_for_user"
assert serialized["result"]["clarification_draft"]["source"] == "send_message"
assert serialized["result"]["clarification_draft"]["message"] == "Choose A or B"
assert (
serialized["result"]["clarification_draft"]["event_id"]
== runtime.outbound_messages[-1]["event_id"]
)
requests = serialized["result"]["clarification_draft"]["requests"]
assert isinstance(requests, list)
assert requests and all(isinstance(item, dict) for item in requests)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("handler_factory", "serialize_method"),
[
(lambda: DatabaseTraceHandler(1), "_serialize_data_for_json"),
(lambda: TaskEventTraceHandler(1), "_serialize_data"),
(lambda: SharedWebSocketTracer(ws=None, task_id=1), "_serialize_data"),
],
ids=[
"database_trace_handler",
"websocket_trace_handler",
"shared_websocket_tracer",
],
)
async def test_pattern_end_trace_payload_without_to_dict_collapses_to_stub(
handler_factory: Any, serialize_method: str, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Removing ``ClarificationDraft.to_dict`` collapses the whole trace
payload to the ``_serialization_error`` stub.
This is the failure-direction guard for the survival cell above: the
serializers have no branch for a bare dataclass, so without ``to_dict``
the draft reaches ``json.dumps`` unconverted, the dump raises, and the
handler replaces the ENTIRE event payload with the three-key stub.
"""
tracer = RecordingTracer()
runtime = PatternRuntime(execution_id="exec-trace-stub", tracer=tracer)
pattern = ReActPattern(max_iterations=2)
context = ExecutionContext(execution_id="exec-trace-stub")
context.add_user_message("Ask")
result = await pattern.run(
context=context, tools=[], llm=_send_message_llm(), runtime=runtime
)
assert result["status"] == "waiting_for_user"
end_events = [
event
for event in tracer.events
if event["event_type"].action == TraceAction.END and "result" in event["data"]
]
payload = end_events[0]["data"]
monkeypatch.delattr(ClarificationDraft, "to_dict", raising=True)
handler = handler_factory()
serialized = getattr(handler, serialize_method)(payload)
assert "_serialization_error" in serialized
assert "result" not in serialized