Enterprise agent platform for building, deploying, and operating AI teammates.
"""Owner/actor regression pins for the WebSocket control handlers.
An admin may operate on another user's task (admin bypass), but the agent
runtime must run as the task OWNER, not the admin. A non-admin who is not the
owner must be refused before any runtime is built. These pin the pause / resume
handlers directly (the focused unit tests cover get_agent_for_task in
isolation; here we exercise the handlers end to end).
"""
from __future__ import annotations
import asyncio
import logging
import re
import threading
import time
from collections.abc import Callable
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Any, TypeVar
from unittest.mock import ANY, AsyncMock, MagicMock, patch
import pytest
from sqlalchemy import create_engine
from sqlalchemy.exc import IntegrityError
from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import QueuePool
from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from tests.shared.execution_scope import register_scope_resolver
from tests.web.services.active_interaction_read_shared import PRE_CHANGE_EQUIVALENT
from tests.web.services.interaction_rollout_policy_shared import (
install_interaction_rollout_policy,
)
from tests.web.services.task_lease_shared import (
live_task_lease as live_task_lease_fixture,
)
from xagent.core.agent.checkpoint import (
CheckpointAccessRefusedError,
CheckpointCorruptError,
CheckpointUnavailableError,
)
from xagent.core.agent.runner import (
UserMessageInjectionOutcome,
UserMessageInjectionRejectedError,
)
from xagent.core.execution_scope import (
ExecutionScope,
)
from xagent.web.api import websocket as websocket_api
from xagent.web.api.websocket import (
_make_command_reply,
handle_chat_message,
handle_pause_task,
handle_resume_task,
send_message_delivery,
)
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.database import Base, get_db, get_engine, init_db
from xagent.web.models.task import Task, TaskStatus, TraceEvent
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.task_interaction import TaskInteractionRequest
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.models.user import User
from xagent.web.services import ops_signals
from xagent.web.services import task_command_execution as command_execution_service
from xagent.web.services import task_execution as task_execution_service
from xagent.web.services import task_orchestrator
from xagent.web.services.chat_history_service import (
DELIVERY_FAILED,
DELIVERY_OUTCOME_UNKNOWN,
DELIVERY_PENDING,
)
from xagent.web.services.managed_file_ref import (
DurableObjectIntegrityError,
DurableStorageOperationError,
)
from xagent.web.services.task_command_execution import (
_claim_user_message_delivery_isolated,
_execute_durable_task_command,
handle_task_message,
make_delivery_notifier,
pause_task,
resume_task,
)
from xagent.web.services.task_command_transport import (
COMMAND_COMPLETED,
ClaimedTaskCommand,
TaskCommandKind,
TaskCommandRejected,
)
from xagent.web.services.task_execution import (
ResumeReservationOutcome,
_restore_resumed_task_lease_to_prior_status,
_waiting_or_paused_event_fields,
background_task_manager,
execute_resume_background,
)
from xagent.web.services.task_execution_controller import StaleTaskRunError
from xagent.web.services.task_interaction_close import (
ActiveInteractionRead,
ActiveInteractionUnavailable,
)
from xagent.web.services.task_lease_service import (
TaskLease,
current_task_lease,
get_runner_id,
)
live_task_lease = live_task_lease_fixture
@pytest.fixture()
def db_session(tmp_path):
init_db(db_url=f"sqlite:///{tmp_path / 'owner_actor.db'}")
db = next(get_db())
try:
yield db
finally:
db.close()
Base.metadata.drop_all(bind=get_engine())
def _user(db, username, *, is_admin=False) -> User:
u = User(username=username, password_hash="x", is_admin=is_admin)
db.add(u)
db.commit()
db.refresh(u)
return u
def _task(db, owner_id: int, status: TaskStatus = TaskStatus.RUNNING) -> Task:
t = Task(
user_id=owner_id,
title="t",
description="d",
status=status,
execution_mode="balanced",
source="sdk",
)
db.add(t)
db.commit()
db.refresh(t)
return t
def _seed_active_interaction_row(
db: Session, *, task_id: int, run_id: str, idempotency_key: str
) -> int:
"""One legal active TaskInteractionRequest row, for the replay-skips-
close tests below. Mirrors test_a2a_api.py's local, single-purpose row
builder of the same name rather than sharing it across test files."""
anchor = TraceEvent(
task_id=task_id,
event_id=f"anchor-{idempotency_key}",
event_type="agent_execution_checkpoint",
timestamp=datetime.now(timezone.utc),
data={},
)
db.add(anchor)
db.flush()
row = TaskInteractionRequest(
task_id=task_id,
run_id=run_id,
kind="clarification",
protocol_version=1,
status="active",
active_slot=1,
origin="sdk",
request_payload={"prompt": "example"},
request_idempotency_key=idempotency_key,
resume_trace_event_id=int(anchor.id),
resume_event_id="resume-event-1",
resume_execution_id="resume-execution-1",
resume_locator_format="trace_event_pk_v1",
resume_checkpoint_type="agent_execution_checkpoint",
resume_run_partition=run_id,
expires_at=datetime.now(timezone.utc) + timedelta(minutes=15),
)
db.add(row)
db.commit()
db.refresh(row)
return int(row.id)
# Anti-hang bounds, not latency assertions. Nothing here is measuring speed:
# these handlers finish in milliseconds, and the budgets only exist so a
# deadlock fails as a test error instead of hanging the run. They are therefore
# set far above any plausible runtime on a contended CI runner. A wait on a
# signal crossing the loop/worker-thread boundary must expire *before* the
# handler deadline, so a stall is reported by the specific assertion that owns
# the signal rather than by a bare TimeoutError on the whole handler.
_THREAD_SIGNAL_DEADLINE_SECONDS = 10.0
_HANDLER_DEADLINE_SECONDS = 30.0
_T = TypeVar("_T")
class _EventLoopLivenessProbe:
"""Deterministic replacement for counting event-loop ticks.
Wrapping a synchronous step in :meth:`gate` makes that step queue an answer
on the event loop and then block until the loop delivers it. Off the loop
(in a worker thread) the loop runs the callback on its next pass. On the
loop thread the queued callback cannot run, because the loop thread is the
one blocking, so the wait expires. The verdict therefore depends on where
the work ran, not on how fast the machine is.
"""
def __init__(self) -> None:
# Captured here rather than inside the gate: the gated step may run in
# a worker thread, where get_running_loop() raises.
self._loop = asyncio.get_running_loop()
self._answered = threading.Event()
self.step_ran = False
self.loop_ran_during_step = False
def gate(self, step: Callable[..., _T]) -> Callable[..., _T]:
"""Wrap ``step`` so it holds until the event loop answers."""
def gated(*args: Any, **kwargs: Any) -> _T:
assert not self.step_ran, "the probe gates a single call"
self.step_ran = True
self._loop.call_soon_threadsafe(self._answered.set)
self.loop_ran_during_step = self._answered.wait(
timeout=_THREAD_SIGNAL_DEADLINE_SECONDS
)
return step(*args, **kwargs)
return gated
def assert_loop_stayed_responsive(self, what: str) -> None:
assert self.step_ran, f"{what} never ran"
assert self.loop_ran_during_step, f"{what} blocked the asyncio event loop"
@pytest.mark.asyncio
async def test_rejected_delivery_serializes_an_explicit_outcome() -> None:
ws_manager = MagicMock(send_personal_message=AsyncMock())
with patch("xagent.web.api.websocket.manager", ws_manager):
await send_message_delivery(
MagicMock(),
client_message_id="rejected-turn",
turn_id="rejected-turn",
accepted=False,
rejection_outcome="not_accepted",
)
payload = ws_manager.send_personal_message.await_args.args[0]
assert payload["type"] == "message_rejected"
assert payload["rejection_outcome"] == "not_accepted"
with pytest.raises(ValueError, match="requires an explicit rejection outcome"):
await send_message_delivery(
MagicMock(),
client_message_id="missing-outcome-turn",
turn_id="missing-outcome-turn",
accepted=False,
)
def _register_current_resume(task_id: int) -> None:
current = asyncio.current_task()
assert current is not None
background_task_manager.resume_tasks[task_id] = current
def _patched_manager_and_agent():
"""Return (patches contextmanagers, captured) wiring get_agent_manager +
the module ``manager`` so the handler can run without real IO."""
captured: dict = {}
agent_service = MagicMock()
agent_service.pause_execution = AsyncMock(return_value={"status": "paused"})
agent_service.resume_execution = AsyncMock()
agent_service.supports_live_control = MagicMock(return_value=False)
async def _get_agent_for_task(
task_id, db, *, user=None, task_owner_user_id=None, **_kwargs
):
captured["task_owner_user_id"] = task_owner_user_id
return agent_service
mgr = MagicMock()
mgr.get_agent_for_task = AsyncMock(side_effect=_get_agent_for_task)
ws_manager = MagicMock()
ws_manager.send_personal_message = AsyncMock()
ws_manager.broadcast_to_task = AsyncMock()
return captured, agent_service, mgr, ws_manager
@pytest.mark.asyncio
async def test_chat_admin_append_to_other_users_task_claims_as_owner(
db_session,
) -> None:
"""The original #587 regression: an admin appending through
``handle_chat_message`` to a task owned by another user. The bug was the
atomic claim using the actor id, so the owner's appendable task failed with
``TaskTurnNotFoundError``. Pin that ``begin_turn`` is invoked with
``task_owner_user_id == task.user_id`` (the owner), not the admin actor.
"""
owner = _user(db_session, "owner")
admin = _user(db_session, "admin", is_admin=True)
# COMPLETED -> the WS path treats the follow-up as an APPEND turn.
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
ws_manager = MagicMock()
ws_manager.broadcast_to_task = AsyncMock()
ws_manager.send_personal_message = AsyncMock()
begin_turn = AsyncMock()
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.begin_turn",
new=begin_turn,
),
):
await handle_chat_message(
MagicMock(),
int(task.id),
{
"message": "follow-up",
"client_message_id": "client-turn-1",
"user": admin,
"files": [],
},
)
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if begin_turn.await_count:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("durable admin message was not dispatched in time")
begin_turn.assert_awaited_once()
assert begin_turn.await_args.kwargs["task_owner_user_id"] == int(owner.id)
assert begin_turn.await_args.kwargs["payload"].turn_id == "client-turn-1"
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
assert accepted[0]["client_message_id"] == "client-turn-1"
assert accepted[0]["turn_id"] == "client-turn-1"
@pytest.mark.asyncio
async def test_chat_without_an_actor_uses_the_authentication_contract() -> None:
ws_manager = MagicMock(send_personal_message=AsyncMock())
with patch("xagent.web.api.websocket.manager", ws_manager):
await handle_chat_message(
MagicMock(),
7,
{
"message": "do not echo this request",
"client_message_id": "unauthenticated-turn",
"files": [],
},
)
payloads = [
call.args[0] for call in ws_manager.send_personal_message.call_args_list
]
assert payloads
assert all(
payload["error_code"] == "authentication_required" for payload in payloads
)
assert all(
payload["message"] == "Authentication is required to send this message."
for payload in payloads
)
assert "do not echo this request" not in repr(payloads)
@pytest.mark.asyncio
async def test_chat_non_owner_uses_the_neutral_unavailable_contract(db_session) -> None:
owner = _user(db_session, "chat-access-owner")
stranger = _user(db_session, "chat-access-stranger")
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
ws_manager = MagicMock(send_personal_message=AsyncMock())
with patch("xagent.web.api.websocket.manager", ws_manager):
await handle_chat_message(
MagicMock(),
int(task.id),
{
"message": "try another owner's task",
"client_message_id": "access-denied-turn",
"user": stranger,
"files": [],
},
)
payloads = [
call.args[0] for call in ws_manager.send_personal_message.call_args_list
]
assert payloads
assert all(payload["error_code"] == "task_unavailable" for payload in payloads)
assert all(
payload["message"] == "Task is no longer available." for payload in payloads
)
assert f"Task {task.id} does not belong to you" not in repr(payloads)
@pytest.mark.asyncio
async def test_unserialized_chat_non_owner_keeps_the_neutral_contract(
db_session,
) -> None:
owner = _user(db_session, "direct-chat-access-owner")
stranger = _user(db_session, "direct-chat-access-stranger")
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
ws_manager = MagicMock(send_personal_message=AsyncMock())
with patch("xagent.web.api.websocket.manager", ws_manager):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "try direct execution",
"client_message_id": "direct-access-denied-turn",
"user": stranger,
"files": [],
},
)
payloads = [
call.args[0] for call in ws_manager.send_personal_message.call_args_list
]
assert payloads
assert all(payload["error_code"] == "task_unavailable" for payload in payloads)
assert all(
payload["message"] == "Task is no longer available." for payload in payloads
)
@pytest.mark.asyncio
async def test_chat_invalid_shape_uses_the_invalid_message_contract() -> None:
ws_manager = MagicMock(send_personal_message=AsyncMock())
with patch("xagent.web.api.websocket.manager", ws_manager):
await handle_task_message(
_make_command_reply(MagicMock()),
7,
{
"message": {"operator_detail": "must not escape"},
"client_message_id": "invalid-message-turn",
"user": SimpleNamespace(id=1, is_admin=False),
"files": [],
},
)
payloads = [
call.args[0] for call in ws_manager.send_personal_message.call_args_list
]
assert payloads
assert all(payload["error_code"] == "invalid_message" for payload in payloads)
assert all(
payload["message"] == "The message format is invalid." for payload in payloads
)
assert "operator_detail" not in repr(payloads)
@pytest.mark.asyncio
async def test_chat_new_turn_releases_request_transaction_before_orchestrator(
db_session,
) -> None:
"""Turn preparation must release its worker checkout before the claim."""
from xagent.web.models.database import get_session_local
owner = _user(db_session, "new-turn-owner")
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
owner_id = int(owner.id)
task_id = int(task.id)
db_session.close()
event_loop_thread = threading.get_ident()
preparation_threads: list[int] = []
session_closed_before_begin: list[bool] = []
closed_sessions: list[bool] = []
SessionLocal = get_session_local()
prepare_turn = command_execution_service._prepare_task_message_sync
class TrackingSessionContext:
def __init__(self) -> None:
self.session = SessionLocal()
def __enter__(self):
return self.session
def __exit__(self, _exc_type, _exc, _traceback) -> None:
self.session.close()
closed_sessions.append(True)
def tracking_session_factory():
return TrackingSessionContext()
def tracked_prepare_turn(**kwargs):
preparation_threads.append(threading.get_ident())
return prepare_turn(**kwargs)
async def begin_turn(**_kwargs):
session_closed_before_begin.append(bool(closed_sessions))
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
with (
patch(
"xagent.web.services.task_command_execution._prepare_task_message_sync",
side_effect=tracked_prepare_turn,
),
patch(
"xagent.web.services.task_command_execution.get_session_local",
return_value=tracking_session_factory,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.begin_turn",
side_effect=begin_turn,
),
):
await handle_task_message(
_make_command_reply(MagicMock()),
task_id,
{
"message": "follow-up",
"client_message_id": "pool-boundary-turn",
"user": SimpleNamespace(id=owner_id, is_admin=False),
"files": [],
},
)
assert preparation_threads
assert preparation_threads[0] != event_loop_thread
assert session_closed_before_begin == [True]
@pytest.mark.asyncio
async def test_chat_turn_releases_one_slot_pool_before_task_info_broadcast(
tmp_path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Task-info snapshot checkout must not overlap turn preparation."""
from xagent.web.models import database as database_module
engine = create_engine(
f"sqlite:///{tmp_path / 'websocket-turn-broadcast.db'}",
connect_args={"check_same_thread": False},
poolclass=QueuePool,
pool_size=1,
max_overflow=0,
pool_timeout=0.25,
)
Base.metadata.create_all(bind=engine)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
with SessionLocal() as db:
owner = User(
username="broadcast-pool-owner",
password_hash="x",
is_admin=False,
)
db.add(owner)
db.flush()
task = Task(
user_id=int(owner.id),
title="broadcast pool task",
description="d",
status=TaskStatus.COMPLETED,
execution_mode="balanced",
source="sdk",
)
db.add(task)
db.commit()
actor_user_id = int(owner.id)
task_id = int(task.id)
def local_get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
class RecordingWebSocket:
def __init__(self) -> None:
self.messages: list[str] = []
async def send_text(self, message: str) -> None:
self.messages.append(message)
websocket = RecordingWebSocket()
local_manager = websocket_api.ConnectionManager()
local_manager.register_connection(websocket, task_id) # type: ignore[arg-type]
begin_turn = AsyncMock()
monkeypatch.setattr(websocket_api, "get_db", local_get_db)
monkeypatch.setattr(
command_execution_service, "get_session_local", lambda: SessionLocal
)
monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
prepare_turn = command_execution_service._prepare_task_message_sync
monkeypatch.setattr(websocket_api, "manager", local_manager)
monkeypatch.setattr(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.begin_turn",
begin_turn,
)
# The gate holds preparation open long enough for loop blocking to be
# observable. Overlap detection is a separate mechanism: the one-slot pool
# raises on a task-info checkout that collides with preparation.
probe = _EventLoopLivenessProbe()
monkeypatch.setattr(
command_execution_service,
"_prepare_task_message_sync",
probe.gate(prepare_turn),
)
try:
await asyncio.wait_for(
handle_task_message(
_make_command_reply(websocket), # type: ignore[arg-type]
task_id,
{
"message": "follow-up",
"client_message_id": "broadcast-pool-turn",
"user": SimpleNamespace(id=actor_user_id, is_admin=False),
"files": [],
},
),
timeout=_HANDLER_DEADLINE_SECONDS,
)
finally:
engine.dispose()
probe.assert_loop_stayed_responsive("WebSocket preparation")
begin_turn.assert_awaited_once()
@pytest.mark.asyncio
async def test_pause_accepted_wait_releases_one_slot_pool_before_previous_run(
tmp_path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Previous-run settlement must acquire the pool while this handler waits."""
from xagent.web.models import database as database_module
engine = create_engine(
f"sqlite:///{tmp_path / 'websocket-turn-pause.db'}",
connect_args={"check_same_thread": False},
poolclass=QueuePool,
pool_size=1,
max_overflow=0,
pool_timeout=0.25,
)
Base.metadata.create_all(bind=engine)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
with SessionLocal() as db:
owner = User(
username="pause-pool-owner",
password_hash="x",
is_admin=False,
)
db.add(owner)
db.flush()
task = Task(
user_id=int(owner.id),
title="pause pool task",
description="d",
status=TaskStatus.RUNNING,
execution_mode="balanced",
source="sdk",
run_id="pause-pool-run",
runner_id="pause-pool-runner",
control_state="pause_requested",
)
db.add(task)
db.commit()
actor_user_id = int(owner.id)
task_id = int(task.id)
def local_get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
async def settle_previous(_task_id: int) -> None:
def settle_sync() -> None:
with SessionLocal() as db:
current = db.query(Task).filter(Task.id == task_id).one()
current.status = TaskStatus.PAUSED
current.control_state = "paused"
db.commit()
await asyncio.to_thread(settle_sync)
background_manager = MagicMock()
background_manager.wait_for_previous = AsyncMock(side_effect=settle_previous)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
begin_turn = AsyncMock()
monkeypatch.setattr(websocket_api, "get_db", local_get_db)
monkeypatch.setattr(
command_execution_service, "get_session_local", lambda: SessionLocal
)
monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
prepare_turn = command_execution_service._prepare_task_message_sync
monkeypatch.setattr(websocket_api, "manager", ws_manager)
monkeypatch.setattr(
task_execution_service,
"background_task_manager",
background_manager,
)
monkeypatch.setattr(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.begin_turn",
begin_turn,
)
task_execution_service._mark_task_pause_accepted(task_id)
probe = _EventLoopLivenessProbe()
monkeypatch.setattr(
command_execution_service,
"_prepare_task_message_sync",
probe.gate(prepare_turn),
)
try:
await asyncio.wait_for(
handle_task_message(
_make_command_reply(MagicMock()),
task_id,
{
"message": "continue after pause",
"client_message_id": "pause-pool-turn",
"user": SimpleNamespace(id=actor_user_id, is_admin=False),
"files": [],
},
),
timeout=_HANDLER_DEADLINE_SECONDS,
)
finally:
task_execution_service._clear_task_pause_accepted(task_id)
engine.dispose()
probe.assert_loop_stayed_responsive("pause settlement preparation")
# The settlement itself is guarded by the one-slot pool: it opens its own
# session, so it can only succeed if the handler released its checkout
# before awaiting the previous run.
background_manager.wait_for_previous.assert_awaited_once_with(task_id)
begin_turn.assert_awaited_once()
assert begin_turn.await_args.kwargs["kind"].value == "append"
@pytest.mark.asyncio
async def test_missing_task_create_commits_claim_message_file_and_prelease_together(
tmp_path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The replacement task is already a fully claimed CREATE at first send."""
from xagent.web.models import database as database_module
engine = create_engine(
f"sqlite:///{tmp_path / 'websocket-missing-create.db'}",
connect_args={"check_same_thread": False},
poolclass=QueuePool,
pool_size=1,
max_overflow=0,
pool_timeout=0.25,
)
Base.metadata.create_all(bind=engine)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
file_path = tmp_path / "atomic.txt"
file_path.write_text("atomic file")
with SessionLocal() as db:
owner = User(
username="missing-create-owner",
password_hash="x",
is_admin=False,
)
db.add(owner)
db.flush()
file_record = UploadedFile(
file_id="missing-create-file",
user_id=int(owner.id),
filename="atomic.txt",
storage_path=str(file_path),
mime_type="text/plain",
file_size=file_path.stat().st_size,
)
db.add(file_record)
db.commit()
owner_id = int(owner.id)
observed_task_ids: list[int] = []
sent_messages: list[dict] = []
class AtomicCreateManager:
def move_connection(self, _websocket: object, new_task_id: int) -> None:
observed_task_ids.append(new_task_id)
async def send_personal_message(self, event: dict, _websocket: object) -> None:
sent_messages.append(event)
if event.get("type") != "task_id_updated":
return
assert engine.pool.checkedout() == 0
new_task_id = int(event["new_task_id"])
with SessionLocal() as db:
task = db.query(Task).filter(Task.id == new_task_id).one()
message = (
db.query(TaskChatMessage)
.filter(TaskChatMessage.task_id == new_task_id)
.one()
)
uploaded = (
db.query(UploadedFile)
.filter(UploadedFile.file_id == "missing-create-file")
.one()
)
assert task.status == TaskStatus.RUNNING
assert task.runner_id
assert task.run_id
assert task.lease_expires_at is not None
assert message.content == "start atomically"
assert message.delivery_status == DELIVERY_PENDING
assert uploaded.task_id == new_task_id
async def broadcast_to_task(self, _event: dict, _task_id: int) -> None:
assert engine.pool.checkedout() == 0
ws_manager = AtomicCreateManager()
schedule = AsyncMock()
begin_turn = AsyncMock()
monkeypatch.setattr(
command_execution_service, "get_session_local", lambda: SessionLocal
)
monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
monkeypatch.setattr(websocket_api, "manager", ws_manager)
monkeypatch.setattr(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.schedule_claimed_create_turn",
schedule,
)
monkeypatch.setattr(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.begin_turn",
begin_turn,
)
try:
await handle_task_message(
_make_command_reply(MagicMock()),
987654,
{
"message": "start atomically",
"client_message_id": "missing-create-turn",
"user": SimpleNamespace(id=owner_id, is_admin=False),
"files": [
{
"file_id": "missing-create-file",
"name": "atomic.txt",
}
],
},
)
finally:
engine.dispose()
assert observed_task_ids
schedule.assert_awaited_once()
assert schedule.await_args.kwargs["task_id"] == observed_task_ids[0]
begin_turn.assert_not_awaited()
assert any(event.get("type") == "message_accepted" for event in sent_messages)
@pytest.mark.asyncio
async def test_missing_task_file_bind_race_rolls_back_the_whole_create(
tmp_path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A lost file claim cannot expose a Task without its first turn."""
from xagent.web.models import database as database_module
engine = create_engine(
f"sqlite:///{tmp_path / 'websocket-missing-bind-race.db'}",
connect_args={"check_same_thread": False},
poolclass=QueuePool,
pool_size=1,
max_overflow=0,
pool_timeout=0.25,
)
Base.metadata.create_all(bind=engine)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
file_path = tmp_path / "raced.txt"
file_path.write_text("raced file")
with SessionLocal() as db:
owner = User(
username="missing-bind-race-owner",
password_hash="x",
is_admin=False,
)
db.add(owner)
db.flush()
db.add(
UploadedFile(
file_id="missing-bind-race-file",
user_id=int(owner.id),
filename="raced.txt",
storage_path=str(file_path),
mime_type="text/plain",
file_size=file_path.stat().st_size,
)
)
db.commit()
owner_id = int(owner.id)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
schedule = AsyncMock()
monkeypatch.setattr(
command_execution_service, "get_session_local", lambda: SessionLocal
)
monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
monkeypatch.setattr(websocket_api, "manager", ws_manager)
monkeypatch.setattr(
task_orchestrator,
"bind_turn_files_no_commit",
MagicMock(return_value=["missing-bind-race-file"]),
)
monkeypatch.setattr(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.schedule_claimed_create_turn",
schedule,
)
try:
await handle_task_message(
_make_command_reply(MagicMock()),
654321,
{
"message": "rollback raced create",
"client_message_id": "missing-bind-race-turn",
"user": SimpleNamespace(id=owner_id, is_admin=False),
"files": [
{
"file_id": "missing-bind-race-file",
"name": "raced.txt",
}
],
},
)
with SessionLocal() as db:
assert (
db.query(Task)
.filter(
Task.user_id == owner_id,
Task.title.like("Chat: rollback raced create%"),
)
.count()
== 0
)
assert db.query(TaskChatMessage).count() == 0
uploaded = (
db.query(UploadedFile)
.filter(UploadedFile.file_id == "missing-bind-race-file")
.one()
)
assert uploaded.task_id is None
finally:
engine.dispose()
schedule.assert_not_awaited()
@pytest.mark.asyncio
async def test_missing_task_send_failure_leaves_only_ttl_owned_running_claim(
tmp_path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
from xagent.web.models import database as database_module
engine = create_engine(
f"sqlite:///{tmp_path / 'websocket-missing-send-failure.db'}",
connect_args={"check_same_thread": False},
poolclass=QueuePool,
pool_size=1,
max_overflow=0,
pool_timeout=0.25,
)
Base.metadata.create_all(bind=engine)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
with SessionLocal() as db:
owner = User(
username="missing-send-failure-owner",
password_hash="x",
is_admin=False,
)
db.add(owner)
db.commit()
owner_id = int(owner.id)
ws_manager = MagicMock()
ws_manager.send_personal_message = AsyncMock(
side_effect=RuntimeError("socket send failed")
)
ws_manager.broadcast_to_task = AsyncMock()
schedule = AsyncMock()
monkeypatch.setattr(
command_execution_service, "get_session_local", lambda: SessionLocal
)
monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
monkeypatch.setattr(websocket_api, "manager", ws_manager)
monkeypatch.setattr(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.schedule_claimed_create_turn",
schedule,
)
try:
with pytest.raises(RuntimeError, match="socket send failed"):
await handle_task_message(
_make_command_reply(MagicMock()),
876543,
{
"message": "send may fail",
"client_message_id": "missing-send-failure-turn",
"user": SimpleNamespace(id=owner_id, is_admin=False),
"files": [],
},
)
with SessionLocal() as db:
task = (
db.query(Task)
.filter(
Task.user_id == owner_id,
Task.title.like("Chat: send may fail%"),
)
.one()
)
message = (
db.query(TaskChatMessage)
.filter(TaskChatMessage.task_id == int(task.id))
.one()
)
assert task.status == TaskStatus.RUNNING
assert task.runner_id
assert task.run_id
assert task.lease_expires_at is not None
assert message.delivery_status == DELIVERY_FAILED
finally:
engine.dispose()
schedule.assert_not_awaited()
@pytest.mark.asyncio
async def test_missing_task_cancel_after_atomic_create_never_leaves_pending(
tmp_path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
from xagent.web.models import database as database_module
engine = create_engine(
f"sqlite:///{tmp_path / 'websocket-missing-cancel.db'}",
connect_args={"check_same_thread": False},
poolclass=QueuePool,
pool_size=1,
max_overflow=0,
pool_timeout=0.25,
)
Base.metadata.create_all(bind=engine)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
with SessionLocal() as db:
owner = User(
username="missing-cancel-owner",
password_hash="x",
is_admin=False,
)
db.add(owner)
db.commit()
owner_id = int(owner.id)
worker_finished = threading.Event()
release_worker = threading.Event()
prepare_turn = command_execution_service._prepare_task_message_sync
def blocked_prepare_turn(**kwargs):
preparation = prepare_turn(**kwargs)
worker_finished.set()
assert release_worker.wait(timeout=_THREAD_SIGNAL_DEADLINE_SECONDS)
return preparation
schedule = AsyncMock()
monkeypatch.setattr(
command_execution_service, "get_session_local", lambda: SessionLocal
)
monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
monkeypatch.setattr(
command_execution_service,
"_prepare_task_message_sync",
blocked_prepare_turn,
)
monkeypatch.setattr(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.schedule_claimed_create_turn",
schedule,
)
handler = asyncio.create_task(
handle_task_message(
_make_command_reply(MagicMock()),
765432,
{
"message": "cancel after commit",
"client_message_id": "missing-cancel-turn",
"user": SimpleNamespace(id=owner_id, is_admin=False),
"files": [],
},
)
)
try:
assert await asyncio.to_thread(
worker_finished.wait, _THREAD_SIGNAL_DEADLINE_SECONDS
)
handler.cancel()
release_worker.set()
with pytest.raises(asyncio.CancelledError):
await handler
with SessionLocal() as db:
task = (
db.query(Task)
.filter(
Task.user_id == owner_id,
Task.title.like("Chat: cancel after commit%"),
)
.one()
)
message = (
db.query(TaskChatMessage)
.filter(TaskChatMessage.task_id == int(task.id))
.one()
)
assert task.status == TaskStatus.RUNNING
assert task.runner_id
assert task.run_id
assert task.lease_expires_at is not None
assert message.delivery_status == DELIVERY_PENDING
finally:
release_worker.set()
engine.dispose()
schedule.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("reason", "expected_code", "expected_message"),
[
(
"actor_task_reuse_unsupported",
"message_continuation_unsupported",
"Task does not support message continuation.",
),
(
"workforce_archived",
"workforce_archived",
"This workforce has been archived. Unarchive and publish it before "
"starting a new conversation, or select an active workforce.",
),
(
"workforce_config_changed",
"workforce_unavailable",
"This workforce conversation can no longer accept messages; "
"please start a new conversation.",
),
(
"workforce_run_not_found",
"workforce_unavailable",
"This workforce conversation can no longer accept messages; "
"please start a new conversation.",
),
(
"workforce_run_not_active",
"workforce_unavailable",
"This workforce conversation can no longer accept messages; "
"please start a new conversation.",
),
],
)
async def test_chat_turn_rejection_payload_is_loaded_off_loop(
db_session,
reason: str,
expected_code: str,
expected_message: str,
) -> None:
from xagent.web.services.task_orchestrator import TaskTurnError
owner = _user(db_session, "busy-payload-owner")
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
event_loop_thread = threading.get_ident()
payload_threads: list[int] = []
payload_messages: list[str] = []
payload_codes: list[str | None] = []
def read_payload(_task_id, default_message, *_args, **_kwargs):
payload_threads.append(threading.get_ident())
payload_messages.append(default_message)
payload_codes.append(_kwargs.get("error_code"))
return {
"type": "agent_error",
"message": "busy",
"error_code": _kwargs.get("error_code"),
}
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.begin_turn",
side_effect=TaskTurnError(reason),
),
patch(
"xagent.web.services.task_command_execution._read_task_error_payload_isolated",
side_effect=read_payload,
),
patch(
"xagent.web.services.task_command_execution._task_error_payload",
side_effect=AssertionError("payload query ran on the event loop"),
),
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "follow-up",
"client_message_id": "busy-payload-turn",
"user": owner,
"files": [],
},
)
assert len(payload_threads) == 1
assert payload_threads[0] != event_loop_thread
assert payload_codes == [expected_code]
assert payload_messages == [expected_message]
broadcast = ws_manager.broadcast_to_task.await_args.args[0]
assert broadcast["error_code"] == expected_code
rejected = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_rejected"
]
assert rejected[0]["error_code"] == expected_code
@pytest.mark.asyncio
async def test_chat_without_client_id_uses_durable_command_id_as_turn_id(
db_session,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
begin_turn = AsyncMock()
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.begin_turn",
new=begin_turn,
),
):
await handle_chat_message(
MagicMock(),
int(task.id),
{"message": "server generated identity", "user": owner, "files": []},
)
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if begin_turn.await_count:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("durable message was not dispatched in time")
db_session.expire_all()
command = (
db_session.query(TaskExecutionCommand)
.filter(TaskExecutionCommand.task_id == int(task.id))
.one()
)
assert command.command_id.startswith("message:")
assert command.payload["client_message_id"] == command.command_id
assert begin_turn.await_args.kwargs["payload"].turn_id == command.command_id
@pytest.mark.asyncio
async def test_running_chat_message_is_persisted_before_resume(
live_task_lease, db_session
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "live-runner"
task.run_id = "live-run"
db_session.commit()
live_lease = live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
observed_leases: list[TaskLease | None] = []
async def post_user_message(*_args, **_kwargs) -> UserMessageInjectionOutcome:
observed_leases.append(current_task_lease())
return UserMessageInjectionOutcome.POSTED_FRESH
agent.post_user_message = AsyncMock(side_effect=post_user_message)
mgr = MagicMock(get_agent_for_task=AsyncMock(return_value=agent))
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
resume_bg = AsyncMock()
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.execute_resume_background", resume_bg
),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
):
await handle_chat_message(
MagicMock(),
int(task.id),
{
"message": "Use the audio tool",
"client_message_id": "live-turn-1",
"user": owner,
"files": [],
},
)
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if bg_mgr.register_reserved_resume.call_count:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("durable live message was not dispatched in time")
stored = (
db_session.query(TaskChatMessage)
.filter(
TaskChatMessage.task_id == int(task.id),
TaskChatMessage.role == "user",
)
.one()
)
assert stored.content == "Use the audio tool"
assert stored.turn_id == "live-turn-1"
assert agent.post_user_message.await_args.kwargs["turn_id"] == "live-turn-1"
assert observed_leases == [live_lease]
bg_mgr.register_reserved_resume.assert_called_once()
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
assert accepted[0]["client_message_id"] == "live-turn-1"
@pytest.mark.asyncio
async def test_legacy_continuation_runtime_fails_closed_without_side_effects(
db_session,
) -> None:
owner = _user(db_session, "legacy-continuation-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "runner-a"
task.run_id = "run-a"
db_session.commit()
legacy_pattern = MagicMock()
legacy_pattern.request_continuation = MagicMock()
agent = MagicMock()
agent.supports_live_control.return_value = False
agent.get_dag_pattern.return_value = legacy_pattern
mgr = MagicMock(get_agent_for_task=AsyncMock(return_value=agent))
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_command_execution.mark_user_message_delivery_sync",
) as mark_delivery,
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "must not enter legacy continuation",
"client_message_id": "legacy-continuation-1",
"user": owner,
"files": [],
},
)
legacy_pattern.request_continuation.assert_not_called()
mark_delivery.assert_not_called()
sent_messages = [
call.args[0] for call in ws_manager.send_personal_message.call_args_list
]
assert any(
message.get("message") == "Task does not support message continuation."
and message.get("error_code") == "message_continuation_unsupported"
for message in sent_messages
)
rejection = next(
message
for message in sent_messages
if message.get("type") == "message_rejected"
)
assert rejection["error_code"] == "message_continuation_unsupported"
@pytest.mark.asyncio
async def test_running_chat_message_uses_one_offloop_scope_and_no_request_session(
db_session,
) -> None:
"""The MESSAGE -> live resume handoff must not keep the request Session
while agent construction or resume coordination awaits.
"""
owner = _user(db_session, "scope-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
scope = ExecutionScope(
sandbox_key_suffix="tenant-a", workspace_segments=("tenant-a",)
)
main_thread_id = threading.get_ident()
resolver_threads: list[int] = []
def resolver(resolved_task_id: str) -> ExecutionScope:
assert resolved_task_id == str(task.id)
resolver_threads.append(threading.get_ident())
return scope
register_scope_resolver(resolver)
original_get_db = get_db
tracked_sessions: list[SimpleNamespace] = []
class TrackingSession:
def __init__(self, inner: object) -> None:
self.inner = inner
self.state = SimpleNamespace(closed=False)
tracked_sessions.append(self.state)
def __getattr__(self, name: str) -> object:
return getattr(self.inner, name)
def close(self) -> None:
self.state.closed = True
self.inner.close() # type: ignore[attr-defined]
def tracked_get_db():
inner_generator = original_get_db()
tracked = TrackingSession(next(inner_generator))
try:
yield tracked
finally:
tracked.close()
inner_generator.close()
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
)
manager_calls: list[tuple[object, dict]] = []
claim_threads: list[int] = []
def claim_delivery(**kwargs: object):
claim_threads.append(threading.get_ident())
return _claim_user_message_delivery_isolated(**kwargs) # type: ignore[arg-type]
async def get_agent_for_task(_task_id: int, db: object, **kwargs: object) -> object:
manager_calls.append(
(
db,
{
**kwargs,
"request_sessions_closed": all(
session.closed for session in tracked_sessions
),
},
)
)
return agent
agent_manager = MagicMock(
get_agent_for_task=AsyncMock(side_effect=get_agent_for_task)
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
resume_bg = AsyncMock()
bg_manager = MagicMock()
bg_manager.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_manager.running_tasks.get.return_value = None
try:
with (
patch("xagent.web.api.websocket.get_db", side_effect=tracked_get_db),
patch(
"xagent.web.services.task_command_execution._claim_user_message_delivery_isolated",
side_effect=claim_delivery,
),
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=agent_manager,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.execute_resume_background",
resume_bg,
),
patch(
"xagent.web.services.task_execution.background_task_manager", bg_manager
),
patch(
"xagent.web.api.websocket.task_execution_controller.transition",
new=AsyncMock(
return_value=SimpleNamespace(
run_id="live-run", status=TaskStatus.RUNNING
)
),
),
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "continue safely",
"client_message_id": "scope-live-turn",
"user": owner,
"files": [],
"_durable_ack_sent": True,
},
)
await asyncio.sleep(0)
finally:
register_scope_resolver(None)
assert len(manager_calls) == 1
manager_db, manager_kwargs = manager_calls[0]
assert manager_db is None
assert manager_kwargs["request_sessions_closed"] is True
assert manager_kwargs["task_setup_snapshot"] is not None
assert manager_kwargs["resolved_execution_scope"] is scope
assert resolver_threads and resolver_threads[0] != main_thread_id
assert len(resolver_threads) == 1
assert claim_threads and claim_threads[0] != main_thread_id
assert resume_bg.await_args.kwargs["resolved_execution_scope"] is scope
# The paused task row's own source must reach the resume, not the
# ``execute_resume_background`` default of ``None`` -- see
# ``test_execution_scope_turn_wiring.test_resume_task_forwards_the_task_rows_source``
# for the sibling resume-command call site.
assert resume_bg.await_args.kwargs["trusted_task_source"] == task.source
assert task.source == "sdk"
@pytest.mark.asyncio
async def test_deferred_chat_message_is_acked_after_durable_command_commit(
db_session,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.NOT_POSTED
)
mgr = MagicMock(get_agent_for_task=AsyncMock(return_value=agent))
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
resume_bg = AsyncMock()
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
websocket = MagicMock()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.execute_resume_background", resume_bg
),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
):
await handle_chat_message(
websocket,
int(task.id),
{
"message": "Wait for the checkpoint",
"client_message_id": "deferred-turn-1",
"user": owner,
"files": [],
},
)
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
db_session.expire_all()
stored_command = (
db_session.query(TaskExecutionCommand)
.filter_by(task_id=int(task.id), command_id="deferred-turn-1")
.one()
)
if (
stored_command.status == "pending"
and int(stored_command.attempt_count or 0) >= 1
and resume_bg.await_count == 1
):
break
await asyncio.sleep(0.01)
else:
raise AssertionError("deferred command claim was not released in time")
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
assert accepted[0]["client_message_id"] == "deferred-turn-1"
assert stored_command.status == "pending"
assert not any(
call.args[0].get("type") == "message_rejected"
for call in ws_manager.send_personal_message.call_args_list
)
kwargs = resume_bg.call_args.kwargs
assert kwargs["delivery_already_dispatched"] is False
assert kwargs["delivery_notifier"] is None
@pytest.mark.asyncio
async def test_live_lease_injection_degrades_to_deferred_on_checkpoint_unavailable(
live_task_lease,
db_session,
) -> None:
"""A checkpoint read failure during live injection must fold into the
same posted=False deferred-delivery path as no exact lease at all --
not a raw exception and not a rejection."""
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "unavailable-runner"
task.run_id = "unavailable-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
side_effect=CheckpointUnavailableError("checkpoint query failed")
)
mgr = MagicMock(get_agent_for_task=AsyncMock(return_value=agent))
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
resume_bg = AsyncMock()
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
websocket = MagicMock()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.execute_resume_background", resume_bg
),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
):
await handle_chat_message(
websocket,
int(task.id),
{
"message": "Wait for the checkpoint",
"client_message_id": "unavailable-turn-1",
"user": owner,
"files": [],
},
)
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
db_session.expire_all()
stored_command = (
db_session.query(TaskExecutionCommand)
.filter_by(task_id=int(task.id), command_id="unavailable-turn-1")
.one()
)
if (
stored_command.status == "pending"
and int(stored_command.attempt_count or 0) >= 1
and resume_bg.await_count == 1
):
break
await asyncio.sleep(0.01)
else:
raise AssertionError("deferred command claim was not released in time")
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
assert accepted[0]["client_message_id"] == "unavailable-turn-1"
assert stored_command.status == "pending"
assert not any(
call.args[0].get("type") == "message_rejected"
for call in ws_manager.send_personal_message.call_args_list
)
kwargs = resume_bg.call_args.kwargs
assert kwargs["delivery_already_dispatched"] is False
assert kwargs["delivery_notifier"] is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
("error", "sender_error_code", "sender_message"),
[
(
CheckpointCorruptError("all matching rows undecodable"),
"task_checkpoint_unreadable",
"The task's saved progress could not be read.",
),
(
CheckpointAccessRefusedError("active lease is not bound to this reader"),
"task_checkpoint_unreadable",
"The task's saved progress could not be read.",
),
(
DurableObjectIntegrityError(
"checksum mismatch",
storage_key="users/1/uploads/corrupt.txt",
),
"message_attachment_corrupt",
"A stored file for this message failed its integrity check and must be re-uploaded.",
),
(
DurableStorageOperationError(
"provider unavailable",
storage_key="users/1/uploads/unavailable.txt",
),
"message_attachment_unavailable",
"A stored file for this message could not be read. Please try again.",
),
],
ids=[
"checkpoint-corrupt",
"checkpoint-access-refused",
"attachment-corrupt",
"attachment-unavailable",
],
)
@pytest.mark.parametrize(
"durable_ack_sent",
[False, True],
ids=["live-ack", "suppressed-ack"],
)
async def test_durable_failure_keeps_detail_sender_only(
live_task_lease,
db_session,
error: Exception,
sender_error_code: str,
sender_message: str,
durable_ack_sent: bool,
) -> None:
"""Answer a durable turn failure without disclosing it to subscribers."""
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "rejected-runner"
task.run_id = "rejected-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(side_effect=error)
mgr = MagicMock(get_agent_for_task=AsyncMock(return_value=agent))
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
resume_bg = AsyncMock()
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
origin_socket = MagicMock(name="origin-socket")
payload = {
"message": "Wait for the checkpoint",
"client_message_id": "rejected-turn-1",
"user": owner,
"files": [],
}
if durable_ack_sent:
payload["_durable_ack_sent"] = True
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.execute_resume_background", resume_bg
),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
):
await handle_task_message(
_make_command_reply(origin_socket),
int(task.id),
payload,
)
resume_bg.assert_not_awaited()
bg_mgr.release_resume_reservation.assert_called_once_with(int(task.id))
rejected = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_rejected"
]
if durable_ack_sent:
assert not rejected
sender_errors = [
call
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "error"
]
assert len(sender_errors) == 1
assert sender_errors[0].args[0]["error_code"] == sender_error_code
assert sender_errors[0].args[0]["message"] == sender_message
assert sender_errors[0].args[1] is origin_socket
else:
assert len(rejected) == 1
assert rejected[0]["client_message_id"] == "rejected-turn-1"
assert rejected[0]["error_code"] == sender_error_code
assert rejected[0]["message"] == sender_message
assert rejected[0]["rejection_outcome"] == "not_accepted"
task_errors = [
call.args[0]
for call in ws_manager.broadcast_to_task.call_args_list
if call.args[0].get("type") in {"error", "agent_error", "task_error"}
]
assert task_errors
assert all(
payload["error_code"] == "task_execution_failed"
and payload["message"] == websocket_api.CLIENT_SAFE_TASK_FAILURE
for payload in task_errors
)
db_session.expire_all()
stored = (
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == "rejected-turn-1")
.one()
)
assert stored.delivery_status == DELIVERY_FAILED
@pytest.mark.asyncio
async def test_attachment_bind_race_keeps_specific_failure_on_origin_lane(
db_session,
tmp_path,
) -> None:
"""A post-prepare bind loss must not disclose sender state to subscribers."""
owner = _user(db_session, "bind-race-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "bind-race-runner"
task.run_id = "bind-race-run"
file_path = tmp_path / "bind-race.txt"
file_path.write_text("raced attachment")
db_session.add(
UploadedFile(
file_id="bind-race-file",
user_id=int(owner.id),
task_id=int(task.id),
filename="bind-race.txt",
storage_path=str(file_path),
mime_type="text/plain",
file_size=file_path.stat().st_size,
)
)
db_session.commit()
class RecordingSocket:
def __init__(self) -> None:
self.messages: list[dict[str, Any]] = []
async def send_text(self, payload: str) -> None:
import json
self.messages.append(json.loads(payload))
origin = RecordingSocket()
subscriber = RecordingSocket()
connection_manager = websocket_api.ConnectionManager()
connection_manager.register_connection(origin, int(task.id)) # type: ignore[arg-type]
connection_manager.register_connection(subscriber, int(task.id)) # type: ignore[arg-type]
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent_manager = MagicMock(get_agent_for_task=AsyncMock(return_value=agent))
bg_manager = MagicMock()
bg_manager.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_manager.running_tasks.get.return_value = None
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=agent_manager,
),
patch("xagent.web.api.websocket.manager", connection_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_manager),
patch(
"xagent.web.services.task_command_execution.bind_turn_files_no_commit",
return_value=["bind-race-file"],
),
):
await handle_task_message(
_make_command_reply(origin), # type: ignore[arg-type]
int(task.id),
{
"message": "Use my attachment",
"client_message_id": "bind-race-turn",
"user": SimpleNamespace(id=int(owner.id), is_admin=False),
"files": [
{
"file_id": "bind-race-file",
"name": "bind-race.txt",
}
],
},
)
assert any(
event.get("type") == "message_rejected"
and event.get("error_code") == "message_attachment_unavailable"
for event in origin.messages
)
subscriber_failures = [
event
for event in subscriber.messages
if event.get("type") in {"error", "agent_error", "task_error"}
]
assert subscriber_failures
assert all(
event.get("error_code") == "task_execution_failed"
and event.get("message") == websocket_api.CLIENT_SAFE_TASK_FAILURE
for event in subscriber_failures
)
@pytest.mark.asyncio
async def test_resume_registration_failure_keeps_injected_delivery_pending(
live_task_lease,
db_session,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "registration-runner"
task.run_id = "registration-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
)
mgr = MagicMock(get_agent_for_task=AsyncMock(return_value=agent))
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
bg_mgr.register_reserved_resume.side_effect = RuntimeError("reservation lost")
bg_handle = MagicMock()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.execute_resume_background", AsyncMock()
),
patch(
"xagent.web.api.websocket.asyncio.create_task",
return_value=bg_handle,
),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
):
await handle_chat_message(
MagicMock(),
int(task.id),
{
"message": "Apply this safely",
"client_message_id": "registration-failure-turn",
"user": owner,
"files": [],
},
)
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if bg_handle.cancel.called:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("resume registration failure was not handled in time")
bg_handle.cancel.assert_called_once()
db_session.expire_all()
stored = (
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == "registration-failure-turn")
.one()
)
assert stored.delivery_status == DELIVERY_PENDING
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
@pytest.mark.asyncio
async def test_live_marker_failure_after_registered_handoff_is_still_accepted(
live_task_lease,
db_session,
) -> None:
owner = _user(db_session, "marker-failure-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "marker-failure-runner"
task.run_id = "marker-failure-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_execution.execute_resume_background", AsyncMock()
),
patch(
"xagent.web.services.task_command_execution.mark_user_message_delivery_sync",
side_effect=RuntimeError("marker unavailable"),
),
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "Apply this once",
"client_message_id": "marker-failure-turn",
"user": owner,
"files": [],
},
)
bg_mgr.register_reserved_resume.assert_called_once()
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
"active_interaction_read,expected_interaction_id", PRE_CHANGE_EQUIVALENT
)
async def test_live_resume_reads_the_interaction_row_before_injecting(
live_task_lease,
db_session,
active_interaction_read: ActiveInteractionRead,
expected_interaction_id: int | None,
caplog: pytest.LogCaptureFixture,
) -> None:
"""The close is keyed on the row observed *before* the injection, and
only the ordering makes that true -- see task_interaction_close's
module docstring. The read also sits before the ``posted`` fork, so the
deferred branch carries the same observation instead of taking one of
its own even later.
Parametrized over every state PRE_CHANGE_EQUIVALENT enumerates (Found,
Absent, and both Unavailable reasons): this site's translation to the
``int | None`` the close call takes must produce the same result for
all three states that it did before this became a three-state read,
regardless of which reason an unavailable read carries.
That equivalence alone would also hold for a two-way ``Found`` versus
everything-else fold, so the cells also assert the log line this site
emits on -- and only on -- the ``ActiveInteractionUnavailable``
branch, carrying that state's own reason word. A fold that dropped the
third branch leaves both unavailable cells without their line.
"""
owner = _user(db_session, "close-order-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "close-order-runner"
task.run_id = "close-order-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
order: list[str] = []
def record_read(_task_id: int) -> ActiveInteractionRead:
order.append("read")
return active_interaction_read
async def record_injection(
*_args: object, **_kwargs: object
) -> UserMessageInjectionOutcome:
order.append("inject")
return UserMessageInjectionOutcome.POSTED_FRESH
agent.post_user_message = AsyncMock(side_effect=record_injection)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_execution.execute_resume_background", AsyncMock()
),
patch(
"xagent.web.services.task_command_execution.active_interaction_id_sync",
side_effect=record_read,
),
patch(
"xagent.web.services.task_command_execution.close_legacy_resume_interaction_sync",
return_value=1,
) as close_mock,
caplog.at_level(logging.INFO, logger=command_execution_service.__name__),
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "Apply this once",
"client_message_id": "close-order-turn",
"user": owner,
"files": [],
},
)
assert order == ["read", "inject"]
close_mock.assert_called_once_with(
task_id=int(task.id),
run_id="close-order-run",
interaction_id=expected_interaction_id,
)
unavailable_lines = [
record.getMessage()
for record in caplog.records
if "active interaction read unavailable" in record.getMessage()
]
if isinstance(active_interaction_read, ActiveInteractionUnavailable):
assert unavailable_lines == [
"active interaction read unavailable "
f"(reason={active_interaction_read.reason}) for task_id={int(task.id)}; "
"the legacy resume close will match no row"
]
else:
assert unavailable_lines == []
def _live_injection_doubles(
outcome: UserMessageInjectionOutcome,
) -> tuple[MagicMock, MagicMock, MagicMock]:
"""An agent whose live injection reports ``outcome``, plus the
WebSocket and background managers ``handle_task_message`` needs to
reach it."""
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(return_value=outcome)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
return agent, ws_manager, bg_mgr
async def _send_live_message(
task_id: int,
owner: User,
*,
client_message_id: str,
agent: MagicMock,
ws_manager: MagicMock,
bg_mgr: MagicMock,
) -> None:
"""Drive ``handle_task_message`` with the real interaction reader and
the real close."""
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_execution.execute_resume_background", AsyncMock()
),
):
await handle_task_message(
_make_command_reply(MagicMock()),
task_id,
{
"message": "Use staging",
"client_message_id": client_message_id,
"user": owner,
"files": [],
},
)
def _sent_error_frames(ws_manager: MagicMock) -> list[dict]:
return [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "error"
]
def _interaction_and_marker(
db: Session, *, task_id: int, row_id: int
) -> tuple[tuple[str, str | None, int | None], int | None]:
db.expire_all()
row = db.get(TaskInteractionRequest, row_id)
task = db.get(Task, task_id)
return (row.status, row.terminal_reason, row.active_slot), (
task.interaction_protocol_version
)
@pytest.mark.asyncio
async def test_a_chat_message_with_an_active_row_is_injected_and_closes_it_under_native_shared_execution(
live_task_lease,
db_session,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The online injection site reads and closes the active row the same
way in a process that can publish native rows: the message is injected
once, and the row it answered is closed as answered through a legacy
resume with the task's marker cleared. Neither the reader nor the close
is stubbed."""
install_interaction_rollout_policy(
monkeypatch, mode="native", shared_task_execution=True
)
owner = _user(db_session, "native-close-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "native-close-runner"
task.run_id = "native-close-run"
task.interaction_protocol_version = 1
db_session.commit()
live_task_lease(db_session, task)
task_id = int(task.id)
row_id = _seed_active_interaction_row(
db_session,
task_id=task_id,
run_id="native-close-run",
idempotency_key="native-close-q1",
)
agent, ws_manager, bg_mgr = _live_injection_doubles(
UserMessageInjectionOutcome.POSTED_FRESH
)
await _send_live_message(
task_id,
owner,
client_message_id="native-close-turn",
agent=agent,
ws_manager=ws_manager,
bg_mgr=bg_mgr,
)
assert _sent_error_frames(ws_manager) == []
agent.post_user_message.assert_awaited_once()
assert _interaction_and_marker(db_session, task_id=task_id, row_id=row_id) == (
("terminated", "answered_via_legacy_resume", None),
None,
)
@pytest.mark.asyncio
async def test_a_row_left_by_an_unknown_injection_refuses_a_bare_resume_and_closes_on_the_next_fresh_message(
live_task_lease,
db_session,
monkeypatch: pytest.MonkeyPatch,
request: pytest.FixtureRequest,
) -> None:
"""An injection whose outcome is unknown leaves the active row and the
marker in place, because only a fresh post proves the message reached
the run. That row then refuses a resume carrying no receipt, and the
next message that is posted fresh closes it.
Between the refused resume and the last message the test itself writes
the task back to the running state the online injection site needs;
the task's paused state before the resume is written by the test too.
"""
# The refused resume registers this process-global signal; /health's
# own tests assert its payload exactly, so it must not outlive the test.
request.addfinalizer(
lambda: ops_signals.clear_degradation(
ops_signals.INTERACTION_LEGACY_RESUME_SHIM
)
)
install_interaction_rollout_policy(
monkeypatch, mode="legacy", shared_task_execution=True
)
owner = _user(db_session, "unknown-close-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "unknown-close-runner"
task.run_id = "unknown-close-run"
task.interaction_protocol_version = 1
db_session.commit()
live_task_lease(db_session, task)
task_id = int(task.id)
row_id = _seed_active_interaction_row(
db_session,
task_id=task_id,
run_id="unknown-close-run",
idempotency_key="unknown-close-q1",
)
agent, ws_manager, bg_mgr = _live_injection_doubles(
UserMessageInjectionOutcome.OUTCOME_UNKNOWN
)
await _send_live_message(
task_id,
owner,
client_message_id="unknown-close-turn-1",
agent=agent,
ws_manager=ws_manager,
bg_mgr=bg_mgr,
)
agent.post_user_message.assert_awaited_once()
assert _interaction_and_marker(db_session, task_id=task_id, row_id=row_id) == (
("active", None, 1),
1,
)
db_session.get(Task, task_id).status = TaskStatus.PAUSED
db_session.commit()
with patch("xagent.web.api.websocket.manager", ws_manager):
result = await command_execution_service.resume_task(
_make_command_reply(MagicMock()),
task_id,
{"user": SimpleNamespace(id=int(owner.id), is_admin=False)},
)
assert result.outcome is command_execution_service.ResumeCommandOutcome.REJECTED
assert result.reason_code == "interaction_pending"
assert _interaction_and_marker(db_session, task_id=task_id, row_id=row_id) == (
("active", None, 1),
1,
)
db_session.get(Task, task_id).status = TaskStatus.RUNNING
db_session.commit()
agent, ws_manager, bg_mgr = _live_injection_doubles(
UserMessageInjectionOutcome.POSTED_FRESH
)
await _send_live_message(
task_id,
owner,
client_message_id="unknown-close-turn-2",
agent=agent,
ws_manager=ws_manager,
bg_mgr=bg_mgr,
)
agent.post_user_message.assert_awaited_once()
assert _interaction_and_marker(db_session, task_id=task_id, row_id=row_id) == (
("terminated", "answered_via_legacy_resume", None),
None,
)
@pytest.mark.asyncio
async def test_live_injection_skips_the_close_on_a_replayed_turn_id(
live_task_lease,
db_session,
) -> None:
"""A replayed turn id short-circuits inside AgentRunner.inject_user_message
and is reported back as POSTED_REPLAY, not the same truthy value a
fresh write produces. The interaction row this test seeds is not the
question the replay answered, so the online injection site must leave
it untouched: still active, uncleared marker, and the close statement
must not run at all. That is asserted directly on the mock: replacing
the guard with `if True:` calls the close for real and turns
assert_not_called red. No DB-state assertion is made here -- with the
close function mocked out, nothing writes to the row or the marker, so
"still active" would hold no matter what the guard did."""
owner = _user(db_session, "close-replay-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "close-replay-runner"
task.run_id = "close-replay-run"
task.interaction_protocol_version = 1
db_session.commit()
live_task_lease(db_session, task)
task_id = int(task.id)
_seed_active_interaction_row(
db_session,
task_id=task_id,
run_id="close-replay-run",
idempotency_key="close-replay-q1",
)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_REPLAY
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_execution.execute_resume_background", AsyncMock()
),
patch(
"xagent.web.services.task_command_execution.close_legacy_resume_interaction_sync",
) as close_mock,
):
await handle_task_message(
_make_command_reply(MagicMock()),
task_id,
{
"message": "a retried delivery",
"client_message_id": "close-replay-turn",
"user": owner,
"files": [],
},
)
agent.post_user_message.assert_awaited_once()
close_mock.assert_not_called()
@pytest.mark.asyncio
async def test_live_close_failure_after_registered_handoff_is_still_accepted(
live_task_lease,
db_session,
) -> None:
"""A legacy resume interaction close failure must not turn an already
registered, already accepted resume handoff into a rejected one."""
owner = _user(db_session, "close-failure-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "close-failure-runner"
task.run_id = "close-failure-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_execution.execute_resume_background", AsyncMock()
),
patch(
"xagent.web.services.task_command_execution.close_legacy_resume_interaction_sync",
side_effect=RuntimeError("interaction close unavailable"),
) as close_mock,
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "Apply this once",
"client_message_id": "close-failure-turn",
"user": owner,
"files": [],
},
)
bg_mgr.register_reserved_resume.assert_called_once()
close_mock.assert_called_once_with(
task_id=int(task.id), run_id="close-failure-run", interaction_id=None
)
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
@pytest.mark.asyncio
async def test_live_close_cancellation_does_not_abort_registered_handoff(
live_task_lease,
db_session,
) -> None:
"""Same guarantee as the failure case above, for the CancelledError
branch specifically -- deleting that branch must turn this red."""
owner = _user(db_session, "close-cancel-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "close-cancel-runner"
task.run_id = "close-cancel-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
def raise_cancelled(*_args: object, **_kwargs: object) -> int:
raise asyncio.CancelledError
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_execution.execute_resume_background", AsyncMock()
),
patch(
"xagent.web.services.task_command_execution.close_legacy_resume_interaction_sync",
side_effect=raise_cancelled,
) as close_mock,
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "Apply this once",
"client_message_id": "close-cancel-turn",
"user": owner,
"files": [],
},
)
bg_mgr.register_reserved_resume.assert_called_once()
close_mock.assert_called_once_with(
task_id=int(task.id), run_id="close-cancel-run", interaction_id=None
)
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
def test_lease_restore_fence_miss_leaves_the_marker_untouched(db_session) -> None:
"""When the exact-lease fence excludes every row (this call lost the
race for the row), the marker clear must not run at all -- not even
the no-op form. Flipping the ``if restored:`` guard to unconditional
would clear a marker this call never had authority to touch.
"""
owner = _user(db_session, "restore-fence-miss-owner")
task = _task(db_session, owner.id, status=TaskStatus.WAITING_FOR_USER)
task.runner_id = "current-runner"
task.run_id = "run-fence-miss"
task.interaction_protocol_version = 1
db_session.commit()
stale_lease = TaskLease(
task_id=int(task.id), runner_id="a-different-runner", run_id="run-fence-miss"
)
restored = _restore_resumed_task_lease_to_prior_status(
stale_lease, status=TaskStatus.WAITING_FOR_USER
)
assert restored is False
db_session.expire_all()
refreshed = db_session.query(Task).filter(Task.id == task.id).one()
assert refreshed.interaction_protocol_version == 1
def test_lease_restore_clears_the_marker_once_no_active_row_remains(
db_session,
) -> None:
"""The mirror case: the fence matches (this call owns the row) and no
active interaction row remains, so the marker is stale and gets
zeroed."""
owner = _user(db_session, "restore-clears-marker-owner")
task = _task(db_session, owner.id, status=TaskStatus.WAITING_FOR_USER)
task.runner_id = "current-runner"
task.lease_attempt_id = "restore-attempt"
task.run_id = "run-clears-marker"
task.interaction_protocol_version = 1
db_session.commit()
lease = TaskLease(
task_id=int(task.id),
runner_id="current-runner",
run_id="run-clears-marker",
attempt_id="restore-attempt",
)
restored = _restore_resumed_task_lease_to_prior_status(
lease, status=TaskStatus.WAITING_FOR_USER
)
assert restored is True
db_session.expire_all()
refreshed = db_session.query(Task).filter(Task.id == task.id).one()
assert refreshed.interaction_protocol_version is None
def test_live_claim_unique_loser_returns_the_committed_winner(
db_session,
) -> None:
owner = _user(db_session, "live-unique-loser-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Apply exactly once",
message_type="user_message",
turn_id="live-unique-loser-turn",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
with patch(
"xagent.web.services.task_command_execution.claim_user_message_delivery_no_commit",
side_effect=IntegrityError(
"INSERT task_chat_messages",
{},
RuntimeError("unique constraint"),
),
):
claim = _claim_user_message_delivery_isolated(
task_id=int(task.id),
task_owner_user_id=int(owner.id),
content="Apply exactly once",
attachments=None,
file_ids=[],
turn_id="live-unique-loser-turn",
expected_run_id=None,
expected_status=TaskStatus.RUNNING,
)
assert claim.claimed is False
assert claim.payload_matches is True
assert claim.pending is True
assert claim.failed is False
def test_live_claim_unique_loser_returns_conflicting_winner_with_files(
db_session,
) -> None:
owner = _user(db_session, "live-conflicting-unique-loser-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Winner payload",
message_type="user_message",
turn_id="live-conflicting-unique-loser-turn",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
with patch(
"xagent.web.services.task_command_execution.claim_user_message_delivery_no_commit",
side_effect=IntegrityError(
"INSERT task_chat_messages",
{},
RuntimeError("unique constraint"),
),
):
claim = _claim_user_message_delivery_isolated(
task_id=int(task.id),
task_owner_user_id=int(owner.id),
content="Conflicting loser payload",
attachments=[{"file_id": "loser-file"}],
file_ids=["loser-file"],
turn_id="live-conflicting-unique-loser-turn",
expected_run_id=None,
expected_status=TaskStatus.RUNNING,
)
assert claim.claimed is False
assert claim.payload_matches is False
assert claim.pending is True
assert claim.failed is False
@pytest.mark.asyncio
async def test_message_handoff_registers_the_minted_run_not_the_stale_one(
db_session,
) -> None:
"""A legacy NULL-run_id row gets a fresh run at the transition.
``apply_task_control_transition`` mints a run id when the row has none,
so the value read from the routing snapshot before that call is ``None``
while the task now runs under a uuid. Registering the coordinator under
the stale value makes a later RESUME asking about the real run read a
live resume as ``RESERVATION_HELD``, defer sixty times, and terminally
fail a resume that was already succeeding.
"""
owner = _user(db_session, "minted-run-owner")
# WAITING_FOR_USER, because that is what routes a chat message into the
# live-control resume handoff (``_task_status_uses_live_control``).
task = _task(db_session, owner.id, status=TaskStatus.WAITING_FOR_USER)
task.run_id = None
task.control_state = "waiting_for_user"
db_session.commit()
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
# No exact live lease, so the handoff defers the message to the resume
# owner instead of injecting it -- the path that reaches the transition.
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.NOT_POSTED
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
registered_run_ids: list[str | None] = []
registered_handles: list[asyncio.Task] = []
def register_resume(
_task_id: int,
task_handle: asyncio.Task,
*,
run_id: str | None = None,
) -> None:
registered_run_ids.append(run_id)
registered_handles.append(task_handle)
async def resume_forever(*_args, **_kwargs) -> None:
await asyncio.Event().wait()
bg_mgr.register_reserved_resume.side_effect = register_resume
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_execution.execute_resume_background",
side_effect=resume_forever,
) as resume_bg,
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "resume a legacy row",
"client_message_id": "minted-run-turn",
"user": owner,
"files": [],
},
)
# The coroutine is recorded at construction, before the loop runs it.
resume_expected_run_id = resume_bg.call_args.kwargs["expected_run_id"]
for handle in registered_handles:
handle.cancel()
await asyncio.gather(*registered_handles, return_exceptions=True)
db_session.expire_all()
stored = db_session.get(Task, int(task.id))
assert stored is not None
minted = stored.run_id
assert minted is not None, "the transition should have minted a run id"
assert registered_run_ids == [minted], (
"the coordinator must be registered under the run the task is "
"actually executing, not the NULL the snapshot carried"
)
# And the execution has to agree: a None here would reach
# acquire_task_lease_no_commit and mint a *second* run, putting the row,
# the registration and the lease on three different answers.
assert resume_expected_run_id == minted
@pytest.mark.asyncio
async def test_live_marker_cancellation_does_not_cancel_registered_handoff(
live_task_lease,
db_session,
) -> None:
owner = _user(db_session, "marker-cancellation-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "marker-cancellation-runner"
task.run_id = "marker-cancellation-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
marker_started = threading.Event()
marker_release = threading.Event()
registered_tasks: list[asyncio.Task] = []
background_cancelled = asyncio.Event()
def mark_delivery(*_args, **_kwargs) -> None:
marker_started.set()
assert marker_release.wait(timeout=_THREAD_SIGNAL_DEADLINE_SECONDS)
async def resume_forever(*_args, **_kwargs) -> None:
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
background_cancelled.set()
raise
def register_resume(
_task_id: int,
task_handle: asyncio.Task,
*,
run_id: str | None = None,
) -> None:
assert run_id == "marker-cancellation-run"
registered_tasks.append(task_handle)
bg_mgr.register_reserved_resume.side_effect = register_resume
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_execution.execute_resume_background",
side_effect=resume_forever,
),
patch(
"xagent.web.services.task_command_execution.mark_user_message_delivery_sync",
side_effect=mark_delivery,
),
):
handling = asyncio.get_running_loop().create_task(
handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "Apply despite disconnect",
"client_message_id": "marker-cancellation-turn",
"user": owner,
"files": [],
},
)
)
assert await asyncio.to_thread(
marker_started.wait, _THREAD_SIGNAL_DEADLINE_SECONDS
)
handling.cancel()
marker_release.set()
with pytest.raises(asyncio.CancelledError):
await handling
assert len(registered_tasks) == 1
assert registered_tasks[0].cancelled() is False
assert background_cancelled.is_set() is False
bg_mgr.release_resume_reservation.assert_not_called()
registered_tasks[0].cancel()
with pytest.raises(asyncio.CancelledError):
await registered_tasks[0]
def test_live_claim_reconciles_a_commit_acknowledgement_failure(
db_session,
monkeypatch: pytest.MonkeyPatch,
) -> None:
owner = _user(db_session, "live-ack-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
original_commit = Session.commit
commit_raised = False
def acknowledge_then_disconnect(session: Session) -> None:
nonlocal commit_raised
original_commit(session)
if session is not db_session and not commit_raised:
commit_raised = True
raise ConnectionError("commit acknowledgement lost")
monkeypatch.setattr(Session, "commit", acknowledge_then_disconnect)
claim = _claim_user_message_delivery_isolated(
task_id=int(task.id),
task_owner_user_id=int(owner.id),
content="Apply safely",
attachments=None,
file_ids=[],
turn_id="live-ack-turn",
expected_run_id=None,
expected_status=TaskStatus.RUNNING,
)
assert commit_raised is True
assert claim.claimed is True
db_session.expire_all()
assert (
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == "live-ack-turn")
.one()
.delivery_status
== DELIVERY_PENDING
)
def test_waiting_for_user_claim_reconciles_a_commit_acknowledgement_failure(
db_session,
monkeypatch: pytest.MonkeyPatch,
) -> None:
owner = _user(db_session, "waiting-ack-owner")
task = _task(db_session, owner.id, status=TaskStatus.WAITING_FOR_USER)
task.run_id = "waiting-ack-run"
db_session.commit()
original_commit = Session.commit
commit_raised = False
def acknowledge_then_disconnect(session: Session) -> None:
nonlocal commit_raised
original_commit(session)
if session is not db_session and not commit_raised:
commit_raised = True
raise ConnectionError("commit acknowledgement lost")
monkeypatch.setattr(Session, "commit", acknowledge_then_disconnect)
claim = _claim_user_message_delivery_isolated(
task_id=int(task.id),
task_owner_user_id=int(owner.id),
content="Answer the pending question",
attachments=None,
file_ids=[],
turn_id="waiting-ack-turn",
expected_run_id="waiting-ack-run",
expected_status=TaskStatus.WAITING_FOR_USER,
)
assert commit_raised is True
assert claim.claimed is True
db_session.expire_all()
stored = (
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == "waiting-ack-turn")
.one()
)
assert stored.delivery_status == DELIVERY_PENDING
def test_websocket_commit_reconciliation_read_failure_stays_unknown(
monkeypatch: pytest.MonkeyPatch,
) -> None:
session = MagicMock()
session.query.side_effect = RuntimeError("reconciliation database unavailable")
monkeypatch.setattr(
command_execution_service,
"get_session_local",
lambda: lambda: session,
)
monkeypatch.setattr(command_execution_service.time, "sleep", MagicMock())
assert (
command_execution_service._reconcile_command_acceptance_graph(
task_id=123,
task_owner_user_id=456,
turn_id="read-failure-turn",
content="Accepted",
file_ids=[],
expected_run_id="run",
expected_status=TaskStatus.RUNNING,
)
is False
)
assert session.query.call_count == 3
def test_websocket_commit_reconciliation_rejects_failed_delivery(
db_session,
) -> None:
owner = _user(db_session, "failed-reconciliation-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.run_id = "failed-reconciliation-run"
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Accepted",
message_type="user_message",
turn_id="failed-reconciliation-turn",
delivery_status=DELIVERY_FAILED,
)
)
db_session.commit()
assert (
command_execution_service._reconcile_command_acceptance_graph(
task_id=int(task.id),
task_owner_user_id=int(owner.id),
turn_id="failed-reconciliation-turn",
content="Accepted",
file_ids=[],
expected_run_id="failed-reconciliation-run",
expected_status=TaskStatus.RUNNING,
)
is False
)
def test_missing_task_prepare_reconciles_a_commit_acknowledgement_failure(
db_session,
monkeypatch: pytest.MonkeyPatch,
) -> None:
owner = _user(db_session, "missing-ack-owner")
original_commit = Session.commit
commit_raised = False
def acknowledge_then_disconnect(session: Session) -> None:
nonlocal commit_raised
original_commit(session)
if session is not db_session and not commit_raised:
commit_raised = True
raise ConnectionError("commit acknowledgement lost")
monkeypatch.setattr(Session, "commit", acknowledge_then_disconnect)
preparation = command_execution_service._prepare_task_message_sync(
requested_task_id=987654,
actor_user_id=int(owner.id),
actor_is_admin=False,
user_message="start after acknowledgement loss",
raw_context={},
raw_files=[],
client_message_id="missing-ack-turn",
turn_id="missing-ack-turn",
durable_attempt_count=1,
durable_target_run_id=None,
pause_accepted=False,
)
assert commit_raised is True
assert preparation.task_created is True
db_session.expire_all()
assert (
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == "missing-ack-turn")
.one()
.delivery_status
== DELIVERY_PENDING
)
def test_prepare_task_message_omits_kb_instruction_for_agent_builder_agents(
db_session,
tmp_path,
) -> None:
"""Task-path agents no longer mount the KB authoring tools (#2219).
A saved agent carrying the ``agent-builder`` skill used to get a system
prompt ordering ``create_knowledge_base_from_file`` for every upload.
That tool is only mounted by the builder chat now, so the task path must
not name it anywhere the model reads.
"""
from xagent.web.models.agent import Agent
owner = _user(db_session, "builder-skill-owner")
agent_row = Agent(
user_id=int(owner.id),
name="builder-skill-agent",
skills=["agent-builder"],
)
db_session.add(agent_row)
db_session.commit()
db_session.refresh(agent_row)
task = _task(db_session, int(owner.id))
task.agent_id = int(agent_row.id)
db_session.commit()
upload_path = tmp_path / "faq.txt"
upload_path.write_text("faq")
db_session.add(
UploadedFile(
file_id="builder-skill-file",
user_id=int(owner.id),
task_id=None,
filename="faq.txt",
storage_path=str(upload_path),
mime_type="text/plain",
file_size=3,
)
)
db_session.commit()
preparation = command_execution_service._prepare_task_message_sync(
requested_task_id=int(task.id),
actor_user_id=int(owner.id),
actor_is_admin=False,
user_message="build a knowledge base from this",
raw_context={},
raw_files=[{"file_id": "builder-skill-file"}],
client_message_id=None,
turn_id="builder-skill-turn",
durable_attempt_count=1,
durable_target_run_id=None,
pause_accepted=False,
)
system_prompt = preparation.execution_context["system_prompt"]
assert "## UPLOADED FILES" in system_prompt
assert "create_knowledge_base_from_file" not in system_prompt
assert "builder-skill-file" in preparation.user_message_for_llm
assert "create_knowledge_base_from_file" not in preparation.user_message_for_llm
def test_missing_task_prepare_keeps_an_absent_commit_outcome_unknown(
db_session,
monkeypatch: pytest.MonkeyPatch,
) -> None:
owner = _user(db_session, "missing-unknown-owner")
def lose_commit_before_acknowledgement(_session: Session) -> None:
raise ConnectionError("commit acknowledgement lost")
monkeypatch.setattr(Session, "commit", lose_commit_before_acknowledgement)
with pytest.raises(command_execution_service._TaskCommandCommitOutcomeUnknown):
command_execution_service._prepare_task_message_sync(
requested_task_id=987655,
actor_user_id=int(owner.id),
actor_is_admin=False,
user_message="unknown acceptance",
raw_context={},
raw_files=[],
client_message_id="missing-unknown-turn",
turn_id="missing-unknown-turn",
durable_attempt_count=1,
durable_target_run_id=None,
pause_accepted=False,
)
@pytest.mark.asyncio
async def test_live_control_delivery_failure_pool_timeout_is_not_retried(
live_task_lease,
db_session,
) -> None:
"""One failed DELIVERY_FAILED checkout still produces one rejection ack."""
owner = _user(db_session, "pool-timeout-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "pool-timeout-runner"
task.run_id = "pool-timeout-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
side_effect=UserMessageInjectionRejectedError("inject failed")
)
mgr = MagicMock(get_agent_for_task=AsyncMock(return_value=agent))
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
mark_delivery = MagicMock(
side_effect=SQLAlchemyTimeoutError("delivery pool exhausted")
)
error_payload_reader = MagicMock()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_command_execution.mark_user_message_delivery_sync",
mark_delivery,
),
patch(
"xagent.web.services.task_command_execution._read_task_error_payload_isolated",
error_payload_reader,
),
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "apply once",
"client_message_id": "live-control-pool-timeout",
"user": owner,
"files": [],
},
)
mark_delivery.assert_called_once_with(
int(task.id), "live-control-pool-timeout", DELIVERY_FAILED
)
error_payload_reader.assert_not_called()
rejected = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_rejected"
]
assert len(rejected) == 1
assert rejected[0]["client_message_id"] == "live-control-pool-timeout"
assert "inject failed" not in repr(rejected[0])
# A proven-absent write is a not-accepted delivery, but the failed
# persistence leaves the sender unable to tell, so no new-id retry.
assert rejected[0]["error_code"] == "message_delivery_failed"
assert rejected[0]["rejection_outcome"] == "outcome_unknown"
assert not rejected[0].get("retry_with_new_id")
@pytest.mark.asyncio
async def test_delivery_failure_persistence_drains_before_cancellation(
live_task_lease,
db_session,
) -> None:
owner = _user(db_session, "delivery-cancellation-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "delivery-cancellation-runner"
task.run_id = "delivery-cancellation-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
side_effect=UserMessageInjectionRejectedError("inject failed")
)
agent_manager = MagicMock(get_agent_for_task=AsyncMock(return_value=agent))
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_manager = MagicMock()
bg_manager.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
persistence_started = threading.Event()
allow_persistence = threading.Event()
persistence_finished = threading.Event()
def blocking_mark_delivery(*_args, **_kwargs) -> None:
persistence_started.set()
assert allow_persistence.wait(timeout=_THREAD_SIGNAL_DEADLINE_SECONDS)
persistence_finished.set()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=agent_manager,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_manager),
patch(
"xagent.web.services.task_command_execution.mark_user_message_delivery_sync",
side_effect=blocking_mark_delivery,
),
):
handling = asyncio.create_task(
handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "apply once",
"client_message_id": "delivery-cancellation-turn",
"user": owner,
"files": [],
},
)
)
# Without the assert a slow runner silently proceeds to cancel before
# persistence started, which changes what the rest of the test proves.
assert await asyncio.to_thread(
persistence_started.wait, _THREAD_SIGNAL_DEADLINE_SECONDS
)
handling.cancel()
await asyncio.sleep(0)
assert not handling.done()
allow_persistence.set()
with pytest.raises(asyncio.CancelledError):
await handling
assert persistence_finished.is_set()
@pytest.mark.asyncio
async def test_retried_durable_message_is_accepted_without_reexecution(
db_session,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Already delivered",
message_type="user_message",
turn_id="stable-turn-1",
)
)
db_session.commit()
agent_manager = MagicMock()
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=agent_manager,
),
patch("xagent.web.api.websocket.manager", ws_manager),
):
await handle_chat_message(
MagicMock(),
int(task.id),
{
"message": "Already delivered",
"client_message_id": "stable-turn-1",
"user": owner,
"files": [],
},
)
agent_manager.get_agent_for_task.assert_not_called()
stored = (
db_session.query(TaskChatMessage)
.filter(
TaskChatMessage.task_id == int(task.id),
TaskChatMessage.turn_id == "stable-turn-1",
)
.all()
)
assert len(stored) == 1
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
@pytest.mark.asyncio
async def test_same_id_resend_runs_one_turn(live_task_lease, db_session) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id, task.run_id = "live-runner", "live-run"
db_session.commit()
live_task_lease(db_session, task)
release = asyncio.Event()
async def post_user_message(*_args, **_kwargs) -> UserMessageInjectionOutcome:
await release.wait()
return UserMessageInjectionOutcome.POSTED_FRESH
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(side_effect=post_user_message)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(), send_personal_message=AsyncMock()
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
frame = {"message": "Use the tool", "client_message_id": "resend-1", "files": []}
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.execute_resume_background", AsyncMock()
),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
):
# The resend arrives while the first delivery is still being applied.
for _ in range(2):
await handle_chat_message(
MagicMock(), int(task.id), {**frame, "user": owner}
)
release.set()
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
db_session.expire_all()
command = db_session.query(TaskExecutionCommand).one()
if command.status == "completed":
break
await asyncio.sleep(0.01)
assert command.status == "completed"
types = [c.args[0]["type"] for c in ws_manager.send_personal_message.call_args_list]
assert types.count("message_accepted") == 2
assert db_session.query(TaskChatMessage).filter_by(turn_id="resend-1").count() == 1
assert command.attempt_count == 1
assert agent.post_user_message.await_count == 1
bg_mgr.register_reserved_resume.assert_called_once()
@pytest.mark.asyncio
async def test_reusing_client_id_with_different_content_is_rejected(
db_session,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Original content",
message_type="user_message",
turn_id="stable-turn-1",
)
)
db_session.commit()
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
begin_turn = AsyncMock()
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.begin_turn",
new=begin_turn,
),
):
await handle_chat_message(
MagicMock(),
int(task.id),
{
"message": "Different content",
"client_message_id": "stable-turn-1",
"user": owner,
"files": [],
},
)
begin_turn.assert_not_awaited()
rejected = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_rejected"
]
assert len(rejected) == 1
assert rejected[0]["error_code"] == "message_id_conflict"
assert rejected[0]["retry_with_new_id"] is True
assert rejected[0]["rejection_outcome"] == "not_accepted"
@pytest.mark.asyncio
async def test_failed_durable_delivery_is_not_silently_accepted(db_session) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.FAILED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Retry after checkpoint failure",
message_type="user_message",
turn_id="failed-turn-1",
delivery_status=DELIVERY_FAILED,
)
)
db_session.commit()
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
begin_turn = AsyncMock()
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_orchestrator.TaskTurnOrchestrator.begin_turn",
new=begin_turn,
),
):
await handle_chat_message(
MagicMock(),
int(task.id),
{
"message": "Retry after checkpoint failure",
"client_message_id": "failed-turn-1",
"user": owner,
"files": [],
},
)
begin_turn.assert_not_awaited()
rejected = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_rejected"
]
assert len(rejected) == 1
assert rejected[0]["error_code"] == "message_delivery_failed"
assert rejected[0]["retry_with_new_id"] is True
assert rejected[0]["rejection_outcome"] == "not_accepted"
@pytest.mark.asyncio
async def test_pending_same_id_delivery_reports_unknown_outcome(db_session) -> None:
owner = _user(db_session, "pending-owner")
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Pending guidance",
message_type="user_message",
turn_id="pending-turn-1",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
with patch("xagent.web.api.websocket.manager", ws_manager):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "Pending guidance",
"client_message_id": "pending-turn-1",
"user": owner,
"files": [],
},
)
rejected = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_rejected"
]
assert len(rejected) == 1
assert rejected[0]["client_message_id"] == "pending-turn-1"
assert rejected[0]["error_code"] == "guidance_in_progress"
assert rejected[0]["rejection_outcome"] == "outcome_unknown"
@pytest.mark.asyncio
async def test_pause_admin_on_other_users_task_runs_as_owner(db_session) -> None:
owner = _user(db_session, "owner")
admin = _user(db_session, "admin", is_admin=True)
task = _task(db_session, owner.id)
captured, agent, mgr, ws_manager = _patched_manager_and_agent()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
):
await handle_pause_task(MagicMock(), int(task.id), {"user": admin})
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if "task_owner_user_id" in captured:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("durable pause command was not dispatched in time")
# Built and paused as the OWNER, not the admin actor.
assert captured["task_owner_user_id"] == int(owner.id)
agent.pause_execution.assert_awaited_once()
@pytest.mark.asyncio
async def test_durable_pause_propagates_stale_run_error(db_session) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id)
_captured, _agent, mgr, ws_manager = _patched_manager_and_agent()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_command_execution._apply_pause_requested_isolated",
side_effect=StaleTaskRunError("run rotated"),
),
pytest.raises(StaleTaskRunError, match="run rotated"),
):
await pause_task(
_make_command_reply(MagicMock()),
int(task.id),
{"user": owner, "_durable_ack_sent": True},
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("settled_status", "applied"),
[
# The interrupted run paused itself before the fenced write ran.
(TaskStatus.PAUSED, True),
# The run really ended; the pause did not take effect.
(TaskStatus.FAILED, False),
],
)
async def test_durable_pause_after_run_settles_before_fenced_write(
db_session, settled_status: TaskStatus, applied: bool
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id)
task.run_id = "run-a"
db_session.commit()
task_id = int(task.id)
_captured, agent, mgr, ws_manager = _patched_manager_and_agent()
async def interrupt_and_settle():
# The interrupt reaches the run, which settles before the fenced
# write, so the real RUNNING-only UPDATE matches no row.
row = db_session.get(Task, task_id)
row.status = settled_status
db_session.commit()
return {"status": "paused"}
agent.pause_execution = AsyncMock(side_effect=interrupt_and_settle)
message_data = {"user": owner, "_durable_ack_sent": True}
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_command_execution.publish_task_event",
new=AsyncMock(),
) as published,
):
await pause_task(_make_command_reply(MagicMock()), task_id, message_data)
agent.pause_execution.assert_awaited_once()
# Nothing is left pending: the run has already ended, so a marker would
# hold back messages on a later resumed run.
assert not task_execution_service._is_task_pause_accepted(task_id)
assert not [
call
for call in published.await_args_list
if call.args[0].get("type") == "task_pause_requested"
]
if applied:
assert "_durable_command_error" not in message_data
else:
assert message_data["_durable_command_error"] == (
"Task finished before the pause request was applied"
)
@pytest.mark.asyncio
async def test_durable_resume_propagates_stale_run_error(db_session) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
_captured, agent, mgr, ws_manager = _patched_manager_and_agent()
agent.supports_live_control.return_value = True
bg_mgr = MagicMock()
bg_mgr.resume_admission_state.return_value = None
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.api.websocket.task_execution_controller.transition",
new=AsyncMock(side_effect=StaleTaskRunError("run rotated")),
),
pytest.raises(StaleTaskRunError, match="run rotated"),
):
await resume_task(
_make_command_reply(MagicMock()),
int(task.id),
{"user": owner, "_durable_ack_sent": True},
)
bg_mgr.release_resume_reservation.assert_called_once_with(int(task.id))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("kind", "handler_name"),
[
(TaskCommandKind.PAUSE, "pause_task"),
(TaskCommandKind.RESUME, "resume_task"),
],
)
async def test_durable_control_converts_handler_stale_run_to_terminal_rejection(
db_session,
kind: TaskCommandKind,
handler_name: str,
) -> None:
owner = _user(db_session, f"durable-{kind.value}-stale-owner")
task = _task(db_session, owner.id)
task.runner_id = None
task.run_id = "run-a"
db_session.commit()
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=int(owner.id),
command_id=f"{kind.value}-stale-handler",
kind=kind,
payload={"type": f"{kind.value}_task"},
target_run_id="run-a",
attempt_count=1,
)
with (
patch.object(websocket_api.manager, "connections_for_task", return_value=[]),
patch(
f"xagent.web.services.task_command_execution.{handler_name}",
new=AsyncMock(side_effect=StaleTaskRunError("run rotated")),
),
pytest.raises(TaskCommandRejected) as exc_info,
):
await _execute_durable_task_command(command)
assert exc_info.value.reason == "stale_run"
assert "run rotated" in str(exc_info.value)
def _durable_pause_command(task: Task, owner: User) -> ClaimedTaskCommand:
return ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=int(owner.id),
command_id="pause-personal-reply-routing",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
target_run_id="run-a",
attempt_count=1,
)
@pytest.mark.asyncio
async def test_durable_command_targets_the_registered_origin_not_list_order(
db_session,
) -> None:
"""Origin is never inferred from connection order (#1514 round 6).
This test previously pinned the opposite: "the first real connection is
picked". That guess could hand a personal reply - including raw error
text - to whichever socket happened to be first, e.g. an anonymous
public visitor. A durable command now targets the socket registered as
its origin at enqueue; without a registration it gets the discarding
sink even when real connections exist."""
owner = _user(db_session, "broadcast-skip-owner")
task = _task(db_session, owner.id)
task.runner_id = None
task.run_id = "run-a"
db_session.commit()
command = _durable_pause_command(task, owner)
first_real = SimpleNamespace() # would have been picked by the old guess
origin = SimpleNamespace()
websocket_api._command_origins.register(command.command_id, origin, int(task.id))
try:
with (
patch.object(
websocket_api.manager,
"is_connection_registered",
return_value=True,
),
patch(
"xagent.web.services.task_command_execution.pause_task",
new=AsyncMock(return_value=None),
) as handler,
):
await _execute_durable_task_command(command)
with patch.object(
websocket_api.manager, "send_personal_message", new=AsyncMock()
) as send_reply:
payload = {"type": "task_paused", "task_id": int(task.id)}
await handler.await_args.args[0](payload)
send_reply.assert_awaited_once_with(payload, origin)
# Without a registration, real connections in the list are ignored.
websocket_api._command_origins.discard_command(command.command_id, int(task.id))
with (
patch.object(
websocket_api.manager,
"connections_for_task",
return_value=[first_real],
),
patch(
"xagent.web.services.task_command_execution.pause_task",
new=AsyncMock(return_value=None),
) as handler,
):
await _execute_durable_task_command(command)
assert handler.await_args.args[0] is websocket_api.discard_command_reply
finally:
websocket_api._command_origins.discard_command(command.command_id, int(task.id))
@pytest.mark.asyncio
async def test_durable_command_falls_back_to_discarding_reply_when_all_connections_are_broadcast_only(
db_session,
) -> None:
"""When every registered connection for the task is broadcast-only
(e.g. only v1 SSE streams are attached, no real WebSocket), there is
no valid personal-reply target, so the handler gets the discarding
sink -- same as when the list is empty."""
owner = _user(db_session, "broadcast-only-fallback-owner")
task = _task(db_session, owner.id)
task.runner_id = None
task.run_id = "run-a"
db_session.commit()
command = _durable_pause_command(task, owner)
broadcast_only = SimpleNamespace(is_broadcast_only=True)
with (
patch.object(
websocket_api.manager,
"connections_for_task",
return_value=[broadcast_only],
),
patch(
"xagent.web.services.task_command_execution.pause_task",
new=AsyncMock(return_value=None),
) as handler,
):
await _execute_durable_task_command(command)
assert handler.await_args.args[0] is websocket_api.discard_command_reply
@pytest.mark.asyncio
async def test_durable_command_falls_back_to_discarding_reply_when_no_connections(
db_session,
) -> None:
"""An empty connection list still routes to the discarding sink,
unchanged by the broadcast-only filter."""
owner = _user(db_session, "no-connections-fallback-owner")
task = _task(db_session, owner.id)
task.runner_id = None
task.run_id = "run-a"
db_session.commit()
command = _durable_pause_command(task, owner)
with (
patch.object(websocket_api.manager, "connections_for_task", return_value=[]),
patch(
"xagent.web.services.task_command_execution.pause_task",
new=AsyncMock(return_value=None),
) as handler,
):
await _execute_durable_task_command(command)
assert handler.await_args.args[0] is websocket_api.discard_command_reply
@pytest.mark.asyncio
async def test_pause_non_owner_non_admin_is_refused(db_session) -> None:
owner = _user(db_session, "owner")
stranger = _user(db_session, "stranger") # not admin, not owner
task = _task(db_session, owner.id)
captured, agent, mgr, ws_manager = _patched_manager_and_agent()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
):
# The handler authorizes the task away and handles the denial
# internally; the point is that no owner runtime is built / paused.
await handle_pause_task(MagicMock(), int(task.id), {"user": stranger})
assert "task_owner_user_id" not in captured
agent.pause_execution.assert_not_awaited()
@pytest.mark.asyncio
async def test_resume_admin_on_other_users_task_runs_as_owner(db_session) -> None:
owner = _user(db_session, "owner")
admin = _user(db_session, "admin", is_admin=True)
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
captured, agent, mgr, ws_manager = _patched_manager_and_agent()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
):
await handle_resume_task(MagicMock(), int(task.id), {"user": admin})
# ``dispatch_task_command_promptly`` may detach the durable resume
# command after its 50ms deadline, so the captured agent build can
# land after handle_resume_task returns (same pattern as the pause
# test above).
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if "task_owner_user_id" in captured:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("durable resume command was not dispatched in time")
# Command dispatch may detach after its short prompt deadline. Wait for
# the worker to reach agent construction instead of racing that
# documented durable-dispatch boundary.
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if "task_owner_user_id" in captured:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("resume command did not reach agent construction")
assert captured["task_owner_user_id"] == int(owner.id)
@pytest.mark.asyncio
async def test_running_resume_completes_as_explicit_idempotent_success(
db_session,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.run_id = "run-current"
task.state_version = 7
task.control_state = "running"
task.runner_id = get_runner_id()
task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=1)
db_session.commit()
_captured, agent, mgr, ws_manager = _patched_manager_and_agent()
agent.supports_live_control = MagicMock(return_value=True)
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
):
await handle_resume_task(MagicMock(), int(task.id), {"user": owner})
command = None
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
db_session.expire_all()
command = (
db_session.query(TaskExecutionCommand)
.filter(
TaskExecutionCommand.task_id == int(task.id),
TaskExecutionCommand.kind == TaskCommandKind.RESUME.value,
)
.one_or_none()
)
if command is not None and command.status == COMMAND_COMPLETED:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("idempotent resume did not complete in time")
assert command is not None
assert command.result["resume_outcome"] == "already_in_progress"
# The durable row records an idempotent success, but the client that
# asked still gets the task's state tuple: the resume control only
# renders while the client believes the task is paused, so a silent
# completion would leave a stale client clicking resume forever. This is
# the payload shape a resume correction has always carried.
payload = None
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
for call in ws_manager.send_personal_message.await_args_list:
if call.args and "task" in call.args[0]:
payload = call.args[0]
break
if payload is not None:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("idempotent resume correction did not arrive in time")
# The handler deliberately supplies no state tuple: its own setup
# snapshot can be stale by the time this branch fires, and
# ``send_personal_message`` attaches the live row precisely when the
# producer supplied none. The attachment is pinned by
# ``test_idempotent_resume_frame_is_enriched_with_the_live_row``.
assert payload["task"] == {"id": int(task.id)}
@pytest.mark.asyncio
async def test_resume_live_control_admin_runs_background_as_owner(db_session) -> None:
"""Live-control resume schedules ``execute_resume_background``; when an
admin resumes another user's task it must run with the OWNER's
UserContext, i.e. ``task_owner_user_id`` is the owner, not the admin."""
owner = _user(db_session, "owner")
admin = _user(db_session, "admin", is_admin=True)
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
captured, agent, mgr, ws_manager = _patched_manager_and_agent()
agent.supports_live_control = MagicMock(return_value=True)
resume_bg = AsyncMock()
transition = AsyncMock(
return_value=SimpleNamespace(
run_id="run-from-resume-transition",
status=TaskStatus.PAUSED,
)
)
bg_mgr = MagicMock()
bg_mgr.running_tasks.get = MagicMock(return_value=None)
bg_mgr.resume_admission_state.return_value = None
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.execute_resume_background", resume_bg
),
patch(
"xagent.web.api.websocket.task_execution_controller.transition",
new=transition,
),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
):
await handle_resume_task(MagicMock(), int(task.id), {"user": admin})
# ``dispatch_task_command_promptly`` may detach the durable resume
# command after its 50ms deadline, so the captured agent build and
# the background-resume scheduling can land after handle_resume_task
# returns (same pattern as the pause test above).
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if "task_owner_user_id" in captured and resume_bg.call_count:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("durable resume command was not dispatched in time")
# Durable dispatch intentionally detaches after its short prompt
# deadline. Keep the patched runtime owners in place until the command
# reaches agent construction instead of racing that boundary under
# parallel test load.
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if "task_owner_user_id" in captured:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("resume command did not reach agent construction")
# Agent built as owner, and the background resume runs as owner.
assert captured["task_owner_user_id"] == int(owner.id)
resume_bg.assert_called_once()
assert resume_bg.call_args.kwargs["task_owner_user_id"] == int(owner.id)
assert resume_bg.call_args.kwargs["expected_run_id"] == "run-from-resume-transition"
bg_mgr.try_reserve_resume.assert_called_once_with(
int(task.id),
expected_run_id=None,
)
bg_mgr.register_reserved_resume.assert_called_once()
@pytest.mark.asyncio
async def test_resume_registration_failure_cancels_coordinator(db_session) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
captured, agent, mgr, ws_manager = _patched_manager_and_agent()
agent.supports_live_control = MagicMock(return_value=True)
bg_mgr = MagicMock()
bg_mgr.resume_admission_state.return_value = None
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
bg_mgr.register_reserved_resume.side_effect = RuntimeError("reservation lost")
bg_handle = MagicMock()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.execute_resume_background", MagicMock()
),
patch(
"xagent.web.api.websocket.asyncio.create_task",
return_value=bg_handle,
),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
):
await handle_resume_task(MagicMock(), int(task.id), {"user": owner})
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if bg_handle.cancel.called:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("resume command did not finish in time")
bg_handle.cancel.assert_called_once()
@pytest.mark.asyncio
async def test_execute_resume_background_rejects_owner_mismatch(db_session) -> None:
"""``execute_resume_background`` runs the resume under
``UserContext(task_owner_user_id)``. If a caller passes an owner id that
disagrees with the task row, the symmetric guard (same as
``execute_task_background``) must fire before the agent resumes, so the
runtime never executes as the wrong user."""
owner = _user(db_session, "owner")
task = _task(db_session, owner.id)
agent = MagicMock()
agent.resume_execution_by_id = AsyncMock()
ws_manager = MagicMock()
ws_manager.broadcast_to_task = AsyncMock()
with (
patch(
"xagent.web.services.task_execution.stop_task_lease_heartbeat",
new=AsyncMock(),
),
patch("xagent.web.api.websocket.manager", ws_manager),
):
_register_current_resume(int(task.id))
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id) + 999, # != task owner
)
# Guard fired before the resume ran -- nothing executed as the wrong user.
agent.resume_execution_by_id.assert_not_awaited()
error_types = {
msg.get("type")
for (msg, _tid) in (
call.args for call in ws_manager.broadcast_to_task.call_args_list
)
if isinstance(msg, dict)
}
assert "task_error" in error_types
task_errors = [
call.args[0]
for call in ws_manager.broadcast_to_task.call_args_list
if call.args[0].get("type") == "task_error"
]
assert task_errors[0]["message"] == websocket_api.CLIENT_SAFE_TASK_FAILURE
assert task_errors[0]["error"] == websocket_api.CLIENT_SAFE_TASK_FAILURE
assert not _mentions_number(task_errors[0], int(owner.id) + 999)
def _mentions_number(payload: Any, number: int) -> bool:
"""Whether ``number`` appears as a value or a whole token in ``payload``.
A substring search over ``repr(payload)`` also matches digits inside
unrelated values, such as the event's float timestamp, so it fails
intermittently.
"""
if isinstance(payload, bool):
return False
if isinstance(payload, int):
return payload == number
if isinstance(payload, str):
return re.search(rf"(?<!\d){number}(?!\d)", payload) is not None
if isinstance(payload, dict):
return any(
_mentions_number(key, number) or _mentions_number(value, number)
for key, value in payload.items()
)
if isinstance(payload, (list, tuple, set)):
return any(_mentions_number(item, number) for item in payload)
return False
def test_mentions_number_ignores_digits_inside_other_values() -> None:
# The CI timestamp that made a repr() substring check report a leak.
assert not _mentions_number({"timestamp": 1790404311.310002}, 1000)
assert not _mentions_number({"message": "task 51000 failed"}, 1000)
assert _mentions_number({"message": "owner 1000 mismatch"}, 1000)
assert _mentions_number({"details": [{"owner_id": 1000}]}, 1000)
@pytest.mark.asyncio
async def test_execute_resume_background_persists_assistant_for_live_turn(
db_session,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
agent = MagicMock()
context = SimpleNamespace(
messages=[
SimpleNamespace(
role="user",
metadata={"turn_id": "live-turn-1"},
)
]
)
agent.resume_execution_by_id = AsyncMock(
return_value={
"status": "completed",
"success": True,
"output": "Guidance applied",
"agent_result": {"context": context},
}
)
ws_manager = MagicMock(broadcast_to_task=AsyncMock())
with patch("xagent.web.api.websocket.manager", ws_manager):
_register_current_resume(int(task.id))
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
)
db_session.expire_all()
stored = (
db_session.query(TaskChatMessage)
.filter(
TaskChatMessage.task_id == int(task.id),
TaskChatMessage.role == "assistant",
)
.one()
)
assert stored.content == "Guidance applied"
assert stored.turn_id == "live-turn-1"
db_session.refresh(task)
assert task.status == TaskStatus.COMPLETED
assert task.output == "Guidance applied"
@pytest.mark.asyncio
async def test_execute_resume_background_persists_missing_checkpoint_failure(
db_session,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
agent = MagicMock(
resume_execution_by_id=AsyncMock(return_value=None),
)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="deferred-failure-turn",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
ws_manager = MagicMock(broadcast_to_task=AsyncMock())
with patch("xagent.web.api.websocket.manager", ws_manager):
_register_current_resume(int(task.id))
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
delivery_turn_id="deferred-failure-turn",
)
db_session.expire_all()
db_session.refresh(task)
assert task.status == TaskStatus.FAILED
assert "No resumable execution checkpoint" in str(task.error_message)
delivery = (
db_session.query(TaskChatMessage)
.filter(
TaskChatMessage.turn_id == "deferred-failure-turn",
TaskChatMessage.role == "user",
)
.one()
)
assert delivery.delivery_status == DELIVERY_FAILED
failures = [
call.args[0]
for call in ws_manager.broadcast_to_task.call_args_list
if call.args[0].get("type") == "task_error"
]
assert len(failures) == 1
assert failures[0]["task"]["status"] == TaskStatus.FAILED.value
@pytest.mark.asyncio
async def test_resume_background_settles_running_prior_status_on_checkpoint_unavailable(
db_session,
) -> None:
"""A RUNNING prior status is never a valid restore target.
``release_task_lease_no_commit`` refuses to release a lease back to
RUNNING. A resume that steals an abandoned lease from a task whose row
was still RUNNING (no active runner, expired TTL) must therefore fall
through to the ordinary settle/FAILED path on a checkpoint read failure,
not attempt a "restore to prior status" that can only dead-end with the
lease stuck unreleased until TTL recovery.
"""
owner = _user(db_session, "running-prior-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
assert task.runner_id is None
assert task.lease_expires_at is None
agent = MagicMock(
resume_execution_by_id=AsyncMock(
side_effect=CheckpointUnavailableError("checkpoint query failed")
),
)
ws_manager = MagicMock(broadcast_to_task=AsyncMock())
with patch("xagent.web.api.websocket.manager", ws_manager):
_register_current_resume(int(task.id))
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
)
db_session.expire_all()
db_session.refresh(task)
assert task.status == TaskStatus.FAILED
assert task.runner_id is None
failures = [
call.args[0]
for call in ws_manager.broadcast_to_task.call_args_list
if call.args[0].get("type") == "task_error"
]
assert len(failures) == 1
def test_waiting_or_paused_event_fields_shared_by_both_call_sites() -> None:
"""The live-lease restore broadcast and the historical-replay status
reassertion both compute their event type/message off this one helper;
pin its output so a change to either site's vocabulary is caught here
rather than only in one of the two integration tests."""
assert _waiting_or_paused_event_fields(TaskStatus.WAITING_FOR_USER) == (
"task_waiting_for_user",
"Task waiting for user response",
)
assert _waiting_or_paused_event_fields(TaskStatus.PAUSED) == (
"task_paused",
"Task paused",
)
@pytest.mark.parametrize(
("prior_status", "expected_event_type"),
[
(TaskStatus.PAUSED, "task_paused"),
(TaskStatus.WAITING_FOR_USER, "task_waiting_for_user"),
],
)
@pytest.mark.asyncio
async def test_resume_background_broadcasts_corrective_event_after_restore(
db_session,
prior_status: TaskStatus,
expected_event_type: str,
) -> None:
"""After a checkpoint-unavailable restore, clients that saw the optimistic
RUNNING transition need the prior status re-asserted -- reusing the same
event vocabulary the historical-replay path uses for PAUSED/WAITING_FOR_USER."""
owner = _user(db_session, "restore-broadcast-owner")
task = _task(db_session, owner.id, status=prior_status)
task.error_message = "earlier attempt failed"
task.output = "prior turn answer"
db_session.commit()
agent = MagicMock(
resume_execution_by_id=AsyncMock(
side_effect=CheckpointUnavailableError("checkpoint query failed")
),
)
ws_manager = MagicMock(broadcast_to_task=AsyncMock())
with patch("xagent.web.api.websocket.manager", ws_manager):
_register_current_resume(int(task.id))
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
)
db_session.expire_all()
db_session.refresh(task)
assert task.status == prior_status
assert task.error_message is None # named: restore clears stale error
assert task.output == "prior turn answer" # named: restore preserves output
corrective = [
call.args[0]
for call in ws_manager.broadcast_to_task.call_args_list
if call.args[0].get("type") == expected_event_type
]
assert len(corrective) == 1
assert corrective[0]["type"] == expected_event_type
assert corrective[0]["status"] == prior_status.value
assert "state_version" in corrective[0]
assert not any(
call.args[0].get("type") == "task_error"
for call in ws_manager.broadcast_to_task.call_args_list
)
@pytest.mark.parametrize(
("settled", "expected_error_broadcasts"),
[(True, 1), (False, 0)],
)
@pytest.mark.asyncio
async def test_resume_failure_broadcasts_only_after_exact_settlement(
db_session,
settled: bool,
expected_error_broadcasts: int,
) -> None:
secret = "resume-provider-secret"
owner = _user(db_session, "resume-settlement-owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
agent = MagicMock(
resume_execution_by_id=AsyncMock(
side_effect=RuntimeError(f"resume failed: {secret}")
),
)
events: list[str] = []
error_payloads: list[dict] = []
def settle(*_args, **_kwargs) -> bool:
events.append("settle")
return settled
async def broadcast(payload, *_args, **_kwargs) -> None:
if payload.get("type") == "task_error":
events.append("broadcast")
error_payloads.append(payload)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(side_effect=broadcast),
)
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution._settle_resumed_task_lease",
side_effect=settle,
),
):
_register_current_resume(int(task.id))
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
)
assert events == ["settle"] + (["broadcast"] * expected_error_broadcasts)
if settled:
assert error_payloads[0]["message"] == websocket_api.CLIENT_SAFE_TASK_FAILURE
assert error_payloads[0]["error"] == websocket_api.CLIENT_SAFE_TASK_FAILURE
assert secret not in repr(error_payloads[0])
@pytest.mark.asyncio
async def test_resume_failure_rejection_redacts_exception_text(db_session) -> None:
secret = "resume-rejection-secret"
owner = _user(db_session, "resume-rejection-owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="resume-rejection-turn",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
agent = MagicMock(
resume_execution_by_id=AsyncMock(
side_effect=RuntimeError(f"resume failed: {secret}")
),
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
with patch("xagent.web.api.websocket.manager", ws_manager):
_register_current_resume(int(task.id))
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
delivery_turn_id="resume-rejection-turn",
delivery_notifier=make_delivery_notifier(
_make_command_reply(MagicMock()), "resume-rejection-turn"
),
)
rejected = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_rejected"
]
assert len(rejected) == 1
assert rejected[0]["message"] == task_execution_service.CLIENT_SAFE_VALIDATION_ERROR
assert secret not in repr(rejected[0])
def _fenced_live_runner(execution_id: str):
"""A real runner whose live context is fenced by an earlier unknown write."""
from xagent.core.agent.context import ContextManager
from xagent.core.agent.context.execution import context_checkpoint_gate
from xagent.core.agent.runner import AgentRunner
manager = ContextManager()
context = manager.create_context(execution_id)
context.add_user_message("original")
context_checkpoint_gate(context).injection_uncertain = True
tracer = SimpleNamespace(
load_latest_checkpoint=AsyncMock(return_value=None), checkpoint=AsyncMock()
)
runner = AgentRunner(
SimpleNamespace(llm=None), tracer=tracer, context_manager=manager
)
# The fenced run is still active, so the fence rejects instead of evicting.
runner._active_controls[execution_id] = SimpleNamespace(
runtime=SimpleNamespace(last_checkpoint=None, request_interrupt=lambda *_: None)
)
return runner, manager
def _injecting_through(runner):
async def post_user_message(execution_id, **kwargs):
result = await runner.inject_user_message(execution_id, **kwargs)
assert result.outcome is UserMessageInjectionOutcome.REJECTED_RETRYABLE
runner.tracer.checkpoint.assert_not_awaited()
return result.outcome
return post_user_message
@pytest.mark.asyncio
async def test_deferred_injection_rejected_by_fence_pauses_without_resuming(
db_session,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="deferred-fenced-reject",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
runner, manager = _fenced_live_runner(str(task.id))
agent = MagicMock(
post_user_message=AsyncMock(side_effect=_injecting_through(runner)),
resume_execution_by_id=AsyncMock(),
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
published = AsyncMock()
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
patch("xagent.web.services.task_events.publish_task_event", published),
):
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"files": [],
"turn_id": "deferred-fenced-reject",
},
delivery_turn_id="deferred-fenced-reject",
delivery_notifier=make_delivery_notifier(
_make_command_reply(MagicMock()), "deferred-fenced-reject"
),
)
paused = [
call.args[0]
for call in published.await_args_list
if call.args[0].get("type") == "task_paused"
]
assert [event["message"] for event in paused] == [
"Input was not accepted; execution paused"
]
delivery_events = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") in {"message_accepted", "message_rejected"}
]
assert [event["type"] for event in delivery_events] == ["message_rejected"]
assert delivery_events[0]["retry_with_new_id"] is True
assert delivery_events[0]["rejection_outcome"] == "not_accepted"
assert delivery_events[0]["error_code"] == "message_delivery_failed"
agent.resume_execution_by_id.assert_not_awaited()
db_session.expire_all()
delivery = (
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == "deferred-fenced-reject")
.one()
)
assert delivery.delivery_status == DELIVERY_FAILED
task_row = db_session.get(Task, int(task.id))
assert task_row.status == TaskStatus.PAUSED
assert task_row.runner_id is None
assert not any(
call.args[0].get("type") == "task_error"
for call in ws_manager.broadcast_to_task.call_args_list
)
@pytest.mark.asyncio
@pytest.mark.parametrize("unknown", [False, True])
async def test_deferred_injection_failure_rejects_before_any_acceptance(
db_session,
unknown,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="deferred-injection-failure",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
observed_leases: list[TaskLease | None] = []
async def post_user_message(*_args, **_kwargs):
observed_leases.append(current_task_lease())
return UserMessageInjectionOutcome.OUTCOME_UNKNOWN if unknown else False
agent = MagicMock(
post_user_message=AsyncMock(side_effect=post_user_message),
resume_execution_by_id=AsyncMock(),
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
):
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"files": [],
"turn_id": "deferred-injection-failure",
},
delivery_turn_id="deferred-injection-failure",
delivery_notifier=make_delivery_notifier(
_make_command_reply(MagicMock()), "deferred-injection-failure"
),
)
delivery_events = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") in {"message_accepted", "message_rejected"}
]
assert [event["type"] for event in delivery_events] == ["message_rejected"]
assert bool(delivery_events[0].get("retry_with_new_id")) is (not unknown)
assert delivery_events[0]["rejection_outcome"] == (
"outcome_unknown" if unknown else "not_accepted"
)
agent.resume_execution_by_id.assert_not_awaited()
assert len(observed_leases) == 1
assert observed_leases[0] is not None
assert observed_leases[0].task_id == int(task.id)
assert observed_leases[0].runner_id
assert observed_leases[0].run_id
db_session.expire_all()
delivery = (
db_session.query(TaskChatMessage)
.filter(
TaskChatMessage.turn_id == "deferred-injection-failure",
TaskChatMessage.role == "user",
)
.one()
)
assert delivery.delivery_status == (
DELIVERY_OUTCOME_UNKNOWN if unknown else DELIVERY_FAILED
)
assert db_session.get(Task, int(task.id)).status == (
TaskStatus.PAUSED if unknown else TaskStatus.FAILED
)
@pytest.mark.asyncio
async def test_deferred_injection_marker_failure_does_not_abort_resume(
db_session,
) -> None:
owner = _user(db_session, "deferred-marker-owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="deferred-marker-turn",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
context = SimpleNamespace(
messages=[
SimpleNamespace(
role="user",
metadata={"turn_id": "deferred-marker-turn"},
)
]
)
agent = MagicMock(
post_user_message=AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
),
resume_execution_by_id=AsyncMock(
return_value={
"status": "completed",
"success": True,
"output": "Applied",
"agent_result": {"context": context},
}
),
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
def mark_delivery(_task_id: int, _turn_id: str, status: str):
if status == command_execution_service.DELIVERY_DISPATCHED:
raise RuntimeError("delivery marker unavailable")
return None
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
patch(
"xagent.web.services.task_execution.mark_user_message_delivery_sync",
side_effect=mark_delivery,
) as mark_mock,
):
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"files": [],
"turn_id": "deferred-marker-turn",
},
delivery_turn_id="deferred-marker-turn",
delivery_notifier=make_delivery_notifier(
_make_command_reply(MagicMock()), "deferred-marker-turn"
),
)
mark_mock.assert_any_call(
int(task.id),
"deferred-marker-turn",
command_execution_service.DELIVERY_DISPATCHED,
)
agent.resume_execution_by_id.assert_awaited_once_with(
str(task.id), metadata={"task_source": None, "run_id": ANY}
)
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
db_session.refresh(task)
assert task.status == TaskStatus.COMPLETED
@pytest.mark.asyncio
async def test_deferred_injection_marker_cancellation_does_not_abort_resume(
db_session,
) -> None:
owner = _user(db_session, "deferred-marker-cancel-owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="deferred-marker-cancel-turn",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
context = SimpleNamespace(
messages=[
SimpleNamespace(
role="user",
metadata={"turn_id": "deferred-marker-cancel-turn"},
)
]
)
agent = MagicMock(
post_user_message=AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
),
resume_execution_by_id=AsyncMock(
return_value={
"status": "completed",
"success": True,
"output": "Applied",
"agent_result": {"context": context},
}
),
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
def mark_delivery(_task_id: int, _turn_id: str, status: str):
if status == command_execution_service.DELIVERY_DISPATCHED:
raise asyncio.CancelledError
return None
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
patch(
"xagent.web.services.task_execution.mark_user_message_delivery_sync",
side_effect=mark_delivery,
) as mark_mock,
):
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"files": [],
"turn_id": "deferred-marker-cancel-turn",
},
delivery_turn_id="deferred-marker-cancel-turn",
delivery_notifier=make_delivery_notifier(
_make_command_reply(MagicMock()), "deferred-marker-cancel-turn"
),
)
mark_mock.assert_any_call(
int(task.id),
"deferred-marker-cancel-turn",
command_execution_service.DELIVERY_DISPATCHED,
)
agent.resume_execution_by_id.assert_awaited_once_with(
str(task.id), metadata={"task_source": None, "run_id": ANY}
)
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
db_session.refresh(task)
assert task.status == TaskStatus.COMPLETED
@pytest.mark.asyncio
async def test_deferred_injection_close_failure_does_not_abort_resume(
db_session,
) -> None:
"""Same guarantee as the deferred delivery-marker failure case above,
for the legacy resume interaction close issued right before it."""
owner = _user(db_session, "deferred-close-failure-owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="deferred-close-failure-turn",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
context = SimpleNamespace(
messages=[
SimpleNamespace(
role="user",
metadata={"turn_id": "deferred-close-failure-turn"},
)
]
)
agent = MagicMock(
post_user_message=AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
),
resume_execution_by_id=AsyncMock(
return_value={
"status": "completed",
"success": True,
"output": "Applied",
"agent_result": {"context": context},
}
),
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
# Recorded, not asserted, inside the side_effect below: the production
# code that calls close_legacy_resume_interaction_sync wraps it in
# ``except Exception:``, which would silently swallow an AssertionError
# raised from inside this callback the same way it swallows the
# RuntimeError this test is otherwise driving. The real assertions run
# after the `with` block, outside that handler's reach.
observed_close_calls: list[tuple[int, str, str | None]] = []
def fail_close(*, task_id: int, run_id: str, interaction_id: int | None) -> int:
# A fresh session, not db_session: this runs inside the worker
# thread run_db_io_cancellation_safe schedules it on, while the
# test's own db_session sits unused on the main thread -- sharing
# one Session across that handoff is not a pattern this suite uses
# elsewhere.
probe = next(get_db())
try:
live_run_id = probe.query(Task).filter(Task.id == task_id).one().run_id
finally:
probe.close()
observed_close_calls.append((task_id, run_id, live_run_id))
raise RuntimeError("interaction close unavailable")
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
patch(
"xagent.web.services.task_execution.close_legacy_resume_interaction_sync",
side_effect=fail_close,
),
):
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"files": [],
"turn_id": "deferred-close-failure-turn",
},
delivery_turn_id="deferred-close-failure-turn",
delivery_notifier=make_delivery_notifier(
_make_command_reply(MagicMock()), "deferred-close-failure-turn"
),
)
agent.resume_execution_by_id.assert_awaited_once_with(
str(task.id), metadata={"task_source": None, "run_id": ANY}
)
assert len(observed_close_calls) == 1
called_task_id, called_run_id, live_run_id = observed_close_calls[0]
assert called_task_id == int(task.id)
assert called_run_id
assert called_run_id == live_run_id
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
@pytest.mark.asyncio
async def test_deferred_injection_closes_the_row_the_online_handler_observed(
db_session,
) -> None:
"""The deferred path takes no observation of its own. Its injection is
later still than the online one, so a read taken here would be even
further past the point where the answered row is identifiable. The
online handler's pre-injection observation travels in
pending_user_message and is what the close is keyed on."""
owner = _user(db_session, "deferred-carry-owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="deferred-carry-turn",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
context = SimpleNamespace(
messages=[
SimpleNamespace(role="user", metadata={"turn_id": "deferred-carry-turn"})
]
)
agent = MagicMock(
post_user_message=AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
),
resume_execution_by_id=AsyncMock(
return_value={
"status": "completed",
"success": True,
"output": "Applied",
"agent_result": {"context": context},
}
),
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
patch(
"xagent.web.services.task_interaction_close.active_interaction_id_sync",
side_effect=AssertionError("the deferred path must not read its own"),
) as read_mock,
patch(
"xagent.web.services.task_execution.close_legacy_resume_interaction_sync",
return_value=1,
) as close_mock,
):
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"files": [],
"turn_id": "deferred-carry-turn",
"interaction_id": 9876,
},
delivery_turn_id="deferred-carry-turn",
delivery_notifier=make_delivery_notifier(
_make_command_reply(MagicMock()), "deferred-carry-turn"
),
)
close_mock.assert_called_once()
assert close_mock.call_args.kwargs["task_id"] == int(task.id)
assert close_mock.call_args.kwargs["interaction_id"] == 9876
read_mock.assert_not_called()
agent.resume_execution_by_id.assert_awaited_once_with(
str(task.id), metadata={"task_source": None, "run_id": ANY}
)
db_session.refresh(task)
assert task.status == TaskStatus.COMPLETED
assert any(
call.args[0].get("type") == "message_accepted"
for call in ws_manager.send_personal_message.call_args_list
)
@pytest.mark.asyncio
async def test_deferred_injection_skips_the_close_on_a_replayed_turn_id(
db_session,
) -> None:
"""A cross-run retry of this background task re-enters with the same
pending_user_message. post_user_message short-circuits the repeated
turn id and reports POSTED_REPLAY, so the close must be skipped
entirely -- not just matched against the carried (stale) interaction
id, but never attempted. The row this test seeds is the question the
resumed agent has staged *since* the first attempt (a different id
from the one carried in pending_user_message), so a site that
re-derived its close key by reading the current active row instead of
using the carried one would retire this live question; asserting
active_interaction_id_sync is never called pins that the deferred path
still takes no read of its own, replay or not -- the same technique
the carried-vs-observed test above uses. close_legacy_resume_interaction_sync
is asserted uncalled directly: replacing the guard with `if True:`
calls it for real and turns this assertion red on its own."""
owner = _user(db_session, "deferred-replay-owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
task.interaction_protocol_version = 1
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="deferred-replay-turn",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
task_id = int(task.id)
# The question staged since the first attempt -- a different row from
# the one named in pending_user_message below.
_seed_active_interaction_row(
db_session,
task_id=task_id,
run_id="deferred-replay-run",
idempotency_key="deferred-replay-q1",
)
context = SimpleNamespace(
messages=[
SimpleNamespace(role="user", metadata={"turn_id": "deferred-replay-turn"})
]
)
agent = MagicMock(
post_user_message=AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_REPLAY
),
resume_execution_by_id=AsyncMock(
return_value={
"status": "completed",
"success": True,
"output": "Applied",
"agent_result": {"context": context},
}
),
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
patch(
"xagent.web.services.task_interaction_close.active_interaction_id_sync",
side_effect=AssertionError("the deferred path must not read its own"),
) as read_mock,
patch(
"xagent.web.services.task_execution.close_legacy_resume_interaction_sync",
) as close_mock,
):
await execute_resume_background(
task_id=task_id,
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"files": [],
"turn_id": "deferred-replay-turn",
# The stale id carried from the first attempt -- not the
# row seeded above, which is what a re-read would find.
"interaction_id": 424242,
},
delivery_turn_id="deferred-replay-turn",
delivery_notifier=make_delivery_notifier(
_make_command_reply(MagicMock()), "deferred-replay-turn"
),
)
agent.post_user_message.assert_awaited_once()
close_mock.assert_not_called()
read_mock.assert_not_called()
agent.resume_execution_by_id.assert_awaited_once_with(
str(task.id), metadata={"task_source": None, "run_id": ANY}
)
db_session.refresh(task)
assert task.status == TaskStatus.COMPLETED
assert any(
call.args[0].get("type") == "message_accepted"
for call in ws_manager.send_personal_message.call_args_list
)
@pytest.mark.asyncio
async def test_deferred_injection_close_cancellation_does_not_abort_resume(
db_session,
) -> None:
"""CancelledError branch specifically -- deleting it must turn this red."""
owner = _user(db_session, "deferred-close-cancel-owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="deferred-close-cancel-turn",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
context = SimpleNamespace(
messages=[
SimpleNamespace(
role="user",
metadata={"turn_id": "deferred-close-cancel-turn"},
)
]
)
agent = MagicMock(
post_user_message=AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
),
resume_execution_by_id=AsyncMock(
return_value={
"status": "completed",
"success": True,
"output": "Applied",
"agent_result": {"context": context},
}
),
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
# Recorded, not asserted, inside the side_effect below -- see the sibling
# failure-case test's comment: an AssertionError raised from inside this
# callback would be swallowed by the production code's own
# ``except Exception:`` the same way it swallows an ordinary failure.
observed_close_calls: list[tuple[int, str, str | None]] = []
def raise_cancelled(
*, task_id: int, run_id: str, interaction_id: int | None
) -> int:
probe = next(get_db())
try:
live_run_id = probe.query(Task).filter(Task.id == task_id).one().run_id
finally:
probe.close()
observed_close_calls.append((task_id, run_id, live_run_id))
raise asyncio.CancelledError
with (
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
patch(
"xagent.web.services.task_execution.close_legacy_resume_interaction_sync",
side_effect=raise_cancelled,
),
):
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"files": [],
"turn_id": "deferred-close-cancel-turn",
},
delivery_turn_id="deferred-close-cancel-turn",
delivery_notifier=make_delivery_notifier(
_make_command_reply(MagicMock()), "deferred-close-cancel-turn"
),
)
agent.resume_execution_by_id.assert_awaited_once_with(
str(task.id), metadata={"task_source": None, "run_id": ANY}
)
assert len(observed_close_calls) == 1
called_task_id, called_run_id, live_run_id = observed_close_calls[0]
assert called_task_id == int(task.id)
assert called_run_id
assert called_run_id == live_run_id
accepted = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") == "message_accepted"
]
assert len(accepted) == 1
@pytest.mark.asyncio
async def test_deferred_injection_rejects_before_post_when_lease_is_denied(
db_session,
) -> None:
owner = _user(db_session, "owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id="deferred-lease-denied",
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
agent = MagicMock(
post_user_message=AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
),
resume_execution_by_id=AsyncMock(),
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
with (
patch(
"xagent.web.services.task_execution._acquire_resume_task_lease",
return_value=None,
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
):
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"files": [],
"turn_id": "deferred-lease-denied",
},
delivery_turn_id="deferred-lease-denied",
delivery_notifier=make_delivery_notifier(
_make_command_reply(MagicMock()), "deferred-lease-denied"
),
)
agent.post_user_message.assert_not_awaited()
delivery_events = [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") in {"message_accepted", "message_rejected"}
]
assert [event["type"] for event in delivery_events] == ["message_rejected"]
assert delivery_events[0]["retry_with_new_id"] is True
db_session.expire_all()
delivery = (
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == "deferred-lease-denied")
.one()
)
assert delivery.delivery_status == DELIVERY_FAILED
@pytest.mark.asyncio
async def test_resume_non_owner_non_admin_is_refused(db_session) -> None:
owner = _user(db_session, "owner")
stranger = _user(db_session, "stranger")
task = _task(db_session, owner.id)
captured, agent, mgr, ws_manager = _patched_manager_and_agent()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=mgr,
),
patch("xagent.web.api.websocket.manager", ws_manager),
):
await handle_resume_task(MagicMock(), int(task.id), {"user": stranger})
# Authorized away before any runtime is built; an error is sent back.
assert "task_owner_user_id" not in captured
ws_manager.send_personal_message.assert_awaited()
@pytest.mark.asyncio
async def test_durable_attachment_failure_keeps_the_storage_key_off_the_socket(
db_session,
caplog,
) -> None:
"""A stored-file fault must not send the storage key to the client.
Attachment preparation runs in the handler's *outer* scope, so this fault
surfaces before the inner agent-execution arms and was answered with
``str(exc)``, which then read
``Failed to restore durable object: users/<id>/uploads/...`` and embedded the
owning user's id. Same defect as the model-facing leak in #1467, one
transport over.
The key has since moved off the message onto ``storage_key``, so ``str(exc)``
no longer carries it and this arm is no longer the only thing standing
between the fault and the client. Both still matter: this pins the arm's
fixed message, and ``test_the_wrap_keeps_the_storage_key_out_of_its_own_message``
pins the exception. Neither alone would have caught both rounds of this.
What this proves is the *rejection frame*: it asserts one was sent, so the
negative assertions below cannot pass over an empty list. The broadcast
assertion is defence in depth -- this path does not broadcast, so it holds
vacuously and exists to fail if a future edit starts. The persisted
rejection is not reached: ``finish_delivery_failure`` only writes when
``delivery_claimed`` is set, and that comes from
``preparation.delivery_claimed``, which never gets assigned when preparation
is what raised. Do not read this test as covering all three egresses.
"""
import logging
from xagent.web.services.managed_file_ref import DurableStorageOperationError
owner = _user(db_session, "durable-leak-owner")
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
owner_id = int(owner.id)
task_id = int(task.id)
db_session.close()
storage_key = f"users/{owner_id}/uploads/8ac1f2/quarterly-report.xlsx"
class _ProviderThrottled(RuntimeError):
pass
def failing_prepare(**_kwargs):
wrap = DurableStorageOperationError(
"Failed to restore durable object", storage_key=storage_key
)
wrap.__cause__ = _ProviderThrottled("SlowDown: reduce your request rate")
raise wrap
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
logger_name = command_execution_service.__name__
with (
patch(
"xagent.web.services.task_command_execution._prepare_task_message_sync",
side_effect=failing_prepare,
),
patch("xagent.web.api.websocket.manager", ws_manager),
caplog.at_level(logging.WARNING, logger=logger_name),
):
with pytest.raises(DurableStorageOperationError):
await handle_task_message(
_make_command_reply(MagicMock()),
task_id,
{
"message": "with an attachment",
"client_message_id": "durable-leak-probe",
"user": SimpleNamespace(id=owner_id, is_admin=False),
"files": ["8ac1f2"],
},
)
# The rejection frame must actually have gone out, or every negative
# assertion below would hold over nothing and this test would pass while
# answering the client with anything at all.
frames = [str(call) for call in ws_manager.send_personal_message.await_args_list]
assert any("message_rejected" in frame for frame in frames), frames
# Every outbound egress: nothing may carry the key or the provider text.
outbound = frames + [
str(call) for call in ws_manager.broadcast_to_task.await_args_list
]
for payload in outbound:
assert storage_key not in payload
assert f"users/{owner_id}" not in payload
assert "Failed to restore durable object" not in payload
assert "SlowDown" not in payload
# The server-side record keeps the whole chain, exactly once.
fault_lines = [
logging.Formatter("%(message)s").format(entry)
for entry in caplog.records
if entry.name == logger_name
and "Durable storage unavailable" in entry.getMessage()
]
assert len(fault_lines) == 1, fault_lines
assert "during websocket chat turn preparation" in fault_lines[0]
assert storage_key in fault_lines[0]
assert "_ProviderThrottled" in fault_lines[0]
@pytest.mark.asyncio
async def test_a_dispatch_fault_is_labelled_with_the_message_that_failed(
db_session,
caplog,
) -> None:
"""The endpoint arm must name the message it was applying, not "chat turn".
That arm guards the whole receive-loop dispatch, so it sees faults from
every message type whose handler lets one through. It logged a fixed
``"websocket chat turn"``, mislabelling all of them in the single line meant
to be authoritative about what failed.
``resume_task`` rather than ``execute_task``, and the difference matters:
``handle_execute_task`` ends in ``except RuntimeError`` with no re-raise, so
a durable fault there never reaches this arm at all. An earlier version of
this test mocked that handler and asserted a label for a path production
cannot take -- green, and describing nothing.
``handle_resume_task`` propagates, so mocking it to raise stands in for a
real fault, and
``test_a_type_is_unlabelled_only_because_its_handler_swallows`` is what keeps
that distinction true rather than remembered.
Driven through the real endpoint, because neither the label map nor its
agreement with the cascade would notice this arm going back to a constant.
"""
import json
import logging
from fastapi import WebSocketDisconnect
from xagent.web.services.managed_file_ref import DurableStorageOperationError
owner = _user(db_session, "dispatch-label-owner")
task = _task(db_session, owner.id)
owner_id = int(owner.id)
task_id = int(task.id)
db_session.close()
websocket = MagicMock()
websocket.accept = AsyncMock()
websocket.receive_text = AsyncMock(
side_effect=[json.dumps({"type": "resume_task"}), WebSocketDisconnect()]
)
ws_manager = MagicMock(
register_connection=MagicMock(),
disconnect=MagicMock(),
send_personal_message=AsyncMock(),
broadcast_to_task=AsyncMock(),
)
logger_name = websocket_api.__name__
with (
patch.object(websocket_api, "manager", ws_manager),
patch.object(
websocket_api,
"get_authenticated_user",
AsyncMock(return_value=SimpleNamespace(id=owner_id, is_admin=False)),
),
patch.object(websocket_api, "handle_status_request", AsyncMock()),
patch.object(
websocket_api,
"handle_resume_task",
AsyncMock(
side_effect=DurableStorageOperationError(
"Failed to restore durable object",
storage_key=f"users/{owner_id}/uploads/a/b.txt",
)
),
),
caplog.at_level(logging.WARNING, logger=logger_name),
):
await websocket_api.websocket_chat_endpoint(websocket, task_id, None)
fault_lines = [
entry.getMessage()
for entry in caplog.records
if entry.name == logger_name
and "Durable storage unavailable" in entry.getMessage()
]
assert len(fault_lines) == 1, fault_lines
assert "during websocket resume_task" in fault_lines[0]
assert "chat turn" not in fault_lines[0]
# The key still reaches the log, from the attribute rather than the message.
assert f"storage_key=users/{owner_id}/uploads/a/b.txt" in fault_lines[0]
@pytest.mark.asyncio
async def test_a_durable_integrity_fault_is_answered_as_corruption_not_an_outage(
db_session,
caplog,
) -> None:
"""The integrity subclass through the real cascade, not just the parent.
``test_durable_attachment_failure_keeps_the_storage_key_off_the_socket``
injects only ``DurableStorageOperationError``, so it would pass with these
two arms swapped -- the parent would catch the subclass and tell the client
to retry something retrying cannot fix, while emitting a transient-outage
warning over the permanent-corruption ERROR already logged upstream.
Ordering is checked across all twelve pairs by
``test_the_integrity_arm_precedes_its_parent_at_every_site``; this is what
proves this pair's arms also produce the right answers.
"""
import logging
from xagent.web.services.managed_file_ref import (
FILE_INTEGRITY_REUPLOAD_MESSAGE,
DurableObjectIntegrityError,
)
owner = _user(db_session, "integrity-answer-owner")
task = _task(db_session, owner.id, status=TaskStatus.COMPLETED)
owner_id = int(owner.id)
task_id = int(task.id)
db_session.close()
def failing_prepare(**_kwargs):
raise DurableObjectIntegrityError(
FILE_INTEGRITY_REUPLOAD_MESSAGE,
storage_key="users/7/uploads/8ac1f2/corrupt.txt",
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
logger_name = command_execution_service.__name__
with (
patch(
"xagent.web.services.task_command_execution._prepare_task_message_sync",
side_effect=failing_prepare,
),
patch("xagent.web.api.websocket.manager", ws_manager),
caplog.at_level(logging.WARNING, logger=logger_name),
):
with pytest.raises(DurableObjectIntegrityError):
await handle_task_message(
_make_command_reply(MagicMock()),
task_id,
{
"message": "with a corrupted attachment",
"client_message_id": "integrity-probe",
"user": SimpleNamespace(id=owner_id, is_admin=False),
"files": ["8ac1f2"],
},
)
frames = [str(call) for call in ws_manager.send_personal_message.await_args_list]
assert any("message_rejected" in frame for frame in frames), frames
# Told to re-upload, not to retry: retrying cannot repair corruption.
assert any("integrity check" in frame for frame in frames), frames
assert not any("temporarily unavailable" in frame for frame in frames), frames
assert not [
entry
for entry in caplog.records
if entry.name == logger_name
and "Durable storage unavailable" in entry.getMessage()
], "an integrity fault emitted an outage warning -- the arms are misordered"
def _seed_deferred_delivery(db_session: Session, suffix: str) -> tuple[User, Task, str]:
owner = _user(db_session, f"deferred-unknown-{suffix}-owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
turn_id = f"deferred-unknown-{suffix}"
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
content="Deferred guidance",
message_type="user_message",
turn_id=turn_id,
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
return owner, task, turn_id
def _delivery_status(db_session: Session, turn_id: str) -> str:
db_session.expire_all()
return str(
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == turn_id, TaskChatMessage.role == "user")
.one()
.delivery_status
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("message", "files", "stored_content"),
[
("Deferred guidance", [], "Deferred guidance"),
("Deferred guidance", [{"file_id": "original-file"}], "Deferred guidance"),
("", [{"file_id": "original-file"}], "Uploaded file(s)"),
],
)
async def test_websocket_retry_preserves_unknown_delivery(
db_session, message, files, stored_content
):
from xagent.web.api.websocket import handle_chat_message
owner, task, turn_id = _seed_deferred_delivery(db_session, "websocket-unknown")
row = (
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == turn_id)
.one()
)
row.delivery_status = DELIVERY_OUTCOME_UNKNOWN
row.attachments = files
row.content = stored_content
db_session.commit()
ws = MagicMock(send_personal_message=AsyncMock())
with patch("xagent.web.api.websocket.manager", ws):
await handle_chat_message(
MagicMock(),
int(task.id),
{
"message": message,
"files": files,
"client_message_id": turn_id,
"user": owner,
},
)
ack = _delivery_acks(ws)[0]
assert ack["rejection_outcome"] == "outcome_unknown"
assert not ack.get("retry_with_new_id")
@pytest.mark.asyncio
async def test_durable_unknown_notifies_origin_without_client_resend(db_session):
owner, task, turn_id = _seed_deferred_delivery(db_session, "durable-origin")
# A registered background owner has already settled the original command.
row = (
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == turn_id)
.one()
)
row.delivery_status = DELIVERY_OUTCOME_UNKNOWN
db_session.commit()
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=int(owner.id),
command_id=turn_id,
kind=TaskCommandKind.MESSAGE,
payload={
"type": "chat_message",
"message": "Deferred guidance",
"client_message_id": turn_id,
"files": [],
},
target_run_id=None,
attempt_count=2,
)
origin = SimpleNamespace()
websocket_api._command_origins.register(turn_id, origin, int(task.id))
try:
with (
patch.object(
websocket_api.manager, "is_connection_registered", return_value=True
),
patch.object(
websocket_api.manager, "send_personal_message", new=AsyncMock()
) as send,
patch.object(
websocket_api.manager, "broadcast_to_task", new=AsyncMock()
) as broadcast,
):
result = await command_execution_service._execute_and_report_task_command(
command
)
assert result["delivery_outcome"] == DELIVERY_OUTCOME_UNKNOWN
assert send.await_count == 2
ack, recipient = send.await_args_list[0].args
assert recipient is origin
assert ack["type"] == "message_rejected"
assert ack["error_code"] == "message_outcome_unknown"
assert ack["rejection_outcome"] == "outcome_unknown"
assert not ack.get("retry_with_new_id")
notice, recipient = send.await_args_list[1].args
assert recipient is origin
assert notice["type"] == "error"
assert notice["task_id"] == int(task.id)
assert notice["client_message_id"] == turn_id
assert notice["error_code"] == "message_outcome_unknown"
assert not notice.get("retry_with_new_id")
broadcast.assert_not_awaited()
assert not websocket_api._command_origins.has(turn_id, int(task.id))
finally:
websocket_api._command_origins.discard_command(turn_id, int(task.id))
@pytest.mark.asyncio
@pytest.mark.parametrize(
"original_files",
[
[{"file_id": "file-a"}, {"file_id": "file-a"}],
[{"file_id": "file-a"}, {"file_id": "missing-file"}],
],
)
@pytest.mark.parametrize("conflicting_retry", [False, True])
async def test_unknown_retry_compares_original_command_before_normalized_attachments(
db_session,
original_files,
conflicting_retry,
):
owner, task, turn_id = _seed_deferred_delivery(db_session, "normalized-retry")
payload = {
"message": "Deferred guidance",
"files": original_files,
"client_message_id": turn_id,
}
enqueued = websocket_api._enqueue_websocket_task_command_sync(
task_id=int(task.id),
actor_user_id=int(owner.id),
actor_is_admin=False,
command_id=turn_id,
kind=TaskCommandKind.MESSAGE,
payload=payload,
allow_missing_task=False,
)
assert enqueued.created
# Preparation persists only resolved, deduplicated attachment IDs.
row = (
db_session.query(TaskChatMessage)
.filter(TaskChatMessage.turn_id == turn_id)
.one()
)
row.attachments = [{"file_id": "file-a"}]
row.delivery_status = DELIVERY_OUTCOME_UNKNOWN
db_session.commit()
retry = dict(payload, user=owner)
if conflicting_retry:
retry["files"] = [{"file_id": "different-file"}]
ws = MagicMock(send_personal_message=AsyncMock())
with (
patch.object(websocket_api, "manager", ws),
patch.object(
websocket_api, "dispatch_task_command_promptly", new=AsyncMock()
) as dispatch,
):
await websocket_api.handle_chat_message(MagicMock(), int(task.id), retry)
ack = _delivery_acks(ws)[0]
assert ack["rejection_outcome"] == (
"not_accepted" if conflicting_retry else "outcome_unknown"
)
assert bool(ack.get("retry_with_new_id")) is conflicting_retry
assert _delivery_status(db_session, turn_id) == DELIVERY_OUTCOME_UNKNOWN
dispatch.assert_not_awaited()
@pytest.mark.asyncio
async def test_live_injection_outcome_unknown_is_not_acknowledged_or_reposted(
live_task_lease,
db_session,
) -> None:
owner = _user(db_session, "outcome-unknown-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "outcome-unknown-runner"
task.run_id = "outcome-unknown-run"
db_session.commit()
live_task_lease(db_session, task)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.OUTCOME_UNKNOWN
)
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
resume_background_mock = AsyncMock()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_execution.execute_resume_background",
resume_background_mock,
),
patch(
"xagent.web.services.task_command_execution.close_legacy_resume_interaction_sync",
) as close_mock,
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "Unconfirmed write",
"client_message_id": "outcome-unknown-turn",
"user": owner,
"files": [],
},
)
close_mock.assert_not_called()
bg_mgr.register_reserved_resume.assert_not_called()
bg_mgr.release_resume_reservation.assert_called_once_with(int(task.id))
resume_background_mock.assert_not_called()
ack = _delivery_acks(ws_manager)[0]
assert ack["type"] == "message_rejected"
assert ack["error_code"] == "message_outcome_unknown"
assert ack["rejection_outcome"] == "outcome_unknown"
assert not ack.get("retry_with_new_id")
assert (
_delivery_status(db_session, "outcome-unknown-turn") == DELIVERY_OUTCOME_UNKNOWN
)
# Live injection still has no task handoff until R1 enables the producer.
assert db_session.get(Task, int(task.id)).status == TaskStatus.RUNNING
@pytest.mark.asyncio
async def test_live_injection_rejected_by_fence_is_retryable_not_unknown(
live_task_lease,
db_session,
) -> None:
owner = _user(db_session, "fenced-reject-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "fenced-reject-runner"
task.run_id = "fenced-reject-run"
db_session.commit()
live_task_lease(db_session, task)
runner, context_manager = _fenced_live_runner(str(task.id))
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(side_effect=_injecting_through(runner))
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
resume_background_mock = AsyncMock()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
patch(
"xagent.web.services.task_execution.execute_resume_background",
resume_background_mock,
),
patch(
"xagent.web.services.task_command_execution.close_legacy_resume_interaction_sync",
) as close_mock,
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "Fenced input",
"client_message_id": "fenced-reject-turn",
"user": owner,
"files": [],
},
)
agent.post_user_message.assert_awaited_once()
close_mock.assert_not_called()
bg_mgr.register_reserved_resume.assert_not_called()
bg_mgr.release_resume_reservation.assert_called_once_with(int(task.id))
resume_background_mock.assert_not_called()
acks = _delivery_acks(ws_manager)
assert len(acks) == 1
ack = acks[0]
assert ack["type"] == "message_rejected"
assert ack["error_code"] == "message_delivery_failed"
assert ack["rejection_outcome"] == "not_accepted"
assert ack["retry_with_new_id"] is True
assert _delivery_status(db_session, "fenced-reject-turn") == DELIVERY_FAILED
assert not any(
call.args[0].get("type") in {"task_error", "error"}
for call in ws_manager.broadcast_to_task.call_args_list
)
assert db_session.get(Task, int(task.id)).status == TaskStatus.RUNNING
def _delivery_acks(ws_manager: MagicMock) -> list[dict[str, Any]]:
return [
call.args[0]
for call in ws_manager.send_personal_message.call_args_list
if call.args[0].get("type") in {"message_accepted", "message_rejected"}
]
@pytest.mark.parametrize("missing_actor_subject", [False, True])
def test_existing_command_payload_comparison_is_read_only(
db_session, missing_actor_subject
):
from sqlalchemy import event
from xagent.web.services.task_command_transport import (
existing_task_command_payload_matches,
)
owner, task, turn_id = _seed_deferred_delivery(db_session, "readonly")
payload = {"message": "Deferred guidance", "files": []}
websocket_api._enqueue_websocket_task_command_sync(
task_id=int(task.id),
actor_user_id=int(owner.id),
actor_is_admin=False,
command_id=turn_id,
kind=TaskCommandKind.MESSAGE,
payload=payload,
allow_missing_task=False,
)
db_session.refresh(owner)
if missing_actor_subject:
owner.actor_subject = None
db_session.commit()
task_id, owner_id = int(task.id), int(owner.id)
task.title = "pending caller-owned write"
statements = []
def require_select(_conn, _cursor, statement, _parameters, _context, _many):
statements.append(statement)
assert statement.lstrip().upper().startswith("SELECT"), statement
engine = db_session.get_bind()
event.listen(engine, "before_cursor_execute", require_select)
try:
matches = existing_task_command_payload_matches(
db_session,
task_id=task_id,
actor_user_id=owner_id,
command_id=turn_id,
kind=TaskCommandKind.MESSAGE,
payload=payload,
)
finally:
event.remove(engine, "before_cursor_execute", require_select)
assert statements
assert matches is (not missing_actor_subject)
assert task in db_session.dirty
if missing_actor_subject:
assert owner.actor_subject is None
db_session.rollback()
@pytest.mark.asyncio
async def test_unknown_retry_during_task_deletion_does_not_enqueue(db_session):
from xagent.web.services import task_command_transport as transport
owner, task, turn_id = _seed_deferred_delivery(db_session, "deleted-retry")
task_id, owner_id = int(task.id), int(owner.id)
payload = {
"message": "Deferred guidance",
"files": [],
"client_message_id": turn_id,
}
websocket_api._enqueue_websocket_task_command_sync(
task_id=task_id,
actor_user_id=owner_id,
actor_is_admin=False,
command_id=turn_id,
kind=TaskCommandKind.MESSAGE,
payload=payload,
allow_missing_task=False,
)
row = db_session.query(TaskChatMessage).filter_by(turn_id=turn_id).one()
row.delivery_status = DELIVERY_OUTCOME_UNKNOWN
db_session.commit()
load_actor_subject = transport._load_actor_subject
def delete_after_command_read(db, actor_user_id):
db_session.query(Task).filter_by(id=task_id).delete(synchronize_session=False)
db_session.commit()
return load_actor_subject(db, actor_user_id)
ws = MagicMock(send_personal_message=AsyncMock())
with (
patch.object(
transport, "_load_actor_subject", side_effect=delete_after_command_read
),
patch.object(
websocket_api,
"enqueue_task_command",
side_effect=AssertionError("retry must only read"),
),
patch.object(
websocket_api, "dispatch_task_command_promptly", new=AsyncMock()
) as dispatch,
patch.object(websocket_api, "manager", ws),
):
await websocket_api.handle_chat_message(
MagicMock(), task_id, dict(payload, user=owner)
)
ack = _delivery_acks(ws)[0]
assert ack["error_code"] == "message_outcome_unknown"
assert ack["rejection_outcome"] == "outcome_unknown"
assert not ack.get("retry_with_new_id")
dispatch.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["unknown", "cancelled", "exception"])
async def test_deferred_unknown_pauses_without_resuming(db_session, failure):
owner, task, turn_id = _seed_deferred_delivery(db_session, "pause-unknown")
post = AsyncMock(return_value=UserMessageInjectionOutcome.OUTCOME_UNKNOWN)
store = _CommittedInjectionStore()
runner, context = _runner_for_cancelled_injection(task.id, store)
async def failed_write(**payload):
if failure == "cancelled":
raise asyncio.CancelledError()
store.load_latest_checkpoint = AsyncMock(
side_effect=RuntimeError("read unavailable")
)
raise RuntimeError("lost ack")
store.checkpoint = failed_write
async def inject(execution_id, **kwargs):
return (await runner.inject_user_message(execution_id, **kwargs)).outcome
post.side_effect = inject
agent = MagicMock(post_user_message=post, resume_execution_by_id=AsyncMock())
notify = AsyncMock()
with patch.object(
task_execution_service.background_task_manager, "promote_resume_task"
):
operation = execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"turn_id": turn_id,
},
delivery_turn_id=turn_id,
delivery_notifier=notify,
)
if failure == "cancelled":
with pytest.raises(asyncio.CancelledError):
await operation
else:
await operation
assert _delivery_status(db_session, turn_id) == DELIVERY_OUTCOME_UNKNOWN
db_session.refresh(task)
assert task.status == TaskStatus.PAUSED
assert task.runner_id is None
agent.resume_execution_by_id.assert_not_awaited()
assert notify.await_args.kwargs["error_code"] == "message_outcome_unknown"
assert not notify.await_args.kwargs["retry_with_new_id"]
@pytest.mark.asyncio
async def test_deferred_accepted_then_cancelled_is_accepted_not_unknown(db_session):
"""A cancellation in the callback window cannot make a durable turn unknown."""
owner, task, turn_id = _seed_deferred_delivery(db_session, "accepted-cancel")
store = _CommittedInjectionStore()
runner, context = _runner_for_cancelled_injection(task.id, store)
async def write(**payload):
store.payload = payload
async def callback(**kwargs):
raise asyncio.CancelledError()
store.checkpoint = write
runner.callbacks = [SimpleNamespace(on_user_message_posted=callback)]
async def inject(execution_id, **kwargs):
return (await runner.inject_user_message(execution_id, **kwargs)).outcome
agent = MagicMock(
post_user_message=AsyncMock(side_effect=inject),
resume_execution_by_id=AsyncMock(),
)
notify = AsyncMock()
try:
with patch.object(
task_execution_service.background_task_manager, "promote_resume_task"
):
with pytest.raises(asyncio.CancelledError):
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"turn_id": turn_id,
},
delivery_turn_id=turn_id,
delivery_notifier=notify,
)
assert [m.content for m in context.messages] == ["Deferred guidance"]
assert (
_delivery_status(db_session, turn_id)
== command_execution_service.DELIVERY_DISPATCHED
)
assert notify.await_args.kwargs["accepted"] is True
assert not any(
call.kwargs.get("rejection_outcome") for call in notify.await_args_list
)
db_session.refresh(task)
# Not paused as an unknown input: the cancellation settles normally.
assert task.status == TaskStatus.FAILED
assert task.runner_id is None
agent.resume_execution_by_id.assert_not_awaited()
finally:
runner.context_manager.remove_context(str(task.id))
@pytest.mark.asyncio
@pytest.mark.parametrize("prior_status", [TaskStatus.RUNNING, TaskStatus.FAILED])
async def test_uncertain_settlement_preserves_terminal_result(db_session, prior_status):
from xagent.web.services.task_orchestrator import pause_unknown_task_lease
owner = _user(db_session, "unknown-settle-owner")
task = _task(db_session, owner.id, status=TaskStatus.PAUSED)
lease = task_execution_service._acquire_resume_task_lease(
int(task.id), int(owner.id), None
)
assert lease is not None
db_session.refresh(task)
task.status = prior_status
task.control_state = prior_status.value
if prior_status == TaskStatus.FAILED:
task.error_message = "explicit cancellation"
task.agent_config = {"a2a_state": "TASK_STATE_CANCELED"}
db_session.commit()
assert await pause_unknown_task_lease(lease)
db_session.refresh(task)
assert task.status == (
TaskStatus.PAUSED if prior_status == TaskStatus.RUNNING else TaskStatus.FAILED
)
assert task.runner_id is None
if prior_status == TaskStatus.FAILED:
assert task.error_message == "explicit cancellation"
class _CommittedInjectionStore:
"""Commit, then block so the caller can cancel before receiving a result."""
def __init__(self):
self.payload = None
self.committed = asyncio.Event()
async def checkpoint(self, **payload):
self.payload = payload
self.committed.set()
await asyncio.Event().wait()
async def load_latest_checkpoint(self, execution_id):
return self.payload
def _runner_for_cancelled_injection(task_id, store):
from xagent.core.agent.agent import Agent
from xagent.core.agent.context import ExecutionContext
from xagent.core.agent.runner import AgentRunner, ExecutionControl
from xagent.core.agent.runtime import PatternRuntime
execution_id = str(task_id)
runner = AgentRunner(agent=Agent(name="writer", patterns=[]), tracer=store)
context = ExecutionContext(execution_id=execution_id)
runner.context_manager.set_context(context)
runner._active_controls[execution_id] = ExecutionControl(
runtime=PatternRuntime(execution_id=execution_id, tracer=store),
task=None,
)
return runner, context
@pytest.mark.asyncio
@pytest.mark.parametrize("cancel_at", ["write", "callback"])
async def test_live_committed_write_cancellation_preserves_unknown(
db_session, live_task_lease, cancel_at
):
owner = _user(db_session, "cancel-live-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = "cancel-live-runner"
task.run_id = "cancel-live-run"
db_session.commit()
live_task_lease(db_session, task)
turn_id = "cancel-live-turn"
store = _CommittedInjectionStore()
runner, context = _runner_for_cancelled_injection(task.id, store)
if cancel_at == "callback":
async def write(**payload):
store.payload = payload
async def callback(**kwargs):
store.committed.set()
await asyncio.Event().wait()
store.checkpoint = write
runner.callbacks = [SimpleNamespace(on_user_message_posted=callback)]
async def post(execution_id, **kwargs):
return (await runner.post_user_message(execution_id, **kwargs)).outcome
agent = MagicMock(post_user_message=AsyncMock(side_effect=post))
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
reply = AsyncMock()
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
):
operation = asyncio.create_task(
handle_task_message(
reply,
int(task.id),
{
"message": "guidance",
"client_message_id": turn_id,
"user": owner,
"files": [],
},
)
)
try:
await asyncio.wait_for(store.committed.wait(), DB_PROGRESS_TIMEOUT)
operation.cancel()
with pytest.raises(asyncio.CancelledError):
await operation
assert len(context.messages) == (1 if cancel_at == "callback" else 0)
ack = reply.await_args.args[0]
if cancel_at == "callback":
# Accepted before the cancellation: never reported unknown.
assert (
_delivery_status(db_session, turn_id)
== command_execution_service.DELIVERY_DISPATCHED
)
assert ack["type"] == "message_accepted"
assert not ack.get("rejection_outcome")
else:
assert _delivery_status(db_session, turn_id) == DELIVERY_OUTCOME_UNKNOWN
assert ack["rejection_outcome"] == "outcome_unknown"
assert not ack.get("retry_with_new_id")
bg_mgr.register_reserved_resume.assert_not_called()
bg_mgr.release_resume_reservation.assert_called_with(int(task.id))
finally:
operation.cancel()
await asyncio.gather(operation, return_exceptions=True)
runner.context_manager.remove_context(str(task.id))
@pytest.mark.asyncio
async def test_deferred_lease_loss_keeps_committed_write_unknown(db_session):
from xagent.web.services.task_lease_service import TaskLeaseHeartbeatOutcome
owner, task, turn_id = _seed_deferred_delivery(db_session, "lease-loss")
store = _CommittedInjectionStore()
runner, context = _runner_for_cancelled_injection(task.id, store)
async def post(execution_id, **kwargs):
return (await runner.post_user_message(execution_id, **kwargs)).outcome
async def lose_lease(*args):
await asyncio.wait_for(store.committed.wait(), DB_PROGRESS_TIMEOUT)
db_session.refresh(task)
task.runner_id = "replacement-owner"
db_session.commit()
return TaskLeaseHeartbeatOutcome(lease_lost=True)
agent = MagicMock(
post_user_message=AsyncMock(side_effect=post),
resume_execution_by_id=AsyncMock(),
)
notify = AsyncMock()
try:
with (
patch.object(
task_execution_service,
"run_task_lease_heartbeat",
side_effect=lose_lease,
),
patch.object(
task_execution_service.background_task_manager, "promote_resume_task"
),
):
await asyncio.wait_for(
execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"turn_id": turn_id,
},
delivery_turn_id=turn_id,
delivery_notifier=notify,
),
DB_PROGRESS_TIMEOUT,
)
assert context.messages == []
assert _delivery_status(db_session, turn_id) == DELIVERY_OUTCOME_UNKNOWN
db_session.refresh(task)
assert task.runner_id == "replacement-owner"
assert task.status == TaskStatus.RUNNING
agent.resume_execution_by_id.assert_not_awaited()
assert notify.await_args.kwargs["rejection_outcome"] == "outcome_unknown"
finally:
runner.context_manager.remove_context(str(task.id))
def _not_accepted_runner(task_id, cause):
"""A real runner whose next injection is provably not written.
``absent``: the write fails and the authoritative read-back finds no
checkpoint holding the turn. ``fenced``: an earlier uncertain write left
the context fenced, so the runner writes nothing and rejects.
"""
from xagent.core.agent.context.execution import context_checkpoint_gate
store = SimpleNamespace(
checkpoint=AsyncMock(),
load_latest_checkpoint=AsyncMock(return_value=None),
)
runner, context = _runner_for_cancelled_injection(task_id, store)
if cause == "absent":
store.checkpoint.side_effect = RuntimeError("lost write")
else:
context_checkpoint_gate(context).injection_uncertain = True
return runner, context, store
@pytest.mark.asyncio
@pytest.mark.parametrize("cause", ["absent", "fenced"])
async def test_live_not_accepted_real_runner_is_retryable_without_task_failure(
live_task_lease, db_session, cause
) -> None:
from xagent.core.agent.context.execution import context_checkpoint_gate
owner = _user(db_session, f"live-not-accepted-{cause}-owner")
task = _task(db_session, owner.id, status=TaskStatus.RUNNING)
task.runner_id = f"live-not-accepted-{cause}-runner"
task.run_id = f"live-not-accepted-{cause}-run"
db_session.commit()
live_task_lease(db_session, task)
turn_id = f"live-not-accepted-{cause}"
runner, context, store = _not_accepted_runner(task.id, cause)
runner.pause = MagicMock(side_effect=runner.pause)
async def post(execution_id, **kwargs):
return (await runner.post_user_message(execution_id, **kwargs)).outcome
agent = MagicMock(post_user_message=AsyncMock(side_effect=post))
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
ws_manager = MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
)
bg_mgr = MagicMock()
bg_mgr.try_reserve_resume.return_value = ResumeReservationOutcome.RESERVED
bg_mgr.running_tasks.get.return_value = None
try:
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(
get_agent_for_task=AsyncMock(return_value=agent)
),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch("xagent.web.services.task_execution.background_task_manager", bg_mgr),
):
await handle_task_message(
_make_command_reply(MagicMock()),
int(task.id),
{
"message": "guidance",
"client_message_id": turn_id,
"user": owner,
"files": [],
},
)
acks = _delivery_acks(ws_manager)
assert len(acks) == 1
assert acks[0]["type"] == "message_rejected"
assert acks[0]["error_code"] == "message_delivery_failed"
assert acks[0]["rejection_outcome"] == "not_accepted"
assert acks[0]["retry_with_new_id"] is True
assert _delivery_status(db_session, turn_id) == DELIVERY_FAILED
# Only this input failed; the running task is not reported as failed.
assert not any(
call.args[0].get("type") in {"task_error", "agent_error", "error"}
for call in ws_manager.broadcast_to_task.call_args_list
)
bg_mgr.register_reserved_resume.assert_not_called()
bg_mgr.release_resume_reservation.assert_called_with(int(task.id))
assert context.messages == []
assert store.checkpoint.await_count == (1 if cause == "absent" else 0)
if cause == "absent":
# A proven absence lifts the fence and leaves the live run alone.
assert not context_checkpoint_gate(context).injection_uncertain
runner.pause.assert_not_called()
db_session.refresh(task)
assert task.status == TaskStatus.RUNNING
finally:
runner.context_manager.remove_context(str(task.id))
@pytest.mark.asyncio
@pytest.mark.parametrize(
"cause,prior_status",
[
("absent", TaskStatus.PAUSED),
("absent", TaskStatus.WAITING_FOR_USER),
("fenced", TaskStatus.WAITING_FOR_USER),
],
)
async def test_deferred_not_accepted_real_runner_is_never_failed_or_resumed(
db_session, cause, prior_status
) -> None:
owner, task, turn_id = _seed_deferred_delivery(
db_session, f"not-accepted-{cause}-{prior_status.value}"
)
task.status = prior_status
task.control_state = prior_status.value
db_session.commit()
runner, context, store = _not_accepted_runner(task.id, cause)
async def inject(execution_id, **kwargs):
return (await runner.inject_user_message(execution_id, **kwargs)).outcome
agent = MagicMock(
post_user_message=AsyncMock(side_effect=inject),
resume_execution_by_id=AsyncMock(),
)
notify = AsyncMock()
try:
with patch.object(
task_execution_service.background_task_manager, "promote_resume_task"
):
await execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"turn_id": turn_id,
},
delivery_turn_id=turn_id,
delivery_notifier=notify,
)
assert context.messages == []
assert _delivery_status(db_session, turn_id) == DELIVERY_FAILED
assert notify.await_args.kwargs["accepted"] is False
assert notify.await_args.kwargs["rejection_outcome"] == "not_accepted"
assert notify.await_args.kwargs["retry_with_new_id"] is True
agent.resume_execution_by_id.assert_not_awaited()
db_session.refresh(task)
# A fenced run stays paused for the user; a proven absence restores
# the prior resting status. Neither fails the task.
assert task.status == (TaskStatus.PAUSED if cause == "fenced" else prior_status)
assert task.runner_id is None
finally:
runner.context_manager.remove_context(str(task.id))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("canceller", "slow_outcome_write", "hung_client"),
[
("a2a", False, False),
("a2a", True, False),
("a2a", True, True),
("external", False, False),
("internal", False, False),
],
ids=["a2a", "a2a-slow", "a2a-slow-hung-client", "external", "internal"],
)
async def test_explicit_cancel_wins_over_uncertain_deferred_injection(
db_session, monkeypatch, caplog, canceller, slow_outcome_write, hung_client
) -> None:
"""An explicit stop fails the task; only a non-explicit cancel pauses it.
Either way the uncertain delivery stays outcome_unknown. In the slow case
the write outlasts the A2A cancel's 0.5s wait, so cancel_task cancels the
resume again while it records the outcome; the client still hears it.
"""
from xagent.web.models.agent import Agent
from xagent.web.services.a2a_task_cancel import cancel_a2a_task
from xagent.web.services.external_task_cancel import (
EXTERNAL_TURN_INTERRUPTED_MESSAGE,
cancel_external_task_unserialized,
)
caplog.set_level(logging.INFO, logger=task_execution_service.__name__)
owner, task, turn_id = _seed_deferred_delivery(db_session, f"cancel-{canceller}")
agent_row = Agent(user_id=int(owner.id), name=f"cancel-{canceller}")
db_session.add(agent_row)
db_session.commit()
source = "sdk" if canceller == "internal" else canceller
task.agent_id = int(agent_row.id)
task.source = source
db_session.commit()
store = _CommittedInjectionStore()
runner, _context = _runner_for_cancelled_injection(task.id, store)
async def inject(execution_id, **kwargs):
return (await runner.inject_user_message(execution_id, **kwargs)).outcome
agent = MagicMock(
post_user_message=AsyncMock(side_effect=inject),
resume_execution_by_id=AsyncMock(),
)
notify = AsyncMock()
if hung_client:
async def never_sent(**_: Any) -> None:
# A client that stops reading: the send never completes.
await asyncio.Event().wait()
notify.side_effect = never_sent
monkeypatch.setattr(
task_execution_service, "_DEFERRED_DELIVERY_NOTICE_TIMEOUT_SECONDS", 0.1
)
mark = task_execution_service.mark_user_message_delivery_sync
def mark_delivery(task_id, turn_id, status, *args, **kwargs):
if slow_outcome_write and status == DELIVERY_OUTCOME_UNKNOWN:
time.sleep(1.0)
return mark(task_id, turn_id, status, *args, **kwargs)
manager = task_execution_service.BackgroundTaskManager()
try:
with (
patch.object(task_execution_service, "background_task_manager", manager),
patch.object(
task_execution_service, "mark_user_message_delivery_sync", mark_delivery
),
):
assert (
manager.try_reserve_resume(
int(task.id), expected_run_id=task_execution_service.ANY_RESUME_RUN
)
is ResumeReservationOutcome.RESERVED
)
operation = asyncio.create_task(
execute_resume_background(
task_id=int(task.id),
agent_service=agent,
task_owner_user_id=int(owner.id),
pending_user_message={
"execution_message": "Deferred guidance",
"display_message": "Deferred guidance",
"turn_id": turn_id,
},
delivery_turn_id=turn_id,
delivery_notifier=notify,
trusted_task_source=source,
)
)
manager.register_reserved_resume(int(task.id), operation, run_id=None)
# The write committed and has not been acknowledged: uncertain.
await asyncio.wait_for(store.committed.wait(), DB_PROGRESS_TIMEOUT)
db_session.refresh(task)
assert task.status == TaskStatus.RUNNING
run_id, version = task.run_id, int(task.state_version or 0)
if canceller == "a2a":
await cancel_a2a_task(
task_id=int(task.id),
agent_id=int(agent_row.id),
expected_run_id=run_id,
expected_state_version=version,
)
elif canceller == "external":
await cancel_external_task_unserialized(
task_id=int(task.id),
agent_id=int(agent_row.id),
expected_run_id=run_id,
expected_state_version=version,
turn_id=turn_id,
)
else:
# Shutdown or lease loss: not a stop request.
operation.cancel()
[outcome] = await asyncio.wait_for(
asyncio.gather(operation, return_exceptions=True), DB_PROGRESS_TIMEOUT
)
assert isinstance(outcome, asyncio.CancelledError)
if slow_outcome_write:
assert "Cancellation timeout for task" in caplog.text
assert _delivery_status(db_session, turn_id) == DELIVERY_OUTCOME_UNKNOWN
notify.assert_awaited_once()
assert notify.await_args.kwargs["rejection_outcome"] == "outcome_unknown"
assert not notify.await_args.kwargs["retry_with_new_id"]
agent.resume_execution_by_id.assert_not_awaited()
db_session.refresh(task)
assert task.runner_id is None
if canceller == "internal":
assert task.status == TaskStatus.PAUSED
else:
assert task.status == TaskStatus.FAILED
assert task.run_id == run_id
if canceller == "external":
assert task.error_message == EXTERNAL_TURN_INTERRUPTED_MESSAGE
else:
assert task.agent_config["a2a_state"] == "TASK_STATE_CANCELED"
finally:
runner.context_manager.remove_context(str(task.id))