Enterprise agent platform for building, deploying, and operating AI teammates.
"""Reply acceptance, exact handoff, and cross-process preparation results."""
import asyncio
from dataclasses import replace
from unittest.mock import AsyncMock
import pytest
from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from tests.web.services.admission_capacity_shared import (
release_bucket,
saturate_bucket,
)
from tests.web.services.coordinator_command_shared import (
claim_task_command,
settle_command,
)
from xagent.web.models.agent import Agent
from xagent.web.models.database import Base, get_engine, get_session_local, init_db
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.user import User
from xagent.web.services import (
task_command_transport,
task_coordinator_runtime,
task_coordinator_service,
task_resume,
)
from xagent.web.services import task_resume_command as module
from xagent.web.services.task_command_execution import execute_durable_task_command
from xagent.web.services.task_command_transport import TaskCommandRejected
from xagent.web.services.task_resume import TaskReplyInput, TaskResumeBusyError
def owner_lease(task_id):
with get_session_local()() as db, db.begin():
task_coordinator_service.recover_expired_idle_task_lease_no_commit(db, task_id)
lease = task_coordinator_service.acquire_task_lease_no_commit(
db, task_id, runner_id="worker-1"
)
if lease is None:
task = db.get(Task, task_id)
assert task.runner_id == "worker-1"
lease = task_coordinator_service.TaskLease(
task_id, task.runner_id, task.lease_attempt_id
)
return lease
def commit_handoff(command):
return module._handoff(command, owner_lease(command.task_id))
@pytest.fixture
async def reply(tmp_path, monkeypatch):
monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "true")
monkeypatch.setattr(task_coordinator_runtime, "get_runner_id", lambda: "worker-1")
init_db(db_url=f"sqlite:///{tmp_path / 'reply.db'}")
with get_session_local()() as db:
user = User(username="owner", password_hash="unused")
db.add(user)
db.flush()
agent = Agent(user_id=user.id, name="reply")
db.add(agent)
db.flush()
task = Task(
user_id=user.id,
agent_id=agent.id,
title="Reply",
source="sdk",
status=TaskStatus.WAITING_FOR_USER,
run_id="run-1",
state_version=1,
control_state="waiting_for_user",
)
db.add(task)
db.commit()
ctx = TaskReplyInput(
task_id=task.id,
agent_id=agent.id,
task_owner_user_id=user.id,
actor_user_id=user.id,
run_id="run-1",
status=TaskStatus.WAITING_FOR_USER,
text="answer",
)
yield ctx
await task_coordinator_runtime.close_task_coordinators()
Base.metadata.drop_all(bind=get_engine())
async def claim(ctx):
command_id = module._admit_reply(ctx, "sdk", "", "reply-1")
with get_session_local()() as db:
return await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
def test_reply_is_durable_without_acquiring_a_request_lease(reply):
command_id = module._admit_reply(reply, "sdk", "", "reply-1")
with get_session_local()() as db:
task = db.get(Task, reply.task_id)
command = db.get(TaskExecutionCommand, command_id)
assert task.runner_id is None
assert task.status == TaskStatus.WAITING_FOR_USER
assert task.control_state == "waiting_for_user"
assert command.payload["text"] == "answer"
with pytest.raises(TaskResumeBusyError):
module._admit_reply(reply, "sdk", "", "reply-2")
async def test_reply_handoff_and_command_completion_are_one_transaction(
reply, monkeypatch
):
command = await claim(reply)
monkeypatch.setattr(
module, "finish_task_command_no_commit", lambda *args, **kwargs: False
)
with pytest.raises(TaskCommandRejected):
commit_handoff(command)
with get_session_local()() as db:
task = db.get(Task, reply.task_id)
assert task.runner_id == "worker-1"
assert task.control_state == "waiting_for_user"
assert db.get(TaskExecutionCommand, command.id).status == "processing"
async def test_old_reply_claim_cannot_acquire_or_restore_a_new_attempt(reply):
command = await claim(reply)
old = replace(command, attempt_count=command.attempt_count - 1)
with pytest.raises(TaskCommandRejected):
commit_handoff(old)
from xagent.web.services.task_command_transport import fail_task_command
assert not await settle_command(
old,
lambda: fail_task_command(
old.id,
"worker-1",
"stale",
force_terminal=True,
expected_attempt_count=old.attempt_count,
),
)
with get_session_local()() as db:
assert db.get(Task, reply.task_id).control_state == "waiting_for_user"
payload, lease, owner, state = commit_handoff(command)
assert lease.run_id == reply.run_id
assert state["lease_attempt_id"] == lease.attempt_id
assert module._read_reply_outcome(command.id) is None
module._record_outcome(command, state, "accepted", owner_lease(command.task_id))
assert module._read_reply_outcome(command.id)["outcome"] == "accepted"
@pytest.mark.asyncio
async def test_worker_uses_preacquired_lease_and_stable_turn_identity(
reply, monkeypatch
):
command = await claim(reply)
prepare = AsyncMock()
monkeypatch.setattr(task_resume, "resume_task_reply", prepare)
await execute_durable_task_command(command)
assert prepare.await_count == 1
assert prepare.call_args.kwargs["turn_id"] == command.command_id
assert prepare.call_args.kwargs["preacquired_lease"].run_id == "run-1"
assert module._read_reply_outcome(command.id)["outcome"] == "accepted"
async def test_reply_command_is_completed_before_its_injection_starts(
reply, monkeypatch
):
"""The worker's handoff marks the RESUME_INPUT command completed and
commits before it calls into the reply's injection. The claim query
returns the claimed command before the handoff; at the injection it no
longer does and the stored command is completed, so the same command
cannot reach the injection a second time. The injection is reached with
the command id as its turn id."""
command = await claim(reply)
def stored_status_and_claimable() -> tuple[str, bool]:
with get_session_local()() as db:
return (
db.get(TaskExecutionCommand, command.id).status,
task_command_transport._claimable_query(
db, runner_id="worker-1", command_db_id=command.id
).first()
is not None,
)
assert stored_status_and_claimable() == (
task_command_transport.COMMAND_PROCESSING,
True,
)
observed: list[tuple[str, bool, str]] = []
async def at_injection_entry(ctx, *, preacquired_lease, preacquired_state, turn_id):
observed.append((*stored_status_and_claimable(), turn_id))
monkeypatch.setattr(task_resume, "resume_task_reply", at_injection_entry)
await execute_durable_task_command(command)
assert observed == [
(task_command_transport.COMMAND_COMPLETED, False, command.command_id)
], observed
@pytest.mark.parametrize("replaced", [False, True])
async def test_terminal_reply_failure_restores_only_its_admission(reply, replaced):
from xagent.web.services.task_command_transport import (
MAX_COMMAND_FAILURES,
fail_task_command,
)
command = await claim(reply)
with get_session_local()() as db:
db.get(TaskExecutionCommand, command.id).failure_count = (
MAX_COMMAND_FAILURES - 1
)
if replaced:
db.get(Task, reply.task_id).state_version += 1
db.commit()
assert await settle_command(
command,
lambda: fail_task_command(
command.id,
"worker-1",
"unavailable",
expected_attempt_count=command.attempt_count,
),
)
await task_coordinator_runtime.close_task_coordinators()
with get_session_local()() as db:
task = db.get(Task, reply.task_id)
assert task.runner_id is None
assert task.control_state == "waiting_for_user"
assert module._read_reply_outcome(command.id)["outcome"] == "unavailable"
@pytest.mark.parametrize(
"invalid",
[{"version": True}, {"agent_id": 0}, {"run_id": ""}, {"prior_status": "paused"}],
)
async def test_invalid_reply_payload_cannot_acquire_lease(reply, invalid):
command = await claim(reply)
command = replace(command, payload={**command.payload, **invalid})
with pytest.raises(TaskCommandRejected):
commit_handoff(command)
with get_session_local()() as db:
assert db.get(Task, reply.task_id).status == TaskStatus.WAITING_FOR_USER
@pytest.mark.parametrize(
"source,status",
[
("sdk", TaskStatus.WAITING_FOR_USER),
("a2a", TaskStatus.WAITING_FOR_USER),
("a2a", TaskStatus.PAUSED),
],
)
async def test_reply_waits_for_previous_execution_to_release_lease(
reply, source, status
):
from datetime import datetime, timedelta, timezone
from xagent.web.services.task_lease_service import TaskLease
from xagent.web.services.task_orchestrator import settle_task_lease_isolated
ctx = replace(reply, status=status)
with get_session_local()() as db:
task = db.get(Task, ctx.task_id)
task.source = source
task.status = status
task.control_state = status.value
task.runner_id = "old-worker"
task.lease_attempt_id = "old-attempt"
task.lease_expires_at = datetime.now(timezone.utc) + timedelta(seconds=60)
task.last_heartbeat_at = datetime.now(timezone.utc)
db.commit()
command_id = module._admit_reply(ctx, source, "message", "reply")
with get_session_local()() as db:
assert db.query(TaskExecutionCommand).count() == 1
assert db.get(Task, ctx.task_id).lease_attempt_id == "old-attempt"
assert (
await claim_task_command(db, runner_id="worker-1", command_db_id=command_id)
is None
)
assert db.get(Task, ctx.task_id).control_state == status.value
assert settle_task_lease_isolated(
TaskLease(
task_id=ctx.task_id,
runner_id="old-worker",
run_id=ctx.run_id,
attempt_id="old-attempt",
)
)
command_id = module._admit_reply(ctx, source, "message", "reply")
with get_session_local()() as db:
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
_, lease, _, _ = commit_handoff(command)
assert lease.run_id == ctx.run_id
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["sdk", "a2a"])
@pytest.mark.parametrize("unavailable", [False, True])
async def test_input_review_rejection_restores_waiting_reply(
reply, monkeypatch, source, unavailable
):
from types import SimpleNamespace
from xagent.core.agent import Agent as RuntimeAgent
from xagent.core.agent import ExecutionContext
from xagent.core.agent.runner import AgentRunner
from xagent.core.guardrails import input as input_review
from xagent.web.services import task_event_bridge
with get_session_local()() as db:
db.get(Task, reply.task_id).source = source
db.commit()
runner = AgentRunner(agent=RuntimeAgent(name="reviewed", patterns=[], llm=None))
context = ExecutionContext(execution_id=str(reply.task_id))
runner.context_manager.set_context(context)
hook = AsyncMock(
return_value=input_review.InputReviewResult(
refusal="Custom refusal.", unavailable=unavailable
)
)
monkeypatch.setattr(input_review, "_review", hook)
manager = SimpleNamespace(get_agent_for_task=AsyncMock(return_value=runner))
monkeypatch.setattr(
task_resume.agent_runtime_service, "get_agent_manager", lambda: manager
)
message_id = "message" if source == "a2a" else ""
command_id = module._admit_reply(reply, source, message_id, "original")
with get_session_local()() as db:
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
await execute_durable_task_command(command)
result = module._read_reply_outcome(command_id)
assert result["outcome"] == (
"input_review_unavailable" if unavailable else "input_blocked"
)
assert result["refusal"] == "Custom refusal."
retry_id = module._admit_reply(reply, source, message_id, "original")
assert (retry_id != command_id) is (unavailable and source == "a2a")
if retry_id == command_id:
monkeypatch.setattr(
task_event_bridge, "_bridge", SimpleNamespace(require_ready=lambda: None)
)
monkeypatch.setattr(module, "notify_task_command_dispatcher", lambda: None)
with pytest.raises(
input_review.InputReviewRejected, match="Custom refusal"
) as exc:
await module.enqueue_resume_input(
reply, source=source, message_id=message_id, command_id="original"
)
assert exc.value.unavailable is unavailable
with get_session_local()() as db:
assert db.get(Task, reply.task_id).status == TaskStatus.WAITING_FOR_USER
assert context.messages == []
hook.assert_awaited_once()
request = hook.await_args.args[0]
assert request.user_id == reply.task_owner_user_id
assert request.source == "reply"
@pytest.mark.asyncio
@pytest.mark.parametrize("review_failure", [False, True])
async def test_a2a_retries_checkpoint_read_failure_with_same_message_id(
reply, monkeypatch, review_failure
):
from types import SimpleNamespace
from xagent.core.agent.checkpoint import CheckpointUnavailableError
from xagent.core.agent.runner import UserMessageInjectionOutcome
from xagent.core.guardrails.input import InputReviewRejected
from xagent.web.services.task_lease_service import stop_task_lease_heartbeat
with get_session_local()() as db:
db.get(Task, reply.task_id).source = "a2a"
db.commit()
post = AsyncMock(
side_effect=[
InputReviewRejected("Try later.", unavailable=True)
if review_failure
else CheckpointUnavailableError("temporary read outage"),
UserMessageInjectionOutcome.POSTED_FRESH,
]
)
manager = SimpleNamespace(
get_agent_for_task=AsyncMock(
return_value=SimpleNamespace(post_user_message=post)
)
)
monkeypatch.setattr(
task_resume.agent_runtime_service, "get_agent_manager", lambda: manager
)
async def schedule(**kwargs):
await stop_task_lease_heartbeat(
kwargs["heartbeat_task"], kwargs["heartbeat_stop"]
)
scheduled = AsyncMock(side_effect=schedule)
monkeypatch.setattr(task_resume, "_schedule_waiting_a2a_resume", scheduled)
original_id = module._admit_reply(reply, "a2a", "message-1", "original-command")
with get_session_local()() as db:
first = await claim_task_command(
db, runner_id="worker-1", command_db_id=original_id
)
original_version = db.get(
TaskExecutionCommand, original_id
).target_state_version
await execute_durable_task_command(first)
assert module._read_reply_outcome(original_id)["outcome"] == (
"input_review_unavailable" if review_failure else "retryable_unavailable"
)
with get_session_local()() as db:
task = db.get(Task, reply.task_id)
assert task.runner_id == "worker-1"
assert task.control_state == "waiting_for_user"
retry_id = module._admit_reply(reply, "a2a", "message-1", "original-command")
assert retry_id != original_id
assert (
module._admit_reply(reply, "a2a", "message-1", "original-command") == retry_id
)
with get_session_local()() as db:
second = await claim_task_command(
db, runner_id="worker-1", command_db_id=retry_id
)
await execute_durable_task_command(second)
assert module._read_reply_outcome(retry_id)["outcome"] == "accepted"
assert (
module._admit_reply(reply, "a2a", "message-1", "original-command") == retry_id
)
assert post.await_count == 2
assert {call.kwargs["turn_id"] for call in post.await_args_list} == {
f"a2a:{reply.task_id}:message-1"
}
scheduled.assert_awaited_once()
with get_session_local()() as db:
assert db.query(TaskExecutionCommand).count() == 2
assert (
db.get(TaskExecutionCommand, original_id).target_state_version
== original_version
)
@pytest.mark.parametrize("outcome", ["unavailable", "accepted", "not_resumable"])
async def test_a2a_does_not_retry_unknown_or_terminal_outcome(reply, outcome):
with get_session_local()() as db:
db.get(Task, reply.task_id).source = "a2a"
db.commit()
command_id = module._admit_reply(reply, "a2a", "message", "original")
with get_session_local()() as db:
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
_, lease, _, state = commit_handoff(command)
assert task_resume._restore_a2a_resume_prelease_sync(
lease, status=TaskStatus.WAITING_FOR_USER
)
module._record_outcome(command, state, outcome, owner_lease(command.task_id))
assert module._admit_reply(reply, "a2a", "message", "original") == command_id
with get_session_local()() as db:
assert db.query(TaskExecutionCommand).count() == 1
assert (
await claim_task_command(db, runner_id="worker-1", command_db_id=command_id)
is None
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failure_stage", ["after_injection", "lease_restore", "unknown_result"]
)
async def test_a2a_unsafe_checkpoint_failure_cannot_create_retry(
reply, monkeypatch, failure_stage
):
from types import SimpleNamespace
from xagent.core.agent.checkpoint import CheckpointUnavailableError
from xagent.core.agent.runner import UserMessageInjectionOutcome
with get_session_local()() as db:
db.get(Task, reply.task_id).source = "a2a"
db.commit()
post = AsyncMock(return_value=UserMessageInjectionOutcome.POSTED_FRESH)
manager = SimpleNamespace(
get_agent_for_task=AsyncMock(
return_value=SimpleNamespace(post_user_message=post)
)
)
monkeypatch.setattr(
task_resume.agent_runtime_service, "get_agent_manager", lambda: manager
)
failure = CheckpointUnavailableError("temporary outage")
if failure_stage == "unknown_result":
post.return_value = UserMessageInjectionOutcome.OUTCOME_UNKNOWN
elif failure_stage == "after_injection":
monkeypatch.setattr(
task_resume, "_schedule_waiting_a2a_resume", AsyncMock(side_effect=failure)
)
else:
post.side_effect = failure
monkeypatch.setattr(
task_resume,
"_restore_a2a_resume_prelease_isolated",
AsyncMock(return_value=False),
)
command_id = module._admit_reply(reply, "a2a", "message", "original")
with get_session_local()() as db:
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
await execute_durable_task_command(command)
assert module._read_reply_outcome(command_id)["outcome"] == (
"unavailable" if failure_stage == "lease_restore" else "unknown"
)
assert module._admit_reply(reply, "a2a", "message", "original") == command_id
post.assert_awaited_once()
with get_session_local()() as db:
assert db.query(TaskExecutionCommand).count() == 1
@pytest.mark.parametrize(
"source,status",
[
("sdk", TaskStatus.WAITING_FOR_USER),
("a2a", TaskStatus.WAITING_FOR_USER),
("a2a", TaskStatus.PAUSED),
],
)
async def test_reply_reclaims_expired_resting_lease_atomically(
reply, monkeypatch, source, status
):
from datetime import datetime, timedelta, timezone
from xagent.web.services.task_lease_service import (
TaskLease,
TaskLeaseRefreshState,
refresh_task_lease,
)
from xagent.web.services.task_orchestrator import settle_task_lease_isolated
ctx = replace(reply, status=status)
# Use the new worker's runner ID too: the attempt must fence an old
# execution even when the process identity is reused.
old_lease = TaskLease(
task_id=ctx.task_id,
runner_id="worker-1",
run_id=ctx.run_id,
attempt_id="old-attempt",
)
expired_at = datetime.now(timezone.utc) - timedelta(minutes=10)
with get_session_local()() as db:
task = db.get(Task, ctx.task_id)
task.source = source
task.status = status
task.control_state = status.value
task.runner_id = old_lease.runner_id
task.lease_attempt_id = old_lease.attempt_id
task.lease_expires_at = expired_at
original_version = task.state_version
db.commit()
def fail_staging(*args, **kwargs):
raise RuntimeError("command staging failed")
with monkeypatch.context() as patch:
patch.setattr(module, "stage_task_command", fail_staging)
with pytest.raises(RuntimeError, match="command staging failed"):
module._admit_reply(ctx, source, "message", "reply")
with get_session_local()() as db:
task = db.get(Task, ctx.task_id)
assert task.runner_id == old_lease.runner_id
assert task.lease_attempt_id == old_lease.attempt_id
assert task.lease_expires_at.replace(tzinfo=timezone.utc) == expired_at
assert task.control_state == status.value
assert task.state_version == original_version
assert db.query(TaskExecutionCommand).count() == 0
command_id = module._admit_reply(ctx, source, "message", "reply")
with get_session_local()() as db:
task = db.get(Task, ctx.task_id)
assert task.runner_id == old_lease.runner_id
assert task.lease_attempt_id == old_lease.attempt_id
assert task.control_state == status.value
assert task.state_version == original_version
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
_, lease, _, state = commit_handoff(command)
assert lease.run_id == ctx.run_id
assert lease.attempt_id != old_lease.attempt_id
assert not settle_task_lease_isolated(old_lease)
with get_session_local()() as db:
assert refresh_task_lease(db, old_lease) == TaskLeaseRefreshState.LOST
task = db.get(Task, ctx.task_id)
assert task.lease_attempt_id == lease.attempt_id
assert task.state_version == state["state_version"]
assert task.control_state == "running"
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["sdk", "a2a"])
async def test_following_command_waits_for_reply_registration(
reply, monkeypatch, source
):
import asyncio
import threading
from xagent.web.services import task_command_execution
from xagent.web.services.task_command_transport import (
TaskCommandKind,
stage_task_command,
)
with get_session_local()() as db:
db.get(Task, reply.task_id).source = source
db.commit()
command_id = module._admit_reply(reply, source, "message", "reply")
with get_session_local()() as db:
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
committed, release = threading.Event(), threading.Event()
original = module._handoff
registered = False
def slow_handoff(command, lease):
handoff = original(command, lease)
committed.set()
assert release.wait(DB_PROGRESS_TIMEOUT)
return handoff
async def prepare(*args, **kwargs):
nonlocal registered
registered = True
async def followup(command):
assert registered
return {}
monkeypatch.setattr(module, "_handoff", slow_handoff)
monkeypatch.setattr(
task_resume,
"resume_task_reply" if source == "sdk" else "resume_a2a_task",
prepare,
)
effects = AsyncMock(side_effect=followup)
monkeypatch.setattr(
task_command_execution, "_execute_and_report_task_command", effects
)
resume = asyncio.create_task(
task_command_execution.execute_durable_task_command(command)
)
next_task = None
try:
assert await asyncio.to_thread(committed.wait, 5)
with get_session_local()() as db:
staged = stage_task_command(
db,
task_id=reply.task_id,
actor_user_id=reply.task_owner_user_id,
command_id="followup",
kind=TaskCommandKind.MESSAGE,
payload={"message": "next"},
)
db.commit()
following = await claim_task_command(
db, runner_id="worker-1", command_db_id=staged.staged_db_id
)
assert following is not None
next_task = asyncio.create_task(
task_command_execution.execute_durable_task_command(following)
)
await asyncio.sleep(0.05)
effects.assert_not_awaited()
release.set()
await asyncio.wait_for(asyncio.gather(resume, next_task), DB_PROGRESS_TIMEOUT)
effects.assert_awaited_once()
finally:
release.set()
await asyncio.gather(
resume, *([next_task] if next_task else []), return_exceptions=True
)
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["sdk", "a2a"])
async def test_reply_timeout_retains_accepted_command_for_later_execution(
reply, monkeypatch, source
):
from unittest.mock import Mock
from xagent.web.services import task_event_bridge
monkeypatch.setattr(task_event_bridge, "_bridge", Mock())
monkeypatch.setattr(module, "get_task_reply_wait_timeout_seconds", lambda: 0.01)
with get_session_local()() as db:
db.get(Task, reply.task_id).source = source
db.commit()
with pytest.raises(task_resume.TaskResumeOutcomeUnknownError):
await module.enqueue_resume_input(
reply, source=source, message_id="message", command_id="accepted"
)
with get_session_local()() as db:
row = db.query(TaskExecutionCommand).one()
assert row.status == "pending"
assert db.get(Task, reply.task_id).control_state == "waiting_for_user"
command_id = row.id
assert module._admit_reply(reply, source, "message", "accepted") == command_id
with pytest.raises(TaskResumeBusyError):
module._admit_reply(reply, source, "message", "duplicate")
prepare = AsyncMock()
monkeypatch.setattr(
task_resume,
"resume_task_reply" if source == "sdk" else "resume_a2a_task",
prepare,
)
with get_session_local()() as db:
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
await execute_durable_task_command(command)
prepare.assert_awaited_once()
assert module._read_reply_outcome(command_id)["outcome"] == "accepted"
with get_session_local()() as db:
assert (
await claim_task_command(db, runner_id="worker-1", command_db_id=command_id)
is None
)
@pytest.mark.parametrize("source", ["sdk", "a2a"])
async def test_unknown_reply_is_settled_paused_and_replayed(reply, monkeypatch, source):
from types import SimpleNamespace
from xagent.core.agent.runner import UserMessageInjectionOutcome
with get_session_local()() as db:
db.get(Task, reply.task_id).source = source
db.commit()
post = AsyncMock(return_value=UserMessageInjectionOutcome.OUTCOME_UNKNOWN)
manager = SimpleNamespace(
get_agent_for_task=AsyncMock(
return_value=SimpleNamespace(post_user_message=post)
)
)
monkeypatch.setattr(
task_resume.agent_runtime_service, "get_agent_manager", lambda: manager
)
schedule = AsyncMock()
monkeypatch.setattr(task_resume, "_schedule_waiting_reply_resume", schedule)
monkeypatch.setattr(task_resume, "_schedule_waiting_a2a_resume", schedule)
command_id = module._admit_reply(reply, source, "message", "unknown-reply")
with get_session_local()() as db:
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
await execute_durable_task_command(command)
assert module._read_reply_outcome(command_id)["outcome"] == "unknown"
assert module._admit_reply(reply, source, "message", "unknown-reply") == command_id
with get_session_local()() as db:
assert db.get(Task, reply.task_id).status == TaskStatus.PAUSED
schedule.assert_not_awaited()
post.assert_awaited_once()
@pytest.mark.parametrize("source", ["sdk", "a2a"])
async def test_fenced_reply_rejection_settles_busy_without_resuming(
reply, monkeypatch, source
):
from types import SimpleNamespace
from xagent.core.agent.runner import UserMessageInjectionOutcome
with get_session_local()() as db:
db.get(Task, reply.task_id).source = source
db.commit()
post = AsyncMock(return_value=UserMessageInjectionOutcome.REJECTED_RETRYABLE)
manager = SimpleNamespace(
get_agent_for_task=AsyncMock(
return_value=SimpleNamespace(post_user_message=post)
)
)
monkeypatch.setattr(
task_resume.agent_runtime_service, "get_agent_manager", lambda: manager
)
schedule = AsyncMock()
monkeypatch.setattr(task_resume, "_schedule_waiting_reply_resume", schedule)
monkeypatch.setattr(task_resume, "_schedule_waiting_a2a_resume", schedule)
command_id = module._admit_reply(reply, source, "message", "fenced-reply")
with get_session_local()() as db:
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
await execute_durable_task_command(command)
# Nothing was written, so the outcome is a known rejection, not "unknown".
assert module._read_reply_outcome(command_id)["outcome"] == "busy"
with get_session_local()() as db:
task = db.get(Task, reply.task_id)
assert task.status == TaskStatus.WAITING_FOR_USER
assert task.control_state == "waiting_for_user"
schedule.assert_not_awaited()
post.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize("install", ["legacy", "classifier"])
async def test_capacity_blocked_reply_is_acknowledged_as_queued_without_waiting(
reply, monkeypatch, request, install
):
source = "sdk"
from unittest.mock import Mock
from xagent.web.services import task_event_bridge
from xagent.web.services import task_execution_admission as admission
monkeypatch.setattr(task_event_bridge, "_bridge", Mock())
def hook(db, command):
return admission.AdmissionPolicy("tenant:batch", 1, 20)
if install == "legacy":
admission.set_task_admission_hook(hook)
request.addfinalizer(lambda: admission.set_task_admission_hook(None))
else:
admission.set_command_classifier(
lambda db, command, view: admission.CommandClassification(
admission=hook(db, command)
),
admission_enabled=True,
)
request.addfinalizer(admission.reset_command_classifier)
holding = saturate_bucket(reply.task_owner_user_id, reply.agent_id)
async def submit():
return await asyncio.wait_for(
module.enqueue_resume_input(
reply, source=source, message_id="message", command_id="queued"
),
10,
)
queued = await submit()
assert queued.queued is True
assert (queued.command_id, queued.run_id) == ("queued", "run-1")
assert queued.control_state == "waiting_for_user"
with get_session_local()() as db:
row = db.query(TaskExecutionCommand).filter_by(command_id="queued").one()
assert (row.status, row.attempt_count, row.defer_count) == ("pending", 0, 0)
# The snapshot reports the acceptance version, which is the task's.
assert queued.state_version == row.target_state_version == 1
assert db.get(Task, reply.task_id).control_state == "waiting_for_user"
command_id = row.id
# Same-ID replay converges on the same durable command and stays queued.
assert (await submit()).queued is True
with get_session_local()() as db:
assert (
db.query(TaskExecutionCommand).filter_by(kind="resume_input").count() == 1
)
assert (
await claim_task_command(db, runner_id="worker-1", command_db_id=command_id)
is None
)
with get_session_local()() as db:
row = db.get(TaskExecutionCommand, command_id)
assert (row.status, row.attempt_count, row.defer_count) == ("pending", 0, 0)
release_bucket(holding)
prepare = AsyncMock()
monkeypatch.setattr(
task_resume,
"resume_task_reply" if source == "sdk" else "resume_a2a_task",
prepare,
)
with get_session_local()() as db:
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
assert command is not None
await execute_durable_task_command(command)
prepare.assert_awaited_once()
# Once executed, the same ID replays the stored outcome instead of queueing.
accepted = await submit()
assert accepted.queued is False
assert (accepted.command_id, accepted.run_id) == ("queued", "run-1")
@pytest.mark.asyncio
async def test_ungoverned_reply_keeps_waiting_for_its_outcome(reply, monkeypatch):
from unittest.mock import Mock
from xagent.web.services import task_event_bridge
monkeypatch.setattr(task_event_bridge, "_bridge", Mock())
monkeypatch.setattr(module, "get_task_reply_wait_timeout_seconds", lambda: 0.05)
with pytest.raises(task_resume.TaskResumeOutcomeUnknownError):
await module.enqueue_resume_input(
reply, source="sdk", message_id="", command_id="slow-worker"
)
@pytest.mark.asyncio
async def test_reply_stopped_by_a_control_while_queued_replays_as_not_accepted(
reply, monkeypatch, request
):
from unittest.mock import Mock
from xagent.web.services import task_event_bridge
from xagent.web.services import task_execution_admission as admission
from xagent.web.services.task_command_transport import (
TaskCommandKind,
stage_task_command,
)
monkeypatch.setattr(task_event_bridge, "_bridge", Mock())
admission.set_task_admission_hook(
lambda db, command: admission.AdmissionPolicy("tenant:batch", 1, 20)
)
request.addfinalizer(lambda: admission.set_task_admission_hook(None))
saturate_bucket(reply.task_owner_user_id, reply.agent_id)
queued = await asyncio.wait_for(
module.enqueue_resume_input(
reply, source="sdk", message_id="", command_id="stopped"
),
10,
)
assert queued.queued is True
# A cancel control invalidates the waiting reply before it ever ran.
with get_session_local()() as db:
cancel = stage_task_command(
db,
task_id=reply.task_id,
actor_user_id=reply.task_owner_user_id,
command_id="cancel-1",
kind=TaskCommandKind.CANCEL,
payload={},
)
db.get(Task, reply.task_id).state_version += 1
db.flush()
admission.settle_cancelled_admissions(db, cancel.staged_db_id)
db.commit()
row = db.query(TaskExecutionCommand).filter_by(command_id="stopped").one()
assert row.status == "failed"
assert row.result == {"rejection_reason": "cancelled_before_admission"}
# Nothing was injected: the stored answer is definite, not unknown.
with pytest.raises(task_resume.TaskResumeNotAcceptedError):
await asyncio.wait_for(
module.enqueue_resume_input(
reply, source="sdk", message_id="", command_id="stopped"
),
10,
)
def governed(request, policy=None):
from xagent.web.services import task_execution_admission as admission
admission.set_task_admission_hook(
lambda db, command: policy or admission.AdmissionPolicy("tenant:batch", 1, 20)
)
request.addfinalizer(lambda: admission.set_task_admission_hook(None))
@pytest.mark.asyncio
async def test_governed_reply_with_free_capacity_waits_for_its_outcome(
reply, monkeypatch, request
):
"""An installed policy alone is not a reason to answer queued."""
from unittest.mock import Mock
from xagent.web.services import task_event_bridge
monkeypatch.setattr(task_event_bridge, "_bridge", Mock())
monkeypatch.setattr(module, "get_task_reply_wait_timeout_seconds", lambda: 0.05)
governed(request)
with pytest.raises(task_resume.TaskResumeOutcomeUnknownError):
await module.enqueue_resume_input(
reply, source="sdk", message_id="", command_id="free"
)
with get_session_local()() as db:
row = db.query(TaskExecutionCommand).filter_by(command_id="free").one()
assert row.status == "pending"
@pytest.mark.asyncio
async def test_governed_a2a_reply_keeps_waiting_while_capacity_is_occupied(
reply, monkeypatch, request
):
"""A2A has no queued projection, so its reply keeps the outcome wait."""
from unittest.mock import Mock
from xagent.web.services import task_event_bridge
monkeypatch.setattr(task_event_bridge, "_bridge", Mock())
monkeypatch.setattr(module, "get_task_reply_wait_timeout_seconds", lambda: 0.05)
governed(request)
with get_session_local()() as db:
db.get(Task, reply.task_id).source = "a2a"
db.commit()
saturate_bucket(reply.task_owner_user_id, reply.agent_id)
with pytest.raises(task_resume.TaskResumeOutcomeUnknownError):
await module.enqueue_resume_input(
reply, source="a2a", message_id="message", command_id="a2a-queued"
)
with get_session_local()() as db:
row = db.query(TaskExecutionCommand).filter_by(command_id="a2a-queued").one()
assert (row.status, row.attempt_count, row.defer_count) == ("pending", 0, 0)
def test_older_unheld_waiter_blocks_a_newer_reply(reply, request):
from xagent.web.services import task_execution_admission as admission
governed(request)
saturate_bucket(reply.task_owner_user_id, reply.agent_id, held=False)
command_id = module._admit_reply(reply, "sdk", "", "behind")
with get_session_local()() as db:
# No slot is occupied; FIFO order alone keeps the reply waiting.
assert admission.waiting_for_capacity(db, command_id) is True
def test_startup_pacing_blocks_a_reply_until_its_next_start(reply, request):
from xagent.web.services import task_execution_admission as admission
from xagent.web.services.task_admission_pacing import (
StartupPacing,
reserve_startup,
)
governed(
request,
admission.AdmissionPolicy("tenant:batch", 1, 20, pacing=StartupPacing(3600, 1)),
)
command_id = module._admit_reply(reply, "sdk", "", "paced")
with get_session_local()() as db:
assert admission.waiting_for_capacity(db, command_id) is False
# Another start consumed the only burst; the next one is an hour away.
assert reserve_startup(db, "tenant:batch")
db.commit()
assert admission.waiting_for_capacity(db, command_id) is True
@pytest.mark.asyncio
async def test_unknown_tool_effect_is_terminal_reply_outcome(reply, monkeypatch):
from types import SimpleNamespace
from xagent.core.agent.checkpoint import UnknownToolEffectError
with get_session_local()() as db:
db.get(Task, reply.task_id).source = "a2a"
db.commit()
post = AsyncMock(side_effect=UnknownToolEffectError("unknown external effect"))
manager = SimpleNamespace(
get_agent_for_task=AsyncMock(
return_value=SimpleNamespace(post_user_message=post)
)
)
monkeypatch.setattr(
task_resume.agent_runtime_service, "get_agent_manager", lambda: manager
)
command_id = module._admit_reply(reply, "a2a", "message", "original")
with get_session_local()() as db:
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=command_id
)
await execute_durable_task_command(command)
assert module._read_reply_outcome(command_id)["outcome"] == "not_resumable"
assert module._admit_reply(reply, "a2a", "message", "original") == command_id
post.assert_awaited_once()