Enterprise agent platform for building, deploying, and operating AI teammates.
from __future__ import annotations
import asyncio
import logging
import os
import time
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack
from dataclasses import FrozenInstanceError, is_dataclass
from datetime import datetime, timedelta
from threading import Barrier, Event, get_ident
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from sqlalchemy import create_engine, text
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import QueuePool
from tests.shared.async_waits import DB_PROGRESS_TIMEOUT, eventually
from tests.web.pool_contention_shared import (
GUARD_TIMEOUT,
LOOP_LIVENESS_TICKS,
assert_pool_checkout_off_loop,
gated_pool_checkout,
wait_for_ticks,
)
from tests.web.services.task_lease_shared import (
live_task_lease as live_task_lease_fixture,
)
from xagent.core.agent.runner import UserMessageInjectionOutcome
from xagent.db.sqlite import apply_sqlite_concurrency_pragmas
from xagent.web.api import websocket as websocket_api
from xagent.web.models import database as database_module
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.database import (
Base,
get_db,
get_engine,
get_session_local,
init_db,
)
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.task_command_terminal_event import TaskCommandTerminalEvent
from xagent.web.models.user import User
from xagent.web.services import task_command_execution as command_execution_service
from xagent.web.services import task_command_transport as task_command_transport_module
from xagent.web.services import task_execution as task_execution_service
from xagent.web.services.task_command_execution import (
_load_command_message_delivery_status,
execute_durable_task_command,
)
from xagent.web.services.task_command_transport import (
COMMAND_COMPLETED,
COMMAND_FAILED,
COMMAND_PENDING,
MAX_COMMAND_DEFERS,
MAX_COMMAND_FAILURES,
ClaimedTaskCommand,
TaskCommandConflictKind,
TaskCommandDeferred,
TaskCommandKind,
TaskCommandOwnerStateError,
TaskCommandRejected,
TaskCommandTaskMissing,
_claim_heartbeat,
claim_task_command,
classify_task_command_conflict,
defer_task_command,
dispatch_one_task_command,
dispatch_task_command_promptly,
enqueue_task_command,
fail_task_command,
finish_task_command,
load_task_command,
max_command_defers,
notify_task_command_dispatcher,
renew_task_command_claim,
retry_failed_task_command,
stage_task_command,
start_task_command_dispatcher,
stop_task_command_dispatcher,
task_has_live_foreign_runner,
task_has_live_runner,
)
live_task_lease = live_task_lease_fixture
@pytest.fixture()
def db_session(tmp_path):
init_db(db_url=f"sqlite:///{tmp_path / 'task-commands.db'}")
db = next(get_db())
try:
yield db
finally:
db.close()
Base.metadata.drop_all(bind=get_engine())
@pytest.fixture()
def queue_pool_command_db(tmp_path):
"""A real one-slot QueuePool for dispatcher checkout contention."""
engine = create_engine(
f"sqlite:///{tmp_path / 'task-command-queue-pool.db'}",
connect_args={"check_same_thread": False},
poolclass=QueuePool,
pool_size=1,
max_overflow=0,
pool_timeout=0.2,
)
Base.metadata.create_all(bind=engine)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
try:
yield engine, SessionLocal
finally:
engine.dispose()
@pytest.fixture()
def low_timeout_sqlite_engine(tmp_path):
"""A file-backed SQLite engine with a short busy_timeout for lock tests.
init_db does not expose apply_sqlite_concurrency_pragmas's busy_timeout_ms
parameter (it always uses the 5s production default), so a lock-path test
that needs a bounded wait builds its own engine directly instead of going
through init_db.
"""
engine = create_engine(
f"sqlite:///{tmp_path / 'task-command-lock-path.db'}",
connect_args={"check_same_thread": False},
)
# Keep SQLite lock waits short without asserting wall-clock performance.
apply_sqlite_concurrency_pragmas(engine, busy_timeout_ms=200)
Base.metadata.create_all(bind=engine)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
try:
yield engine, SessionLocal
finally:
engine.dispose()
@pytest.fixture()
def postgres_task_command_sessions():
"""A real PostgreSQL sessionmaker plus one running task's id.
Used only by the truly-concurrent raced-duplicate test: SQLite serializes
all writers through one database-wide lock, so a second writer's insert
can never be genuinely in flight at the same instant as the first's --
it simply cannot begin until the first's transaction ends. PostgreSQL's
per-row locking lets two inserts targeting the same unique key both be
open at once, with the second blocking until the first resolves.
"""
url = os.getenv("XAGENT_TEST_POSTGRES_URL")
if not url:
pytest.skip("XAGENT_TEST_POSTGRES_URL is not set")
# init_db rebinds the module-global engine/sessionmaker in place, and
# every other fixture and test in this file depends on that global
# pointing at its own sqlite engine -- the prior binding must be restored
# on exit rather than left pointed at this fixture's postgres engine.
prior_engine = database_module._engine
prior_session_local = database_module._SessionLocal
init_db(db_url=url)
engine = get_engine()
Base.metadata.drop_all(bind=engine)
Base.metadata.create_all(bind=engine)
SessionLocal = get_session_local()
try:
with SessionLocal() as db:
user, task = _create_running_task(db)
task.runner_id = None
task.lease_expires_at = None
db.commit()
task_id = int(task.id)
actor_id = int(user.id)
yield SessionLocal, task_id, actor_id
finally:
Base.metadata.drop_all(bind=engine)
database_module._engine = prior_engine
database_module._SessionLocal = prior_session_local
def _create_running_task(db) -> tuple[User, Task]:
user = User(username="command-user", password_hash="hash", is_admin=False)
db.add(user)
db.commit()
task = Task(
user_id=user.id,
title="Durable commands",
description="Durable commands",
status=TaskStatus.RUNNING,
execution_mode="auto",
run_id="run-1",
runner_id="runner-a",
lease_expires_at=datetime.utcnow() + timedelta(minutes=1),
)
db.add(task)
db.commit()
db.refresh(task)
return user, task
def test_enqueue_is_committed_and_idempotent(db_session) -> None:
user, task = _create_running_task(db_session)
payload = {"type": "pause_task"}
first = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="pause-1",
kind=TaskCommandKind.PAUSE,
payload=payload,
)
duplicate = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="pause-1",
kind=TaskCommandKind.PAUSE,
payload=payload,
)
conflict = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="pause-1",
kind=TaskCommandKind.RESUME,
payload={"type": "resume_task"},
)
assert first.created is True
assert duplicate.command_id == first.command_id
assert duplicate.created is False
assert duplicate.payload_matches is True
assert conflict.payload_matches is False
assert db_session.query(TaskExecutionCommand).count() == 1
def test_reused_actor_id_does_not_match_a_legacy_command(db_session) -> None:
user, task = _create_running_task(db_session)
actor_id = int(user.id)
task_id = int(task.id)
legacy = TaskExecutionCommand(
task_id=task_id,
actor_user_id=actor_id,
actor_subject=f"legacy-user-id:{actor_id}",
command_id="legacy-actor-idempotency",
kind=TaskCommandKind.PAUSE.value,
payload={"type": "pause_task"},
)
db_session.add(legacy)
db_session.commit()
assert user.actor_subject != legacy.actor_subject
duplicate = stage_task_command(
db_session,
task_id=task_id,
actor_user_id=actor_id,
command_id=legacy.command_id,
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
classification = classify_task_command_conflict(
db_session,
task_id=task_id,
command_id=legacy.command_id,
actor_user_id=actor_id,
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
assert duplicate.created is False
assert duplicate.payload_matches is False
assert classification.kind is TaskCommandConflictKind.RACED_DUPLICATE
assert classification.raced is not None
assert classification.raced.payload_matches is False
def test_live_run_command_stays_with_owner_until_task_lease_expires(
db_session,
) -> None:
user, task = _create_running_task(db_session)
command = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="message-1",
kind=TaskCommandKind.MESSAGE,
payload={"type": "chat", "message": "new guidance"},
)
assert claim_task_command(db_session, runner_id="runner-b") is None
owner_claim = claim_task_command(db_session, runner_id="runner-a")
assert owner_claim is not None
assert owner_claim.id == command.command_id
# Simulate the owner crashing after claim. Once both the command claim and
# task lease expire, another worker can replay the same durable command.
row = db_session.query(TaskExecutionCommand).filter_by(id=command.command_id).one()
row.claim_expires_at = datetime.utcnow() - timedelta(seconds=1)
task.lease_expires_at = datetime.utcnow() - timedelta(seconds=1)
db_session.commit()
recovered = claim_task_command(db_session, runner_id="runner-b")
assert recovered is not None
assert recovered.id == command.command_id
assert recovered.attempt_count == 2
def test_reassigned_command_routes_only_to_the_current_live_owner(db_session) -> None:
user, task = _create_running_task(db_session)
command = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="owner-takeover",
kind=TaskCommandKind.CANCEL,
payload={"agent_id": 1},
)
task.runner_id = "runner-b"
task.lease_expires_at = datetime.utcnow() + timedelta(minutes=1)
db_session.commit()
assert claim_task_command(db_session, runner_id="runner-c") is None
current_owner_claim = claim_task_command(db_session, runner_id="runner-b")
assert current_owner_claim is not None
assert current_owner_claim.id == command.command_id
def test_same_command_row_has_a_single_concurrent_claim_winner(db_session) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
command = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="concurrent-claim",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
barrier = Barrier(2)
def claim(runner_id: str) -> int | None:
SessionLocal = get_session_local()
with SessionLocal() as db:
barrier.wait()
claimed = claim_task_command(
db,
runner_id=runner_id,
command_db_id=command.command_id,
)
return claimed.id if claimed is not None else None
with ThreadPoolExecutor(max_workers=2) as executor:
first = executor.submit(claim, "runner-a")
second = executor.submit(claim, "runner-b")
winners = [value for value in (first.result(), second.result()) if value]
assert winners == [command.command_id]
def test_stale_attempt_cannot_mutate_reclaimed_command(db_session) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="reclaimed-generation",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
first = claim_task_command(
db_session,
runner_id="runner-a",
command_db_id=enqueued.command_id,
)
assert first is not None
row = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert row is not None
row.claim_expires_at = datetime.utcnow() - timedelta(seconds=1)
db_session.commit()
second = claim_task_command(
db_session,
runner_id="runner-a",
command_db_id=enqueued.command_id,
)
assert second is not None
assert second.attempt_count == first.attempt_count + 1
assert not renew_task_command_claim(
enqueued.command_id,
"runner-a",
expected_attempt_count=first.attempt_count,
)
assert not fail_task_command(
enqueued.command_id,
"runner-a",
"stale failure",
expected_attempt_count=first.attempt_count,
)
assert not defer_task_command(
enqueued.command_id,
"runner-a",
"stale deferral",
expected_attempt_count=first.attempt_count,
)
assert not finish_task_command(
enqueued.command_id,
"runner-a",
expected_attempt_count=first.attempt_count,
)
db_session.expire_all()
row = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert row is not None
assert row.status == "processing"
assert row.claimed_by == "runner-a"
assert row.attempt_count == second.attempt_count
assert (
db_session.query(TaskCommandTerminalEvent)
.filter(TaskCommandTerminalEvent.task_command_id == enqueued.command_id)
.count()
== 0
)
def test_live_foreign_runner_is_rechecked_before_cancel(db_session) -> None:
_user, task = _create_running_task(db_session)
assert task_has_live_foreign_runner(int(task.id), runner_id="runner-b") is True
assert task_has_live_foreign_runner(int(task.id), runner_id="runner-a") is False
assert task_has_live_runner(int(task.id), expected_run_id="run-1") is True
assert task_has_live_runner(int(task.id), expected_run_id="run-2") is False
task.lease_expires_at = datetime.utcnow() - timedelta(seconds=1)
db_session.commit()
assert task_has_live_foreign_runner(int(task.id), runner_id="runner-b") is False
assert task_has_live_runner(int(task.id), expected_run_id="run-1") is False
@pytest.mark.asyncio
@pytest.mark.parametrize(
("payload", "error"),
[
({}, "Agent ID is missing or null in cancel command payload"),
({"agent_id": None}, "Agent ID is missing or null in cancel command payload"),
({"agent_id": "invalid"}, "Agent ID 'invalid' is invalid"),
],
)
async def test_cancel_command_rejects_invalid_agent_id_payload(
db_session,
payload,
error: str,
) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="invalid-cancel",
kind=TaskCommandKind.CANCEL,
payload=payload,
target_run_id=None,
attempt_count=1,
)
with pytest.raises(ValueError, match=error):
await execute_durable_task_command(command)
@pytest.mark.asyncio
async def test_cancel_command_defers_on_a_live_foreign_owner(db_session) -> None:
user, task = _create_running_task(db_session)
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="misrouted-cancel",
kind=TaskCommandKind.CANCEL,
payload={"agent_id": 1},
target_run_id=None,
attempt_count=1,
)
with pytest.raises(TaskCommandDeferred, match="active task lease owner"):
await execute_durable_task_command(command)
@pytest.mark.asyncio
async def test_pause_command_defers_on_a_live_foreign_owner(db_session) -> None:
user, task = _create_running_task(db_session)
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="misrouted-pause",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
target_run_id=None,
attempt_count=1,
)
with pytest.raises(TaskCommandDeferred, match="active task lease owner"):
await execute_durable_task_command(command)
@pytest.mark.asyncio
async def test_resume_completes_when_the_target_run_has_a_live_foreign_owner(
db_session,
) -> None:
user, task = _create_running_task(db_session)
task.control_state = "running"
db_session.commit()
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="idempotent-foreign-resume",
kind=TaskCommandKind.RESUME,
payload={"type": "resume_task"},
target_run_id="run-1",
attempt_count=1,
)
result = await execute_durable_task_command(command)
assert result is not None
assert result["resume_outcome"] == "already_in_progress"
@pytest.mark.asyncio
async def test_resume_defers_when_a_settling_turn_still_holds_a_foreign_lease(
db_session,
) -> None:
"""RESUME still defers for a foreign lease in a state it cannot classify.
The idempotency evidence only settles RUNNING rows. A turn that has
committed WAITING_FOR_USER but whose lease columns have not been cleared
yet is not evidence that anything is resuming, so the command must defer
rather than schedule into the previous owner's live lease.
"""
user, task = _create_running_task(db_session)
task.status = TaskStatus.WAITING_FOR_USER
task.control_state = "waiting_for_user"
db_session.commit()
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="resume-into-foreign-lease",
kind=TaskCommandKind.RESUME,
payload={"type": "resume_task"},
target_run_id="run-1",
attempt_count=1,
)
with pytest.raises(TaskCommandDeferred):
await execute_durable_task_command(command)
@pytest.mark.asyncio
async def test_cancel_command_does_not_require_persisted_actor(db_session) -> None:
_user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=None,
command_id="cancel-with-deleted-actor",
kind=TaskCommandKind.CANCEL,
payload={},
target_run_id=None,
attempt_count=1,
)
with (
patch.object(command_execution_service, "_load_command_actor") as load_actor,
pytest.raises(ValueError, match="Agent ID is missing"),
):
await execute_durable_task_command(command)
load_actor.assert_not_called()
@pytest.mark.asyncio
async def test_pause_command_with_no_actor_fails_the_way_f1_identified(
db_session,
) -> None:
"""PR #1060 review, F1: unlike CANCEL (see the sibling test above),
PAUSE DOES require a persisted actor -- _load_command_actor(None)
raises. This is exactly the failure the stale-preview-run reaper's
dispatch hit before the fix: it called
pause_workforce_tasks_after_archive with actor_user_id=None for every
reaped run, so every reaped run's "cancel" never actually paused the
running task."""
_user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=None,
command_id="pause-with-no-actor",
kind=TaskCommandKind.PAUSE,
payload={},
target_run_id=None,
attempt_count=1,
)
with pytest.raises(ValueError, match="Task command has no actor"):
await execute_durable_task_command(command)
@pytest.mark.asyncio
async def test_pause_command_with_a_real_actor_gets_past_the_actor_check(
db_session,
) -> None:
"""PR #1060 review, F1 fix: WorkforceRunPauseTarget now carries the
run's own owner (WorkforceRun.user_id, nullable=False) as
actor_user_id, so the reaper's PAUSE dispatch no longer hits the
failure above. Whatever _handle_pause_task_unserialized does next is
exercised by other PAUSE tests; the only thing under test here is
getting past the actor-loading gate without F1's specific failure."""
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
command = ClaimedTaskCommand(
id=2,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="pause-with-real-actor",
kind=TaskCommandKind.PAUSE,
payload={},
target_run_id=None,
attempt_count=1,
)
try:
await execute_durable_task_command(command)
except Exception as exc:
assert "Task command has no actor" not in str(exc)
@pytest.mark.asyncio
async def test_only_terminal_command_failure_is_broadcast(db_session) -> None:
_user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
transient = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=None,
command_id="transient-cancel-failure",
kind=TaskCommandKind.CANCEL,
payload={},
target_run_id=None,
attempt_count=1,
failure_count=0,
)
terminal = ClaimedTaskCommand(
id=2,
task_id=int(task.id),
actor_user_id=None,
command_id="terminal-cancel-failure",
kind=TaskCommandKind.CANCEL,
payload={},
target_run_id=None,
attempt_count=1,
failure_count=MAX_COMMAND_FAILURES - 1,
)
with patch.object(
websocket_api.manager,
"broadcast_to_task",
new=AsyncMock(),
) as broadcast:
with pytest.raises(ValueError, match="Agent ID is missing"):
await execute_durable_task_command(transient)
broadcast.assert_not_awaited()
with pytest.raises(ValueError, match="Agent ID is missing"):
await execute_durable_task_command(terminal)
broadcast.assert_awaited_once()
event, event_task_id = broadcast.await_args.args
assert event_task_id == int(task.id)
assert event["type"] == "agent_error"
assert event["command_id"] == "terminal-cancel-failure"
@pytest.mark.asyncio
async def test_final_command_deferral_is_broadcast(db_session) -> None:
_user, task = _create_running_task(db_session)
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=None,
command_id="terminal-cancel-defer",
kind=TaskCommandKind.CANCEL,
payload={"agent_id": 1},
target_run_id=None,
attempt_count=1,
defer_count=max_command_defers() - 1,
)
with patch.object(
websocket_api.manager,
"broadcast_to_task",
new=AsyncMock(),
) as broadcast:
with pytest.raises(TaskCommandDeferred, match="active task lease owner"):
await execute_durable_task_command(command)
broadcast.assert_awaited_once()
@pytest.mark.asyncio
async def test_command_is_rejected_after_run_rotation(db_session) -> None:
user, task = _create_running_task(db_session)
task.run_id = "run-2"
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
command = ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="stale-pause",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
target_run_id="run-1",
attempt_count=1,
)
with pytest.raises(TaskCommandRejected, match="Task run changed"):
await execute_durable_task_command(command)
@pytest.mark.asyncio
async def test_stale_run_rejection_reason_is_persisted(db_session) -> None:
user, task = _create_running_task(db_session)
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="stale-pause-result",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
task.run_id = "run-2"
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
assert await dispatch_one_task_command(
execute_durable_task_command,
command_db_id=enqueued.command_id,
)
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == COMMAND_FAILED
assert stored.result == {"rejection_reason": "stale_run"}
def test_later_command_cannot_overtake_unfinished_command(db_session) -> None:
user, task = _create_running_task(db_session)
first = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="pause-ordered",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
second = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="resume-ordered",
kind=TaskCommandKind.RESUME,
payload={"type": "resume_task"},
)
claimed_first = claim_task_command(db_session, runner_id="runner-a")
assert claimed_first is not None
assert claimed_first.id == first.command_id
assert (
claim_task_command(
db_session,
runner_id="runner-a",
command_db_id=second.command_id,
)
is None
)
assert finish_task_command(first.command_id, "runner-a") is True
claimed_second = claim_task_command(db_session, runner_id="runner-a")
assert claimed_second is not None
assert claimed_second.id == second.command_id
@pytest.mark.asyncio
async def test_dispatch_claims_and_completes_once(db_session) -> None:
user, task = _create_running_task(db_session)
# Untargeted commands may be consumed by the current test process.
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="cancel-once",
kind=TaskCommandKind.CANCEL,
payload={"agent_id": 1},
)
seen: list[int] = []
async def execute(command):
seen.append(command.id)
return {"ok": True}
assert await dispatch_one_task_command(execute, command_db_id=enqueued.command_id)
assert not await dispatch_one_task_command(
execute, command_db_id=enqueued.command_id
)
db_session.expire_all()
stored = (
db_session.query(TaskExecutionCommand).filter_by(id=enqueued.command_id).one()
)
assert seen == [enqueued.command_id]
assert stored.status == COMMAND_COMPLETED
assert stored.result == {"ok": True}
@pytest.mark.asyncio
async def test_deferred_handoff_retries_without_consuming_failure_budget(
db_session,
) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="deferred-message",
kind=TaskCommandKind.MESSAGE,
payload={"type": "chat", "message": "wait"},
)
async def defer(_command):
raise TaskCommandDeferred("checkpoint is not ready")
assert await dispatch_one_task_command(defer, command_db_id=enqueued.command_id)
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == "pending"
assert stored.attempt_count == 1
assert stored.failure_count == 0
assert stored.defer_count == 1
assert stored.claim_expires_at is not None
stored.claim_expires_at = datetime.utcnow() - timedelta(seconds=1)
db_session.commit()
async def finish(_command):
return {"applied": True}
assert await dispatch_one_task_command(finish, command_db_id=enqueued.command_id)
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == COMMAND_COMPLETED
assert stored.attempt_count == 2
assert stored.failure_count == 0
assert stored.defer_count == 1
@pytest.mark.asyncio
async def test_deferred_message_eventually_fails_and_unblocks_cancel(
db_session,
) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
message = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="stuck-message",
kind=TaskCommandKind.MESSAGE,
payload={"type": "chat", "message": "wait forever"},
)
cancel = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="cancel-after-stuck-message",
kind=TaskCommandKind.CANCEL,
payload={"agent_id": 1},
)
row = db_session.get(TaskExecutionCommand, message.command_id)
assert row is not None
row.defer_count = max_command_defers() - 1
db_session.commit()
async def defer(_command):
raise TaskCommandDeferred("checkpoint never became ready")
assert await dispatch_one_task_command(defer, command_db_id=message.command_id)
db_session.expire_all()
row = db_session.get(TaskExecutionCommand, message.command_id)
assert row is not None
assert row.status == COMMAND_FAILED
assert row.failure_count == 0
assert row.defer_count == max_command_defers()
event = (
db_session.query(TaskCommandTerminalEvent)
.filter(TaskCommandTerminalEvent.task_command_id == message.command_id)
.one()
)
assert event.outcome == COMMAND_FAILED
assert event.outcome_version == row.attempt_count
cancel_claim = claim_task_command(db_session)
assert cancel_claim is not None
assert cancel_claim.id == cancel.command_id
@pytest.mark.asyncio
async def test_real_failures_use_a_separate_bounded_budget(db_session) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
command = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="last-failure",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
claimed = claim_task_command(db_session, runner_id="runner-a")
assert claimed is not None
row = db_session.get(TaskExecutionCommand, command.command_id)
assert row is not None
row.failure_count = MAX_COMMAND_FAILURES - 1
db_session.commit()
assert fail_task_command(command.command_id, "runner-a", "still broken") is True
db_session.expire_all()
row = db_session.get(TaskExecutionCommand, command.command_id)
assert row is not None
assert row.status == COMMAND_FAILED
assert row.failure_count == MAX_COMMAND_FAILURES
def test_failed_command_retry_preserves_immutable_target(db_session) -> None:
user, task = _create_running_task(db_session)
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="retry-terminal-command",
kind=TaskCommandKind.CANCEL,
payload={"agent_id": 1},
)
row = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert row is not None
row.status = COMMAND_FAILED
row.failure_count = MAX_COMMAND_FAILURES
row.defer_count = max_command_defers()
row.error = "temporary cancellation failure"
row.completed_at = datetime.utcnow()
db_session.commit()
assert retry_failed_task_command(
db_session,
enqueued.command_id,
)
db_session.expire_all()
row = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert row is not None
assert row.status == "pending"
assert row.failure_count == 0
assert row.defer_count == 0
assert row.error is None
assert row.completed_at is None
assert row.target_run_id == "run-1"
assert row.target_runner_id == "runner-a"
@pytest.mark.asyncio
async def test_recovery_dispatches_committed_message_across_run_rotation(
live_task_lease,
db_session,
) -> None:
user, task = _create_running_task(db_session)
task.input = "already committed"
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="committed-turn",
kind=TaskCommandKind.MESSAGE,
payload={
"type": "chat",
"message": "already committed",
"client_message_id": "committed-turn",
"files": [],
},
)
first_claim = claim_task_command(db_session, runner_id="runner-a")
assert first_claim is not None
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(user.id),
role="user",
content="already committed",
message_type="user_message",
turn_id="committed-turn",
delivery_status="pending",
)
)
row = db_session.query(TaskExecutionCommand).filter_by(id=enqueued.command_id).one()
row.claim_expires_at = datetime.utcnow() - timedelta(seconds=1)
# MESSAGE commands represent user intent for the task rather than a control
# mutation on one run, so recovery deliberately applies them after rotation.
task.run_id = "run-2"
task.lease_expires_at = datetime.utcnow() - timedelta(seconds=1)
db_session.commit()
live_task_lease(db_session, task)
runtime_agent = MagicMock()
runtime_agent.supports_live_control.return_value = True
runtime_agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
)
runtime_manager = MagicMock(
get_agent_for_task=AsyncMock(return_value=runtime_agent)
)
allow_resume_to_finish = asyncio.Event()
async def hold_resume(*args, **kwargs) -> None:
await allow_resume_to_finish.wait()
resume = AsyncMock(side_effect=hold_resume)
resume_task = None
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=runtime_manager,
),
patch.object(task_execution_service, "execute_resume_background", new=resume),
):
try:
assert await dispatch_one_task_command(
execute_durable_task_command,
command_db_id=enqueued.command_id,
)
# A completed mock can already have been unregistered by the time
# command dispatch returns. Hold execution to inspect live ownership.
resume_task = (
task_execution_service.background_task_manager.resume_tasks.get(
int(task.id)
)
)
assert resume_task is not None
assert not resume_task.done()
finally:
allow_resume_to_finish.set()
if resume_task is not None:
await asyncio.wait_for(resume_task, timeout=1)
assert (
int(task.id) not in task_execution_service.background_task_manager.resume_tasks
)
assert (
int(task.id) not in task_execution_service.background_task_manager.running_tasks
)
db_session.expire_all()
messages = (
db_session.query(TaskChatMessage)
.filter_by(task_id=int(task.id), turn_id="committed-turn")
.all()
)
assert len(messages) == 1
assert messages[0].delivery_status == "dispatched"
command = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert command is not None
assert command.status == COMMAND_COMPLETED
assert task.run_id == "run-2"
assert (
runtime_manager.get_agent_for_task.await_args.kwargs[
"connector_runtime_turn_id"
]
== "committed-turn"
)
runtime_agent.post_user_message.assert_awaited_once()
assert runtime_agent.post_user_message.await_args.kwargs["turn_id"] == (
"committed-turn"
)
resume.assert_awaited_once()
assert resume.await_args.kwargs["expected_run_id"] == "run-2"
@pytest.mark.asyncio
async def test_dispatcher_recovers_command_that_predates_worker_start(
db_session,
) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="startup-recovery",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
applied = asyncio.Event()
async def execute(command):
assert command.id == enqueued.command_id
applied.set()
return None
def completed() -> bool:
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
return stored is not None and stored.status == COMMAND_COMPLETED
start_task_command_dispatcher(execute)
try:
# Anti-hang bounds: the claim and completion each wait on a real
# SQLite commit, which a loaded CI runner can stall for seconds.
await asyncio.wait_for(applied.wait(), timeout=DB_PROGRESS_TIMEOUT)
await eventually(completed)
finally:
await stop_task_command_dispatcher()
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == COMMAND_COMPLETED
@pytest.mark.asyncio
async def test_dispatcher_recovers_unrelated_tasks_concurrently(db_session) -> None:
user, first_task = _create_running_task(db_session)
first_task.runner_id = None
first_task.lease_expires_at = None
second_task = Task(
user_id=user.id,
title="Second durable command",
description="Second durable command",
status=TaskStatus.RUNNING,
execution_mode="auto",
run_id="run-2",
runner_id=None,
lease_expires_at=None,
)
db_session.add(second_task)
db_session.commit()
first = enqueue_task_command(
db_session,
task_id=int(first_task.id),
actor_user_id=int(user.id),
command_id="slow-recovery",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
second = enqueue_task_command(
db_session,
task_id=int(second_task.id),
actor_user_id=int(user.id),
command_id="independent-recovery",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
release_first = asyncio.Event()
second_started = asyncio.Event()
async def execute(command):
if command.id == first.command_id:
await release_first.wait()
elif command.id == second.command_id:
second_started.set()
return None
start_task_command_dispatcher(execute)
try:
# This is an anti-hang bound, not a latency assertion. The dispatcher
# normally starts both tasks immediately, but the complete web suite
# can heavily contend for CI workers and SQLite connections.
await asyncio.wait_for(second_started.wait(), timeout=10)
release_first.set()
finally:
release_first.set()
await stop_task_command_dispatcher()
def test_load_task_command_returns_an_immutable_primitive_snapshot(db_session) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="detached-status",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
row = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert row is not None
row.attempt_count = 3
row.failure_count = 2
row.defer_count = 1
row.result = {"rejection_reason": "stale_run", "mutable": ["ignored"]}
db_session.commit()
stored = load_task_command(enqueued.command_id)
assert stored is not None
assert is_dataclass(stored)
assert not isinstance(stored, TaskExecutionCommand)
assert stored.command_db_id == enqueued.command_id
assert stored.status == "pending"
assert stored.error is None
assert stored.rejection_reason == "stale_run"
assert stored.result == {"rejection_reason": "stale_run"}
stored.result["rejection_reason"] = "mutated"
assert stored.result == {"rejection_reason": "stale_run"}
assert stored.attempt_count == 3
assert stored.failure_count == 2
assert stored.defer_count == 1
with pytest.raises(FrozenInstanceError):
stored.status = "failed" # type: ignore[misc]
def test_legacy_delivery_status_preserves_none(db_session) -> None:
user, task = _create_running_task(db_session)
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(user.id),
role="user",
content="legacy",
message_type="user_message",
turn_id="legacy-delivery",
delivery_status=None,
)
)
db_session.commit()
assert (
_load_command_message_delivery_status(int(task.id), "legacy-delivery") is None
)
@pytest.mark.asyncio
async def test_dispatch_claim_pool_wait_does_not_block_event_loop(
queue_pool_command_db,
monkeypatch,
) -> None:
engine, SessionLocal = queue_pool_command_db
with SessionLocal() as seed_db:
user, task = _create_running_task(seed_db)
task.runner_id = None
task.lease_expires_at = None
seed_db.commit()
enqueued = enqueue_task_command(
seed_db,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="contended-dispatch-claim",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
session_threads: list[int] = []
def session_factory():
session_threads.append(get_ident())
return SessionLocal()
monkeypatch.setattr(database_module, "_SessionLocal", session_factory)
held_connection = engine.connect()
loop_thread = get_ident()
ticker_stop = asyncio.Event()
ticks = 0
async def ticker() -> None:
nonlocal ticks
while not ticker_stop.is_set():
await asyncio.sleep(0.01)
ticks += 1
async def execute(command: ClaimedTaskCommand) -> None:
assert isinstance(command, ClaimedTaskCommand)
with gated_pool_checkout(engine) as gate:
ticker_task = asyncio.create_task(ticker())
dispatch_task = asyncio.create_task(
dispatch_one_task_command(execute, command_db_id=enqueued.command_id)
)
try:
await gate.wait_until_contending()
observed = await wait_for_ticks(lambda: ticks)
assert observed >= LOOP_LIVENESS_TICKS, (
"command claim QueuePool checkout blocked the event loop"
)
assert not dispatch_task.done()
finally:
held_connection.close()
gate.let_through()
ticker_stop.set()
await asyncio.wait_for(
asyncio.gather(dispatch_task, ticker_task, return_exceptions=True),
timeout=GUARD_TIMEOUT,
)
assert dispatch_task.result()
assert session_threads
assert session_threads[0] != loop_thread
@pytest.mark.asyncio
async def test_command_handler_pool_timeout_does_not_checkout_again(
queue_pool_command_db,
monkeypatch,
caplog,
) -> None:
"""A handler checkout timeout leaves its durable claim for expiry/retry."""
engine, SessionLocal = queue_pool_command_db
with SessionLocal() as seed_db:
user, task = _create_running_task(seed_db)
task.runner_id = None
task.lease_expires_at = None
seed_db.commit()
enqueued = enqueue_task_command(
seed_db,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="handler-pool-timeout",
kind=TaskCommandKind.RESUME,
payload={"type": "resume_task"},
)
task_id = int(task.id)
monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
held_connections = []
checkout_probe = ExitStack()
def checkout_from_exhausted_pool() -> None:
with SessionLocal() as db:
db.execute(text("SELECT 1")).scalar()
async def execute(_command: ClaimedTaskCommand) -> None:
held_connections.append(engine.connect())
checkout_probe.enter_context(assert_pool_checkout_off_loop(engine))
await asyncio.to_thread(checkout_from_exhausted_pool)
caplog.set_level(
logging.ERROR,
logger="xagent.web.services.task_command_transport",
)
try:
assert await dispatch_one_task_command(
execute,
command_db_id=enqueued.command_id,
)
finally:
try:
checkout_probe.close()
finally:
for connection in held_connections:
connection.close()
with SessionLocal() as verify_db:
stored = verify_db.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == "processing"
assert stored.claim_expires_at is not None
assert f"task_id={task_id}" in caplog.text
assert "component=task-command-handler" in caplog.text
assert "retaining command claim for expiry" in caplog.text
@pytest.mark.asyncio
@pytest.mark.parametrize(
("executor_outcome", "disposition_name"),
[
("success", "finish_task_command"),
("failure", "fail_task_command"),
("deferred", "defer_task_command"),
],
)
async def test_final_disposition_pool_timeout_retains_claim_for_ttl(
db_session,
monkeypatch,
caplog,
executor_outcome: str,
disposition_name: str,
) -> None:
"""A final write timeout must not trigger another checkout or lose fencing."""
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id=f"{executor_outcome}-disposition-timeout",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
disposition_calls: list[tuple[str, str, int]] = []
def pool_timeout(*args, **kwargs) -> bool:
disposition_calls.append(
(
disposition_name,
str(args[1]),
int(kwargs["expected_attempt_count"]),
)
)
raise SQLAlchemyTimeoutError("synthetic final disposition pool timeout")
def unexpected_disposition(*_args, **_kwargs) -> bool:
raise AssertionError("pool timeout triggered a second disposition checkout")
for candidate in (
"finish_task_command",
"fail_task_command",
"defer_task_command",
):
monkeypatch.setattr(
task_command_transport_module,
candidate,
pool_timeout if candidate == disposition_name else unexpected_disposition,
)
async def execute(_command: ClaimedTaskCommand) -> None:
if executor_outcome == "failure":
raise RuntimeError("executor failed")
if executor_outcome == "deferred":
raise TaskCommandDeferred("handoff not ready")
caplog.set_level(
logging.ERROR,
logger="xagent.web.services.task_command_transport",
)
assert await dispatch_one_task_command(
execute,
command_db_id=enqueued.command_id,
)
assert len(disposition_calls) == 1
observed_disposition, observed_runner_id, observed_attempt = disposition_calls[0]
assert observed_disposition == disposition_name
assert observed_attempt == 1
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == "processing"
assert stored.claimed_by == observed_runner_id
assert stored.claim_expires_at is not None
assert stored.attempt_count == 1
assert "component=task-command-disposition" in caplog.text
assert f"disposition={disposition_name}" in caplog.text
@pytest.mark.asyncio
async def test_real_final_disposition_pool_timeout_stays_off_event_loop(
queue_pool_command_db,
monkeypatch,
caplog,
) -> None:
"""A real one-slot pool timeout leaves the exact processing claim intact."""
engine, SessionLocal = queue_pool_command_db
with SessionLocal() as seed_db:
user, task = _create_running_task(seed_db)
task.runner_id = None
task.lease_expires_at = None
seed_db.commit()
enqueued = enqueue_task_command(
seed_db,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="real-finish-disposition-timeout",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
held_connections = []
checkout_probe = ExitStack()
async def execute(_command: ClaimedTaskCommand) -> dict[str, bool]:
held_connections.append(engine.connect())
checkout_probe.enter_context(assert_pool_checkout_off_loop(engine))
return {"ok": True}
caplog.set_level(
logging.ERROR,
logger="xagent.web.services.task_command_transport",
)
try:
assert await dispatch_one_task_command(
execute,
command_db_id=enqueued.command_id,
)
finally:
try:
checkout_probe.close()
finally:
for connection in held_connections:
connection.close()
with SessionLocal() as verify_db:
stored = verify_db.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == "processing"
assert stored.claimed_by is not None
assert stored.claim_expires_at is not None
assert stored.attempt_count == 1
assert "component=task-command-disposition" in caplog.text
assert "disposition=finish_task_command" in caplog.text
assert "retaining command claim for expiry" in caplog.text
@pytest.mark.asyncio
@pytest.mark.parametrize("heartbeat_failure", ["pool_timeout", "claim_lost"])
async def test_dispatch_skips_disposition_after_unhealthy_heartbeat(
db_session,
monkeypatch,
caplog,
heartbeat_failure: str,
) -> None:
"""An unresolved heartbeat cannot be followed by another DB checkout."""
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id=f"heartbeat-{heartbeat_failure}",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
class HeartbeatOutcome:
claim_lost = heartbeat_failure == "claim_lost"
pool_timeout = (
SQLAlchemyTimeoutError("heartbeat pool timeout")
if heartbeat_failure == "pool_timeout"
else None
)
requires_ttl_recovery = claim_lost or pool_timeout is not None
async def unhealthy_heartbeat(
_command_db_id: int,
_runner_id: str,
_attempt_count: int,
stop_event: asyncio.Event,
) -> HeartbeatOutcome:
await stop_event.wait()
return HeartbeatOutcome()
def unexpected_disposition(*_args, **_kwargs) -> bool:
raise AssertionError("unhealthy heartbeat triggered a final checkout")
monkeypatch.setattr(
task_command_transport_module,
"_claim_heartbeat",
unhealthy_heartbeat,
)
for candidate in (
"finish_task_command",
"fail_task_command",
"defer_task_command",
):
monkeypatch.setattr(
task_command_transport_module,
candidate,
unexpected_disposition,
)
caplog.set_level(
logging.ERROR,
logger="xagent.web.services.task_command_transport",
)
assert await dispatch_one_task_command(
lambda _command: asyncio.sleep(0),
command_db_id=enqueued.command_id,
)
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == "processing"
assert stored.claimed_by is not None
assert stored.claim_expires_at is not None
assert "component=task-command-heartbeat" in caplog.text
assert "retaining command claim for expiry" in caplog.text
@pytest.mark.asyncio
async def test_dispatcher_worker_isolates_one_command_error(
monkeypatch,
caplog,
) -> None:
"""One command failure must not terminate its long-lived dispatcher worker."""
recovered = asyncio.Event()
calls = 0
async def dispatch(_executor, *, command_db_id=None) -> bool:
nonlocal calls
del command_db_id
calls += 1
if calls == 1:
raise RuntimeError("single command failure")
recovered.set()
return False
monkeypatch.setattr(
task_command_transport_module,
"dispatch_one_task_command",
dispatch,
)
monkeypatch.setattr(
task_command_transport_module,
"_dispatcher_wakeup",
asyncio.Event(),
)
monkeypatch.setattr(
task_command_transport_module,
"DISPATCHER_IDLE_SECONDS",
0.01,
)
caplog.set_level(
logging.ERROR,
logger="xagent.web.services.task_command_transport",
)
worker = asyncio.create_task(
task_command_transport_module._run_task_command_dispatcher_worker(
lambda _command: asyncio.sleep(0)
)
)
try:
await asyncio.wait_for(recovered.wait(), timeout=GUARD_TIMEOUT)
finally:
worker.cancel()
await asyncio.gather(worker, return_exceptions=True)
with pytest.raises(asyncio.CancelledError):
worker.result()
assert calls >= 2
assert "component=task-command-dispatcher" in caplog.text
assert "single command failure" in caplog.text
def test_dispatcher_failure_backoff_grows_exponentially_to_cap(monkeypatch) -> None:
monkeypatch.setattr(task_command_transport_module, "DISPATCHER_IDLE_SECONDS", 0.5)
monkeypatch.setattr(
task_command_transport_module, "DISPATCHER_FAILURE_BACKOFF_MAX_SECONDS", 30.0
)
delays = [
task_command_transport_module._dispatcher_failure_backoff_seconds(failures)
# 2000 is past 1025, where an unclamped float(1 << exponent) overflows.
for failures in (1, 2, 3, 4, 5, 6, 7, 8, 2000)
]
assert delays == [0.5, 1.0, 2.0, 4.0, 8.0, 16.0, 30.0, 30.0, 30.0]
@pytest.mark.asyncio
async def test_dispatcher_worker_backs_off_and_rate_limits_failure_logs(
monkeypatch,
caplog,
) -> None:
"""Repeated claim failures back off, log once per interval, and reset."""
# fail x3, idle success, fail x1, idle success (stop).
script = [True, True, True, False, True, False]
finished = asyncio.Event()
calls = 0
async def dispatch(_executor, *, command_db_id=None) -> bool:
nonlocal calls
del command_db_id
if calls >= len(script):
return False
step = script[calls]
calls += 1
if calls == len(script):
finished.set()
if step:
raise RuntimeError("database unreachable")
return False
backoff_requests: list[int] = []
def record_backoff(consecutive_failures: int) -> float:
backoff_requests.append(consecutive_failures)
return 0.001
monkeypatch.setattr(
task_command_transport_module, "dispatch_one_task_command", dispatch
)
monkeypatch.setattr(
task_command_transport_module,
"_dispatcher_failure_backoff_seconds",
record_backoff,
)
monkeypatch.setattr(
task_command_transport_module, "_dispatcher_wakeup", asyncio.Event()
)
monkeypatch.setattr(task_command_transport_module, "DISPATCHER_IDLE_SECONDS", 0.001)
monkeypatch.setattr(
task_command_transport_module, "DISPATCHER_FAILURE_LOG_INTERVAL_SECONDS", 3600.0
)
caplog.set_level(logging.INFO, logger="xagent.web.services.task_command_transport")
worker = asyncio.create_task(
task_command_transport_module._run_task_command_dispatcher_worker(
lambda _command: asyncio.sleep(0)
)
)
try:
await asyncio.wait_for(finished.wait(), timeout=GUARD_TIMEOUT)
# Let the final idle claim's bookkeeping run before cancelling.
await asyncio.sleep(0.01)
finally:
worker.cancel()
await asyncio.gather(worker, return_exceptions=True)
assert backoff_requests == [1, 2, 3, 1]
failure_logs = [
record
for record in caplog.records
if record.levelno == logging.ERROR
and "command dispatch failed" in record.getMessage()
]
# The first failure of each outage is logged; repeats inside the interval
# are suppressed, and recovery re-arms logging for the next outage.
assert len(failure_logs) == 2
assert "consecutive_failures=1" in failure_logs[0].getMessage()
assert "consecutive_failures=1" in failure_logs[1].getMessage()
recovered = [
record.getMessage()
for record in caplog.records
if "dispatch recovered after" in record.getMessage()
]
assert recovered == [
"component=task-command-dispatcher dispatch recovered after "
"3 consecutive failure(s)",
"component=task-command-dispatcher dispatch recovered after "
"1 consecutive failure(s)",
]
@pytest.mark.asyncio
async def test_dispatcher_worker_logs_suppressed_count_after_interval(
monkeypatch,
caplog,
) -> None:
"""Once the log interval elapses, the next failure reports what was suppressed."""
finished = asyncio.Event()
calls = 0
clock = [1000.0]
async def dispatch(_executor, *, command_db_id=None) -> bool:
nonlocal calls
del command_db_id
calls += 1
if calls == 4:
# Simulate the log interval elapsing before the fourth failure.
clock[0] += 61.0
if calls >= 5:
finished.set()
return False
raise RuntimeError("database unreachable")
monkeypatch.setattr(
task_command_transport_module, "dispatch_one_task_command", dispatch
)
monkeypatch.setattr(
task_command_transport_module,
"_dispatcher_failure_backoff_seconds",
lambda _failures: 0.001,
)
monkeypatch.setattr(
task_command_transport_module, "_dispatcher_clock", lambda: clock[0]
)
monkeypatch.setattr(
task_command_transport_module, "_dispatcher_wakeup", asyncio.Event()
)
monkeypatch.setattr(task_command_transport_module, "DISPATCHER_IDLE_SECONDS", 0.001)
monkeypatch.setattr(
task_command_transport_module, "DISPATCHER_FAILURE_LOG_INTERVAL_SECONDS", 60.0
)
caplog.set_level(logging.ERROR, logger="xagent.web.services.task_command_transport")
worker = asyncio.create_task(
task_command_transport_module._run_task_command_dispatcher_worker(
lambda _command: asyncio.sleep(0)
)
)
try:
await asyncio.wait_for(finished.wait(), timeout=GUARD_TIMEOUT)
finally:
worker.cancel()
await asyncio.gather(worker, return_exceptions=True)
failure_logs = [
record.getMessage()
for record in caplog.records
if "command dispatch failed" in record.getMessage()
]
assert len(failure_logs) == 2
assert "consecutive_failures=1 suppressed_since_last_log=0" in failure_logs[0]
assert "consecutive_failures=4 suppressed_since_last_log=2" in failure_logs[1]
def _dispatch_diagnostics(db_session, dispatch_task, command_db_id) -> str:
"""Temporary CI diagnostics: why finish_task_command was never reached."""
import io
from xagent.config import (
get_shared_task_execution_enabled,
get_task_execution_role,
get_worker_count,
)
try: # Added on main by the shared-worker admission change.
from xagent.web.models.task_admission import TaskAdmissionTicket
from xagent.web.services import task_execution_admission
except ImportError:
TaskAdmissionTicket = None
task_execution_admission = None
from xagent.web.services.task_execution_host import consumes_task_commands
from xagent.web.services.task_lease_service import get_runner_id
lines = [f"dispatch done={dispatch_task.done()}"]
if dispatch_task.done():
if dispatch_task.cancelled():
lines.append("dispatch cancelled")
elif dispatch_task.exception() is not None:
lines.append(f"dispatch exception={dispatch_task.exception()!r}")
else:
lines.append(f"dispatch result={dispatch_task.result()!r}")
else:
stack = io.StringIO()
dispatch_task.print_stack(file=stack)
lines.append(f"dispatch stack=\n{stack.getvalue()}")
lines.append(
"shared_enabled={} role={} worker_count={} consumes={} runner_id={}".format(
get_shared_task_execution_enabled(),
get_task_execution_role(),
get_worker_count(),
consumes_task_commands(),
get_runner_id(),
)
)
lines.append(
"admission_installed={!r} admission_enabled={!r}".format(
getattr(task_execution_admission, "_installed", "<no admission module>"),
task_execution_admission.admission_enabled()
if task_execution_admission is not None
else "<no admission module>",
)
)
lines.append(
"env="
+ repr(
{
key: value
for key, value in os.environ.items()
if key.startswith("XAGENT_")
and any(
part in key
for part in ("SHARED", "ROLE", "WORKER", "RUNNER", "LEASE")
)
}
)
)
db_session.expire_all()
command = db_session.get(TaskExecutionCommand, command_db_id)
if command is None:
lines.append("command row missing")
else:
lines.append(
"command status={} kind={} attempts={} claimed_by={} "
"claim_expires_at={} retry_available_at={}".format(
command.status,
command.kind,
command.attempt_count,
command.claimed_by,
command.claim_expires_at,
command.retry_available_at,
)
)
task_row = db_session.get(Task, command.task_id)
lines.append(
"task status={} runner_id={} run_id={} lease_attempt_id={} "
"lease_expires_at={}".format(
task_row.status,
task_row.runner_id,
task_row.run_id,
task_row.lease_attempt_id,
task_row.lease_expires_at,
)
)
if TaskAdmissionTicket is not None:
tickets = (
db_session.query(TaskAdmissionTicket)
.filter(TaskAdmissionTicket.command_id == command_db_id)
.count()
)
lines.append(f"admission tickets={tickets}")
lines.append(
"commands on task: "
+ repr(
[
(row.id, row.kind, row.status)
for row in db_session.query(TaskExecutionCommand)
.filter(TaskExecutionCommand.task_id == command.task_id)
.all()
]
)
)
return "finish_task_command not reached within 1s\n" + "\n".join(lines)
@pytest.mark.asyncio
async def test_dispatch_cancellation_drains_inflight_completion_worker(
db_session,
monkeypatch,
) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="cancel-during-command-finish",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
finish_started = Event()
allow_finish = Event()
finish_completed = Event()
def blocking_finish(*_args, **_kwargs) -> bool:
finish_started.set()
assert allow_finish.wait(timeout=DB_PROGRESS_TIMEOUT)
finish_completed.set()
return True
monkeypatch.setattr(
"xagent.web.services.task_command_transport.finish_task_command",
blocking_finish,
)
async def execute(_command: ClaimedTaskCommand) -> None:
return None
dispatch_task = asyncio.create_task(
dispatch_one_task_command(execute, command_db_id=enqueued.command_id)
)
try:
# The claim before this point commits through SQLite; bound only a hang.
await asyncio.wait_for(
asyncio.to_thread(finish_started.wait, DB_PROGRESS_TIMEOUT),
timeout=DB_PROGRESS_TIMEOUT,
)
except TimeoutError:
pytest.fail(
_dispatch_diagnostics(db_session, dispatch_task, enqueued.command_id)
)
dispatch_task.cancel()
try:
await asyncio.sleep(0.02)
assert not dispatch_task.done()
finally:
allow_finish.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(dispatch_task, timeout=DB_PROGRESS_TIMEOUT)
assert finish_completed.is_set()
@pytest.mark.asyncio
async def test_dispatch_cancellation_during_heartbeat_persists_completion(
db_session,
monkeypatch,
) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="cancel-while-stopping-command-heartbeat",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
heartbeat_stopping = asyncio.Event()
allow_heartbeat_to_finish = asyncio.Event()
async def delayed_heartbeat(
_command_db_id: int,
_runner_id: str,
_attempt_count: int,
stop_event: asyncio.Event,
):
await stop_event.wait()
heartbeat_stopping.set()
await allow_heartbeat_to_finish.wait()
return task_command_transport_module.TaskCommandClaimHeartbeatOutcome()
monkeypatch.setattr(
task_command_transport_module,
"_claim_heartbeat",
delayed_heartbeat,
)
dispatch = asyncio.create_task(
dispatch_one_task_command(
lambda _command: asyncio.sleep(0, result={"ok": True}),
command_db_id=enqueued.command_id,
)
)
await heartbeat_stopping.wait()
dispatch.cancel()
await asyncio.sleep(0)
assert not dispatch.done()
allow_heartbeat_to_finish.set()
with pytest.raises(asyncio.CancelledError):
await dispatch
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == COMMAND_COMPLETED
assert stored.result == {"ok": True}
@pytest.mark.asyncio
async def test_dispatch_cancellation_during_heartbeat_preserves_late_write_error(
db_session,
monkeypatch,
) -> None:
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="cancel-before-command-write-error",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
heartbeat_stopping = asyncio.Event()
allow_heartbeat_to_finish = asyncio.Event()
persistence_error = RuntimeError("disposition write failed")
async def delayed_heartbeat(
_command_db_id: int,
_runner_id: str,
_attempt_count: int,
stop_event: asyncio.Event,
):
await stop_event.wait()
heartbeat_stopping.set()
await allow_heartbeat_to_finish.wait()
return task_command_transport_module.TaskCommandClaimHeartbeatOutcome()
def failing_finish(*_args, **_kwargs) -> bool:
raise persistence_error
monkeypatch.setattr(
task_command_transport_module,
"_claim_heartbeat",
delayed_heartbeat,
)
monkeypatch.setattr(
task_command_transport_module,
"finish_task_command",
failing_finish,
)
dispatch = asyncio.create_task(
dispatch_one_task_command(
lambda _command: asyncio.sleep(0, result={"ok": True}),
command_db_id=enqueued.command_id,
)
)
await heartbeat_stopping.wait()
dispatch.cancel()
await asyncio.sleep(0)
assert not dispatch.done()
allow_heartbeat_to_finish.set()
with pytest.raises(asyncio.CancelledError) as exc_info:
await dispatch
assert exc_info.value.__cause__ is persistence_error
@pytest.mark.asyncio
async def test_claim_heartbeat_cancellation_drains_inflight_renewal(
monkeypatch,
) -> None:
renew_started = Event()
allow_renew = Event()
renew_completed = Event()
def blocking_renew(*_args, **_kwargs) -> bool:
renew_started.set()
assert allow_renew.wait(timeout=2)
renew_completed.set()
return True
monkeypatch.setattr(
"xagent.web.services.task_command_transport.get_task_lease_heartbeat_seconds",
lambda: 0.001,
)
monkeypatch.setattr(
"xagent.web.services.task_command_transport.renew_task_command_claim",
blocking_renew,
)
heartbeat = asyncio.create_task(_claim_heartbeat(7, "runner-a", 1, asyncio.Event()))
await asyncio.wait_for(asyncio.to_thread(renew_started.wait, 1), timeout=1)
heartbeat.cancel()
try:
await asyncio.sleep(0.02)
assert not heartbeat.done()
finally:
allow_renew.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(heartbeat, timeout=1)
assert renew_completed.is_set()
@pytest.mark.asyncio
async def test_claim_heartbeat_reports_unresolved_pool_timeout(monkeypatch) -> None:
stop_event = asyncio.Event()
timeout = SQLAlchemyTimeoutError("heartbeat checkout timed out")
def renew(*_args, **_kwargs) -> bool:
stop_event.set()
raise timeout
monkeypatch.setattr(
task_command_transport_module,
"get_task_lease_heartbeat_seconds",
lambda: 0.001,
)
monkeypatch.setattr(
task_command_transport_module,
"renew_task_command_claim",
renew,
)
outcome = await _claim_heartbeat(7, "runner-a", 1, stop_event)
assert outcome.claim_lost is False
assert outcome.pool_timeout is timeout
assert outcome.requires_ttl_recovery is True
@pytest.mark.asyncio
async def test_claim_heartbeat_clears_pool_timeout_after_success(monkeypatch) -> None:
stop_event = asyncio.Event()
attempts = 0
def renew(*_args, **_kwargs) -> bool:
nonlocal attempts
attempts += 1
if attempts == 1:
raise SQLAlchemyTimeoutError("transient heartbeat checkout timeout")
stop_event.set()
return True
monkeypatch.setattr(
task_command_transport_module,
"get_task_lease_heartbeat_seconds",
lambda: 0.001,
)
monkeypatch.setattr(
task_command_transport_module,
"renew_task_command_claim",
renew,
)
outcome = await _claim_heartbeat(7, "runner-a", 1, stop_event)
assert attempts == 2
assert outcome.claim_lost is False
assert outcome.pool_timeout is None
assert outcome.requires_ttl_recovery is False
@pytest.mark.asyncio
async def test_claim_heartbeat_survives_transient_database_error(
monkeypatch,
) -> None:
stop_event = asyncio.Event()
attempts = 0
def renew(
_command_db_id: int,
_runner_id: str,
*,
expected_attempt_count: int | None = None,
) -> bool:
nonlocal attempts
assert expected_attempt_count == 3
attempts += 1
if attempts == 1:
raise RuntimeError("temporary database outage")
stop_event.set()
return True
monkeypatch.setattr(
"xagent.web.services.task_command_transport.get_task_lease_heartbeat_seconds",
lambda: 0.001,
)
monkeypatch.setattr(
"xagent.web.services.task_command_transport.renew_task_command_claim",
renew,
)
await asyncio.wait_for(
_claim_heartbeat(7, "runner-a", 3, stop_event), timeout=GUARD_TIMEOUT
)
assert attempts == 2
@pytest.mark.asyncio
async def test_dispatcher_does_not_erase_wakeup_during_empty_claim(monkeypatch) -> None:
second_claim = asyncio.Event()
calls = 0
notification_delivered = asyncio.Event()
wakeup_at_empty_claim = []
class ObservedWakeup(asyncio.Event):
def set(self):
super().set()
notification_delivered.set()
async def wait(self):
if calls == 1:
wakeup_at_empty_claim.append(self.is_set())
return await super().wait()
wakeup = ObservedWakeup()
async def fake_dispatch(_executor, *, command_db_id=None) -> bool:
nonlocal calls
del command_db_id
calls += 1
if calls == 1:
notify_task_command_dispatcher()
# Deliver the notification while the first claim is still in flight.
await notification_delivered.wait()
return False
second_claim.set()
return False
monkeypatch.setattr(
"xagent.web.services.task_command_transport.dispatch_one_task_command",
fake_dispatch,
)
monkeypatch.setattr(task_command_transport_module, "_dispatcher_wakeup", wakeup)
monkeypatch.setattr(
task_command_transport_module, "_dispatcher_loop", asyncio.get_running_loop()
)
# One worker must observe the notification; a sibling or the periodic poll
# must not make a lost wakeup look like success.
worker = asyncio.create_task(
task_command_transport_module._run_task_command_dispatcher_worker(
lambda _command: asyncio.sleep(0)
)
)
try:
await asyncio.wait_for(second_claim.wait(), timeout=GUARD_TIMEOUT)
finally:
worker.cancel()
await asyncio.gather(worker, return_exceptions=True)
with pytest.raises(asyncio.CancelledError):
worker.result()
assert calls >= 2
assert wakeup_at_empty_claim == [True]
@pytest.mark.asyncio
async def test_prompt_dispatch_observes_late_task_failure(monkeypatch, caplog) -> None:
async def fail_after_handoff(_executor, *, command_db_id=None) -> bool:
del command_db_id
await asyncio.sleep(0.06)
raise RuntimeError("late dispatch failure")
async def execute(_command):
return None
monkeypatch.setattr(
"xagent.web.services.task_command_transport.dispatch_one_task_command",
fail_after_handoff,
)
caplog.set_level(logging.ERROR)
await dispatch_task_command_promptly(execute, command_db_id=1)
for _ in range(100):
if "Detached prompt task command dispatch failed" in caplog.text:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("late prompt dispatch failure was not observed")
assert "late dispatch failure" in caplog.text
def test_enqueue_detects_a_task_deleted_after_the_caller_loaded_it(
db_session,
) -> None:
"""A caller's earlier snapshot must not let a deleted task be enqueued."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
# The task id was validated earlier -- as the websocket transport does
# between its permission check and this enqueue -- and the row is gone by
# the time the command is written.
db_session.query(Task).filter(Task.id == task_id).delete()
db_session.commit()
with pytest.raises(ValueError, match=f"Task {task_id} not found"):
enqueue_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="pause-after-delete",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
assert db_session.query(TaskExecutionCommand).count() == 0
def test_websocket_enqueue_returns_none_when_the_task_vanishes_mid_enqueue(
db_session,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""allow_missing_task routes a mid-enqueue delete to the recovery sentinel.
The task exists at the permission check but is gone by the time the command
is written. The caller must get None so it creates a replacement task rather
than rejecting the delivery.
"""
user, task = _create_running_task(db_session)
task_id = int(task.id)
def raise_missing(*_args, **_kwargs):
raise TaskCommandTaskMissing(f"Task {task_id} not found")
monkeypatch.setattr(websocket_api, "enqueue_task_command", raise_missing)
result = websocket_api._enqueue_websocket_task_command_sync(
task_id=task_id,
actor_user_id=int(user.id),
actor_is_admin=False,
command_id="pause-vanished",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
allow_missing_task=True,
)
assert result is None
def test_websocket_enqueue_rejects_missing_task_with_client_visible_wording(
db_session,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Without allow_missing_task the delivery must be rejected, not silently
swallowed as if the command had been accepted."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
def raise_missing(*_args, **_kwargs):
raise TaskCommandTaskMissing(f"Task {task_id} not found")
monkeypatch.setattr(websocket_api, "enqueue_task_command", raise_missing)
# The rejection is re-raised as ClientVisibleValidationError at this
# boundary (still a ValueError, so pause/resume reject it the same way)
# so the redaction chokepoint keeps "not found" wording on this race
# instead of flattening it to the generic string (#1514 round 5).
with pytest.raises(ValueError) as raised:
websocket_api._enqueue_websocket_task_command_sync(
task_id=task_id,
actor_user_id=int(user.id),
actor_is_admin=False,
command_id="pause-vanished-strict",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
allow_missing_task=False,
)
assert isinstance(raised.value, task_execution_service.ClientVisibleValidationError)
assert str(raised.value) == f"Task {task_id} not found"
def test_task_foreign_key_violation_is_reported_as_a_missing_task(
db_session,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""When the task really is gone, an IntegrityError with no duplicate row is
a task FK failure. It must surface as a missing task so callers reach their
recovery path, rather than raising NoResultFound from the duplicate lookup."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
real_flush = db_session.flush
flush_calls = 0
def flush_raising_fk_error(*args, **kwargs):
# stage_task_command's owner pre-flush is a real, harmless no-op on
# this clean session -- let it through so only the attributable
# command-insert flush is intercepted below.
nonlocal flush_calls
flush_calls += 1
# flush_calls is coupled to stage_task_command's exact two-flush
# sequence -- owner pre-flush first, command-insert flush second.
# If that sequence changes, this injection lands on the wrong flush.
if flush_calls == 1:
return real_flush(*args, **kwargs)
db_session.flush = real_flush
# Delete the task so the post-rollback recheck sees it as absent, the
# way a concurrent delete would.
db_session.rollback()
db_session.query(Task).filter(Task.id == task_id).delete()
db_session.commit()
raise IntegrityError("INSERT", {}, Exception("FOREIGN KEY constraint failed"))
monkeypatch.setattr(db_session, "flush", flush_raising_fk_error)
with pytest.raises(ValueError, match=f"Task {task_id} not found"):
enqueue_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="pause-fk",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
def test_actor_foreign_key_failure_is_not_reported_as_a_missing_task(
db_session,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The row references both tasks and users. An IntegrityError with no
duplicate command does not prove the task caused it: a concurrently deleted
actor fails the users FK while the task is still present. That must not be
translated into missing-task recovery, which would continue with a cached
principal instead of reloading the actor."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
real_flush = db_session.flush
flush_calls = 0
def flush_raising_fk_error(*args, **kwargs):
# stage_task_command's owner pre-flush is a real, harmless no-op on
# this clean session -- let it through so only the attributable
# command-insert flush is intercepted below.
nonlocal flush_calls
flush_calls += 1
# flush_calls is coupled to stage_task_command's exact two-flush
# sequence -- owner pre-flush first, command-insert flush second.
# If that sequence changes, this injection lands on the wrong flush.
if flush_calls == 1:
return real_flush(*args, **kwargs)
db_session.flush = real_flush
raise IntegrityError("INSERT", {}, Exception("FOREIGN KEY constraint failed"))
monkeypatch.setattr(db_session, "flush", flush_raising_fk_error)
with pytest.raises(IntegrityError):
enqueue_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="pause-actor-fk",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
# --- stage_task_command / classify_task_command_conflict (#1073) ---------
def test_stage_rejects_malformed_command_id_before_any_db_write(db_session) -> None:
"""A malformed command_id must fail before touching the database, so a
caller cannot end up with a partially-written row for an id it rejects."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
with pytest.raises(ValueError, match="command_id must be 1-64"):
stage_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="bad id!",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
assert db_session.query(TaskExecutionCommand).count() == 0
def test_stage_strips_whitespace_from_the_staged_command_id(db_session) -> None:
"""A whitespace-padded command_id must stage a row carrying the stripped
id, not the raw padded string."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
staged = stage_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id=" cmd-1 ",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
assert staged.client_command_id == "cmd-1"
db_session.commit()
row = db_session.get(TaskExecutionCommand, staged.staged_db_id)
assert row is not None
assert row.command_id == "cmd-1"
def test_stage_does_not_commit(db_session) -> None:
"""Staging must be observable only inside the caller's own session until
it commits -- a second session must see nothing beforehand."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
staged = stage_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="stage-visibility",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
assert staged.created is True
SessionLocal = get_session_local()
with SessionLocal() as observer:
assert observer.get(TaskExecutionCommand, staged.staged_db_id) is None
db_session.commit()
with SessionLocal() as observer:
row = observer.get(TaskExecutionCommand, staged.staged_db_id)
assert row is not None
assert row.status == COMMAND_PENDING
def test_stage_missing_task_leaves_no_command_pending(db_session) -> None:
"""Staging against a task deleted before the call must not add any
command row to the session -- there is no write for a caller rollback to
even need to undo."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
db_session.query(Task).filter(Task.id == task_id).delete()
db_session.commit()
with pytest.raises(TaskCommandTaskMissing):
stage_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="stage-missing-task",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
assert db_session.query(TaskExecutionCommand).count() == 0
def test_outer_rollback_leaves_no_command(db_session) -> None:
"""A caller that stages a command as part of a larger unit of work and
then rolls back must lose the command along with the rest of that work --
there is no SAVEPOINT isolating the command insert from the caller's own
writes. The sibling write is created before staging begins; created after,
this would only prove a single-row rollback rather than the caller's
whole transaction being undone."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
task.title = "rolled-back-sibling-write"
staged = stage_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="outer-rollback",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
assert staged.created is True
db_session.rollback()
SessionLocal = get_session_local()
with SessionLocal() as observer:
observed_task = observer.get(Task, task_id)
assert observed_task is not None
assert observed_task.title != "rolled-back-sibling-write"
assert (
observer.query(TaskExecutionCommand)
.filter(TaskExecutionCommand.task_id == task_id)
.count()
== 0
)
def test_enqueue_idempotent_hit_issues_no_commit(db_session) -> None:
"""Only the created=True path may end the caller's transaction -- a
duplicate command_id must leave an accompanying uncommitted write
uncommitted, observed from a second session."""
user, task = _create_running_task(db_session)
first = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="no-commit-duplicate",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
assert first.created is True
task.title = "uncommitted sibling write"
duplicate = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="no-commit-duplicate",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
assert duplicate.created is False
SessionLocal = get_session_local()
with SessionLocal() as observer:
observed_task = observer.get(Task, int(task.id))
assert observed_task is not None
assert observed_task.title != "uncommitted sibling write"
# stage_task_command's owner pre-flush already pushed the title change
# above into the open transaction, so it no longer shows up as dirty by
# this point -- a fresh, still-unflushed write on the same session is
# what proves the session was left healthy rather than silently rolled
# back by the idempotent-hit path.
sibling_user = User(
username="idempotent-hit-sibling-write",
password_hash="hash",
is_admin=False,
)
db_session.add(sibling_user)
assert sibling_user in db_session.new
db_session.commit()
with SessionLocal() as observer:
observed_task = observer.get(Task, int(task.id))
assert observed_task is not None
assert observed_task.title == "uncommitted sibling write"
assert (
observer.query(User)
.filter(User.username == "idempotent-hit-sibling-write")
.first()
is not None
)
def test_enqueue_missing_task_issues_no_commit(db_session) -> None:
"""A missing task must not commit -- an accompanying pending write on the
same session must still be observed as uncommitted by another session."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
db_session.query(Task).filter(Task.id == task_id).delete()
db_session.add(
User(username="no-commit-sibling-write", password_hash="hash", is_admin=False)
)
with pytest.raises(TaskCommandTaskMissing):
enqueue_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="no-commit-missing-task",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
SessionLocal = get_session_local()
with SessionLocal() as observer:
assert (
observer.query(User)
.filter(User.username == "no-commit-sibling-write")
.first()
is None
)
def test_owner_pre_flush_failure_raises_owner_state_error(db_session) -> None:
"""A conflict on the caller's own pending write, surfaced by staging's
pre-flush, must be reported apart from a conflict on the command insert
itself so the two are never confused."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
db_session.add(User(username=user.username, password_hash="hash", is_admin=False))
with pytest.raises(TaskCommandOwnerStateError):
stage_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="owner-state-conflict",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
def test_owner_state_error_is_not_an_integrity_error(db_session) -> None:
"""Pinned deliberately: TaskCommandOwnerStateError must not be an
IntegrityError subclass, or enqueue_task_command's
``except IntegrityError`` would wrongly route an owner-write failure
through duplicate/missing-task classification."""
assert not issubclass(TaskCommandOwnerStateError, IntegrityError)
def test_enqueue_does_not_catch_owner_state_error(db_session) -> None:
"""enqueue_task_command's except IntegrityError must not swallow a
pre-flush failure on the caller's own pending write."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
db_session.add(User(username=user.username, password_hash="hash", is_admin=False))
with pytest.raises(TaskCommandOwnerStateError):
enqueue_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="owner-state-conflict-wrapper",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
def test_stage_snapshot_sees_a_bulk_cas_update_pending_in_the_same_session(
db_session,
) -> None:
"""A CAS-then-stage owner updates the row via a synchronize_session=False
bulk statement, which executes immediately. The staging snapshot must
observe it -- an ORM query here would instead return the identity map's
pre-update cached instance."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
updated = (
db_session.query(Task)
.filter(Task.id == task_id, Task.runner_id == "runner-a")
.update({Task.runner_id: "runner-b"}, synchronize_session=False)
)
assert updated == 1
staged = stage_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="cas-bulk-update",
kind=TaskCommandKind.CANCEL,
payload={"agent_id": 1},
)
db_session.commit()
row = db_session.get(TaskExecutionCommand, staged.staged_db_id)
assert row is not None
assert row.target_runner_id == "runner-b"
def test_stage_snapshot_sees_an_orm_attribute_change_pending_in_the_same_session(
db_session,
) -> None:
"""A CAS-then-stage owner instead mutates the loaded ORM instance rather
than issuing a bulk update. With autoflush disabled this is invisible to
a raw Core select until something flushes it -- staging's own owner
pre-flush must be what surfaces it, not the identity map."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
task.runner_id = "runner-c"
# No explicit flush here: stage_task_command's own pre-flush must be what
# pushes this to the database before its Core select runs.
staged = stage_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="cas-orm-attribute",
kind=TaskCommandKind.CANCEL,
payload={"agent_id": 1},
)
db_session.commit()
row = db_session.get(TaskExecutionCommand, staged.staged_db_id)
assert row is not None
assert row.target_runner_id == "runner-c"
def test_interleaved_duplicate_enqueue_reports_the_committed_winner(
db_session,
) -> None:
"""Two real sessions/threads both go through the public enqueue_task_command
entry point. B's idempotency precheck runs and finds nothing before A's
insert commits, so B's own insert genuinely races A's at the flush --
unlike a recipe where A commits first and B's precheck simply finds the
row, this exercises the IntegrityError-then-rollback-then-classify path
with a real concurrent writer."""
user, task = _create_running_task(db_session)
task_id = int(task.id)
actor_id = int(user.id)
SessionLocal = get_session_local()
precheck_done = Event()
release_b = Event()
def enqueue_as_b():
with SessionLocal() as db_b:
real_add = db_b.add
def paused_add(instance):
if isinstance(instance, TaskExecutionCommand):
precheck_done.set()
assert release_b.wait(timeout=10)
real_add(instance)
db_b.add = paused_add
return enqueue_task_command(
db_b,
task_id=task_id,
actor_user_id=actor_id,
command_id="interleaved-duplicate",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
with ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(enqueue_as_b)
assert precheck_done.wait(timeout=10)
with SessionLocal() as db_a:
winner = enqueue_task_command(
db_a,
task_id=task_id,
actor_user_id=actor_id,
command_id="interleaved-duplicate",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
release_b.set()
loser = future.result(timeout=10)
assert winner.created is True
assert loser.created is False
assert loser.command_id == winner.command_id
assert loser.payload_matches is True
assert loser.status == winner.status
@pytest.mark.postgresql
def test_postgres_concurrent_insert_blocks_until_the_winner_resolves(
postgres_task_command_sessions,
) -> None:
"""A genuinely-concurrent variant of the raced-duplicate test: both
writers' inserts are open against PostgreSQL at the same instant, with
B's insert blocked on A's uncommitted row rather than merely interleaved
by a test-code pause. Requires XAGENT_TEST_POSTGRES_URL; skipped
otherwise."""
SessionLocal, task_id, actor_id = postgres_task_command_sessions
a_flushed = Event()
allow_a_commit = Event()
b_blocked_then_resolved = Event()
def run_a():
with SessionLocal() as db_a:
staged = stage_task_command(
db_a,
task_id=task_id,
actor_user_id=actor_id,
command_id="pg-real-race",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
a_flushed.set()
assert allow_a_commit.wait(timeout=10)
db_a.commit()
return staged
def run_b():
assert a_flushed.wait(timeout=10)
with SessionLocal() as db_b:
try:
stage_task_command(
db_b,
task_id=task_id,
actor_user_id=actor_id,
command_id="pg-real-race",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
except IntegrityError:
db_b.rollback()
classification = classify_task_command_conflict(
db_b,
task_id=task_id,
command_id="pg-real-race",
actor_user_id=actor_id,
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
b_blocked_then_resolved.set()
return classification
raise AssertionError("B's insert should have raced A's committed one")
with ThreadPoolExecutor(max_workers=2) as executor:
future_a = executor.submit(run_a)
future_b = executor.submit(run_b)
assert a_flushed.wait(timeout=10)
# Give B's flush time to genuinely be blocked inside PostgreSQL,
# rather than releasing A the instant B's thread has merely started.
time.sleep(0.2)
allow_a_commit.set()
staged_a = future_a.result(timeout=10)
classification = future_b.result(timeout=10)
assert b_blocked_then_resolved.is_set()
assert classification.kind is TaskCommandConflictKind.RACED_DUPLICATE
assert classification.raced is not None
assert classification.raced.command_db_id == staged_a.staged_db_id
@pytest.mark.asyncio
async def test_dispatch_with_staged_id_before_commit_is_noop_and_converges_after_commit(
db_session,
) -> None:
"""A staged id must not be dispatchable before its owning transaction
commits -- the dispatcher claims through its own isolated session, which
cannot see an uncommitted row. Once the owner commits without notifying,
durable polling eventually converges on it. The target task has no earlier
in-flight command, so unfinished-earlier-command ordering cannot also
explain the delay."""
user, task = _create_running_task(db_session)
task.runner_id = None
task.lease_expires_at = None
db_session.commit()
task_id = int(task.id)
staged = stage_task_command(
db_session,
task_id=task_id,
actor_user_id=int(user.id),
command_id="staged-before-commit",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
async def must_not_dispatch(_command):
raise AssertionError("must not dispatch an uncommitted staged command")
assert not await dispatch_one_task_command(
must_not_dispatch, command_db_id=staged.staged_db_id
)
db_session.commit() # deliberately no notify_task_command_dispatcher() call
applied = asyncio.Event()
async def execute_after_commit(command):
assert command.id == staged.staged_db_id
applied.set()
return None
start_task_command_dispatcher(execute_after_commit)
try:
# Detect a stuck dispatcher without timing shared-runner scheduling.
await asyncio.wait_for(applied.wait(), timeout=GUARD_TIMEOUT)
finally:
await stop_task_command_dispatcher()
@pytest.mark.timeout(30)
def test_owner_holding_a_flushed_command_blocks_a_concurrent_claim(
low_timeout_sqlite_engine,
) -> None:
"""Staging is the last write before the owner's commit, so the row it
flushes stays under SQLite's writer lock for the rest of the owner's
transaction -- far longer than the microseconds an insert that committed
itself holds it for. SQLite's writer lock is database-wide, so a concurrent claim
of a different, already-committed command must still wait out
busy_timeout and fail with OperationalError. Once the owner commits,
the same command must be claimable."""
engine, SessionLocal = low_timeout_sqlite_engine
with SessionLocal() as setup_db:
user, task = _create_running_task(setup_db)
task.runner_id = None
task.lease_expires_at = None
setup_db.commit()
task_id = int(task.id)
actor_id = int(user.id)
claimable = enqueue_task_command(
setup_db,
task_id=task_id,
actor_user_id=actor_id,
command_id="lock-path-claimable",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
with SessionLocal() as owner_db:
stage_task_command(
owner_db,
task_id=task_id,
actor_user_id=actor_id,
command_id="lock-path-owner-holds",
kind=TaskCommandKind.CANCEL,
payload={"agent_id": 1},
)
# Deliberately not committed yet -- this is the window under test.
with SessionLocal() as claimant_db:
with pytest.raises(OperationalError, match="database is locked"):
claim_task_command(
claimant_db,
runner_id="lock-path-claimant",
command_db_id=claimable.command_id,
)
owner_db.commit()
with SessionLocal() as claimant_db:
claimed = claim_task_command(
claimant_db,
runner_id="lock-path-claimant",
command_db_id=claimable.command_id,
)
assert claimed is not None
assert claimed.id == claimable.command_id
assert claimed.attempt_count == 1
@pytest.mark.asyncio
async def test_dispatcher_worker_recovers_from_a_lock_timeout_and_claims_once_released(
low_timeout_sqlite_engine,
monkeypatch,
caplog,
) -> None:
"""The dispatcher worker pool must survive the OperationalError above --
logging it and continuing to poll -- and claim the command once the
owner's transaction ends, instead of the worker dying or hanging."""
engine, SessionLocal = low_timeout_sqlite_engine
with SessionLocal() as seed_db:
user, task = _create_running_task(seed_db)
task.runner_id = None
task.lease_expires_at = None
seed_db.commit()
task_id = int(task.id)
actor_id = int(user.id)
enqueued = enqueue_task_command(
seed_db,
task_id=task_id,
actor_user_id=actor_id,
command_id="dispatcher-progress-target",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
monkeypatch.setattr(task_command_transport_module, "DISPATCHER_IDLE_SECONDS", 0.01)
monkeypatch.setattr(
task_command_transport_module, "_dispatcher_wakeup", asyncio.Event()
)
owner_db = SessionLocal()
stage_task_command(
owner_db,
task_id=task_id,
actor_user_id=actor_id,
command_id="dispatcher-progress-lock-holder",
kind=TaskCommandKind.CANCEL,
payload={"agent_id": 1},
)
# Deliberately uncommitted -- holds SQLite's writer lock across the
# dispatcher worker's first claim attempt.
caplog.set_level(logging.ERROR, logger="xagent.web.services.task_command_transport")
applied = asyncio.Event()
async def execute(command):
assert command.id == enqueued.command_id
applied.set()
return None
worker = asyncio.create_task(
task_command_transport_module._run_task_command_dispatcher_worker(execute)
)
try:
# Wait for the worker to actually observe the lock timeout (rather
# than a fixed sleep) before releasing it -- releasing early would
# let the blocked claim succeed once unblocked instead of timing out,
# never exercising the recovery path this test is for. The bound is
# generous because the claim itself runs on the default thread-pool
# executor, whose scheduling delay grows under a heavily loaded test
# run rather than staying tied to busy_timeout.
for _ in range(2000):
if "component=task-command-dispatcher" in caplog.text:
break
await asyncio.sleep(0.01)
else:
raise AssertionError("worker never logged a lock-timeout failure")
assert not applied.is_set(), "must not have claimed while the lock was held"
owner_db.commit()
await asyncio.wait_for(applied.wait(), timeout=20)
finally:
worker.cancel()
with pytest.raises(asyncio.CancelledError):
await worker
owner_db.close()
assert "component=task-command-dispatcher" in caplog.text
def test_enqueue_notifies_only_after_commit(db_session, monkeypatch) -> None:
"""The dispatcher wakeup must fire only once the command is durable --
notifying before commit could wake a dispatcher onto a row it cannot yet
see, wasting the cycle instead of claiming it."""
user, task = _create_running_task(db_session)
order: list[str] = []
real_commit = db_session.commit
def recording_commit():
order.append("commit")
real_commit()
def recording_notify():
order.append("notify")
monkeypatch.setattr(db_session, "commit", recording_commit)
monkeypatch.setattr(
task_command_transport_module,
"notify_task_command_dispatcher",
recording_notify,
)
enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="notify-order",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
assert order == ["commit", "notify"]
def test_defer_budget_is_coupled_to_the_lease_ttl(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The defer budget must outlast the configured lease TTL with margin.
A deferral's canonical wait is an unreleased task lease, which clears
within one TTL; a fixed budget silently smaller than a raised
``XAGENT_TASK_LEASE_TTL_SECONDS`` turned every long park into a terminal
failure for an already-accepted command (xorbitsai/xagent-saas#952 B3).
"""
monkeypatch.setenv("XAGENT_TASK_LEASE_TTL_SECONDS", "300")
assert max_command_defers() == 600
# The historical constant stays as the floor for short TTLs.
monkeypatch.setenv("XAGENT_TASK_LEASE_TTL_SECONDS", "10")
assert max_command_defers() == MAX_COMMAND_DEFERS
# The default (no env var) doubles the historical constant.
monkeypatch.delenv("XAGENT_TASK_LEASE_TTL_SECONDS", raising=False)
assert max_command_defers() == 120
# An invalid value falls back to the default TTL, not the floor.
monkeypatch.setenv("XAGENT_TASK_LEASE_TTL_SECONDS", "not-a-number")
assert max_command_defers() == 120
def test_duplicate_command_cannot_replace_persisted_reply_origin(db_session):
user, task = _create_running_task(db_session)
arguments = dict(
task_id=int(task.id),
actor_user_id=int(user.id),
command_id="routed-pause",
kind=TaskCommandKind.PAUSE,
payload={"type": "pause_task"},
)
original = enqueue_task_command(
db_session, **arguments, reply_host_id="web-a", reply_origin="socket-a"
)
duplicate = enqueue_task_command(
db_session, **arguments, reply_host_id="web-b", reply_origin="socket-b"
)
assert original.created
assert not duplicate.created
assert duplicate.payload_matches
db_session.expire_all()
row = (
db_session.query(TaskExecutionCommand)
.filter_by(command_id="routed-pause")
.one()
)
assert (row.reply_host_id, row.reply_origin) == ("web-a", "socket-a")
@pytest.mark.asyncio
@pytest.mark.parametrize("wait_style", ["shield", "wait_for_previous"])
async def test_dispatcher_worker_survives_cancellation_not_aimed_at_it(
monkeypatch, caplog, wait_style
) -> None:
"""A cancel raised inside dispatch must not end loops nobody cancelled."""
transport = task_command_transport_module
monkeypatch.setattr(transport, "consumes_task_commands", lambda: True)
monkeypatch.setattr(transport, "DISPATCHER_IDLE_SECONDS", 0.01)
foreign: asyncio.Task | None = None
hit_worker: asyncio.Task | None = None
hit_calls = 0
manager = task_execution_service.BackgroundTaskManager()
async def dispatch(_executor, *, command_db_id=None) -> bool:
nonlocal foreign, hit_worker, hit_calls
if hit_worker is None:
hit_worker = asyncio.current_task()
foreign = asyncio.create_task(asyncio.Event().wait())
if wait_style == "shield":
await asyncio.shield(foreign)
else:
manager.running_tasks[1] = foreign
await manager.wait_for_previous(1)
elif asyncio.current_task() is hit_worker:
hit_calls += 1
return False
monkeypatch.setattr(transport, "dispatch_one_task_command", dispatch)
caplog.set_level(logging.ERROR, logger=transport.__name__)
dispatcher = start_task_command_dispatcher(lambda _command: asyncio.sleep(0))
assert dispatcher is not None
try:
await eventually(lambda: foreign is not None)
foreign.cancel()
# The very loop that took the cancellation must dispatch again; other
# loops running meanwhile cannot satisfy this.
await eventually(lambda: hit_calls > 0)
assert hit_worker is not None and not hit_worker.done()
assert not dispatcher.done()
aimed = [
r for r in caplog.records if "not aimed at this worker" in r.getMessage()
]
assert len(aimed) == 1
finally:
await stop_task_command_dispatcher()
assert dispatcher.done()
@pytest.mark.asyncio
async def test_dispatcher_worker_exits_when_it_is_cancelled(monkeypatch) -> None:
"""Cancelling the worker itself ends it even while its wakeup is still live."""
transport = task_command_transport_module
parked = asyncio.Event()
async def dispatch(_executor, *, command_db_id=None) -> bool:
parked.set()
await asyncio.Event().wait()
return False
monkeypatch.setattr(transport, "dispatch_one_task_command", dispatch)
monkeypatch.setattr(transport, "_dispatcher_wakeup", asyncio.Event())
worker = asyncio.create_task(
transport._run_task_command_dispatcher_worker(lambda _command: asyncio.sleep(0))
)
try:
await asyncio.wait_for(parked.wait(), DB_PROGRESS_TIMEOUT)
worker.cancel()
await asyncio.wait({worker}, timeout=DB_PROGRESS_TIMEOUT)
assert worker.cancelled()
finally:
if not worker.done():
# Let a loop that wrongly survived its own cancel return.
monkeypatch.setattr(transport, "_dispatcher_wakeup", None)
worker.cancel()
await asyncio.gather(worker, return_exceptions=True)
class _Boom(BaseException):
"""An exit that is neither a cancellation nor an ordinary exception."""
async def _wait_forever(_executor) -> None:
await asyncio.Event().wait()
@pytest.fixture
async def fake_dispatcher(monkeypatch):
transport = task_command_transport_module
monkeypatch.setattr(transport, "consumes_task_commands", lambda: True)
monkeypatch.setattr(transport, "run_task_command_dispatcher", _wait_forever)
yield transport
await stop_task_command_dispatcher()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("scenario", "expected"),
[
("external_cancel", "ended by a cancellation outside"),
("base_exception", "exited unexpectedly"),
("returns", "returned while still registered"),
("explicit_stop", None),
],
)
async def test_dispatcher_exit_callback_reports_only_unregistered_exits(
fake_dispatcher, monkeypatch, caplog, scenario, expected
) -> None:
async def raises(_executor) -> None:
raise _Boom
async def returns(_executor) -> None:
return None
if scenario == "base_exception":
monkeypatch.setattr(fake_dispatcher, "run_task_command_dispatcher", raises)
elif scenario == "returns":
monkeypatch.setattr(fake_dispatcher, "run_task_command_dispatcher", returns)
caplog.set_level(logging.CRITICAL, logger=fake_dispatcher.__name__)
dispatcher = start_task_command_dispatcher(lambda _command: asyncio.sleep(0))
assert dispatcher is not None
await asyncio.sleep(0)
if scenario == "external_cancel":
dispatcher.cancel()
elif scenario == "explicit_stop":
await stop_task_command_dispatcher()
await asyncio.gather(dispatcher, return_exceptions=True)
await asyncio.sleep(0)
critical = [r for r in caplog.records if r.levelno == logging.CRITICAL]
if expected is None:
assert critical == []
else:
assert len(critical) == 1
assert expected in critical[0].getMessage()
if scenario in ("external_cancel", "base_exception"):
assert critical[0].exc_info is not None
async def _restart_after(transport, dispatcher) -> None:
await eventually(
lambda: transport._dispatcher_task is not dispatcher
and transport._dispatcher_task is not None
and not transport._dispatcher_task.done()
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"scenario",
[
"stop_first",
"stop_with_dispatcher_ending",
"one_restart",
"restart_limit",
"exits_spaced_beyond_window",
"no_dispatcher_none",
"no_dispatcher_done",
"no_dispatcher_foreign_loop",
"supervisor_cancelled",
"explicit_stop",
],
)
async def test_supervise_task_command_dispatcher(
fake_dispatcher, monkeypatch, caplog, scenario
) -> None:
transport = fake_dispatcher
caplog.set_level(logging.INFO, logger=transport.__name__)
executor = lambda _command: asyncio.sleep(0) # noqa: E731
stop = asyncio.Event()
foreign_loop = None
if scenario == "no_dispatcher_none":
assert transport._dispatcher_task is None
elif scenario == "no_dispatcher_foreign_loop":
foreign_loop = asyncio.new_event_loop()
# Only the loop it belongs to and its done state matter to the supervisor.
foreign = foreign_loop.create_future()
monkeypatch.setattr(transport, "_dispatcher_task", foreign)
else:
start_task_command_dispatcher(executor)
if scenario == "no_dispatcher_done":
dispatcher = transport._dispatcher_task
dispatcher.cancel()
await asyncio.gather(dispatcher, return_exceptions=True)
if scenario == "exits_spaced_beyond_window":
monkeypatch.setattr(transport, "DISPATCHER_RESTART_WINDOW_SECONDS", 0.05)
registered = transport._dispatcher_task
supervisor = asyncio.create_task(
transport.supervise_task_command_dispatcher(executor, stop)
)
errors = lambda: [r for r in caplog.records if r.levelno >= logging.ERROR] # noqa: E731
await asyncio.sleep(0) # let the supervisor start watching
try:
if scenario == "stop_first":
stop.set()
await asyncio.wait_for(supervisor, DB_PROGRESS_TIMEOUT)
assert not registered.done()
assert errors() == []
elif scenario == "stop_with_dispatcher_ending":
stop.set()
registered.cancel()
await asyncio.wait_for(supervisor, DB_PROGRESS_TIMEOUT)
await asyncio.sleep(0)
assert [r for r in errors() if r.levelno == logging.ERROR] == []
assert not any("restarting" in r.getMessage() for r in caplog.records)
elif scenario == "one_restart":
registered.cancel()
await _restart_after(transport, registered)
records = caplog.records
critical = next(i for i, r in enumerate(records) if r.levelno == 50)
restart = next(
i for i, r in enumerate(records) if "restart 1 of 3" in r.getMessage()
)
assert critical < restart
stop.set()
await asyncio.wait_for(supervisor, DB_PROGRESS_TIMEOUT)
elif scenario == "restart_limit":
for _ in range(transport.DISPATCHER_RESTART_LIMIT):
current = transport._dispatcher_task
current.cancel()
await _restart_after(transport, current)
transport._dispatcher_task.cancel()
with pytest.raises(RuntimeError, match="4 times within 600 seconds"):
await asyncio.wait_for(supervisor, DB_PROGRESS_TIMEOUT)
restarts = [
r for r in caplog.records if "restarting dispatcher" in r.getMessage()
]
assert len(restarts) == transport.DISPATCHER_RESTART_LIMIT
elif scenario == "exits_spaced_beyond_window":
for _ in range(transport.DISPATCHER_RESTART_LIMIT + 1):
current = transport._dispatcher_task
current.cancel()
await _restart_after(transport, current)
await asyncio.sleep(0.1)
assert not supervisor.done()
stop.set()
await asyncio.wait_for(supervisor, DB_PROGRESS_TIMEOUT)
elif scenario.startswith("no_dispatcher"):
await eventually(
lambda: any(
"no running dispatcher to supervise" in r.getMessage()
for r in caplog.records
)
)
assert [r for r in errors() if r.levelno == logging.ERROR] == []
assert not supervisor.done()
stop.set()
await asyncio.wait_for(supervisor, DB_PROGRESS_TIMEOUT)
assert transport._dispatcher_task is registered
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
assert len(warnings) == 1
elif scenario == "supervisor_cancelled":
supervisor.cancel()
with pytest.raises(asyncio.CancelledError):
await supervisor
assert not registered.done()
await asyncio.sleep(0)
# The supervisor's internal stop.wait() task must be reclaimed.
assert not [
t
for t in asyncio.all_tasks()
if t.get_coro().__qualname__ == "Event.wait"
]
else:
await stop_task_command_dispatcher()
await eventually(
lambda: any(
"stopped explicitly" in r.getMessage() for r in caplog.records
)
)
assert transport._dispatcher_task is None
assert errors() == []
assert not supervisor.done()
stop.set()
await asyncio.wait_for(supervisor, DB_PROGRESS_TIMEOUT)
finally:
stop.set()
supervisor.cancel()
await asyncio.gather(supervisor, return_exceptions=True)
if foreign_loop is not None:
foreign.cancel()
foreign_loop.close()
@pytest.fixture()
def shared_dispatch_db(db_session, monkeypatch):
"""Shared execution enabled, with tasks that any runner may claim from."""
monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "true")
monkeypatch.setattr(task_command_transport_module, "get_runner_id", lambda: "me")
return db_session
def _unowned_task(db, user_id: int, title: str) -> Task:
task = Task(
user_id=user_id,
title=title,
description=title,
status=TaskStatus.RUNNING,
execution_mode="auto",
run_id=None,
runner_id=None,
lease_expires_at=None,
)
db.add(task)
db.commit()
db.refresh(task)
return task
def _insert_command(db, task_id: int, kind: TaskCommandKind, identity: str) -> int:
row = TaskExecutionCommand(
task_id=task_id, command_id=identity, kind=kind.value, payload={}
)
db.add(row)
db.commit()
return int(row.id)
def _front_fields(
candidate: task_command_transport_module.CommandCandidate | None,
) -> tuple[int, int, TaskCommandKind] | None:
"""The command id, task id and kind of a scan result."""
return None if candidate is None else tuple(candidate[:3]) # type: ignore[return-value]
def _claim_candidate() -> tuple[int, int, TaskCommandKind] | None:
return _front_fields(task_command_transport_module._find_command_candidate(None))
def test_reply_first_claim_order_prefers_a_newer_reply_over_an_older_start(
shared_dispatch_db,
) -> None:
db = shared_dispatch_db
user, first_task = _create_running_task(db)
second_task = _unowned_task(db, int(user.id), "second")
first_task.runner_id = None
first_task.lease_expires_at = None
db.commit()
older_start = _insert_command(
db, int(first_task.id), TaskCommandKind.START, "start-a"
)
newer_reply = _insert_command(
db, int(second_task.id), TaskCommandKind.RESUME_INPUT, "reply-b"
)
assert older_start < newer_reply
assert _claim_candidate() == (
newer_reply,
int(second_task.id),
TaskCommandKind.RESUME_INPUT,
)
def test_reply_first_claim_order_keeps_a_tasks_start_ahead_of_its_own_reply(
shared_dispatch_db,
) -> None:
db = shared_dispatch_db
user, task = _create_running_task(db)
task.runner_id = None
task.lease_expires_at = None
db.commit()
start = _insert_command(db, int(task.id), TaskCommandKind.START, "start-same")
_insert_command(db, int(task.id), TaskCommandKind.RESUME_INPUT, "reply-same")
assert _claim_candidate() == (start, int(task.id), TaskCommandKind.START)
@pytest.mark.parametrize(
"kind", [TaskCommandKind.START, TaskCommandKind.RESUME_INPUT], ids=lambda k: k.value
)
def test_reply_first_claim_order_takes_the_older_of_two_commands_of_one_kind(
shared_dispatch_db, kind: TaskCommandKind
) -> None:
db = shared_dispatch_db
user, first_task = _create_running_task(db)
first_task.runner_id = None
first_task.lease_expires_at = None
db.commit()
second_task = _unowned_task(db, int(user.id), "second")
older = _insert_command(db, int(first_task.id), kind, f"{kind.value}-1")
_insert_command(db, int(second_task.id), kind, f"{kind.value}-2")
assert _claim_candidate() == (older, int(first_task.id), kind)
@pytest.mark.parametrize(
"kind", [TaskCommandKind.CANCEL, TaskCommandKind.PAUSE], ids=lambda k: k.value
)
def test_reply_first_claim_order_puts_a_reply_ahead_of_an_older_control_command(
shared_dispatch_db, kind: TaskCommandKind
) -> None:
db = shared_dispatch_db
user, control_task = _create_running_task(db)
control_task.runner_id = None
control_task.lease_expires_at = None
db.commit()
reply_task = _unowned_task(db, int(user.id), "reply")
older_control = _insert_command(
db, int(control_task.id), kind, f"{kind.value}-older"
)
newer_reply = _insert_command(
db, int(reply_task.id), TaskCommandKind.RESUME_INPUT, "reply-newer"
)
assert older_control < newer_reply
assert _claim_candidate() == (
newer_reply,
int(reply_task.id),
TaskCommandKind.RESUME_INPUT,
)
@pytest.mark.parametrize("reply_task_is_active", [False, True])
def test_reply_first_claim_order_is_not_used_while_the_cap_is_closed(
shared_dispatch_db, reply_task_is_active: bool
) -> None:
from xagent.web.services.task_dispatch_cap import (
DispatchCapSnapshot,
DispatchPolicy,
)
db = shared_dispatch_db
user, start_task = _create_running_task(db)
start_task.runner_id = None
start_task.lease_expires_at = None
db.commit()
reply_task = _unowned_task(db, int(user.id), "waiting")
older_start = _insert_command(
db, int(start_task.id), TaskCommandKind.START, "start-active"
)
_insert_command(db, int(reply_task.id), TaskCommandKind.RESUME_INPUT, "reply")
active = {int(start_task.id)} | (
{int(reply_task.id)} if reply_task_is_active else set()
)
closed = DispatchCapSnapshot(
policy=DispatchPolicy(limit=len(active), interactive_reserve=0),
active_ids=frozenset(active),
)
assert _front_fields(
task_command_transport_module._find_command_candidate(None, (), closed)
) == (
older_start,
int(start_task.id),
TaskCommandKind.START,
)
def test_reply_first_claim_order_falls_back_past_a_reply_on_a_busy_task(
shared_dispatch_db,
) -> None:
db = shared_dispatch_db
user, start_task = _create_running_task(db)
start_task.runner_id = None
start_task.lease_expires_at = None
db.commit()
busy_task = _unowned_task(db, int(user.id), "busy")
start = _insert_command(db, int(start_task.id), TaskCommandKind.START, "start")
_insert_command(db, int(busy_task.id), TaskCommandKind.RESUME_INPUT, "reply")
assert _front_fields(
task_command_transport_module._find_command_candidate(
None, (int(busy_task.id),)
)
) == (start, int(start_task.id), TaskCommandKind.START)
def test_reply_first_claim_order_falls_back_past_a_reply_under_an_expired_foreign_owner(
shared_dispatch_db,
) -> None:
db = shared_dispatch_db
user, foreign_task = _create_running_task(db)
foreign_task.runner_id = "someone-else"
foreign_task.lease_expires_at = datetime.utcnow() - timedelta(minutes=1)
db.commit()
start_task = _unowned_task(db, int(user.id), "start")
start = _insert_command(db, int(start_task.id), TaskCommandKind.START, "start")
_insert_command(db, int(foreign_task.id), TaskCommandKind.RESUME_INPUT, "reply")
assert _claim_candidate() == (start, int(start_task.id), TaskCommandKind.START)
def test_find_command_candidate_with_an_explicit_id_skips_the_reply_lookup(
shared_dispatch_db,
) -> None:
from sqlalchemy import event
db = shared_dispatch_db
user, task = _create_running_task(db)
task.runner_id = None
task.lease_expires_at = None
db.commit()
reply = _insert_command(
db, int(task.id), TaskCommandKind.RESUME_INPUT, "reply-explicit"
)
parameters: list[tuple[object, ...]] = []
def record(_conn, _cursor, statement, params, *_args) -> None:
if statement.lstrip().upper().startswith("SELECT"):
parameters.append(tuple(params))
engine = get_engine()
event.listen(engine, "before_cursor_execute", record)
try:
found = task_command_transport_module._find_command_candidate(reply)
finally:
event.remove(engine, "before_cursor_execute", record)
assert _front_fields(found) == (reply, int(task.id), TaskCommandKind.RESUME_INPUT)
# One statement, and the reply-only kind filter is not among its values.
assert len(parameters) == 1
assert TaskCommandKind.RESUME_INPUT.value not in parameters[0]