Enterprise agent platform for building, deploying, and operating AI teammates.
"""Public dispatch keeps command admission and settlement under one task owner."""
import asyncio
from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock, Mock
import pytest
from sqlalchemy import event
from sqlalchemy.orm import sessionmaker
from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from tests.web.services.task_database_shared import engine as engine_fixture
from tests.web.services.task_database_shared import task_id as task_id_fixture
from xagent.web.models import database
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.services import task_command_transport as transport
from xagent.web.services import task_coordinator_runtime as runtime
from xagent.web.services import task_coordinator_service as owners
engine = engine_fixture
task_id = task_id_fixture
@pytest.fixture
async def host(engine, task_id, monkeypatch):
factory = sessionmaker(engine, expire_on_commit=False)
monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "true")
monkeypatch.setattr(database, "_SessionLocal", factory)
monkeypatch.setattr(transport, "get_runner_id", lambda: "worker")
monkeypatch.setattr(runtime, "get_runner_id", lambda: "worker")
# Every renewal is a SQLite write. When commits are slow (CI fsync), a
# renewal interval shorter than the commit time turns the heartbeat into
# a back-to-back writer that starves the dispatcher's claim transaction
# behind SQLite's polling busy handler. Leave the write lock idle for
# longer than that handler's maximum 100ms poll gap between renewals.
monkeypatch.setattr(runtime, "get_task_lease_heartbeat_seconds", lambda: 0.5)
registry = runtime.TaskCoordinatorRegistry(factory)
monkeypatch.setattr(runtime, "get_task_coordinator_registry", lambda: registry)
monkeypatch.setattr(
transport,
"_claim_heartbeat",
Mock(side_effect=AssertionError("independent heartbeat")),
)
try:
yield factory, task_id, registry
finally:
await registry.close()
def enqueue(host, identity="message", **fields):
factory, tid, _ = host
with factory() as db:
row = TaskExecutionCommand(
task_id=tid, command_id=identity, kind="message", payload={}, **fields
)
db.add(row)
db.commit()
return row.id
async def test_duplicate_wakes_apply_once_and_hold_owner_through_receipt(
host, monkeypatch
):
factory, tid, registry = host
cid = enqueue(host)
applied = []
entered, finish = asyncio.Event(), asyncio.Event()
async def execute(command):
coordinator = runtime.current_task_coordinator(tid)
assert coordinator is not None
with factory() as db:
row = db.get(TaskExecutionCommand, cid)
assert row.claimed_by is None and row.claim_expires_at is None
applied.append(command.attempt_count)
entered.set()
await finish.wait()
return {"applied": True}
dispatches = [
asyncio.create_task(transport.dispatch_one_task_command(execute))
for _ in range(3)
]
await asyncio.wait_for(entered.wait(), DB_PROGRESS_TIMEOUT)
finish.set()
await asyncio.wait_for(asyncio.gather(*dispatches), DB_PROGRESS_TIMEOUT)
assert applied == [1]
with factory() as db:
row = db.get(TaskExecutionCommand, cid)
assert row.status == "completed" and row.result == {"applied": True}
async def test_defer_uses_retry_time_without_a_command_lease(host, monkeypatch):
factory, _, _ = host
cid = enqueue(host)
# CI database round trips can consume the whole one-second defer window.
# Control the command clock, not the lease/heartbeat clock, so this checks
# eligibility rather than assuming dispatch itself finishes within 1s.
now = datetime.now(timezone.utc)
monkeypatch.setattr(transport, "_utc_now", lambda: now)
async def defer(command):
raise transport.TaskCommandDeferred("busy")
assert await transport.dispatch_one_task_command(defer)
assert not await transport.dispatch_one_task_command(defer)
with factory() as db:
row = db.get(TaskExecutionCommand, cid)
assert row.status == "pending" and row.attempt_count == row.defer_count == 1
assert row.retry_available_at is not None
due = row.retry_available_at
if due.tzinfo is None: # SQLite omits timezone information.
due = due.replace(tzinfo=timezone.utc)
assert due == now + timedelta(seconds=1)
assert row.claim_expires_at is None and row.claimed_by is None
now += timedelta(seconds=1)
assert await transport.dispatch_one_task_command(defer)
with factory() as db:
assert db.get(TaskExecutionCommand, cid).attempt_count == 2
async def test_old_processing_claim_does_not_block_new_owner(host):
factory, tid, _ = host
cid = enqueue(
host,
status="processing",
attempt_count=1,
claimed_by="retired-worker",
claim_expires_at=datetime.now(timezone.utc) + timedelta(hours=1),
)
async def reconcile(command):
# Processing is replayed with its original identity and a new attempt;
# the existing application handler must reconcile its persisted effects.
assert command.command_id == "message" and command.attempt_count == 2
assert runtime.current_task_coordinator(tid) is not None
return {"reconciled": True}
assert await transport.dispatch_one_task_command(reconcile)
with factory() as db:
assert db.get(TaskExecutionCommand, cid).result == {"reconciled": True}
async def test_replaced_owner_cannot_complete_command(host, monkeypatch):
factory, tid, registry = host
cid = enqueue(host)
# Exercise the settlement fence before heartbeat-driven cancellation can
# race it. Owner-loss cancellation is covered by the runtime tests.
monkeypatch.setattr(registry, "_run_heartbeats", AsyncMock())
async def execute(command):
with factory() as db:
task = db.get(Task, tid)
task.lease_attempt_id = "replacement"
db.commit()
return {"stale": True}
assert await transport.dispatch_one_task_command(execute)
with factory() as db:
row = db.get(TaskExecutionCommand, cid)
assert row.status == "processing" and row.result is None
assert db.get(Task, tid).lease_attempt_id == "replacement"
async def test_foreign_owner_wait_does_not_spend_command_budget(host):
factory, tid, _ = host
cid = enqueue(host)
with factory() as db, db.begin():
assert owners.acquire_task_lease_no_commit(db, tid, runner_id="other")
assert not await transport.dispatch_one_task_command(Mock())
with factory() as db:
row = db.get(TaskExecutionCommand, cid)
assert (row.attempt_count, row.failure_count, row.defer_count) == (0, 0, 0)
async def test_registry_renews_while_handler_runs_without_command_updates(host, engine):
factory, tid, _ = host
enqueue(host)
statements = []
def record(_conn, _cursor, statement, _params, _context, _many):
if statement.lstrip().upper().startswith("UPDATE"):
statements.append(statement)
event.listen(engine, "before_cursor_execute", record)
try:
async def execute(command):
statements.clear()
async def wait_for_renewal():
while not any("last_heartbeat_at" in sql for sql in statements):
await asyncio.sleep(0.01)
# Wait for a renewal, not a fixed number of heartbeat intervals.
await asyncio.wait_for(wait_for_renewal(), DB_PROGRESS_TIMEOUT)
assert not any("task_execution_commands" in sql for sql in statements)
return {}
assert await transport.dispatch_one_task_command(execute)
finally:
event.remove(engine, "before_cursor_execute", record)
async def test_busy_queue_head_does_not_starve_another_task(host):
factory, tid, _ = host
first_id = enqueue(host)
with factory() as db:
other = Task(user_id=db.get(Task, tid).user_id, title="other")
db.add(other)
db.flush()
row = TaskExecutionCommand(
task_id=other.id, command_id="other", kind="message", payload={}
)
db.add(row)
db.commit()
other_id = row.id
entered, release = asyncio.Event(), asyncio.Event()
seen = []
async def execute(command):
seen.append(command.id)
if command.id == first_id:
entered.set()
await release.wait()
return {}
first = asyncio.create_task(transport.dispatch_one_task_command(execute))
try:
await asyncio.wait_for(entered.wait(), DB_PROGRESS_TIMEOUT)
assert await asyncio.wait_for(
transport.dispatch_one_task_command(execute), DB_PROGRESS_TIMEOUT
)
assert seen == [first_id, other_id]
finally:
release.set()
await first
async def test_same_owner_rejects_older_command_attempt(host):
factory, tid, registry = host
cid = enqueue(host)
async def execute(command):
with factory() as db:
db.get(TaskExecutionCommand, cid).attempt_count += 1
db.commit()
assert not transport.finish_task_command(
cid,
"worker",
result={"stale": True},
expected_attempt_count=command.attempt_count,
)
# The synthetic newer attempt has taken responsibility for the row.
assert transport.finish_task_command(
cid,
"worker",
result={"current": True},
expected_attempt_count=command.attempt_count + 1,
)
return transport.SettledTaskCommand()
assert await transport.dispatch_one_task_command(execute)
with factory() as db:
assert db.get(TaskExecutionCommand, cid).result == {"current": True}
async def test_shutdown_drains_command_while_registry_retains_lease(host):
factory, tid, registry = host
enqueue(host)
entered, cleanup, release = asyncio.Event(), asyncio.Event(), asyncio.Event()
async def execute(command):
entered.set()
try:
await asyncio.Event().wait()
finally:
cleanup.set()
await release.wait()
dispatch = asyncio.create_task(transport.dispatch_one_task_command(execute))
await asyncio.wait_for(entered.wait(), DB_PROGRESS_TIMEOUT)
closing = asyncio.create_task(registry.close())
await asyncio.wait_for(cleanup.wait(), DB_PROGRESS_TIMEOUT)
try:
with factory() as db:
before = db.get(Task, tid).last_heartbeat_at
async def wait_for_renewal():
while True:
with factory() as db:
task = db.get(Task, tid)
assert task.runner_id == "worker"
if task.last_heartbeat_at > before:
return
await asyncio.sleep(0.01)
# Wait for a committed renewal, not a fixed database/scheduler latency.
await asyncio.wait_for(wait_for_renewal(), DB_PROGRESS_TIMEOUT)
assert not closing.done()
finally:
release.set()
await asyncio.wait_for(closing, DB_PROGRESS_TIMEOUT)
await asyncio.gather(dispatch, return_exceptions=True)
async def test_unrecoverable_running_owner_does_not_starve_unrelated_commands(
host, monkeypatch
):
factory, tid, registry = host
first_id = enqueue(host)
with factory() as db:
blocked = db.get(Task, tid)
blocked.status = TaskStatus.RUNNING
blocked.runner_id = "retired"
blocked.lease_attempt_id = "ambiguous-checkpoint-owner"
blocked.lease_expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
other = Task(user_id=blocked.user_id, title="runnable")
db.add(other)
db.flush()
row = TaskExecutionCommand(
task_id=other.id, command_id="runnable", kind="message", payload={}
)
db.add(row)
db.commit()
other_id = row.id
seen = []
async def execute(command):
seen.append(command.id)
return {}
ensure = AsyncMock(wraps=registry.ensure)
monkeypatch.setattr(registry, "ensure", ensure)
# Targeted dispatch must not consume some other command.
assert not await transport.dispatch_one_task_command(
execute, command_db_id=first_id
)
assert await transport.dispatch_one_task_command(execute)
assert seen == [other_id]
assert not await transport.dispatch_one_task_command(execute)
assert all(call.args[0] != tid for call in ensure.call_args_list)
with factory() as db:
row = db.get(TaskExecutionCommand, first_id)
assert row.status == "pending"
assert (row.attempt_count, row.failure_count, row.defer_count) == (0, 0, 0)
assert db.get(Task, tid).lease_attempt_id == "ambiguous-checkpoint-owner"
@pytest.mark.parametrize(
("status", "runner", "expired"),
[
(TaskStatus.RUNNING, "worker", True),
(TaskStatus.RUNNING, "retired", None),
(TaskStatus.RUNNING, None, None),
(TaskStatus.PAUSED, "retired", True),
(TaskStatus.WAITING_FOR_USER, "retired", True),
],
)
async def test_candidate_filter_preserves_other_ownership_states(
host, status, runner, expired
):
factory, tid, _ = host
cid = enqueue(host)
with factory() as db, db.begin():
task = db.get(Task, tid)
task.status = status
task.runner_id = runner
task.lease_attempt_id = "existing" if runner else None
task.lease_expires_at = (
datetime.now(timezone.utc) - timedelta(seconds=1) if expired else None
)
assert transport._find_command_candidate(cid)[0] == cid
async def test_recovered_running_task_reenters_dispatch_without_blacklist(host):
from xagent.web.services.task_lease_recovery import (
recover_task_lease_candidate_isolated,
)
from xagent.web.services.task_lease_service import get_expired_task_lease_candidates
factory, tid, _ = host
cid = enqueue(host)
now = datetime.now(timezone.utc)
with factory() as db, db.begin():
task = db.get(Task, tid)
task.status = TaskStatus.RUNNING
task.runner_id = "retired"
task.lease_attempt_id = "expired-owner"
task.lease_expires_at = now - timedelta(seconds=1)
execute = AsyncMock(return_value={"applied": True})
assert not await transport.dispatch_one_task_command(execute)
with factory() as db:
candidates = get_expired_task_lease_candidates(db, cutoff=now, limit=1)
assert len(candidates) == 1
assert (
recover_task_lease_candidate_isolated(candidates[0], recovered_at=now)
== TaskStatus.FAILED
)
assert await transport.dispatch_one_task_command(execute, command_db_id=cid)
execute.assert_awaited_once()
async def test_owner_acquisition_race_still_serves_next_candidate(host, monkeypatch):
factory, tid, registry = host
first_id = enqueue(host)
with factory() as db, db.begin():
other = Task(user_id=db.get(Task, tid).user_id, title="runnable")
db.add(other)
db.flush()
row = TaskExecutionCommand(
task_id=other.id, command_id="runnable", kind="message", payload={}
)
db.add(row)
db.flush()
other_id = row.id
ensure = registry.ensure
attempts = []
async def race_acquisition(task_id):
attempts.append(task_id)
if task_id == tid:
# The candidate was unowned when selected, but another process
# wins before this dispatcher can acquire it.
with factory() as db, db.begin():
assert owners.acquire_task_lease_no_commit(
db, tid, runner_id="other-worker"
)
return await ensure(task_id)
monkeypatch.setattr(registry, "ensure", race_acquisition)
execute = AsyncMock(return_value={})
assert await transport.dispatch_one_task_command(execute)
assert attempts[0] == tid and len(attempts) == 2
assert execute.await_args.args[0].id == other_id
assert tid not in registry._coordinators
with factory() as db:
row = db.get(TaskExecutionCommand, first_id)
assert (row.status, row.attempt_count, row.defer_count) == ("pending", 0, 0)