Enterprise agent platform for building, deploying, and operating AI teammates.
"""Acquisition fencing against real SQLite and PostgreSQL task rows."""
import asyncio
from datetime import timedelta
import pytest
from sqlalchemy.orm import sessionmaker
from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from tests.web.services.task_database_shared import engine as engine_fixture
from tests.web.services.task_database_shared import task_id as task_id_fixture
from xagent.web.models.task import Task, TaskStatus
from xagent.web.services import agent_service_manager as agent_runtime_service
from xagent.web.services import task_execution as task_execution_service
from xagent.web.services import task_lease_service as leases
from xagent.web.services import task_resume
from xagent.web.services.task_command_execution import _task_lease_snapshot
engine = engine_fixture
task_id = task_id_fixture
@pytest.fixture
def lease_database(engine, task_id, monkeypatch):
factory = sessionmaker(engine)
monkeypatch.setattr("xagent.web.models.database._SessionLocal", factory)
monkeypatch.setattr(
"xagent.web.services.trace_handlers.get_db", lambda: iter([factory()])
)
return factory, task_id
def test_failed_release_rolls_back_work_staged_with_it(lease_database):
factory, tid = lease_database
with factory() as db:
old = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
leases.acquire_task_lease(
db, tid, runner_id="worker", expected_run_id=old.run_id
)
before = db.get(Task, tid).title
with factory() as db:
db.get(Task, tid).title = "staged by a superseded holder"
db.flush()
assert not leases.release_task_lease(db, old, status=TaskStatus.PAUSED)
with factory() as db:
assert db.get(Task, tid).title == before
@pytest.mark.parametrize("change", ["none", "state_version", "expired", "waiting"])
def test_valid_renewal_and_settlement_survive_epoch_check(lease_database, change):
factory, tid = lease_database
with factory() as db:
lease = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
row = db.get(Task, tid)
if change == "state_version":
row.state_version += 1
row.control_state = "pause_requested"
elif change == "expired":
row.lease_expires_at = leases.utc_now() - timedelta(seconds=1)
elif change == "waiting":
row.status = TaskStatus.WAITING_FOR_USER
db.commit()
assert leases.refresh_task_lease(db, lease) == (
leases.TaskLeaseRefreshState.SETTLEMENT_READY
if change == "waiting"
else leases.TaskLeaseRefreshState.REFRESHED
)
def test_old_epoch_lost_but_new_epoch_renews(lease_database):
factory, tid = lease_database
with factory() as db:
first = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
second = leases.acquire_task_lease(
db, tid, runner_id="worker", expected_run_id=first.run_id
)
assert leases.refresh_task_lease(db, first) == leases.TaskLeaseRefreshState.LOST
assert (
leases.refresh_task_lease(db, second)
== leases.TaskLeaseRefreshState.REFRESHED
)
db.get(Task, tid).status = TaskStatus.WAITING_FOR_USER
db.commit()
assert leases.refresh_task_lease(db, first) == leases.TaskLeaseRefreshState.LOST
assert (
leases.refresh_task_lease(db, second)
== leases.TaskLeaseRefreshState.SETTLEMENT_READY
)
@pytest.mark.asyncio
async def test_heartbeat_sharing_requires_epoch_key(lease_database, monkeypatch):
factory, tid = lease_database
with factory() as db:
first = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
second = leases.acquire_task_lease(
db, tid, runner_id="worker", expected_run_id=first.run_id
)
monkeypatch.setattr(leases, "get_task_lease_heartbeat_seconds", lambda: 0.005)
manager = leases._TaskLeaseHeartbeatManager(asyncio.get_running_loop())
old = manager.register(first)
new = manager.register(second)
try:
await asyncio.wait_for(old.terminal_event.wait(), timeout=DB_PROGRESS_TIMEOUT)
assert old._entry.outcome.lease_lost
assert old._entry is not new._entry
assert not new.terminal_event.is_set()
assert not new._entry.outcome.lease_lost
finally:
await old.close()
await new.close()
await manager.wait_until_idle()
@pytest.mark.asyncio
async def test_routing_snapshot_requires_registered_holder(lease_database, monkeypatch):
factory, tid = lease_database
with factory() as db:
lease = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
snapshot = _task_lease_snapshot(db.get(Task, tid))
manager = leases._TaskLeaseHeartbeatManager(asyncio.get_running_loop())
monkeypatch.setattr(leases, "_task_lease_heartbeat_manager", manager)
assert leases.registered_task_lease(snapshot) is None
registration = manager.register(lease)
try:
assert leases.registered_task_lease(snapshot) is lease
finally:
await registration.close()
await manager.wait_until_idle()
assert leases.registered_task_lease(snapshot) is None
@pytest.mark.asyncio
async def test_same_epoch_handoff_keeps_shared_heartbeat(lease_database, monkeypatch):
factory, tid = lease_database
with factory() as db:
lease = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
loop = asyncio.get_running_loop()
refreshed = asyncio.Event()
refresh_batch = leases.refresh_task_leases_isolated
def observe_refresh(batch):
result = refresh_batch(batch)
loop.call_soon_threadsafe(refreshed.set)
return result
monkeypatch.setattr(leases, "refresh_task_leases_isolated", observe_refresh)
monkeypatch.setattr(leases, "get_task_lease_heartbeat_seconds", lambda: 0.005)
manager = leases._TaskLeaseHeartbeatManager(loop)
first = manager.register(lease)
successor = manager.register(lease)
try:
assert first._entry is successor._entry
await asyncio.wait_for(refreshed.wait(), timeout=DB_PROGRESS_TIMEOUT)
assert not (await first.close()).lease_lost
refreshed.clear()
await asyncio.wait_for(refreshed.wait(), timeout=DB_PROGRESS_TIMEOUT)
assert not successor.terminal_event.is_set()
assert not (await successor.close()).lease_lost
finally:
await first.close()
await successor.close()
await manager.wait_until_idle()
@pytest.mark.asyncio
async def test_epoch_key_preserves_soft_pool_timeout_and_recovery(monkeypatch):
calls = 0
def refresh(batch):
nonlocal calls
calls += 1
if calls == 1:
raise TimeoutError("simulated pool wait")
return {
leases._task_lease_key(item): leases.TaskLeaseRefreshState.REFRESHED
for item in batch
}
monkeypatch.setattr(leases, "refresh_task_leases_isolated", refresh)
monkeypatch.setattr(leases, "is_database_pool_timeout", lambda error: True)
monkeypatch.setattr(leases, "get_task_lease_heartbeat_seconds", lambda: 0.005)
manager = leases._TaskLeaseHeartbeatManager(asyncio.get_running_loop())
outcomes = []
recovered = asyncio.Event()
settle = manager._settle_refresh_waiter
def observe(entry, waiter, outcome, **kwargs):
settle(entry, waiter, outcome, **kwargs)
outcomes.append(outcome)
if len(outcomes) >= 2:
recovered.set()
monkeypatch.setattr(manager, "_settle_refresh_waiter", observe)
registration = manager.register(
leases.TaskLease(
task_id=1, runner_id="worker", run_id="run", attempt_id="epoch"
)
)
try:
await asyncio.wait_for(recovered.wait(), timeout=5)
assert outcomes[0].pool_timeout is not None
assert not outcomes[0].lease_lost
assert outcomes[1].pool_timeout is None
assert not outcomes[1].lease_lost
assert not registration.terminal_event.is_set()
finally:
await registration.close()
await manager.wait_until_idle()
@pytest.mark.parametrize("identity", ["superseded", "missing"])
@pytest.mark.parametrize(
"writer",
[
"release",
"fail",
"managed",
"finish",
"result",
"resume",
"title",
"a2a_input",
"reply_input",
"checkpoint",
"outbound",
"usage",
],
)
def test_old_acquisition_cannot_commit_any_execution_result(
lease_database, monkeypatch, identity, writer
):
from dataclasses import replace
from xagent.core.agent.checkpoint import CHECKPOINT_EVENT_TYPE, CHECKPOINT_TYPE
from xagent.core.agent.runner import UserMessageInjectionOutcome
from xagent.core.agent.trace import TraceEvent as CoreTraceEvent
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.task import TraceEvent
from xagent.web.models.task_execution_event import TaskExecutionEvent
from xagent.web.services.managed_task_lease import (
finalize_managed_task_lease_result,
)
from xagent.web.services.task_orchestrator import finish_turn
from xagent.web.services.trace_handlers import DatabaseTraceHandler
factory, tid = lease_database
monkeypatch.setattr(task_execution_service, "get_db", lambda: iter([factory()]))
monkeypatch.setattr(task_execution_service, "get_session_local", lambda: factory)
monkeypatch.setattr(task_resume, "get_session_local", lambda: factory)
with factory() as db:
old = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
current = leases.acquire_task_lease(
db, tid, runner_id="worker", expected_run_id=old.run_id
)
old = replace(current, attempt_id=None) if identity == "missing" else old
task = db.get(Task, tid)
user_id = task.user_id
before = (task.title, task.input, task.output, task.error_message)
empty_outputs = task_execution_service._PreparedTaskFileOutputs((), (), ())
with leases.bind_task_lease_context(old), factory() as db:
if writer == "release":
assert not leases.release_task_lease(db, old, status=TaskStatus.COMPLETED)
elif writer == "fail":
assert not leases.fail_and_release_task_lease_no_commit(
db, old, error_message="stale"
)
db.commit()
elif writer == "managed":
assert not finalize_managed_task_lease_result(
db, old, status=TaskStatus.COMPLETED, assistant_content="stale"
)
elif writer == "finish":
assert not finish_turn(db, tid, task_lease=old)
elif writer == "result":
result = task_execution_service._finalize_task_execution_result_isolated(
task_id=tid,
task_user_id=user_id,
pre_run_status=TaskStatus.RUNNING,
result={"status": "completed", "success": True, "output": "stale"},
expected_run_id=old.run_id,
task_lease=old,
resolved_scope_segments=(),
prepared_outputs=empty_outputs,
)
assert result.late_result
elif writer == "resume":
result = task_execution_service._finalize_resumed_task(
tid,
status="completed",
success=True,
output="stale",
task_owner_user_id=user_id,
result={"output": "stale"},
task_lease=old,
prepared_outputs=empty_outputs,
)
assert result["late_result"]
elif writer == "title":
assert not agent_runtime_service._update_task_title_isolated(
tid, "stale", task_lease=old
)
elif writer == "a2a_input":
assert not task_resume._update_a2a_resume_input_sync(
old, "stale", None, UserMessageInjectionOutcome.NOT_POSTED
)
elif writer == "reply_input":
assert not task_resume._update_reply_input_sync(
old, "stale", None, UserMessageInjectionOutcome.POSTED_FRESH
)
elif writer == "usage":
from xagent.web.tracking.task_tracker import (
TokenUsage,
_write_task_usage_sync,
)
assert not _write_task_usage_sync(
tid,
TokenUsage(input_tokens=99),
old.run_id,
old.runner_id,
old.attempt_id,
)
elif writer == "checkpoint":
handler = DatabaseTraceHandler(tid)
event = CoreTraceEvent(
CHECKPOINT_EVENT_TYPE,
task_id=str(tid),
data={
"checkpoint_type": CHECKPOINT_TYPE,
"execution_id": str(tid),
"snapshot": {"label": "stale"},
},
)
with pytest.raises(RuntimeError, match="lease"):
handler._save_trace_event(db, event)
else:
with pytest.raises(RuntimeError):
task_execution_service._persist_agent_outbound_event(
tid,
{
"type": "agent_message",
"event_id": "stale-message",
"data": {"message": "stale", "expect_response": True},
},
)
with factory() as db:
task = db.get(Task, tid)
assert task.status == TaskStatus.RUNNING
assert task.lease_attempt_id == current.attempt_id
assert (task.title, task.input, task.output, task.error_message) == before
assert task.last_checkpoint_event_id is None
assert task.last_checkpoint_trace_event_id is None
assert db.query(TaskChatMessage).count() == 0
assert db.query(TaskExecutionEvent).count() == 0
assert db.query(TraceEvent).count() == 0
def test_expiry_snapshot_cannot_release_a_new_epoch_with_identical_timestamps(
lease_database, monkeypatch
):
factory, tid = lease_database
now = leases.utc_now()
monkeypatch.setattr(leases, "utc_now", lambda: now)
with factory() as db:
first = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
cutoff = now + timedelta(days=1)
candidate = leases.get_expired_task_lease_candidates(
db, cutoff=cutoff, limit=1
)[0]
second = leases.acquire_task_lease(
db, tid, runner_id="worker", expected_run_id=first.run_id
)
assert second.attempt_id != candidate.attempt_id
assert not leases.recover_expired_task_lease_no_commit(
db,
candidate,
status=TaskStatus.FAILED,
recovered_at=cutoff,
error_message="expired",
)
db.commit()
assert db.get(Task, tid).lease_attempt_id == second.attempt_id
assert db.get(Task, tid).status == TaskStatus.RUNNING
def test_acquisition_and_old_renewal_race_preserves_the_winning_epoch(lease_database):
from concurrent.futures import ThreadPoolExecutor
from threading import Barrier
factory, tid = lease_database
with factory() as db:
old = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
barrier = Barrier(2)
def acquire():
with factory() as db:
barrier.wait(timeout=5)
return leases.acquire_task_lease(
db, tid, runner_id="worker", expected_run_id=old.run_id
)
def renew():
with factory() as db:
barrier.wait(timeout=5)
return leases.refresh_task_lease(db, old)
with ThreadPoolExecutor(max_workers=2) as executor:
acquisition = executor.submit(acquire)
renewal = executor.submit(renew)
current = acquisition.result(timeout=10)
assert renewal.result(timeout=10) in {
leases.TaskLeaseRefreshState.REFRESHED,
leases.TaskLeaseRefreshState.LOST,
}
with factory() as db:
assert db.get(Task, tid).lease_attempt_id == current.attempt_id
assert leases.refresh_task_lease(db, old) == leases.TaskLeaseRefreshState.LOST
assert (
leases.refresh_task_lease(db, current)
== leases.TaskLeaseRefreshState.REFRESHED
)
@pytest.mark.asyncio
async def test_cancelled_acquisition_cleanup_cannot_release_its_successor(
lease_database,
):
import threading
factory, tid = lease_database
acquired = threading.Event()
return_lease = threading.Event()
holder = []
cleanup_results = []
def acquire():
with factory() as db:
lease = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
holder.append(lease)
acquired.set()
assert return_lease.wait(timeout=DB_PROGRESS_TIMEOUT)
return lease
def cleanup(lease):
with factory() as db:
cleanup_results.append(
leases.release_task_lease(db, lease, status=TaskStatus.FAILED)
)
operation = asyncio.create_task(
leases.acquire_task_lease_cancellation_safe(acquire, cleanup)
)
try:
assert await asyncio.to_thread(acquired.wait, DB_PROGRESS_TIMEOUT)
operation.cancel()
with factory() as db:
current = leases.acquire_task_lease(
db, tid, runner_id="worker", expected_run_id=holder[0].run_id
)
return_lease.set()
with pytest.raises(asyncio.CancelledError):
await operation
assert cleanup_results == [False]
with factory() as db:
assert db.get(Task, tid).lease_attempt_id == current.attempt_id
assert db.get(Task, tid).status == TaskStatus.RUNNING
finally:
return_lease.set()
await asyncio.gather(operation, return_exceptions=True)
@pytest.mark.asyncio
async def test_delayed_old_heartbeat_batch_cannot_renew_or_cancel_new_registration(
lease_database, monkeypatch
):
import threading
factory, tid = lease_database
with factory() as db:
old_lease = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
batch_ready = threading.Event()
deliver_batch = threading.Event()
original_refresh = leases.refresh_task_leases_isolated
calls = 0
def delayed_refresh(batch):
nonlocal calls
states = original_refresh(batch)
calls += 1
if calls == 1:
batch_ready.set()
assert deliver_batch.wait(timeout=DB_PROGRESS_TIMEOUT)
return states
monkeypatch.setattr(leases, "refresh_task_leases_isolated", delayed_refresh)
monkeypatch.setattr(leases, "get_task_lease_heartbeat_seconds", lambda: 0.005)
manager = leases._TaskLeaseHeartbeatManager(asyncio.get_running_loop())
old = manager.register(old_lease)
new = None
try:
assert await asyncio.to_thread(batch_ready.wait, DB_PROGRESS_TIMEOUT)
with factory() as db:
current = leases.acquire_task_lease(
db, tid, runner_id="worker", expected_run_id=old_lease.run_id
)
new = manager.register(current)
deliver_batch.set()
await asyncio.wait_for(old.terminal_event.wait(), timeout=DB_PROGRESS_TIMEOUT)
assert not new.terminal_event.is_set()
assert not new._entry.outcome.lease_lost
with factory() as db:
assert db.get(Task, tid).lease_attempt_id == current.attempt_id
finally:
deliver_batch.set()
await old.close()
if new is not None:
await new.close()
await manager.wait_until_idle()
@pytest.mark.parametrize("status", [TaskStatus.RUNNING, TaskStatus.WAITING_FOR_USER])
def test_missing_epoch_cannot_renew_or_claim_settlement_ready(lease_database, status):
from dataclasses import replace
factory, tid = lease_database
with factory() as db:
lease = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
db.get(Task, tid).status = status
db.commit()
assert (
leases.refresh_task_lease(db, replace(lease, attempt_id=None))
== leases.TaskLeaseRefreshState.LOST
)
def test_settlement_lock_serializes_same_process_reacquisition(lease_database):
from concurrent.futures import ThreadPoolExecutor
from threading import Event
factory, tid = lease_database
with factory() as db:
old = leases.acquire_task_lease(db, tid, runner_id="worker", new_run=True)
started = Event()
acquired = Event()
def takeover():
with factory() as db:
started.set()
result = leases.acquire_task_lease(
db, tid, runner_id="worker", expected_run_id=old.run_id
)
acquired.set()
return result
with ThreadPoolExecutor(max_workers=1) as executor, factory() as db:
assert leases.lock_task_lease_no_commit(db, old)
future = executor.submit(takeover)
try:
assert started.wait(timeout=5)
assert not acquired.wait(timeout=0.05)
db.get(Task, tid).output = "committed by original owner"
db.commit()
finally:
db.rollback()
current = future.result(timeout=DB_PROGRESS_TIMEOUT)
with factory() as db:
assert db.get(Task, tid).output == "committed by original owner"
assert db.get(Task, tid).lease_attempt_id == current.attempt_id
assert not leases.lock_task_lease_no_commit(db, old)