Enterprise agent platform for building, deploying, and operating AI teammates.
import time
from concurrent.futures import ThreadPoolExecutor
from threading import Event
from uuid import uuid4
import pytest
from sqlalchemy import delete, text, update
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Query, sessionmaker
from tests.web.services.task_database_shared import engine as engine_fixture
from tests.web.services.task_database_shared import task_id as task_id_fixture
from xagent.core.agent.checkpoint import CHECKPOINT_EVENT_TYPE, CHECKPOINT_TYPE
from xagent.core.agent.trace import TraceEvent
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.services import task_execution as execution
from xagent.web.services import task_lease_service as leases
from xagent.web.services import task_orchestrator as orch
from xagent.web.services import trace_handlers as traces
from xagent.web.services import workforce_runtime as workforce
from xagent.web.services.uploaded_file_store import UploadedFileStore
engine = engine_fixture
task_id = task_id_fixture
def make_call(route, factory, tid, uid, lease, output):
empty = execution._PreparedTaskFileOutputs((), (), ())
if route == "finish":
fn = orch.finish_turn
def call():
with factory() as db:
return fn(db, tid, task_lease=lease)
elif route == "result":
fn = execution._finalize_task_execution_result_isolated
def call():
return fn(
task_id=tid,
task_user_id=uid,
pre_run_status=TaskStatus.RUNNING,
result={"status": "completed", "success": True, "output": output},
expected_run_id=lease.run_id,
task_lease=lease,
resolved_scope_segments=(),
prepared_outputs=empty,
)
elif route == "resume":
fn = execution._finalize_resumed_task
def call():
return fn(
tid,
status="completed",
success=True,
output=output,
task_owner_user_id=uid,
result={"output": output},
task_lease=lease,
prepared_outputs=empty,
)
else:
fn = workforce._sync_workforce_run_status_for_task_id
def call():
with factory() as db:
return fn(db, tid, TaskStatus.RUNNING, task_lease=lease)
return call
@pytest.mark.parametrize("route", ["finish", "result", "resume", "workforce"])
def test_actual_entrypoint_lock_and_child_visibility(
engine, task_id, monkeypatch, route
):
if engine.dialect.name != "postgresql":
pytest.skip("PostgreSQL row and FK locks")
factory = sessionmaker(engine)
monkeypatch.setattr(execution, "get_session_local", lambda: factory)
with factory() as db:
lease = leases.acquire_task_lease(db, task_id, runner_id="review", new_run=True)
task = db.get(Task, task_id)
uid = task.user_id
if route == "finish":
task.status = TaskStatus.COMPLETED
db.add(
TaskChatMessage(
task_id=task_id,
user_id=uid,
role="assistant",
message_type="assistant_response",
content="old answer",
)
)
db.commit()
ready, proceed = Event(), Event()
first = Query.first
def pause_first(query):
result = first(query)
if (
result is not None
and isinstance(result, Task)
and not query.session.info.get("paused")
):
query.session.info["paused"] = True
ready.set()
assert proceed.wait(8)
return result
monkeypatch.setattr(Query, "first", pause_first)
output = "[report.txt](file:00000000-0000-0000-0000-000000000001)"
call = make_call(route, factory, task_id, uid, lease, output)
file_id = str(uuid4())
with ThreadPoolExecutor(1) as pool:
future = pool.submit(call)
try:
assert ready.wait(8)
with factory() as contender:
contender.execute(text("SET LOCAL lock_timeout='150ms'"))
with pytest.raises(OperationalError, match="lock timeout"):
contender.execute(
update(Task).where(Task.id == task_id).values(runner_id="other")
)
contender.rollback()
contender.execute(text("SET LOCAL lock_timeout='150ms'"))
with pytest.raises(OperationalError, match="lock timeout"):
contender.execute(delete(Task).where(Task.id == task_id))
contender.rollback()
with factory() as child:
child.execute(text("SET LOCAL lock_timeout='150ms'"))
if route == "finish":
child.add(
TaskChatMessage(
task_id=task_id,
user_id=uid,
role="assistant",
message_type="assistant_response",
content="new answer",
)
)
def insert_and_commit():
if route != "finish":
UploadedFileStore(child).add_already_durable(
UploadedFile(
task_id=task_id,
user_id=uid,
file_id=file_id,
filename="report.txt",
storage_path="/review/" + file_id,
storage_status="available",
storage_key="review/" + file_id,
checksum="review-checksum",
)
)
child.commit()
with pytest.raises(OperationalError, match="lock timeout"):
insert_and_commit()
child.rollback()
finally:
proceed.set()
result = future.result(timeout=8)
with factory() as db:
task = db.get(Task, task_id)
if route == "finish":
assert task.output == "old answer"
elif route == "resume":
assert file_id not in result["output"]
elif route == "result":
assert file_id not in result.ai_response
def test_checkpoint_and_settlement_complete_without_lock_upgrade(
engine, task_id, monkeypatch
):
if engine.dialect.name != "postgresql":
pytest.skip("PostgreSQL row and FK locks")
factory = sessionmaker(engine)
with factory() as db:
lease = leases.acquire_task_lease(db, task_id, runner_id="review", new_run=True)
inserted, attempting, proceed = Event(), Event(), Event()
pids = {}
stage = traces.stage_trace_event_row
def paused_stage(db, *args, **kwargs):
result = stage(db, *args, **kwargs)
pids["checkpoint"] = db.scalar(text("select pg_backend_pid()"))
inserted.set()
assert proceed.wait(8)
return result
monkeypatch.setattr(traces, "stage_trace_event_row", paused_stage)
lock = orch.lock_task_lease_for_settlement_no_commit
def noticed_lock(db, lease):
pids["finalizer"] = db.scalar(text("select pg_backend_pid()"))
attempting.set()
return lock(db, lease)
monkeypatch.setattr(orch, "lock_task_lease_for_settlement_no_commit", noticed_lock)
event = TraceEvent(
CHECKPOINT_EVENT_TYPE,
task_id=str(task_id),
step_id="step",
data={
"checkpoint_type": CHECKPOINT_TYPE,
"execution_id": "review",
"snapshot": {},
},
require_persisted=True,
)
def checkpoint():
with factory() as db, leases.bind_task_lease_context(lease):
db.execute(text("SET LOCAL statement_timeout='5s'"))
traces.DatabaseTraceHandler(task_id)._save_trace_event(db, event)
def finalize():
with factory() as db:
db.execute(text("SET LOCAL statement_timeout='5s'"))
return orch.finish_turn(db, task_id, task_lease=lease)
with ThreadPoolExecutor(2) as pool:
writer = pool.submit(checkpoint)
try:
assert inserted.wait(8)
finisher = pool.submit(finalize)
assert attempting.wait(8)
# Release only after PostgreSQL confirms the finalizer is waiting
# for the checkpoint transaction, rather than guessing timing.
with factory() as observer:
deadline = time.monotonic() + 3
while True:
blockers = observer.scalar(
text("select pg_blocking_pids(:pid)"),
{"pid": pids["finalizer"]},
)
if pids["checkpoint"] in blockers:
break
assert time.monotonic() < deadline
time.sleep(0.005)
finally:
proceed.set()
writer.result(timeout=8)
assert finisher.result(timeout=8)
with factory() as db:
task = db.get(Task, task_id)
assert task.last_checkpoint_event_id == str(event.id)
assert task.last_checkpoint_trace_event_id is not None
assert task.status == TaskStatus.FAILED
@pytest.mark.parametrize("route", ["finish", "result", "resume", "workforce"])
def test_stale_owner_still_rejected(engine, task_id, monkeypatch, route):
factory = sessionmaker(engine)
monkeypatch.setattr(execution, "get_session_local", lambda: factory)
with factory() as db:
old = leases.acquire_task_lease(db, task_id, runner_id="review", new_run=True)
leases.acquire_task_lease(
db, task_id, runner_id="review", expected_run_id=old.run_id
)
task = db.get(Task, task_id)
uid = task.user_id
before = (
task.status,
task.output,
task.runner_id,
task.run_id,
task.lease_attempt_id,
)
result = make_call(route, factory, task_id, uid, old, "stale result")()
if route == "result":
assert result.late_result
elif route == "resume":
assert result["late_result"]
else:
assert result is False
with factory() as db:
task = db.get(Task, task_id)
assert (
task.status,
task.output,
task.runner_id,
task.run_id,
task.lease_attempt_id,
) == before
assert db.query(TaskChatMessage).count() == 0