Enterprise agent platform for building, deploying, and operating AI teammates.
"""Shared START transaction and execution-attempt fences."""
from dataclasses import replace
from datetime import datetime, timedelta, timezone
import pytest
from tests.shared.async_waits import DB_PROGRESS_TIMEOUT, eventually
from tests.web.services.coordinator_command_shared import (
claim_task_command,
settle_command,
)
from xagent.web.models.database import Base, get_engine, get_session_local, init_db
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.user import User
from xagent.web.services import task_coordinator_runtime, task_coordinator_service
from xagent.web.services import task_start_consumer as consumer
from xagent.web.services.task_command_execution import execute_durable_task_command
from xagent.web.services.task_command_transport import (
SettledTaskCommand,
TaskCommandRejected,
)
from xagent.web.services.task_start_protocol import (
TaskStartPayload,
stage_task_start_command,
)
def owner_lease(task_id):
with get_session_local()() as db, db.begin():
task_coordinator_service.recover_expired_idle_task_lease_no_commit(db, task_id)
lease = task_coordinator_service.acquire_task_lease_no_commit(
db, task_id, runner_id="worker-1"
)
if lease is None:
task = db.get(Task, task_id)
assert task.runner_id == "worker-1"
lease = task_coordinator_service.TaskLease(
task_id, task.runner_id, task.lease_attempt_id
)
return lease
def commit_handoff(command):
return consumer._commit_handoff(command, owner_lease(command.task_id))
async def wait_for_handoff(committed, execution):
"""Wait until a START execution has committed its handoff.
Lease acquisition and the handoff are real SQLite commits, so how long
they take is not what these tests assert: bound the wait as a hang
detector, and surface an execution that ended without committing.
"""
await eventually(lambda: committed.is_set() or execution.done())
if not committed.is_set():
await execution
pytest.fail("START execution ended without committing its handoff")
@pytest.fixture
async def accepted(tmp_path, monkeypatch, request):
monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "true")
monkeypatch.setattr(task_coordinator_runtime, "get_runner_id", lambda: "worker-1")
init_db(db_url=f"sqlite:///{tmp_path / 'consumer.db'}")
with get_session_local()() as db:
user = User(username="owner", password_hash="unused")
db.add(user)
db.flush()
task = Task(
user_id=user.id,
title="Shared",
source=getattr(request, "param", "sdk"),
status=TaskStatus.PENDING,
run_id=None,
state_version=1,
control_state="idle",
)
db.add(task)
db.flush()
start = TaskStartPayload(
version=1,
run_id="run-1",
state_version=1,
turn_id="turn-1",
kind="create",
message="Hello",
execution_message="Hello",
file_ids=[],
)
staged = stage_task_start_command(
db, task_id=task.id, actor_user_id=user.id, start=start
)
db.commit()
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=staged.staged_db_id
)
assert command is not None
yield command
await task_coordinator_runtime.close_task_coordinators()
Base.metadata.drop_all(bind=get_engine())
@pytest.mark.asyncio
@pytest.mark.parametrize(
"accepted,interactive", [("internal", True), ("sdk", False)], indirect=["accepted"]
)
async def test_start_handoff_keeps_claimed_inference_priority(
accepted, interactive, monkeypatch
):
from xagent.core.model.chat.routing_context import xinference_interactive
observed = []
async def schedule(**kwargs):
observed.append(xinference_interactive.get())
monkeypatch.setattr(consumer, "_schedule_committed_turn", schedule)
assert accepted.dispatch_class == ("interactive" if interactive else "batch")
await execute_durable_task_command(accepted)
assert observed == [interactive]
assert not xinference_interactive.get()
def test_handoff_commits_lease_and_completion_together(accepted):
handoff = commit_handoff(accepted)
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
command = db.get(TaskExecutionCommand, accepted.id)
assert task.run_id == "run-1"
assert task.state_version == 2
assert task.runner_id == "worker-1"
assert task.lease_attempt_id == handoff.claimed.task_lease.attempt_id
assert command.status == "completed"
assert command.result["lease_attempt_id"] == task.lease_attempt_id
with pytest.raises(TaskCommandRejected):
commit_handoff(accepted)
def test_completion_fence_failure_rolls_back_lease(accepted, monkeypatch):
monkeypatch.setattr(
consumer, "finish_task_command_no_commit", lambda *a, **kw: False
)
with pytest.raises(TaskCommandRejected):
commit_handoff(accepted)
with get_session_local()() as db:
assert db.get(Task, accepted.task_id).status == TaskStatus.PENDING
assert db.get(TaskExecutionCommand, accepted.id).status == "processing"
def test_old_command_attempt_cannot_acquire_execution(accepted):
with get_session_local()() as db:
db.get(TaskExecutionCommand, accepted.id).attempt_count += 1
db.commit()
with pytest.raises(TaskCommandRejected):
commit_handoff(accepted)
with get_session_local()() as db:
assert db.get(Task, accepted.task_id).status == TaskStatus.PENDING
assert db.get(TaskExecutionCommand, accepted.id).status == "processing"
def test_obsolete_command_expiry_does_not_override_task_owner(accepted):
with get_session_local()() as db:
db.get(TaskExecutionCommand, accepted.id).claim_expires_at = datetime.now(
timezone.utc
) - timedelta(seconds=1)
db.commit()
commit_handoff(accepted)
with get_session_local()() as db:
assert db.get(TaskExecutionCommand, accepted.id).status == "completed"
def test_old_run_rejection_does_not_fail_new_run(accepted):
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
task.run_id = "run-2"
task.status = TaskStatus.RUNNING
task.state_version = 2
db.commit()
assert isinstance(commit_handoff(accepted), SettledTaskCommand)
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
assert task.run_id == "run-2"
assert task.status == TaskStatus.RUNNING
assert task.state_version == 2
assert db.get(TaskExecutionCommand, accepted.id).status == "failed"
def test_invalid_current_start_settles_accepted_task(accepted):
broken = replace(accepted, payload={**accepted.payload, "version": 999})
assert isinstance(commit_handoff(broken), SettledTaskCommand)
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
assert task.status == TaskStatus.FAILED
assert task.control_state == "failed"
assert task.lease_attempt_id is not None
assert db.get(TaskExecutionCommand, accepted.id).status == "failed"
def test_stream_reconciliation_reads_current_run_and_output_together(accepted):
from xagent.web.services.task_stream_snapshot import load_task_stream_snapshots
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
task.status = TaskStatus.COMPLETED
task.control_state = "completed"
task.output = "durable complete answer"
task.run_id = "run-1"
db.commit()
snapshots = load_task_stream_snapshots([accepted.task_id])
assert len(snapshots) == 1
assert snapshots[0]["run_id"] == "run-1"
assert snapshots[0]["output"] == "durable complete answer"
assert snapshots[0]["status"] == "completed"
assert load_task_stream_snapshots([]) == []
def test_existing_execution_accepts_without_another_transcript_row(
accepted, monkeypatch
):
from unittest.mock import Mock
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.services import task_event_bridge
from xagent.web.services.task_existing_command import stage_existing_execution
monkeypatch.setattr(task_event_bridge, "_bridge", Mock())
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
task.status = TaskStatus.COMPLETED
task.control_state = "completed"
task.output = "previous answer"
db.get(TaskExecutionCommand, accepted.id).status = "completed"
db.commit()
previous_owner = (task.runner_id, task.lease_attempt_id)
owner_id = task.user_id
before = db.query(TaskChatMessage).filter_by(task_id=task.id).count()
command_db_id = stage_existing_execution(
task_id=accepted.task_id,
task_owner_user_id=owner_id,
actor_user_id=owner_id,
task_description="description",
context={"execution_mode": "react"},
)
with get_session_local()() as db:
run_id = db.get(TaskExecutionCommand, command_db_id).target_run_id
task = db.get(Task, accepted.task_id)
assert task.run_id != run_id
assert task.output == "previous answer"
assert (task.runner_id, task.lease_attempt_id) == previous_owner
assert db.query(TaskChatMessage).filter_by(task_id=task.id).count() == before
command = (
db.query(TaskExecutionCommand)
.order_by(TaskExecutionCommand.id.desc())
.first()
)
assert command.payload["kind"] == "existing"
assert command.payload["existing_context"]["execution_mode"] == "react"
@pytest.mark.asyncio
async def test_cancellation_after_handoff_still_registers_execution(
accepted, monkeypatch
):
import asyncio
import threading
from unittest.mock import AsyncMock
committed, release = threading.Event(), threading.Event()
original = consumer._commit_handoff
def slow_handoff(command, lease):
handoff = original(command, lease)
committed.set()
assert release.wait(DB_PROGRESS_TIMEOUT)
return handoff
monkeypatch.setattr(consumer, "_commit_handoff", slow_handoff)
schedule = AsyncMock()
monkeypatch.setattr(consumer, "_schedule_committed_turn", schedule)
pending = asyncio.create_task(execute_durable_task_command(accepted))
try:
await wait_for_handoff(committed, pending)
pending.cancel()
await asyncio.sleep(0)
assert not pending.done()
finally:
release.set()
with pytest.raises(asyncio.CancelledError):
await pending
schedule.assert_awaited_once()
with get_session_local()() as db:
assert db.get(TaskExecutionCommand, accepted.id).status == "completed"
def test_shared_acceptance_binds_inputs_and_start_in_one_transaction(
accepted, monkeypatch
):
from unittest.mock import Mock
from cryptography.fernet import Fernet
from xagent.core.tools.adapters.vibe.connector_runtime import ConnectorRef
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.task_runtime_secret import TaskRuntimeSecret
from xagent.web.services import task_event_bridge
from xagent.web.services.task_orchestrator import (
TaskTurnOrchestrator,
TaskTurnPayload,
)
from xagent.web.services.task_runtime_secrets import stage_runtime_values
monkeypatch.setenv("ENCRYPTION_KEY", Fernet.generate_key().decode())
monkeypatch.setattr(task_event_bridge, "get_task_event_bridge", lambda: Mock())
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
previous_owner = (task.runner_id, task.lease_attempt_id)
task.status = TaskStatus.COMPLETED
db.get(TaskExecutionCommand, accepted.id).status = "completed"
db.commit()
stage_runtime_values(
db,
task_id=task.id,
turn_id="next-turn",
values_by_ref={ConnectorRef("mcp", 1): {"secrets": {"key": "synthetic"}}},
)
prepared = TaskTurnOrchestrator.claim_append_turn_no_commit(
db,
task_id=task.id,
task_owner_user_id=task.user_id,
payload=TaskTurnPayload("next", turn_id="next-turn"),
)
row = db.get(TaskExecutionCommand, prepared.command_db_id)
assert row.payload["runtime_values_ref"] == "next-turn"
assert "synthetic" not in str(row.payload)
assert db.query(TaskRuntimeSecret).one().run_id == prepared.run_id
assert (task.runner_id, task.lease_attempt_id) == previous_owner
db.rollback()
with get_session_local()() as db:
assert db.query(TaskRuntimeSecret).count() == 0
assert db.query(TaskChatMessage).filter_by(turn_id="next-turn").count() == 0
assert db.get(TaskExecutionCommand, prepared.command_db_id) is None
@pytest.mark.parametrize("replaced", [False, True])
async def test_terminal_transport_failure_settles_only_its_owned_run(
accepted, replaced
):
from xagent.web.services.task_command_transport import (
MAX_COMMAND_FAILURES,
fail_task_command,
)
with get_session_local()() as db:
db.get(TaskExecutionCommand, accepted.id).failure_count = (
MAX_COMMAND_FAILURES - 1
)
if replaced:
db.get(Task, accepted.task_id).run_id = "replacement"
db.commit()
assert await settle_command(
accepted,
lambda: fail_task_command(
accepted.id,
"worker-1",
"preparation failed",
expected_attempt_count=accepted.attempt_count,
),
)
with get_session_local()() as db:
assert db.get(TaskExecutionCommand, accepted.id).status == "failed"
assert db.get(Task, accepted.task_id).status == (
TaskStatus.PENDING if replaced else TaskStatus.FAILED
)
async def test_replaced_owner_cannot_fail_accepted_run(accepted):
from xagent.web.services.task_command_transport import fail_task_command
with get_session_local()() as db:
db.get(Task, accepted.task_id).lease_attempt_id = "replacement-owner"
db.commit()
assert not await settle_command(
accepted,
lambda: fail_task_command(
accepted.id,
"worker-1",
"stale owner",
force_terminal=True,
expected_attempt_count=accepted.attempt_count,
),
)
with get_session_local()() as db:
assert db.get(Task, accepted.task_id).status == TaskStatus.PENDING
async def test_live_owner_routes_controls_even_without_immutable_target(accepted):
from xagent.web.services.task_command_transport import (
TaskCommandKind,
stage_task_command,
)
commit_handoff(accepted)
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
staged = stage_task_command(
db,
task_id=task.id,
actor_user_id=task.user_id,
command_id="pause",
kind=TaskCommandKind.PAUSE,
payload={},
)
db.get(TaskExecutionCommand, staged.staged_db_id).target_runner_id = None
db.commit()
assert (
await claim_task_command(
db, runner_id="worker-2", command_db_id=staged.staged_db_id
)
is None
)
row = db.get(TaskExecutionCommand, staged.staged_db_id)
assert row.attempt_count == 0
assert (
await claim_task_command(
db, runner_id="worker-1", command_db_id=staged.staged_db_id
)
is not None
)
async def test_worker_death_after_handoff_recovers_without_replaying_start(accepted):
from xagent.web.services.task_lease_recovery import (
recover_expired_task_leases_batch_isolated,
)
commit_handoff(accepted)
with get_session_local()() as db:
db.get(Task, accepted.task_id).lease_expires_at = datetime.now(
timezone.utc
) - timedelta(seconds=1)
db.commit()
recovered = recover_expired_task_leases_batch_isolated(
cutoff=datetime.now(timezone.utc), batch_size=10, after=None
)
assert recovered.recovered == 1
with get_session_local()() as db:
assert db.get(Task, accepted.task_id).status == TaskStatus.FAILED
assert db.get(Task, accepted.task_id).runner_id is None
assert db.get(TaskExecutionCommand, accepted.id).status == "completed"
assert (
await claim_task_command(
db, runner_id="worker-2", command_db_id=accepted.id
)
is None
)
def test_terminal_settlement_deletes_only_accepted_run_values(accepted, monkeypatch):
from cryptography.fernet import Fernet
from xagent.core.tools.adapters.vibe.connector_runtime import ConnectorRef
from xagent.web.models.task_runtime_secret import TaskRuntimeSecret
from xagent.web.services.task_orchestrator import settle_task_lease_isolated
from xagent.web.services.task_runtime_secrets import (
bind_runtime_values_to_run,
stage_runtime_values,
)
monkeypatch.setenv("ENCRYPTION_KEY", Fernet.generate_key().decode())
with get_session_local()() as db:
stage_runtime_values(
db,
task_id=accepted.task_id,
turn_id="turn-1",
values_by_ref={ConnectorRef("mcp", 1): {"secrets": {"key": "synthetic"}}},
)
bind_runtime_values_to_run(
db, task_id=accepted.task_id, turn_id="turn-1", run_id="run-1"
)
db.commit()
handoff = commit_handoff(accepted)
assert settle_task_lease_isolated(
handoff.claimed.task_lease, error_message="execution failed"
)
with get_session_local()() as db:
assert db.query(TaskRuntimeSecret).count() == 0
@pytest.mark.asyncio
async def test_following_command_waits_for_start_registration(accepted, monkeypatch):
import asyncio
import threading
from unittest.mock import AsyncMock
from xagent.web.services import task_command_execution
from xagent.web.services.task_command_transport import (
TaskCommandKind,
stage_task_command,
)
committed, release = threading.Event(), threading.Event()
original = consumer._commit_handoff
registered = False
def slow_handoff(command, lease):
handoff = original(command, lease)
committed.set()
assert release.wait(DB_PROGRESS_TIMEOUT)
return handoff
async def schedule(**kwargs):
nonlocal registered
registered = True
async def followup(command):
assert registered
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
assert task.run_id == accepted.target_run_id
assert task.runner_id == "worker-1"
return {}
monkeypatch.setattr(consumer, "_commit_handoff", slow_handoff)
monkeypatch.setattr(consumer, "_schedule_committed_turn", schedule)
effects = AsyncMock(side_effect=followup)
monkeypatch.setattr(
task_command_execution, "_execute_and_report_task_command", effects
)
start = asyncio.create_task(
task_command_execution.execute_durable_task_command(accepted)
)
next_task = None
try:
await wait_for_handoff(committed, start)
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
staged = stage_task_command(
db,
task_id=task.id,
actor_user_id=task.user_id,
command_id="followup",
kind=TaskCommandKind.MESSAGE,
payload={"message": "next"},
)
db.commit()
command = await claim_task_command(
db, runner_id="worker-1", command_db_id=staged.staged_db_id
)
assert command is not None
next_task = asyncio.create_task(
task_command_execution.execute_durable_task_command(command)
)
await asyncio.sleep(0.05)
effects.assert_not_awaited()
release.set()
await asyncio.wait_for(asyncio.gather(start, next_task), DB_PROGRESS_TIMEOUT)
effects.assert_awaited_once()
finally:
release.set()
await asyncio.gather(
start, *([next_task] if next_task else []), return_exceptions=True
)
def test_completion_wait_recovers_owner_lost_after_real_result_finalization(accepted):
from xagent.web.services.task_completion import _is_run_finished
from xagent.web.services.task_execution import (
_finalize_task_execution_result_isolated,
_PreparedTaskFileOutputs,
)
handoff = commit_handoff(accepted)
result = _finalize_task_execution_result_isolated(
task_id=accepted.task_id,
task_user_id=handoff.task_owner_user_id,
pre_run_status=TaskStatus.RUNNING,
result={"success": True, "output": "finished"},
expected_run_id="run-1",
task_lease=handoff.claimed.task_lease,
resolved_scope_segments=(),
prepared_outputs=_PreparedTaskFileOutputs((), (), ()),
)
assert result.terminal_state_committed
assert not _is_run_finished(accepted.task_id, "run-1")
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
assert task.status == TaskStatus.COMPLETED
assert task.runner_id == "worker-1"
task.lease_expires_at = datetime.now(timezone.utc) - timedelta(minutes=10)
db.commit()
assert not _is_run_finished(accepted.task_id, "run-1")
from xagent.web.services.task_lease_recovery import recover_expired_idle_task_leases
assert recover_expired_idle_task_leases(batch_size=10) == 1
assert _is_run_finished(accepted.task_id, "run-1")
with get_session_local()() as db:
task = db.get(Task, accepted.task_id)
assert task.runner_id is None
assert task.output == "finished"
assert task.status == TaskStatus.COMPLETED