Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

18.5 KB
Raw Blame
"""Combined-mode consumption of ``execute_task``: the shared START handoff with a
local owner, and what the dispatcher tells the client when it fails."""

import asyncio
from unittest.mock import AsyncMock, Mock

import pytest

from xagent.web.models.database import Base, get_engine, get_session_local, init_db
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.task_execution_event import TaskExecutionEvent
from xagent.web.models.user import User
from xagent.web.services import task_command_execution as execution
from xagent.web.services import task_command_transport as transport
from xagent.web.services import task_orchestrator
from xagent.web.services import task_start_consumer as consumer
from xagent.web.services.task_command_transport import (
    ClaimedTaskCommand,
    SettledTaskCommand,
    TaskCommandDeferred,
    TaskCommandRejected,
)
from xagent.web.services.task_existing_command import stage_existing_execution
from xagent.web.services.task_runtime import (
    MCP_RUNTIME_AUTHORIZATION_POLICY_REQUIRED_KEY,
)

RUNNER = "local-1"


def _claim(command_db_id: int) -> ClaimedTaskCommand:
    with get_session_local()() as db:
        command = transport.claim_task_command(
            db, runner_id=RUNNER, command_db_id=command_db_id
        )
    assert command is not None
    return command


def _stage_and_claim(task_id: int, owner_id: int) -> ClaimedTaskCommand:
    return _claim(
        stage_existing_execution(
            task_id=task_id,
            task_owner_user_id=owner_id,
            task_description="saved task",
            context={},
            actor_user_id=owner_id,
        )
    )


@pytest.fixture
def combined(tmp_path, monkeypatch):
    monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "false")
    monkeypatch.setattr(consumer, "get_runner_id", lambda: RUNNER)
    init_db(db_url=f"sqlite:///{tmp_path / 'combined.db'}")
    with get_session_local()() as db:
        user = User(username="owner", password_hash="unused")
        db.add(user)
        db.flush()
        task = Task(
            user_id=user.id,
            title="Existing",
            source="internal",
            status=TaskStatus.COMPLETED,
            run_id="previous-run",
            state_version=3,
            control_state="completed",
        )
        db.add(task)
        db.commit()
        ids = (int(task.id), int(user.id))
    yield ids
    Base.metadata.drop_all(bind=get_engine())


def _task_and_command(task_id: int, command_db_id: int):
    with get_session_local()() as db:
        task = db.get(Task, task_id)
        command = db.get(TaskExecutionCommand, command_db_id)
        return (
            (task.status, task.run_id, task.runner_id, task.state_version),
            (command.status, command.result, command.target_run_id),
        )


def test_handoff_opens_the_staged_run_and_completes_in_one_commit(combined):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)

    handoff = consumer._commit_handoff(command, None)

    (status, run_id, runner, _), (cmd_status, result, target) = _task_and_command(
        task_id, command.id
    )
    # The run is the one the START named, so staged facts and the runner agree.
    assert (status, run_id, runner) == (TaskStatus.RUNNING, target, RUNNER)
    assert cmd_status == "completed"
    assert result["run_id"] == target
    assert handoff.claimed.task_lease.run_id == target
    assert handoff.claimed.task_lease.runner_id == RUNNER


def test_redelivered_start_does_not_run_a_second_time(combined):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    consumer._commit_handoff(command, None)
    # The first run finished and the completion write was reclaimed: the row is
    # a processing START again while the task is no longer RUNNING.
    with get_session_local()() as db:
        db.get(TaskExecutionCommand, command.id).status = "processing"
        db.get(TaskExecutionCommand, command.id).claimed_by = RUNNER
        task = db.get(Task, task_id)
        task.status = TaskStatus.COMPLETED
        task.control_state = "completed"
        db.commit()

    settled = consumer._commit_handoff(command, None)

    assert isinstance(settled, SettledTaskCommand)
    assert settled.result == {"rejection_reason": "start_state_changed"}
    (status, run_id, runner, _), _ = _task_and_command(task_id, command.id)
    assert status == TaskStatus.COMPLETED and run_id == command.target_run_id


def test_start_is_rejected_when_the_task_moved_after_staging(combined):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    with get_session_local()() as db:
        db.get(Task, task_id).state_version = 9
        db.commit()

    settled = consumer._commit_handoff(command, None)

    assert settled.result == {"rejection_reason": "start_state_changed"}
    (status, run_id, runner, _), (cmd_status, _, _) = _task_and_command(
        task_id, command.id
    )
    assert (status, runner, cmd_status) == (TaskStatus.COMPLETED, None, "failed")
    assert run_id == "previous-run"


def test_actor_marked_task_is_rejected_with_its_own_reason(combined):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    with get_session_local()() as db:
        db.get(Task, task_id).agent_config = {
            MCP_RUNTIME_AUTHORIZATION_POLICY_REQUIRED_KEY: True
        }
        db.commit()

    settled = consumer._commit_handoff(command, None)

    assert settled.result == {"rejection_reason": "actor_policy_required"}


def test_start_is_deferred_while_a_local_runner_is_registered(combined):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)

    with pytest.raises(TaskCommandDeferred) as deferred:
        consumer._commit_handoff(command, None, local_run_in_flight=True)

    # Nothing was applied, so the sender may safely retry.
    assert deferred.value.resend_safe is True
    (status, run_id, runner, _), (cmd_status, _, _) = _task_and_command(
        task_id, command.id
    )
    assert (status, run_id, runner) == (TaskStatus.COMPLETED, "previous-run", None)
    assert cmd_status == "processing"


@pytest.mark.asyncio
async def test_start_reads_runner_state_on_the_loop_and_defers(combined, monkeypatch):
    import threading

    from xagent.web.services.task_execution import background_task_manager

    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    loop_thread = threading.get_ident()
    read_on: list[int] = []
    real = consumer.bg_turn_inflight

    def spy(tid):
        read_on.append(threading.get_ident())
        return real(tid)

    monkeypatch.setattr(consumer, "bg_turn_inflight", spy)
    running = asyncio.get_running_loop().create_future()
    background_task_manager.running_tasks[task_id] = running
    try:
        with pytest.raises(TaskCommandDeferred):
            await consumer.execute_task_start(command)
    finally:
        background_task_manager.running_tasks.pop(task_id, None)
        running.cancel()

    assert read_on == [loop_thread]
    # Once the runner is gone the very same START goes through.
    settled = await _start_with_scheduling_stubbed(command)
    assert settled.result == {"run_id": command.target_run_id}


def test_non_existing_start_is_not_consumed_without_a_coordinator(combined):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    payload = {**command.payload, "kind": "create", "existing_context": None}
    with get_session_local()() as db:
        db.get(TaskExecutionCommand, command.id).payload = payload
        db.commit()
    command = ClaimedTaskCommand(**{**command.__dict__, "payload": payload})

    settled = consumer._commit_handoff(command, None)

    assert settled.result == {"rejection_reason": "unsupported_start"}


def test_v2_storage_start_and_run_share_one_run_id(combined):
    task_id, owner_id = combined
    with get_session_local()() as db:
        db.get(Task, task_id).conversation_storage_version = 2
        db.commit()
    command = _stage_and_claim(task_id, owner_id)

    consumer._commit_handoff(command, None)

    with get_session_local()() as db:
        fact = db.query(TaskExecutionEvent).filter_by(kind="input_accepted").one()
        assert fact.run_id == db.get(Task, task_id).run_id


@pytest.mark.asyncio
async def test_start_schedules_the_run_under_the_committed_lease(combined, monkeypatch):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    scheduled: dict = {}

    def fake_schedule(**kwargs):
        scheduled.update(kwargs)
        return asyncio.get_running_loop().create_future()

    monkeypatch.setattr(task_orchestrator, "_schedule_bg", fake_schedule)

    settled = await consumer.execute_task_start(command)

    assert settled.result == {"run_id": command.target_run_id}
    assert scheduled["run_id"] == command.target_run_id
    assert scheduled["task_lease"].run_id == command.target_run_id
    assert scheduled["task_lease"].runner_id == RUNNER


@pytest.mark.asyncio
async def test_scheduling_failure_settles_the_committed_lease(combined, monkeypatch):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    monkeypatch.setattr(
        task_orchestrator, "_schedule_bg", Mock(side_effect=RuntimeError("boom"))
    )
    settle = Mock()
    monkeypatch.setattr(task_orchestrator, "settle_task_lease_isolated", settle)
    monkeypatch.setattr(execution, "publish_task_event", AsyncMock())

    with pytest.raises(consumer.PostCommitStartFailure, match="boom"):
        await consumer.execute_task_start(command)

    settle.assert_called_once()
    assert settle.call_args.args[0].run_id == command.target_run_id


async def _start_with_scheduling_stubbed(command):
    patcher = pytest.MonkeyPatch()
    patcher.setattr(
        task_orchestrator,
        "_schedule_bg",
        lambda **_kw: asyncio.get_running_loop().create_future(),
    )
    try:
        return await consumer.execute_task_start(command)
    finally:
        patcher.undo()


@pytest.mark.asyncio
@pytest.mark.parametrize("budget_spent", [False, True])
async def test_post_commit_scheduling_failure_reaches_the_client_once(
    combined, monkeypatch, budget_spent
):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    if budget_spent:
        command = ClaimedTaskCommand(
            **{**command.__dict__, "failure_count": execution.MAX_COMMAND_FAILURES - 1}
        )
    monkeypatch.setattr(
        task_orchestrator, "_schedule_bg", Mock(side_effect=RuntimeError("boom"))
    )
    monkeypatch.setattr(task_orchestrator, "settle_task_lease_isolated", Mock())
    publish = AsyncMock()
    monkeypatch.setattr(execution, "publish_task_event", publish)

    with pytest.raises(consumer.PostCommitStartFailure) as raised:
        await execution._execute_and_report_local_start(
            command, consumer.execute_task_start
        )

    assert isinstance(raised.value.__cause__, RuntimeError)
    publish.assert_awaited_once()
    event = publish.await_args.args[0]
    assert event["type"] == "agent_error"
    assert event["message_code"] == "task_command_failed"


@pytest.mark.asyncio
async def test_post_commit_failure_is_not_reported_when_settlement_was_fenced(
    combined, monkeypatch
):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    monkeypatch.setattr(
        task_orchestrator, "_schedule_bg", Mock(side_effect=RuntimeError("boom"))
    )
    # False: another writer already settled this run, so no contradicting frame.
    monkeypatch.setattr(
        task_orchestrator, "settle_task_lease_isolated", Mock(return_value=False)
    )
    publish = AsyncMock()
    monkeypatch.setattr(execution, "publish_task_event", publish)

    with pytest.raises(RuntimeError, match="boom") as raised:
        await consumer.execute_task_start(command)

    assert not isinstance(raised.value, consumer.PostCommitStartFailure)
    publish.assert_not_awaited()


@pytest.mark.asyncio
async def test_post_commit_failure_is_reported_even_if_settlement_raises(
    combined, monkeypatch
):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    monkeypatch.setattr(
        task_orchestrator, "_schedule_bg", Mock(side_effect=RuntimeError("boom"))
    )
    monkeypatch.setattr(
        task_orchestrator,
        "settle_task_lease_isolated",
        Mock(side_effect=ConnectionError("commit retries exhausted")),
    )
    publish = AsyncMock()
    monkeypatch.setattr(execution, "publish_task_event", publish)

    with pytest.raises(consumer.PostCommitStartFailure):
        await consumer.execute_task_start(command)

    publish.assert_awaited_once()


@pytest.mark.asyncio
async def test_post_commit_cancellation_is_not_reported_as_a_failure(
    combined, monkeypatch
):
    task_id, owner_id = combined
    command = _stage_and_claim(task_id, owner_id)
    monkeypatch.setattr(
        task_orchestrator, "_schedule_bg", Mock(side_effect=asyncio.CancelledError())
    )
    monkeypatch.setattr(task_orchestrator, "settle_task_lease_isolated", Mock())
    publish = AsyncMock()
    monkeypatch.setattr(execution, "publish_task_event", publish)

    with pytest.raises(asyncio.CancelledError):
        await consumer.execute_task_start(command)

    publish.assert_not_awaited()


@pytest.mark.asyncio
@pytest.mark.parametrize("budget_spent", [False, True])
async def test_local_start_deferral_is_reported_only_when_the_budget_is_spent(
    monkeypatch, budget_spent
):
    finished = Mock()
    monkeypatch.setattr(execution, "finish_task_command_delivery", finished)
    publish = AsyncMock()
    monkeypatch.setattr(execution, "publish_task_event", publish)
    defer_count = transport.max_command_defers() - 1 if budget_spent else 0
    command = ClaimedTaskCommand(**{**_command(0).__dict__, "defer_count": defer_count})

    with pytest.raises(TaskCommandDeferred):
        await execution._execute_and_report_local_start(
            command,
            AsyncMock(side_effect=TaskCommandDeferred("busy", resend_safe=True)),
        )

    assert finished.call_count == int(budget_spent)
    if not budget_spent:
        publish.assert_not_awaited()
        return
    event = publish.await_args.args[0]
    assert event["message_code"] == "task_command_deferred"
    assert event["resend_safe"] is True


@pytest.mark.asyncio
async def test_settled_refusal_broadcast_carries_the_staged_terminal_draft(
    monkeypatch,
):
    monkeypatch.setattr(execution, "finish_task_command_delivery", Mock())
    publish = AsyncMock()
    monkeypatch.setattr(execution, "publish_task_event", publish)
    settled = SettledTaskCommand({"rejection_reason": "start_state_changed"})

    result = await execution._execute_and_report_local_start(
        _command(0), AsyncMock(return_value=settled)
    )

    assert result is settled
    event = publish.await_args.args[0]
    assert event["message_code"] == "task_command_failed"
    assert event["resend_safe"] is False


def _reporting(monkeypatch):
    finished, broadcast = Mock(), AsyncMock()
    monkeypatch.setattr(execution, "finish_task_command_delivery", finished)
    monkeypatch.setattr(execution, "_broadcast_terminal_command_error", broadcast)
    monkeypatch.setattr(
        execution, "_terminal_command_event_draft", AsyncMock(return_value=None)
    )
    monkeypatch.setattr(execution, "bind_terminal_event_draft", Mock())
    return finished, broadcast


def _command(failure_count: int) -> ClaimedTaskCommand:
    return ClaimedTaskCommand(
        id=1,
        task_id=7,
        actor_user_id=1,
        command_id="turn-1",
        kind=transport.TaskCommandKind.START,
        payload={},
        target_run_id="run-1",
        attempt_count=1,
        failure_count=failure_count,
    )


@pytest.mark.asyncio
async def test_final_generic_failure_is_reported_once(monkeypatch):
    finished, broadcast = _reporting(monkeypatch)
    failing = AsyncMock(side_effect=ConnectionError("pool timeout"))

    with pytest.raises(ConnectionError):
        await execution._execute_and_report_local_start(
            _command(execution.MAX_COMMAND_FAILURES - 1), failing
        )

    finished.assert_called_once()
    broadcast.assert_awaited_once()


@pytest.mark.asyncio
async def test_retryable_generic_failure_stays_silent(monkeypatch):
    finished, broadcast = _reporting(monkeypatch)

    with pytest.raises(ConnectionError):
        await execution._execute_and_report_local_start(
            _command(0), AsyncMock(side_effect=ConnectionError("pool timeout"))
        )

    finished.assert_not_called()
    broadcast.assert_not_awaited()


@pytest.mark.asyncio
async def test_failed_broadcast_does_not_replace_the_rejection(monkeypatch):
    _finished, broadcast = _reporting(monkeypatch)
    broadcast.side_effect = RuntimeError("socket gone")
    rejection = TaskCommandRejected("nope", reason="unsupported_start")

    with pytest.raises(TaskCommandRejected) as raised:
        await execution._execute_and_report_local_start(
            _command(0), AsyncMock(side_effect=rejection)
        )

    assert raised.value is rejection


@pytest.mark.asyncio
async def test_lost_claim_is_not_broadcast(monkeypatch):
    finished, broadcast = _reporting(monkeypatch)

    with pytest.raises(TaskCommandRejected):
        await execution._execute_and_report_local_start(
            _command(0),
            AsyncMock(side_effect=TaskCommandRejected("lost", reason="stale_claim")),
        )

    finished.assert_not_called()
    broadcast.assert_not_awaited()


def test_settling_cap_held_commands_fails_a_held_start(combined):
    from xagent.web.services.task_execution_admission import (
        settle_cap_held_commands_no_commit,
    )

    task_id, owner_id = combined
    stage_existing_execution(
        task_id=task_id,
        task_owner_user_id=owner_id,
        task_description="saved task",
        context={},
        actor_user_id=owner_id,
    )
    with get_session_local()() as db:
        control = TaskExecutionCommand(
            task_id=task_id,
            command_id="pause-1",
            kind="pause",
            payload={},
            actor_user_id=owner_id,
            target_state_version=0,
            status="completed",
        )
        db.add(control)
        db.commit()
        control_id = int(control.id)
    with get_session_local()() as db:
        settle_cap_held_commands_no_commit(db, control_id)
        db.commit()

    with get_session_local()() as db:
        start = db.query(TaskExecutionCommand).filter_by(kind="start").one()
        assert start.status == "failed"
        assert start.result == {"rejection_reason": "cancelled_before_admission"}