Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

12.2 KB
Raw Blame
"""Planned-restart execution hold: ledger, combined-mode dispatch cap, tick pause.

The hold is the host seam a thin ingress Gate drives. These tests pin what the
Engine side guarantees on its own: refusal before side effects, a ledger that
covers hand-off windows, and the combined dispatcher applying the existing
closed-cap rules (controls and active-task input pass, new execution waits).
"""

from __future__ import annotations

import asyncio
from typing import Any

import pytest
import sqlalchemy as sa
from sqlalchemy.orm import sessionmaker

from tests.shared.async_waits import DB_PROGRESS_TIMEOUT, eventually
from xagent.web.models import database
from xagent.web.models.database import Base
from xagent.web.models.task import Task
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.user import User

# Imported up front: modules that bind ``get_session_local`` at import time must
# not first load while a test has replaced it (see test_dispatch_cap).
from xagent.web.services import release_hold as release_hold_module
from xagent.web.services import (  # noqa: F401
    task_command_execution,
)
from xagent.web.services import task_command_transport as transport
from xagent.web.services import (  # noqa: F401
    task_coordinator_runtime,
    task_execution,
    task_orchestrator,
    task_resume,
    task_resume_command,
    task_runtime_secrets,
    task_setup_snapshot,
    task_start,
    task_start_consumer,
)
from xagent.web.services.release_hold import ReleaseHold, execution_status
from xagent.web.services.task_execution import BackgroundTaskManager
from xagent.web.services.task_execution_admission import (
    AdmissionQueueFull,
    ExecutionHeld,
)


@pytest.fixture
def hold(monkeypatch: pytest.MonkeyPatch) -> ReleaseHold:
    """A private hold installed as the process singleton for this test."""

    fresh = ReleaseHold()
    monkeypatch.setattr(release_hold_module, "release_hold", fresh)
    monkeypatch.setattr(task_execution, "release_hold", fresh)
    monkeypatch.setattr(transport, "release_hold", fresh)
    return fresh


# --- ledger ------------------------------------------------------------------


def test_admit_refuses_before_the_block_runs_while_held(hold: ReleaseHold) -> None:
    ran: list[str] = []
    hold.close("release r1")
    with pytest.raises(ExecutionHeld) as refused:
        with hold.admit("sdk_create", "agent:1"):
            ran.append("body")
    assert ran == []
    # Every ingress that already projects queue-full handles the refusal.
    assert isinstance(refused.value, AdmissionQueueFull)
    assert hold.try_admit("sdk_create") is None
    assert hold.snapshot()["refused"] == {"sdk_create": 2}
    assert hold.snapshot()["responsibilities"] == []


def test_ledger_counts_admitted_tracked_and_adopted_work(hold: ReleaseHold) -> None:
    async def scenario() -> None:
        admitted = hold.try_admit("external_create", "ca:1")
        assert admitted is not None
        hold.close("release r1")
        # Tracking is allowed while held: it never opens new execution.
        tracked = hold.track("command_dispatch", "7")
        outer = asyncio.get_running_loop().create_future()
        hold.adopt("slack_interaction", "click", outer)
        kinds = {item["kind"] for item in hold.snapshot()["responsibilities"]}
        assert kinds == {"external_create", "command_dispatch", "slack_interaction"}

        admitted.release()
        admitted.release()  # idempotent
        with tracked:
            pass
        outer.set_result(None)
        await asyncio.sleep(0)
        assert hold.snapshot()["responsibilities"] == []

        hold.reopen()
        assert hold.try_admit("external_create") is not None

    asyncio.run(scenario())


async def test_execution_status_reports_each_owner(
    hold: ReleaseHold, monkeypatch: pytest.MonkeyPatch
) -> None:
    manager = BackgroundTaskManager()
    monkeypatch.setattr(task_execution, "background_task_manager", manager)
    assert execution_status()["busy"] is False

    finish = asyncio.Event()
    handle = asyncio.get_running_loop().create_task(finish.wait())
    manager.register_task(41, handle)
    token = hold.track("ws_execute_task", "task:41")
    status = execution_status()
    assert status["busy"] is True
    assert status["counts"]["live_executions"] == 1
    assert status["counts"]["responsibilities"] == 1

    token.release()
    finish.set()
    await handle
    await eventually(lambda: not execution_status()["busy"])


# --- combined-mode dispatcher --------------------------------------------------


class CombinedHost:
    """A combined (non-shared) dispatcher over a SQLite command queue."""

    def __init__(self, factory: sessionmaker, user_id: int) -> None:
        self.factory = factory
        self.user_id = user_id
        self.claimed: list[tuple[int, str]] = []
        self.in_handler = asyncio.Event()
        self.release_handler = asyncio.Event()
        self.block = False
        self._identity = 0

    def new_task(self) -> int:
        with self.factory() as db:
            task = Task(user_id=self.user_id, title="t", description="t")
            db.add(task)
            db.commit()
            return int(task.id)

    def enqueue(self, task_id: int, kind: str) -> int:
        self._identity += 1
        with self.factory() as db:
            row = TaskExecutionCommand(
                task_id=task_id,
                command_id=f"cmd-{self._identity}",
                kind=kind,
                payload={},
            )
            db.add(row)
            db.commit()
            return int(row.id)

    def status_of(self, command_id: int) -> tuple[str, int]:
        with self.factory() as db:
            row = db.get(TaskExecutionCommand, command_id)
            return str(row.status), int(row.attempt_count or 0)

    async def handler(self, command: Any) -> dict[str, Any]:
        self.claimed.append((command.task_id, command.kind.value))
        if self.block:
            self.in_handler.set()
            await self.release_handler.wait()
        return {"ok": True}


@pytest.fixture
async def combined(tmp_path, monkeypatch: pytest.MonkeyPatch, hold: ReleaseHold):
    engine = sa.create_engine(f"sqlite:///{tmp_path / 'combined.db'}")
    Base.metadata.create_all(engine)
    factory = sessionmaker(engine, expire_on_commit=False)
    with factory() as db:
        user = User(username="combined-user", password_hash="unused")
        db.add(user)
        db.commit()
        user_id = int(user.id)
    monkeypatch.delenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", raising=False)
    monkeypatch.delenv("XAGENT_WORKER_COUNT", raising=False)
    monkeypatch.setattr(database, "get_session_local", lambda: factory)
    monkeypatch.setattr(transport, "get_runner_id", lambda: "combined")
    manager = BackgroundTaskManager()
    monkeypatch.setattr(task_execution, "background_task_manager", manager)
    held: list[asyncio.Task] = []
    host = CombinedHost(factory, user_id)
    host.manager = manager  # type: ignore[attr-defined]
    host.held = held  # type: ignore[attr-defined]
    try:
        yield host
    finally:
        for handle in held:
            handle.cancel()
        await asyncio.gather(*held, return_exceptions=True)
        engine.dispose()


async def dispatch(host: CombinedHost, command_db_id: int | None = None) -> bool:
    return await asyncio.wait_for(
        transport.dispatch_one_task_command(host.handler, command_db_id=command_db_id),
        DB_PROGRESS_TIMEOUT,
    )


async def test_combined_dispatch_holds_new_execution_but_applies_controls_and_active_input(
    combined: CombinedHost, hold: ReleaseHold
) -> None:
    host = combined
    idle_task = host.new_task()
    active_task = host.new_task()
    paused_target = host.new_task()

    finish = asyncio.Event()
    live = asyncio.get_running_loop().create_task(finish.wait())
    host.held.append(live)  # type: ignore[attr-defined]
    host.manager.register_task(active_task, live)  # type: ignore[attr-defined]

    new_message = host.enqueue(idle_task, "message")
    resume = host.enqueue(paused_target, "resume")
    hold.close("release r1")

    # The immediate path (``dispatch_task_command_promptly`` names the row):
    # a command that would open an execution is not claimed and stays pending
    # with no attempt counted.
    assert await dispatch(host, new_message) is False
    assert await dispatch(host, resume) is False
    assert host.status_of(new_message) == ("pending", 0)
    assert host.status_of(resume) == ("pending", 0)

    # Input for a task that is already active keeps the cap's existing pass.
    active_input = host.enqueue(active_task, "message")
    assert await dispatch(host) is True
    assert host.claimed == [(active_task, "message")]
    assert host.status_of(active_input)[0] == "completed"

    # A control of an idle task is still applied.
    pause = host.enqueue(paused_target, "pause")
    assert await dispatch(host, pause) is True
    assert host.claimed[-1] == (paused_target, "pause")

    # Reopening (an aborted release) lets the held command run, exactly once.
    hold.reopen()
    assert await dispatch(host, new_message) is True
    assert host.claimed.count((idle_task, "message")) == 1
    assert host.status_of(new_message)[0] == "completed"
    finish.set()


async def test_dispatch_attempt_is_a_responsibility_until_it_returns(
    combined: CombinedHost, hold: ReleaseHold
) -> None:
    host = combined
    host.block = True
    command = host.enqueue(host.new_task(), "message")
    attempt = asyncio.get_running_loop().create_task(dispatch(host, command))
    await asyncio.wait_for(host.in_handler.wait(), DB_PROGRESS_TIMEOUT)
    # Claimed, not yet registered anywhere else: the ledger still shows it.
    hold.close("release r1")
    status = execution_status()
    assert status["busy"] is True
    assert [item["kind"] for item in status["hold"]["responsibilities"]] == [
        "command_dispatch"
    ]
    host.release_handler.set()
    assert await attempt is True
    assert execution_status()["busy"] is False


# --- trigger tick ------------------------------------------------------------------


async def test_trigger_tick_pauses_at_its_boundary_and_resumes(
    hold: ReleaseHold, monkeypatch: pytest.MonkeyPatch
) -> None:
    from xagent.web import app as app_module
    from xagent.web.services import triggers as triggers_module
    from xagent.web.services import workforce_runtime

    scans: list[int] = []
    dispatches: list[int] = []
    tick_started = asyncio.Event()
    tick_release = asyncio.Event()

    def scan(_db: Any) -> list[Any]:
        scans.append(1)
        return []

    async def dispatch_pending(_db: Any, *, limit: int) -> int:
        dispatches.append(limit)
        if len(dispatches) == 1:
            # The first tick is in progress when the hold closes.
            tick_started.set()
            await tick_release.wait()
        return 0

    monkeypatch.setattr(triggers_module, "scan_due_scheduled_triggers", scan)
    monkeypatch.setattr(
        triggers_module, "dispatch_pending_trigger_runs", dispatch_pending
    )
    monkeypatch.setattr(
        workforce_runtime, "reap_stale_preview_workforce_runs", lambda _db: []
    )
    monkeypatch.setattr(app_module, "get_gmail_watch_enabled", lambda: False)
    # Each tick opens (and closes) a session it never queries here.
    monkeypatch.setattr(database, "get_session_local", lambda: sessionmaker())

    loop_task = asyncio.get_running_loop().create_task(
        app_module._run_trigger_dispatcher(poll_interval_seconds=0, batch_size=3)
    )
    try:
        await asyncio.wait_for(tick_started.wait(), DB_PROGRESS_TIMEOUT)
        hold.close("release r1")
        # The in-progress tick is a responsibility until it finishes.
        assert any(
            item["kind"] == "trigger_tick"
            for item in hold.snapshot()["responsibilities"]
        )
        tick_release.set()
        await eventually(lambda: not hold.snapshot()["responsibilities"])
        seen = (len(scans), len(dispatches))
        await asyncio.sleep(0.05)  # negative probe: paused ticks do nothing
        assert (len(scans), len(dispatches)) == seen
        hold.reopen()
        await eventually(lambda: len(dispatches) > seen[1])
    finally:
        loop_task.cancel()
        await asyncio.gather(loop_task, return_exceptions=True)