Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

13.9 KB
Raw Blame
"""Each turn entry point tells the runtime cache when its turn has settled.

``note_runtime_settled`` is the single call the three execution entry points
make from their final cleanup. It must run exactly once per turn on every exit
(normal, failing, cancelled, or failing before the agent ran), and a failure of
the call itself must never change how the turn ended.
"""

from __future__ import annotations

import asyncio
import logging
from datetime import datetime, timezone
from types import SimpleNamespace
from typing import Any, Awaitable, Callable
from unittest.mock import AsyncMock, MagicMock, Mock, patch

import pytest

from tests.web.api.test_execute_task_background_snapshot_passthrough import (
    _build_fake_agent_service,
    _common_patches,
    _make_snapshot,
    _Patches,
)
from xagent.web.models.task import TaskStatus
from xagent.web.services import agent_service_manager, shared_channel_execution
from xagent.web.services.agent_service_manager import AgentServiceManager
from xagent.web.services.task_execution import (
    execute_resume_background,
    execute_task_background,
)
from xagent.web.services.task_execution_context_service import (
    TaskExecutionRecoverySnapshot,
)
from xagent.web.services.task_lease_service import TaskLease
from xagent.web.services.task_settlement_report import SettlementReport
from xagent.web.services.turn_settlement import TurnFacts

HOOK_WARNING = "Settling the cached runtime for task"
OK_RESULT = {"success": True, "output": "ok", "status": "completed"}

# A turn either finishes, fails inside the agent, or is cancelled.
PATHS = ["normal", "exception", "cancellation"]
ENTRIES = [
    "execute_task_background",
    "execute_channel_background",
    "execute_resume_background",
]


def agent_outcome(path: str) -> dict[str, Any] | BaseException:
    return {
        "normal": OK_RESULT,
        "exception": RuntimeError("agent failed"),
        "cancellation": asyncio.CancelledError(),
    }[path]


def expected_outcome(entry: str, path: str) -> type[BaseException] | None:
    """How each entry point ends for the agent outcome (hook working)."""
    if path == "cancellation":
        return asyncio.CancelledError
    if path == "exception" and entry != "execute_resume_background":
        # The resume entry records the failure on the task instead of raising.
        return RuntimeError
    return None


def lease() -> TaskLease:
    return TaskLease(
        task_id=42, runner_id="runner-a", run_id="run-a", attempt_id="attempt-a"
    )


async def drive_task_background(manager: Any, path: str) -> None:
    outcome = agent_outcome(path)
    service = _build_fake_agent_service()
    manager.get_agent_for_task = AsyncMock(return_value=service)
    manager.execute_task = AsyncMock(
        side_effect=outcome if isinstance(outcome, BaseException) else None,
        return_value=outcome if isinstance(outcome, dict) else None,
    )
    with _Patches(_common_patches(MagicMock(), service)):
        await execute_task_background(
            task_id=42,
            user_message="hi",
            context={},
            agent_manager=manager,
            task_owner_user_id=1,
            task_setup_snapshot=_make_snapshot(),
            task_lease=lease(),
            resolved_execution_scope=None,
        )


async def drive_channel_background(
    manager: Any,
    path: str,
    monkeypatch: pytest.MonkeyPatch,
    *,
    fail_before_execute: str | None = None,
    finalize: Callable[..., bool] | None = None,
) -> None:
    outcome = agent_outcome(path)
    tracer = SimpleNamespace(add_handler=Mock(), remove_handler=Mock())

    def stage(name: str) -> Mock:
        return Mock(
            side_effect=RuntimeError(f"{name} failed")
            if fail_before_execute == name
            else None
        )

    service = SimpleNamespace(
        tracer=tracer,
        set_conversation_history=stage("set_conversation_history"),
        set_execution_context_messages=stage("set_execution_context_messages"),
        set_recovered_skill_context=stage("set_recovered_skill_context"),
    )
    manager.get_agent_for_task = AsyncMock(return_value=service)
    manager.execute_task = AsyncMock(
        side_effect=(
            RuntimeError("execute_task failed")
            if fail_before_execute == "execute_task"
            else outcome
            if isinstance(outcome, BaseException)
            else None
        ),
        return_value=outcome if isinstance(outcome, dict) else None,
    )

    async def materialize(_recovery: Any) -> dict[str, Any]:
        if fail_before_execute == "materialize":
            raise RuntimeError("materialize failed")
        return {}

    class Forwarder:
        def __init__(self, *_args: Any, **_kwargs: Any) -> None:
            pass

        async def close(self, *, drain: bool = False) -> None:
            return None

    monkeypatch.setattr(agent_service_manager, "get_agent_manager", lambda: manager)
    monkeypatch.setattr(
        "xagent.web.services.task_execution_context_service."
        "materialize_task_execution_recovery_state",
        materialize,
    )
    monkeypatch.setattr(shared_channel_execution, "ChannelProgressForwarder", Forwarder)
    monkeypatch.setattr(
        shared_channel_execution, "get_task_event_bridge", lambda: MagicMock()
    )
    monkeypatch.setattr(
        "xagent.web.services.execution_result_projection."
        "project_execution_result_for_channel",
        lambda _result: SimpleNamespace(
            task_status=TaskStatus.COMPLETED,
            transcript_content="ok",
            interactions=[],
            message_type="text",
            diagnostic_error=None,
        ),
    )
    monkeypatch.setattr(
        shared_channel_execution, "channel_durable_result", lambda *_a: {}
    )
    monkeypatch.setattr(
        "xagent.web.services.managed_task_lease."
        "finalize_managed_task_lease_result_sync",
        finalize or Mock(return_value=True),
    )
    monkeypatch.setattr(
        shared_channel_execution,
        "run_db_io_cancellation_safe",
        AsyncMock(return_value=True),
    )

    snapshot = SimpleNamespace(
        runtime_user=None,
        conversation_history=(),
        conversation_watermark=None,
        conversation_event_watermark=None,
        execution_recovery=TaskExecutionRecoverySnapshot(),
        task=SimpleNamespace(source="internal", user_id=5),
    )
    await shared_channel_execution.execute_channel_background(
        command=SimpleNamespace(task_id=42, command_id="cmd-1", id=1),
        lease=lease(),
        heartbeat_task=None,
        snapshot=snapshot,
        payload=SimpleNamespace(
            for_agent="hello", transcript_message="hello", attachments=[], file_ids=()
        ),
        turn_facts=TurnFacts(started_at=datetime.now(timezone.utc)),
    )


async def drive_resume_background(
    manager: Any, path: str, monkeypatch: pytest.MonkeyPatch
) -> None:
    outcome = agent_outcome(path)
    heartbeat_stop = asyncio.Event()

    async def heartbeat() -> Any:
        await heartbeat_stop.wait()
        return None

    heartbeat_task = asyncio.create_task(heartbeat())
    service = MagicMock()
    service.resume_execution_by_id = AsyncMock(
        side_effect=outcome if isinstance(outcome, BaseException) else None,
        return_value=outcome if isinstance(outcome, dict) else None,
    )
    monkeypatch.setattr(agent_service_manager, "get_agent_manager", lambda: manager)
    finalized = {
        "task_title": "t",
        "task_description": "x",
        "task_execution_mode": "flash",
        "task_agent_id": None,
        "agent_name": None,
        "agent_logo_url": None,
        "final_status": TaskStatus.COMPLETED.value,
        "lease_released": True,
        "report": SettlementReport(),
        "normalized_outputs": [],
        "output": "ok",
        "late_result": False,
    }
    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                new=MagicMock(),
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=MagicMock(),
            ),
            patch(
                "xagent.web.services.task_execution._finalize_resumed_task",
                return_value=finalized,
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease",
                return_value=True,
            ),
            patch(
                "xagent.web.services.task_execution.load_routine_history_turn_id",
                return_value=None,
            ),
            patch(
                "xagent.web.api.websocket.manager",
                MagicMock(broadcast_to_task=AsyncMock()),
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager"
                ".promote_resume_task"
            ),
        ]
    ):
        try:
            await execute_resume_background(
                task_id=42,
                agent_service=service,
                task_owner_user_id=1,
                expected_run_id="run-a",
                resolved_execution_scope=None,
                preacquired_lease=lease(),
                preacquired_heartbeat_stop=heartbeat_stop,
                preacquired_heartbeat_task=heartbeat_task,
            )
        finally:
            heartbeat_stop.set()
            await asyncio.gather(heartbeat_task, return_exceptions=True)


def make_driver(
    entry: str, monkeypatch: pytest.MonkeyPatch
) -> Callable[[Any, str], Awaitable[None]]:
    if entry == "execute_task_background":
        return drive_task_background
    if entry == "execute_channel_background":
        return lambda manager, path: drive_channel_background(
            manager, path, monkeypatch
        )
    return lambda manager, path: drive_resume_background(manager, path, monkeypatch)


async def run_and_capture(
    driver: Callable[[Any, str], Awaitable[None]], manager: Any, path: str
) -> type[BaseException] | None:
    try:
        await driver(manager, path)
    except BaseException as error:  # noqa: BLE001 -- the outcome is the datum
        return type(error)
    return None


@pytest.mark.parametrize("entry", ENTRIES)
@pytest.mark.parametrize("path", PATHS)
async def test_settled_hook_is_best_effort(entry, path, monkeypatch, caplog) -> None:
    driver = make_driver(entry, monkeypatch)

    working = Mock()
    baseline = await run_and_capture(driver, working, path)

    failing = Mock(note_runtime_settled=Mock(side_effect=RuntimeError("hook broke")))
    with caplog.at_level(logging.WARNING):
        caplog.clear()
        outcome = await run_and_capture(driver, failing, path)

    # The hook failing changes nothing about how the turn ended.
    assert outcome is baseline
    assert baseline is expected_outcome(entry, path)
    failing.note_runtime_settled.assert_called_once_with(42)
    warnings = [r for r in caplog.records if HOOK_WARNING in r.getMessage()]
    assert len(warnings) == 1
    assert warnings[0].levelno == logging.WARNING


@pytest.mark.parametrize("entry", ENTRIES)
@pytest.mark.parametrize("path", PATHS)
async def test_settled_hook_called_once_from_each_entrypoint(
    entry, path, monkeypatch
) -> None:
    manager = Mock()

    await run_and_capture(make_driver(entry, monkeypatch), manager, path)

    manager.note_runtime_settled.assert_called_once_with(42)


@pytest.mark.parametrize(
    "failing_stage",
    [
        "set_conversation_history",
        "materialize",
        "set_execution_context_messages",
        "set_recovered_skill_context",
        "execute_task",
    ],
)
async def test_channel_hook_runs_when_a_stage_before_the_result_fails(
    failing_stage, monkeypatch
) -> None:
    manager = Mock()

    with pytest.raises(RuntimeError, match=failing_stage):
        await drive_channel_background(
            manager, "normal", monkeypatch, fail_before_execute=failing_stage
        )

    manager.note_runtime_settled.assert_called_once_with(42)


async def test_many_settled_turns_stay_bounded() -> None:
    now = [1000.0]
    manager = AgentServiceManager()
    manager._monotonic = lambda: now[0]
    manager.execute_task = AsyncMock(return_value=dict(OK_RESULT))

    async def get_agent_for_task(task_id: int, *_args: Any, **_kwargs: Any) -> Any:
        service = _build_fake_agent_service()
        service.get_execution_status = MagicMock(return_value=None)
        manager._agents[task_id] = service
        manager._agent_build_locks[task_id] = asyncio.Lock()
        manager._agent_run_generations[task_id] = task_id
        manager._agent_last_active_at[task_id] = manager._monotonic()
        return service

    manager.get_agent_for_task = get_agent_for_task  # type: ignore[method-assign]
    floor = agent_service_manager._IDLE_RUNTIME_SOFT_CAP_MIN_AGE_SECONDS
    max_age = agent_service_manager._IDLE_RUNTIME_MAX_AGE_SECONDS
    cap = agent_service_manager._IDLE_RUNTIME_SOFT_CAP
    peak_unexpired = 0

    with _Patches(_common_patches(MagicMock(), None)):
        for task_id in range(1, 501):
            await execute_task_background(
                task_id=task_id,
                user_message="hi",
                context={},
                agent_manager=manager,
                task_owner_user_id=1,
                task_setup_snapshot=_make_snapshot(),
                task_lease=lease(),
                resolved_execution_scope=None,
            )
            idle = [now[0] - manager._agent_last_active_at[t] for t in manager._agents]
            # Only runtimes past the soft cap's age floor are bounded by it.
            assert sum(1 for i in idle if floor <= i < max_age) <= cap
            peak_unexpired = max(peak_unexpired, sum(1 for i in idle if i < max_age))
            now[0] += 0.5

    # The age rule alone would keep every runtime younger than ``max_age``;
    # holding more than the cap proves the cap is what bounds the older ones.
    assert peak_unexpired > cap
    now[0] += 120
    manager._sweep_idle_runtimes(reason="test")
    assert manager._agents == {}
    assert manager._agent_build_locks == {}