Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

24.3 KB
Raw Blame
import asyncio
from types import SimpleNamespace

import pytest
from pydantic import ValidationError

from xagent.core.agent.budget import (
    ExecutionBudget,
    ExecutionBudgetPolicy,
    active_execution_budget,
    budget_llm_kwargs,
    budget_warning_handler,
)
from xagent.core.agent.context import ExecutionContext
from xagent.core.agent.runner import AgentRunner
from xagent.core.agent.runtime import (
    LLMCallInterrupted,
    PatternRuntime,
    ToolCallInterrupted,
)
from xagent.core.model.chat.token_context import (
    TokenContextManager,
    add_token_usage,
    token_usage_observer,
)
from xagent.core.model.chat.types import ChunkType, StreamChunk


class MeteredLLM:
    def __init__(self, tokens=30):
        self.tokens = tokens
        self.calls = []

    async def chat(self, **kwargs):
        self.calls.append(kwargs)
        add_token_usage(
            input_tokens=self.tokens - 1, output_tokens=1, cached_input_tokens=10
        )
        return {"content": "result"}


class LoopPattern:
    status = "running"

    async def run(self, context, runtime, llm=None, **kwargs):
        self.runtime = runtime
        self.context = context
        while not await runtime.should_interrupt():
            await runtime.run_llm_call(
                llm, messages=[{"role": "user", "content": "work"}]
            )
            await runtime.checkpoint("after_call", context=context, pattern=self)
        return {"status": "interrupted", "success": False}


def runner(pattern, llm, policy):
    return AgentRunner(
        SimpleNamespace(patterns=[pattern], tools=[], llm=llm),
        workspace_enabled=False,
        budget_policy_provider=lambda: policy,
    )


@pytest.mark.asyncio
async def test_real_runner_stops_and_warns_once_without_extra_summary_call():
    llm = MeteredLLM()
    pattern = LoopPattern()
    result = await runner(
        pattern, llm, ExecutionBudgetPolicy(max_tokens=100, soft_limit_percent=50)
    ).run("work", outbound_message_handler=lambda payload: None)
    assert len(llm.calls) == 4  # last admitted call overshoots; no fifth call
    assert "Execution token budget" not in str(llm.calls[0])
    assert "Execution token budget" in str(llm.calls[2])
    assert len(pattern.runtime.outbound_messages) == 1
    assert result["termination_reason"] == "token_budget"
    assert result["completion_outcome"] == "blocked"
    assert result["context"].execution_budget["used_tokens"] == 120
    assert pattern.runtime.last_checkpoint["context"]["execution_budget"]["closed"]
    assert active_execution_budget.get() is None
    assert token_usage_observer.get() is None
    assert budget_warning_handler.get() is None


@pytest.mark.asyncio
async def test_unlimited_run_does_not_add_budget_only_tail_checkpoint():
    class RecordingPattern:
        async def run(self, context, runtime, llm, **kwargs):
            self.runtime = runtime
            await runtime.run_llm_call(llm, messages=[])
            context.add_assistant_message("recorded")
            await runtime.checkpoint("final", context=context, pattern=self)
            return {"success": True, "output": "recorded"}

    pattern = RecordingPattern()
    llm = MeteredLLM()
    result = await runner(pattern, llm, ExecutionBudgetPolicy()).run("work")

    assert result["success"]
    assert len(llm.calls) == 1
    assert result["context"].execution_budget is None
    assert pattern.runtime.last_checkpoint["label"] == "final"
    assert pattern.runtime.last_checkpoint["context"]["execution_budget"] is None
    assert not pattern.runtime.outbound_messages


@pytest.mark.asyncio
async def test_resume_cannot_reset_or_relax_budget_but_new_completed_turn_can():
    old_context = ExecutionContext()
    old_context.add_user_message("work", metadata={"turn_id": "one"})
    old_context.execution_budget = ExecutionBudget(
        policy=ExecutionBudgetPolicy(max_tokens=100), used_tokens=100, turn_id="one"
    ).model_dump()
    llm = MeteredLLM()
    pattern = LoopPattern()
    first = await runner(pattern, llm, ExecutionBudgetPolicy(max_tokens=1000)).run(
        None, checkpoint={"context": old_context.to_dict()}
    )
    assert not llm.calls
    assert first["context"].execution_budget["policy"]["max_tokens"] == 100
    completed_context = first["context"]
    completed_context.add_user_message("next", metadata={"turn_id": "two"})
    second = await runner(LoopPattern(), llm, ExecutionBudgetPolicy(max_tokens=60)).run(
        None, checkpoint={"context": completed_context.to_dict()}
    )
    assert len(llm.calls) == 2
    assert second["context"].execution_budget["used_tokens"] == 60


@pytest.mark.asyncio
async def test_user_metadata_cannot_supply_budget_or_reset_consumption():
    pattern = LoopPattern()
    result = await runner(
        pattern, MeteredLLM(), ExecutionBudgetPolicy(max_tokens=30)
    ).run(
        "work",
        metadata={
            "execution_budget": {"used_tokens": 0, "policy": {"max_tokens": 99999}},
            "request_context": {"execution_budget": None},
        },
    )
    assert result["context"].execution_budget["used_tokens"] == 30


@pytest.mark.asyncio
async def test_existing_quota_gate_takes_precedence():
    llm = MeteredLLM()
    result = await runner(LoopPattern(), llm, ExecutionBudgetPolicy(max_tokens=30)).run(
        "work", interrupt_checker=lambda: "quota unavailable"
    )
    assert result["status"] == "interrupted"
    assert "termination_reason" not in result
    assert not llm.calls


@pytest.mark.asyncio
async def test_observer_counts_nested_contexts_and_cached_input_once():
    budget = ExecutionBudget(policy=ExecutionBudgetPolicy(max_tokens=100))
    token = token_usage_observer.set(budget.record_usage)
    try:
        with TokenContextManager():
            add_token_usage(input_tokens=30, output_tokens=5, cached_input_tokens=20)
            with TokenContextManager():
                add_token_usage(input_tokens=40, output_tokens=10)
        assert budget.used_tokens == 85
    finally:
        token_usage_observer.reset(token)


@pytest.mark.asyncio
async def test_exhausted_budget_blocks_llm_and_tool_before_invocation():
    budget = ExecutionBudget(policy=ExecutionBudgetPolicy(max_tokens=1), used_tokens=1)
    token = active_execution_budget.set(budget)
    try:
        llm = MeteredLLM()
        with pytest.raises(LLMCallInterrupted):
            await PatternRuntime().run_llm_call(llm, messages=[])
        with pytest.raises(ToolCallInterrupted):
            await PatternRuntime().run_tool_call(
                lambda: pytest.fail("tool must not run")
            )
        assert not llm.calls
    finally:
        active_execution_budget.reset(token)


@pytest.mark.parametrize(
    "values",
    [
        {"max_tokens": 0},
        {"max_tokens": -1},
        {"max_tokens": True},
        {"soft_limit_percent": 100},
        {"soft_limit_percent": 0},
    ],
)
def test_invalid_policies_are_rejected(values):
    with pytest.raises(ValidationError):
        ExecutionBudgetPolicy(**values)


def test_context_snapshot_does_not_share_budget_containers():
    context = ExecutionContext(
        execution_budget=ExecutionBudget(
            policy=ExecutionBudgetPolicy(max_tokens=100)
        ).model_dump()
    )
    payload = context.to_dict()
    context.execution_budget["used_tokens"] = 50
    assert ExecutionContext.from_dict(payload).execution_budget["used_tokens"] == 0


@pytest.mark.asyncio
async def test_nested_runner_cannot_get_a_separate_allowance():
    llm = MeteredLLM(tokens=30)

    class ParentPattern:
        async def run(self, context, runtime, **kwargs):
            await runtime.run_llm_call(llm, messages=[])
            child = runner(LoopPattern(), llm, ExecutionBudgetPolicy(max_tokens=99999))
            child_result = await child.run("child work")
            assert child_result["termination_reason"] == "token_budget"
            assert not active_execution_budget.get().closed
            with pytest.raises(LLMCallInterrupted):
                await runtime.run_llm_call(llm, messages=[])
            return {"success": False, "status": "interrupted"}

    result = await runner(
        ParentPattern(), llm, ExecutionBudgetPolicy(max_tokens=60)
    ).run("parent work")
    assert len(llm.calls) == 2
    assert result["context"].execution_budget["used_tokens"] == 60


@pytest.mark.asyncio
async def test_parallel_calls_share_usage_and_next_batch_is_stopped():
    llm = MeteredLLM(tokens=30)

    class ParallelPattern:
        async def run(self, context, runtime, **kwargs):
            await asyncio.gather(
                *(runtime.run_llm_call(llm, messages=[]) for _ in range(3))
            )
            with pytest.raises(LLMCallInterrupted):
                await runtime.run_llm_call(llm, messages=[])
            return {"success": False, "status": "interrupted"}

    result = await runner(
        ParallelPattern(), llm, ExecutionBudgetPolicy(max_tokens=60)
    ).run("parallel")
    assert len(llm.calls) == 3  # all three were admitted before usage was reported
    assert result["context"].execution_budget["used_tokens"] == 90


@pytest.mark.asyncio
async def test_budget_blocks_native_stream_before_provider_is_called():
    class StreamLLM:
        async def stream_chat(self, **kwargs):
            pytest.fail("exhausted budget must not call streaming provider")
            yield None

    token = active_execution_budget.set(
        ExecutionBudget(policy=ExecutionBudgetPolicy(max_tokens=1), used_tokens=1)
    )
    try:
        with pytest.raises(LLMCallInterrupted):
            await PatternRuntime().run_streaming_llm_call(StreamLLM(), messages=[])
    finally:
        active_execution_budget.reset(token)


@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["react", "dag"])
async def test_real_patterns_stop_before_executing_over_budget_tool(mode):
    from tests.core.agent.test_react import FakeTool
    from xagent.core.agent import DAGPattern, ExecutionPlan, PlanStep, ReActPattern

    class WorkingLLM(MeteredLLM):
        async def chat(self, **kwargs):
            await super().chat(**kwargs)
            return {
                "tool_calls": [
                    {
                        "id": str(len(self.calls)),
                        "name": "calculator",
                        "args": {"expression": "1 + 1"},
                    }
                ]
            }

    llm = WorkingLLM(tokens=30)
    tool = FakeTool()
    if mode == "react":
        pattern = ReActPattern(max_iterations=10)
    else:
        pattern = DAGPattern(
            plan_generator=lambda **kwargs: ExecutionPlan(
                steps=[
                    PlanStep(id="first", task="Calculate", tool_names=["calculator"])
                ]
            ),
            max_concurrency=1,
        )
    agent_runner = runner(
        pattern, llm, ExecutionBudgetPolicy(max_tokens=60, soft_limit_percent=50)
    )
    agent_runner.agent.tools = [tool]
    result = await agent_runner.run(
        "Use calculator repeatedly", metadata={"output_language": "Chinese"}
    )
    assert result.get("termination_reason") == "token_budget", result
    assert len(llm.calls) == 2
    assert len(tool.calls) == 1
    assert result["context"].execution_budget["used_tokens"] == 60
    assert "本次执行" in result["output"]


@pytest.mark.asyncio
async def test_budget_stop_hands_over_registered_files_only():
    from unittest.mock import AsyncMock

    agent_runner = runner(
        LoopPattern(), MeteredLLM(), ExecutionBudgetPolicy(max_tokens=100)
    )
    runtime = PatternRuntime(budget_owner=True)
    context = ExecutionContext()
    workspace = SimpleNamespace(
        get_output_files=lambda: [
            {"file_id": "saved-file", "filename": "result.csv"},
            {"file_id": None, "filename": "unregistered.csv"},
        ]
    )
    runtime.checkpoint = AsyncMock()
    runtime.checkpoint_context_tail = AsyncMock()
    result = await agent_runner._finish_budget_stop(
        context, runtime, LoopPattern(), workspace
    )
    assert "[result.csv](file:saved-file)" in result["output"]
    assert "unregistered.csv" not in result["output"]
    assert result["completion_outcome"] == "partial"


@pytest.mark.asyncio
async def test_budget_stop_includes_real_workspace_root_but_not_inputs_or_temp(
    tmp_path,
):
    from unittest.mock import AsyncMock

    from xagent.core.workspace import TaskWorkspace

    workspace = TaskWorkspace("budget-stop", base_dir=str(tmp_path))
    for directory, name in (
        (workspace.workspace_dir, "root-result.csv"),
        (workspace.output_dir, "output-result.csv"),
        (workspace.input_dir, "source.csv"),
        (workspace.temp_dir, "scratch.csv"),
    ):
        path = directory / name
        path.write_text("value\n1\n")
        workspace.register_file(str(path))
    (workspace.workspace_dir / "unregistered.csv").write_text("value\n2\n")
    agent_runner = runner(
        LoopPattern(), MeteredLLM(), ExecutionBudgetPolicy(max_tokens=100)
    )
    runtime = PatternRuntime(budget_owner=True)
    runtime.checkpoint = AsyncMock()
    runtime.checkpoint_context_tail = AsyncMock()

    result = await agent_runner._finish_budget_stop(
        ExecutionContext(), runtime, LoopPattern(), workspace
    )

    for name in ("root-result.csv", "output-result.csv"):
        assert f"[{name}](file:" in result["output"]
    for name in ("source.csv", "scratch.csv", "unregistered.csv"):
        assert name not in result["output"]
    assert result["completion_outcome"] == "partial"


@pytest.mark.asyncio
async def test_final_answer_already_paid_for_is_not_discarded_at_limit():
    from xagent.core.agent import ReActPattern

    class AnswerLLM(MeteredLLM):
        async def chat(self, **kwargs):
            await super().chat(**kwargs)
            return {
                "tool_calls": [
                    {
                        "id": "final",
                        "name": "final_answer",
                        "args": {"answer": "The answer is 42.", "outcome": "completed"},
                    }
                ]
            }

    llm = AnswerLLM(tokens=30)
    result = await runner(
        ReActPattern(), llm, ExecutionBudgetPolicy(max_tokens=20)
    ).run("Answer directly")
    assert result["success"]
    assert result.get("termination_reason") != "token_budget"
    assert "42" in str(result)
    assert len(llm.calls) == 1


@pytest.mark.asyncio
async def test_policy_failure_stops_before_any_calls_and_cleans_context():
    llm = MeteredLLM()
    agent_runner = runner(LoopPattern(), llm, ExecutionBudgetPolicy())

    async def unavailable():
        raise RuntimeError("policy unavailable")

    agent_runner.budget_policy_provider = unavailable
    with pytest.raises(RuntimeError, match="policy unavailable"):
        await agent_runner.run("work")
    assert not llm.calls
    assert active_execution_budget.get() is None
    assert token_usage_observer.get() is None


@pytest.mark.asyncio
async def test_pause_and_new_runner_resume_keep_recorded_consumption():
    llm = MeteredLLM(tokens=30)

    class PausePattern:
        async def run(self, context, runtime, **kwargs):
            self.runtime = runtime
            await runtime.run_llm_call(llm, messages=[])
            await runtime.checkpoint(
                "paused", context=context, pattern=self, status="interrupted"
            )
            return {"success": False, "status": "interrupted"}

    pattern = PausePattern()
    await runner(pattern, llm, ExecutionBudgetPolicy(max_tokens=60)).run("work")
    saved = pattern.runtime.last_checkpoint
    assert saved["context"]["execution_budget"]["used_tokens"] == 30
    result = await runner(LoopPattern(), llm, ExecutionBudgetPolicy(max_tokens=60)).run(
        None, checkpoint=saved
    )
    assert len(llm.calls) == 2
    assert result["context"].execution_budget["used_tokens"] == 60


@pytest.mark.asyncio
async def test_new_user_turn_after_failed_checkpoint_gets_fresh_budget():
    context = ExecutionContext()
    context.add_user_message("new request", metadata={"turn_id": "new-turn"})
    context.execution_budget = ExecutionBudget(
        policy=ExecutionBudgetPolicy(max_tokens=30), used_tokens=30, turn_id="old-turn"
    ).model_dump()
    llm = MeteredLLM()
    result = await runner(LoopPattern(), llm, ExecutionBudgetPolicy(max_tokens=30)).run(
        None, checkpoint={"context": context.to_dict(), "status": "failed"}
    )
    assert len(llm.calls) == 1
    assert result["context"].execution_budget["turn_id"] == "new-turn"


@pytest.mark.asyncio
@pytest.mark.parametrize("held_kind", ["llm", "tool", "stream"])
async def test_budget_refusal_does_not_cancel_an_admitted_sibling(held_kind):
    entered, release = asyncio.Event(), asyncio.Event()
    budget = ExecutionBudget(policy=ExecutionBudgetPolicy(max_tokens=30))
    token = active_execution_budget.set(budget)
    usage_token = token_usage_observer.set(budget.record_usage)
    runtime = PatternRuntime()

    async def held_call(**kwargs):
        entered.set()
        await release.wait()
        add_token_usage(input_tokens=10, output_tokens=1)
        return "paid result"

    class HeldStream:
        async def stream_chat(self, **kwargs):
            result = await held_call(**kwargs)
            yield StreamChunk(type=ChunkType.TOKEN, delta=result)
            yield StreamChunk(type=ChunkType.END)

    try:
        if held_kind == "stream":
            call = runtime.run_streaming_llm_call(HeldStream(), messages=[])
        elif held_kind == "llm":
            call = runtime.run_llm_call(SimpleNamespace(chat=held_call), messages=[])
        else:
            call = runtime.run_tool_call(held_call)
        held = asyncio.create_task(call)
        await asyncio.wait_for(entered.wait(), timeout=5)
        await runtime.run_llm_call(MeteredLLM(), messages=[])
        with pytest.raises(LLMCallInterrupted):
            await runtime.run_llm_call(MeteredLLM(), messages=[])
        with pytest.raises(ToolCallInterrupted):
            await runtime.run_tool_call(lambda: pytest.fail("new work admitted"))
        assert runtime.budget_stopped
        assert not await runtime.should_interrupt()
        assert not held.done()
        release.set()
        result = await held
        assert result == "paid result"
        assert budget.used_tokens == 41
    finally:
        release.set()
        await asyncio.gather(held, return_exceptions=True)
        token_usage_observer.reset(usage_token)
        active_execution_budget.reset(token)


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "language", ["English", "Simplified Chinese", "Traditional Chinese"]
)
async def test_nested_soft_warning_is_delivered_by_owner_in_its_language(language):
    messages = []

    class ChildPattern:
        async def run(self, context, runtime, **kwargs):
            assert runtime.outbound_message_handler is None
            await asyncio.gather(
                runtime.run_tool_call(lambda: None),
                runtime.run_tool_call(lambda: None),
            )
            assert not runtime.outbound_messages
            return {"success": True, "output": "child done"}

    class ParentPattern:
        async def run(self, context, runtime, llm, **kwargs):
            await runtime.run_llm_call(llm, messages=[])
            await runner(ChildPattern(), llm, ExecutionBudgetPolicy()).run("child")
            await runtime.run_tool_call(lambda: None)
            assert active_execution_budget.get().soft_notified
            return {"success": True, "output": "done"}

    async def deliver(payload):
        await asyncio.sleep(0)
        messages.append(payload)

    await runner(
        ParentPattern(),
        MeteredLLM(60),
        ExecutionBudgetPolicy(max_tokens=100, soft_limit_percent=50),
    ).run(
        "work", metadata={"output_language": language}, outbound_message_handler=deliver
    )
    assert len(messages) == 1
    assert messages[0]["metadata"]["kind"] == "budget_warning"
    assert "60" in messages[0]["message"]
    expected = {
        "English": "approaching",
        "Simplified Chinese": "本次执行",
        "Traditional Chinese": "本次執行",
    }
    assert expected[language] in messages[0]["message"]
    assert "Prioritize" not in messages[0]["message"]


@pytest.mark.asyncio
async def test_undelivered_soft_warning_does_not_fail_work_or_consume_notice(caplog):
    budget = ExecutionBudget(
        policy=ExecutionBudgetPolicy(max_tokens=100), used_tokens=80
    )
    token = active_execution_budget.set(budget)
    delivered = []

    def broken_handler(payload):
        raise RuntimeError("publish failed")

    runtime = PatternRuntime()
    try:
        assert await runtime.run_tool_call(lambda: "first") == "first"
        assert not budget.soft_notified
        runtime.outbound_message_handler = broken_handler
        assert await runtime.run_tool_call(lambda: "second") == "second"
        assert not budget.soft_notified
        assert "Could not deliver execution budget warning" in caplog.text
        runtime.outbound_message_handler = delivered.append
        assert await runtime.run_tool_call(lambda: "third") == "third"
        assert budget.soft_notified
        assert len(delivered) == 1
    finally:
        active_execution_budget.reset(token)


def test_soft_limit_prompt_prefix_stays_stable_as_usage_grows():
    budget = ExecutionBudget(
        policy=ExecutionBudgetPolicy(max_tokens=100), used_tokens=80
    )
    token = active_execution_budget.set(budget)
    original = {
        "messages": [
            {"role": "system", "content": "stable"},
            {"role": "user", "content": "work"},
        ]
    }
    try:
        first = budget_llm_kwargs(original)
        budget.record_usage(5, 5)
        assert budget_llm_kwargs(original) == first
        assert original["messages"][0]["content"] == "stable"
    finally:
        active_execution_budget.reset(token)


@pytest.mark.asyncio
@pytest.mark.parametrize("reason", ["user_stop", "sibling_usage"])
async def test_warning_delivery_rechecks_admission_after_yield(reason):
    entered, release = asyncio.Event(), asyncio.Event()
    budget = ExecutionBudget(
        policy=ExecutionBudgetPolicy(max_tokens=100), used_tokens=80
    )
    token = active_execution_budget.set(budget)

    async def deliver(payload):
        entered.set()
        await release.wait()

    runtime = PatternRuntime(outbound_message_handler=deliver)
    work = asyncio.create_task(
        runtime.run_tool_call(lambda: pytest.fail("work admitted after stop"))
    )
    try:
        await asyncio.wait_for(entered.wait(), timeout=5)
        if reason == "user_stop":
            runtime.request_interrupt("user stopped")
        else:
            budget.record_usage(20, 0)
        release.set()
        with pytest.raises(ToolCallInterrupted):
            await work
        assert runtime.budget_stopped == (reason == "sibling_usage")
    finally:
        release.set()
        await asyncio.gather(work, return_exceptions=True)
        active_execution_budget.reset(token)


@pytest.mark.asyncio
async def test_budget_handoff_excludes_old_turn_files_but_keeps_resumed_turn_files(
    tmp_path,
):
    import os
    from unittest.mock import AsyncMock

    from xagent.core.workspace import TaskWorkspace

    workspace = TaskWorkspace("turn-files", base_dir=str(tmp_path))
    path = workspace.output_dir / "result.csv"
    path.write_text("old\n")
    workspace.register_file(str(path))
    os.utime(path, (100, 100))
    budget = ExecutionBudget(
        policy=ExecutionBudgetPolicy(max_tokens=10), started_at=200
    )
    runtime = PatternRuntime(budget_owner=True)
    runtime.checkpoint = AsyncMock()
    runtime.checkpoint_context_tail = AsyncMock()
    agent_runner = runner(LoopPattern(), MeteredLLM(), budget.policy)
    token = active_execution_budget.set(budget)
    try:
        result = await agent_runner._finish_budget_stop(
            ExecutionContext(), runtime, LoopPattern(), workspace
        )
        assert result["completion_outcome"] == "blocked"
        assert "result.csv" not in result["output"]
        path.write_text("updated\n")
        os.utime(path, (300, 300))
        # A new process restores the original turn boundary, not the resume time.
        resumed = ExecutionBudget.model_validate(budget.model_dump())
        active_execution_budget.set(resumed)
        result = await agent_runner._finish_budget_stop(
            ExecutionContext(), runtime, LoopPattern(), workspace
        )
        assert result["completion_outcome"] == "partial"
        assert "[result.csv](file:" in result["output"]
        assert resumed.started_at == 200
    finally:
        active_execution_budget.reset(token)