Enterprise agent platform for building, deploying, and operating AI teammates.
from __future__ import annotations
import argparse
import asyncio
import json
import sys
import pytest
from tokenizers import Tokenizer
from tokenizers.models import WordLevel
from tokenizers.pre_tokenizers import WhitespaceSplit
from scripts.benchmarks import agent_cache_benchmark
from scripts.benchmarks.agent_cache_benchmark import (
AuditTool,
AuditWorkload,
BenchmarkLLM,
build_context,
cache_summary,
invoice_amount,
)
from xagent.core.agent import PatternRuntime
from xagent.core.agent.trace import (
TraceAction,
TraceCategory,
TraceEventType,
TraceScope,
)
from xagent.core.model.chat.basic.xinference import XinferenceLLM
from xagent.core.model.chat.types import ChunkType, StreamChunk
@pytest.mark.parametrize("compaction", [False, True])
def test_context_setup_keeps_audit_request_and_recovery_guidance(compaction):
context = build_context("reference-ledger", 42, 131072, compaction=compaction)
request = context.current_user_request_text()
assert "Audit the invoice chain starting with invoice-000042" in request
assert ("read_ledger_invoice" in request) is compaction
assert len(context.messages) == 2
assert "reference-ledger" in context.messages[0].content
def test_audit_requires_real_ordered_receipts_and_retries(tmp_path):
workload = AuditWorkload([1, 100], tmp_path)
assert (
workload.save({"receipts": [], "total_cents": workload.total})["success"]
is False
)
assert (
workload.check(
{
"invoice_id": "invoice-000001",
"amount_cents": 0,
"previous_receipt": "START",
}
)["success"]
is False
)
first = workload.check(
{
"invoice_id": "invoice-000001",
"amount_cents": invoice_amount(1),
"previous_receipt": "START",
}
)
args = {
"invoice_id": "invoice-000100",
"amount_cents": invoice_amount(100),
"previous_receipt": first["receipt"],
}
assert workload.check(args)["retryable"] is True
assert len(workload.receipts) == 1
assert workload.check(args)["success"] is True
assert (
workload.save(
{
"receipts": list(reversed(workload.receipts)),
"total_cents": workload.total,
}
)["success"]
is False
)
result = workload.save(
{"receipts": workload.receipts, "total_cents": workload.total}
)
assert result["saved_to_workspace"] is True
assert json.loads((tmp_path / "audit.json").read_text())["verified"] is True
def test_cache_summary_distinguishes_missing_zero_and_weighted_hits():
assert (
cache_summary([{"usage": {"prompt_tokens": 10}}])["reported_cached_tokens"]
is None
)
missing = {"usage": {"prompt_tokens": 100}}
cold = {
"usage": {"prompt_tokens": 10, "prompt_tokens_details": {"cached_tokens": 0}}
}
hit = {
"usage": {"prompt_tokens": 90, "prompt_tokens_details": {"cached_tokens": 80}},
"adapter_first_payload_s": 2.0,
}
result = cache_summary([missing, cold, hit])
assert result["cache_usage_samples"] == 2
assert result["cached_input_fraction"] == 0.8
assert result["cache_hit_first_payload_median_s"] == 2.0
@pytest.mark.asyncio
@pytest.mark.parametrize("save_audit", [False, True])
async def test_benchmark_run_reconstructs_audit_and_compaction_state(
tmp_path, monkeypatch, save_audit
):
workloads = []
class TrackingWorkload(AuditWorkload):
def __init__(self, **kwargs):
super().__init__(**kwargs)
workloads.append(self)
class AuditLLM:
model_name = "fixture"
def __init__(self, *, context_window, **kwargs):
self.context_window = context_window
self.calls = []
self.summed = False
async def close(self):
pass
async def chat(self, **kwargs):
workload = workloads[-1]
position = len(workload.receipts)
if workload.saved or (self.summed and not save_audit):
name = "final_answer"
args = {
"answer": "Audit saved and verified.",
"response_language": "English",
}
elif position == len(workload.targets) and not self.summed:
name = "sum_amounts"
args = {"amounts_cents": [invoice_amount(i) for i in workload.targets]}
self.summed = True
elif position == len(workload.targets):
name = "save_audit"
args = {"receipts": workload.receipts, "total_cents": workload.total}
else:
name = "check_invoice"
args = {
"invoice_id": f"invoice-{workload.targets[position]:06d}",
"amount_cents": invoice_amount(workload.targets[position]),
"previous_receipt": workload.receipts[-1] if position else "START",
}
return {
"content": "",
"tool_calls": [
{
"id": f"call-{position}-{len(workload.attempts)}-{name}",
"function": {"name": name, "arguments": json.dumps(args)},
}
],
}
forced_thresholds = []
async def compact(runtime, *, context, **kwargs):
if context.compact_config.threshold == 8192:
forced_thresholds.append(context.compact_config.threshold)
await runtime.tracer.trace_event(
TraceEventType(
TraceScope.ACTION, TraceAction.END, TraceCategory.COMPACT
),
data={"removed_count": 1},
)
monkeypatch.setattr(agent_cache_benchmark, "AuditWorkload", TrackingWorkload)
monkeypatch.setattr(agent_cache_benchmark, "BenchmarkLLM", AuditLLM)
monkeypatch.setattr(PatternRuntime, "compact_context_if_needed", compact)
tokenizer = Tokenizer(WordLevel({"[UNK]": 0}, unk_token="[UNK]"))
tokenizer.pre_tokenizer = WhitespaceSplit()
tokenizer_file = tmp_path / "tokenizer.json"
tokenizer.save(str(tokenizer_file))
output = tmp_path / "audit"
report = await agent_cache_benchmark.run(
argparse.Namespace(
output=output,
tokenizer_json=tokenizer_file,
model="fixture",
endpoint="http://127.0.0.1:9997",
api_key_file=None,
context_window=16384,
context_tokens=4200,
rounds=3,
fail_at=2,
resume_after=2,
compact_after=1,
timeout=30,
)
)
assert report["passed"] is save_audit
assert report["resumed"] and report["compaction_attempted"]
assert report["transient_failure_exercised"] and report["sum_tool_exercised"]
assert report["verified_invoices"] == 3
assert report["tool_attempts"] == 4
assert forced_thresholds == [8192]
assert len(report["compactions"]) == 1
original, resumed = workloads
assert original is not resumed and original.receipts is not resumed.receipts
saved = json.loads((output / "audit-state.json").read_text())
assert saved["compaction_attempted"] is True
assert saved["receipts"] == original.receipts == resumed.receipts[:2]
assert (output / "resume-checkpoint.json").is_file()
assert (output / "audit.json").exists() is save_audit
events = [
json.loads(line) for line in (output / "events.jsonl").read_text().splitlines()
]
assert any(
e.get("prompt_prefix", {}).get("change") == "append_only" for e in events
)
tool = AuditTool(resumed, read=True)
assert await tool.run_json_async({"invoice_id": "invoice-000001"}) == {
"success": True,
"invoice_id": "invoice-000001",
"amount_cents": invoice_amount(1),
}
@pytest.mark.asyncio
async def test_benchmark_records_cancelled_and_closed_calls(tmp_path, monkeypatch):
async def chat(self, **kwargs):
raise asyncio.CancelledError()
async def stream(self, **kwargs):
yield StreamChunk(type=ChunkType.TOKEN, content="partial")
monkeypatch.setattr(XinferenceLLM, "chat", chat)
monkeypatch.setattr(XinferenceLLM, "stream_chat", stream)
llm = BenchmarkLLM(output=tmp_path, context_window=16384)
with pytest.raises(asyncio.CancelledError):
await llm.chat()
generator = llm.stream_chat()
await anext(generator)
await generator.aclose()
assert [(row["success"], row["error_type"]) for row in llm.calls] == [
(False, "CancelledError"),
(False, "GeneratorExit"),
]
def test_benchmark_cli_rejects_out_of_range_resume(monkeypatch, tmp_path, capsys):
monkeypatch.setattr(
sys,
"argv",
[
"agent_cache_benchmark.py",
"--endpoint",
"http://127.0.0.1:9997",
"--model",
"fixture",
"--tokenizer-json",
str(tmp_path / "tokenizer.json"),
"--output",
str(tmp_path / "audit"),
"--rounds",
"2",
"--resume-after",
"3",
],
)
with pytest.raises(SystemExit) as error:
agent_cache_benchmark.main()
assert error.value.code == 2
assert "positions between 0 and rounds" in capsys.readouterr().err