Enterprise agent platform for building, deploying, and operating AI teammates.
"""Concurrent real ReAct audits and fixed-arrival foreground requests.
This optional benchmark never launches/reconfigures a server. ``baseline`` only
suppresses agent hints in this harness. ``lifecycle`` and ``lmcache`` send the
same hints; the operator must configure and record the storage backend separately.
"""
from __future__ import annotations
import argparse
import asyncio
import datetime
import hashlib
import json
import math
import statistics
import time
from collections import deque
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
from uuid import uuid4
from scripts.benchmarks import agent_cache_benchmark as audit
from xagent.core.agent import PatternRuntime, ReActPattern
from xagent.core.agent.runner import _session_routing_key
from xagent.core.model.chat.routing_context import xinference_routing_key
ADMISSION_POLICIES = ("foreground-first", "fifo")
def utc() -> str:
return datetime.datetime.now(datetime.timezone.utc).isoformat()
class AdmissionGate:
"""Shared request limit; drain in-flight work before a thermal cooldown.
All agents and foreground calls share this client-side gate. With the
default foreground-first policy, waiting foreground requests take the next
free slot without interrupting work. Only intervals with zero in-flight
requests count as cooling idle time. Per-request admission waits must
never be summed and subtracted from concurrent batch wall time.
"""
def __init__(
self,
limit: int,
telemetry: Path | None = None,
*,
trigger: float = 70,
resume: float = 60,
admission_policy: str = "foreground-first",
):
if admission_policy not in ADMISSION_POLICIES:
raise ValueError(f"Unknown admission policy: {admission_policy!r}")
self.limit = limit
self.telemetry = telemetry
self.trigger = trigger
self.resume = resume
self.active = 0
self.condition = asyncio.Condition()
self.waiters: deque[object] = deque()
self.foreground_waiters: deque[object] = deque()
self.admission_policy = admission_policy
self.cooling = False
self.idle_started: float | None = None
self.cooling_intervals: list[dict[str, Any]] = []
def temperatures(self) -> list[dict[str, Any]]:
if self.telemetry is None:
return []
rows = list(json.loads(self.telemetry.read_text()).values())
now = datetime.datetime.now(datetime.timezone.utc)
if not rows or any(
(now - datetime.datetime.fromisoformat(row["utc"])).total_seconds() > 10
for row in rows
):
raise RuntimeError("Missing or stale GPU telemetry")
return rows
def can_enter(self) -> bool:
rows = self.temperatures()
thermal = any(
row["sw_thermal_throttle"] or row["hw_thermal_throttle"] for row in rows
)
if thermal or any(row["temperature_C"] > self.trigger for row in rows):
self.cooling = True
if self.cooling:
if not self.active and self.idle_started is None:
self.idle_started = time.perf_counter()
if (
not self.active
and not thermal
and all(row["temperature_C"] <= self.resume for row in rows)
):
self.cooling = False
self.finish_idle()
else:
if (
self.idle_started is not None
and time.perf_counter() - self.idle_started > 600
):
raise RuntimeError("GPU cooling exceeded ten minutes")
return False
return self.active < self.limit
def finish_idle(self) -> None:
if self.idle_started is not None:
self.cooling_intervals.append(
{
"elapsed_s": time.perf_counter() - self.idle_started,
"utc_finished": utc(),
}
)
self.idle_started = None
@asynccontextmanager
async def slot(self, *, foreground: bool = False):
requested = time.perf_counter()
ticket = object()
queue = (
self.foreground_waiters
if foreground and self.admission_policy == "foreground-first"
else self.waiters
)
async with self.condition:
queue.append(ticket)
try:
while (self.foreground_waiters or self.waiters)[
0
] is not ticket or not self.can_enter():
try:
await asyncio.wait_for(self.condition.wait(), timeout=0.5)
except asyncio.TimeoutError:
pass
queue.popleft()
self.active += 1
self.condition.notify_all()
except BaseException:
queue.remove(ticket)
self.condition.notify_all()
raise
try:
yield {"admission_wait_s": time.perf_counter() - requested}
finally:
async with self.condition:
self.active -= 1
if self.cooling and not self.active and self.idle_started is None:
self.idle_started = time.perf_counter()
self.condition.notify_all()
class ConcurrentLLM(audit.BenchmarkLLM):
def __init__(
self, *, gate: AdmissionGate, hints: bool, foreground: bool, **kwargs: Any
):
super().__init__(**kwargs)
self.gate = gate
self.hints = hints
self.foreground = foreground
self.protocol: dict[str, bool] = {}
self.admission: dict[str, float] = {}
self.control: list[dict[str, Any]] = []
def _apply_session_affinity(self, config: dict[str, Any]) -> None:
if self.hints:
super()._apply_session_affinity(config)
self.protocol["routing_key_sent"] = "routing_key" in config
def _apply_cache_lifecycle(self, config: dict[str, Any]) -> None:
if self.hints:
super()._apply_cache_lifecycle(config)
self.protocol["cache_session_sent"] = "cache_session" in config
async def _update_cache_session(self, session: dict[str, Any]) -> None:
row = {
"phase": session["phase"],
"revision": session["revision"],
"utc": utc(),
"success": False,
}
try:
await super()._update_cache_session(session)
row["success"] = True
finally:
self.control.append(row)
def record(self, row: dict[str, Any]) -> None:
row.update(self.protocol, **self.admission, utc_recorded=utc())
super().record(row)
async def stream_chat(self, **kwargs: Any):
async with self.gate.slot(foreground=self.foreground) as admission:
self.admission = admission
async for chunk in super().stream_chat(**kwargs):
yield chunk
async def chat(self, **kwargs: Any):
async with self.gate.slot(foreground=self.foreground) as admission:
self.admission = admission
return await super().chat(**kwargs)
class WaitingAuditTool(audit.AuditTool):
def __init__(self, workload: audit.AuditWorkload, *, delay: float, **kwargs: Any):
super().__init__(workload, **kwargs)
self.delay = delay
self.waited = False
async def run_json_async(self, args: dict[str, Any]) -> Any:
result = await super().run_json_async(args)
if (
not self.save
and not self.read
and not self.waited
and result.get("success")
):
self.waited = True
await asyncio.sleep(self.delay)
return result
def model(
args: argparse.Namespace,
output: Path,
gate: AdmissionGate,
*,
hints: bool,
foreground: bool = False,
) -> ConcurrentLLM:
return ConcurrentLLM(
output=output,
gate=gate,
hints=hints,
foreground=foreground,
context_window=args.context_window,
model_name=args.model,
model_uid=args.model,
base_url=args.endpoint,
api_key=args.api_key_file.read_text().strip() if args.api_key_file else None,
default_temperature=0,
default_max_tokens=1024,
timeout=args.timeout,
timeout_config=audit.TimeoutConfig(args.timeout, 60),
abilities=["chat", "tool_calling"],
)
async def run_agent(
args: argparse.Namespace,
index: int,
ledger: str,
rows: int,
gate: AdmissionGate,
scope: str,
) -> dict[str, Any]:
output = args.output / f"agent-{index:02d}"
output.mkdir()
(output / "ledger.txt").write_text(ledger)
workload = audit.AuditWorkload([1, rows], output, fail_at=2)
trace = audit.BenchmarkTrace(output)
runtime = PatternRuntime(tracer=trace)
llm = model(args, output, gate, hints=args.variant != "baseline")
context = audit.build_context(ledger, 1, args.context_window, compaction=False)
context.system_prompt = (
f"Experiment scope {scope}-{index}.\n" + context.system_prompt
)
routing_key = _session_routing_key(context, f"{scope}-{index}")
token = xinference_routing_key.set(routing_key)
delay = args.tool_delays[index % len(args.tool_delays)]
start = time.perf_counter()
started_utc = utc()
result: dict[str, Any] = {}
error = None
try:
result = await ReActPattern(
max_iterations=14,
user_interaction_enabled=False,
repeated_tool_decision_after_consecutive_tool_calls=None,
repeated_tool_decision_after_consecutive_work_tool_calls=None,
).run(
context=context,
tools=[
WaitingAuditTool(workload, delay=delay),
audit.AuditTool(workload, save=True),
audit.SumAmountsTool(),
],
llm=llm,
compact_llm=llm,
runtime=runtime,
)
except Exception as exc:
error = type(exc).__name__
finally:
try:
await runtime.finish_cache_session()
publisher = runtime._cache_session._publisher
if publisher is not None:
await asyncio.wait_for(asyncio.shield(publisher), timeout=2)
finally:
xinference_routing_key.reset(token)
await llm.close()
calls = llm.calls
expected_hints = args.variant != "baseline"
protocol = bool(calls) and all(
call.get("routing_key_sent") is expected_hints
and call.get("cache_session_sent") is expected_hints
for call in calls
)
first_usage = (calls[0].get("usage") or {}) if calls else {}
first_cached = (first_usage.get("prompt_tokens_details") or {}).get("cached_tokens")
waiting = any(row["phase"] == "waiting" and row["success"] for row in llm.control)
finished = any(row["phase"] == "finished" and row["success"] for row in llm.control)
tools = {
row.get("tool_name")
for row in trace.events
if row["category"] == "tool" and row["action"] == "end"
}
saved = json.loads((output / "audit.json").read_text()) if workload.saved else {}
verified = (
bool(saved)
and saved.get("verified") is True
and saved.get("targets") == [1, rows]
and saved.get("receipts") == workload.receipts
and saved.get("total_cents") == workload.total
and len(set(workload.receipts)) == 2
)
report = {
"agent": index,
"scope": f"{scope}-{index}",
"routing_key": routing_key,
"utc_started": started_utc,
"utc_finished": utc(),
"elapsed_s": time.perf_counter() - start,
"tool_delay_s": delay,
"passed": result.get("success") is True
and verified
and protocol
and first_cached == 0
and workload.failed_once
and {"check_invoice", "sum_amounts", "save_audit"} <= tools
and (not expected_hints or waiting and finished),
"audit_verified": verified,
"protocol_valid": protocol,
"initial_cached_tokens": first_cached,
"transient_failure_exercised": workload.failed_once,
"error_type": error,
"waiting_acknowledged": waiting,
"finished_acknowledged": finished,
"control_updates": llm.control,
"calls": calls,
**audit.cache_summary(calls),
}
audit.write_json(output / "report.json", report)
return report
async def run_probe(
args: argparse.Namespace,
index: int,
start: float,
tokenizer: Any,
gate: AdmissionGate,
scope: str,
) -> dict[str, Any]:
scheduled = start + args.probe_start + index * args.probe_interval
await asyncio.sleep(max(0, scheduled - time.perf_counter()))
output = args.output / f"foreground-{index:02d}"
output.mkdir()
llm = model(args, output, gate, hints=False, foreground=True)
text = (
f"Independent request {scope}-{index}.\n"
+ "Inventory and delivery records are complete. " * args.probe_tokens
)
text = tokenizer.decode(tokenizer.encode(text).ids[: args.probe_tokens])
text += "\nReply with OK only."
error = None
passed = False
try:
content = []
async for chunk in llm.stream_chat(
messages=[{"role": "user", "content": text}], max_tokens=8
):
content.append(chunk.content or "")
passed = "OK" in "".join(content).upper()
except Exception as exc:
error = type(exc).__name__
finally:
await llm.close()
cold_probe = len(llm.calls) == 1 and (
((llm.calls[0].get("usage") or {}).get("prompt_tokens_details") or {}).get(
"cached_tokens"
)
== 0
and not llm.calls[0].get("routing_key_sent")
and not llm.calls[0].get("cache_session_sent")
)
report = {
"foreground": index,
"passed": passed and cold_probe,
"error_type": error,
"scheduled_offset_s": scheduled - start,
"utc_finished": utc(),
"scheduled_to_finish_s": time.perf_counter() - scheduled,
"calls": llm.calls,
}
audit.write_json(output / "report.json", report)
return report
def summarize(
agents: list[dict[str, Any]],
probes: list[dict[str, Any]],
elapsed: float,
cooling: float,
) -> dict[str, Any]:
valid_agents = [row for row in agents if row.get("passed")]
latencies = [row["scheduled_to_finish_s"] for row in probes if row.get("passed")]
adjusted = max(0, elapsed - cooling)
return {
"passed": len(valid_agents) == len(agents)
and all(row.get("passed") for row in probes),
"successful_agents": len(valid_agents),
"attempted_agents": len(agents),
"successful_foreground": len(latencies),
"attempted_foreground": len(probes),
"batch_wall_s": elapsed,
"cooling_idle_s": cooling,
"batch_without_cooling_s": adjusted,
"successful_agents_per_minute_wall": len(valid_agents) * 60 / elapsed
if elapsed
else None,
"successful_agents_per_minute_without_cooling": len(valid_agents)
* 60
/ adjusted
if adjusted
else None,
"foreground_scheduled_latency_median_s": statistics.median(latencies)
if latencies
else None,
"foreground_scheduled_latency_max_s": max(latencies) if latencies else None,
}
async def run(args: argparse.Namespace) -> dict[str, Any]:
from tokenizers import Tokenizer
args.output.mkdir(parents=True, exist_ok=False)
tokenizer = Tokenizer.from_file(str(args.tokenizer_json))
ledger, rows = audit.build_ledger(args.context_tokens, tokenizer)
if rows < 2:
raise ValueError("Corpus needs at least two invoices")
scope = uuid4().hex
gate = AdmissionGate(
args.max_inflight,
args.gpu_telemetry_json,
trigger=args.cooldown_trigger,
resume=args.cooldown_resume,
admission_policy=args.admission_policy,
)
source = audit.source_manifest()
source["source_sha256"][
"scripts/benchmarks/agent_cache_concurrency_benchmark.py"
] = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
audit.write_json(args.output / "source.json", source)
start_utc = utc()
start = time.perf_counter()
jobs = [
asyncio.create_task(run_agent(args, index, ledger, rows, gate, scope))
for index in range(args.agents)
]
jobs.extend(
asyncio.create_task(run_probe(args, index, start, tokenizer, gate, scope))
for index in range(args.probes)
)
try:
outcomes = await asyncio.gather(*jobs, return_exceptions=True)
except BaseException:
for job in jobs:
job.cancel()
await asyncio.gather(*jobs, return_exceptions=True)
raise
gate.finish_idle()
elapsed = time.perf_counter() - start
def outcome(value: Any) -> dict[str, Any]:
return (
{"passed": False, "error_type": type(value).__name__}
if isinstance(value, BaseException)
else value
)
agents = [outcome(row) for row in outcomes[: args.agents]]
probes = [outcome(row) for row in outcomes[args.agents :]]
cooling = sum(row["elapsed_s"] for row in gate.cooling_intervals)
report = {
"variant": args.variant,
"source": source,
"scope": scope,
"utc_started": start_utc,
"utc_finished": utc(),
"configuration": {
key: getattr(args, key)
for key in [
"model",
"agents",
"context_tokens",
"context_window",
"max_inflight",
"admission_policy",
"tool_delays",
"probes",
"probe_tokens",
"probe_interval",
"probe_start",
"cooldown_trigger",
"cooldown_resume",
]
},
**summarize(agents, probes, elapsed, cooling),
"cooling_intervals": gate.cooling_intervals,
"agents": agents,
"foreground": probes,
}
audit.write_json(args.output / "report.json", report)
return report
def parser() -> argparse.ArgumentParser:
value = argparse.ArgumentParser(description=__doc__)
value.add_argument("--endpoint", required=True)
value.add_argument("--model", required=True)
value.add_argument("--api-key-file", type=Path)
value.add_argument("--tokenizer-json", type=Path, required=True)
value.add_argument("--output", type=Path, required=True)
value.add_argument(
"--variant", choices=["baseline", "lifecycle", "lmcache"], required=True
)
value.add_argument("--agents", type=int, choices=[2, 4], required=True)
value.add_argument("--context-tokens", type=int, default=65536)
value.add_argument("--context-window", type=int, default=131072)
value.add_argument("--max-inflight", type=int, default=2)
value.add_argument(
"--admission-policy",
choices=ADMISSION_POLICIES,
default="foreground-first",
help="Client queue order. Use fifo as the comparison baseline.",
)
value.add_argument("--tool-delays", type=float, nargs="+", default=[2, 20])
value.add_argument("--probes", type=int, default=8)
value.add_argument("--probe-tokens", type=int, default=8192)
value.add_argument("--probe-interval", type=float, default=10)
value.add_argument("--probe-start", type=float, default=5)
value.add_argument("--gpu-telemetry-json", type=Path)
value.add_argument("--cooldown-trigger", type=float, default=70)
value.add_argument("--cooldown-resume", type=float, default=60)
value.add_argument("--timeout", type=float, default=1200)
return value
def main() -> None:
cli = parser()
args = cli.parse_args()
if (
args.context_tokens <= 4096
or args.context_tokens + 2048 > args.context_window
or args.max_inflight < 1
or args.probes < 1
or args.probe_tokens < 1
or args.probe_tokens + 2048 > args.context_window
or args.probe_interval <= 0
or args.probe_start < 0
or args.timeout <= 0
or not 0 < args.cooldown_resume < args.cooldown_trigger
or any(delay < 0 for delay in args.tool_delays)
or not all(
math.isfinite(value)
for value in [
args.probe_interval,
args.probe_start,
args.timeout,
args.cooldown_trigger,
args.cooldown_resume,
*args.tool_delays,
]
)
):
cli.error(
"Require valid context headroom, positive request limits/intervals, and nonnegative tool delays/start"
)
report = asyncio.run(run(args))
print(
json.dumps(
{
key: value
for key, value in report.items()
if key not in {"agents", "foreground"}
}
),
flush=True,
)
raise SystemExit(0 if report["passed"] else 1)
if __name__ == "__main__":
main()