Enterprise agent platform for building, deploying, and operating AI teammates.
"""Per-process runtime diagnostics line and the execution counts behind it."""
from __future__ import annotations
import asyncio
import gc
import logging
import os
import sys
from types import ModuleType
import pytest
from xagent.web.services import process_diagnostics
from xagent.web.services import task_command_transport as transport
from xagent.web.services.task_dispatch_cap import DispatchPolicy
from xagent.web.services.task_execution import BackgroundTaskManager
ONE_SLOT = DispatchPolicy(limit=1, interactive_reserve=0)
LOGGER = process_diagnostics.__name__
MANAGER_LOGGER = "xagent.web.services.task_execution"
@pytest.fixture
def manager(monkeypatch):
fresh = BackgroundTaskManager()
monkeypatch.setattr(
"xagent.web.services.task_execution.background_task_manager", fresh
)
return fresh
def _counts(manager: BackgroundTaskManager) -> tuple[int, int, int, int]:
return manager.drain_window_counts()
async def _settle() -> None:
for _ in range(3):
await asyncio.sleep(0)
class _FakeClock:
def __init__(self) -> None:
self.now = 1000.0
def __call__(self) -> float:
return self.now
@pytest.mark.asyncio
@pytest.mark.parametrize("busy", [False, True], ids=["idle", "busy"])
async def test_flush_logs_one_line_per_window(manager, monkeypatch, caplog, busy):
monkeypatch.setenv("XAGENT_TASK_EXECUTION_ROLE", "combined")
pending = asyncio.create_task(asyncio.sleep(3600))
done = asyncio.create_task(asyncio.sleep(0))
await done
if busy:
manager.register_task(1, pending)
manager.register_task(2, done)
await _settle()
clock = _FakeClock()
flushes = 0
async def fake_sleep(seconds: float) -> None:
nonlocal flushes
clock.now += seconds
if flushes == 1:
raise asyncio.CancelledError
flushes += 1
caplog.set_level(logging.INFO, logger=LOGGER)
with pytest.raises(asyncio.CancelledError):
await process_diagnostics.run_process_diagnostics(
flush_interval_s=1.0, sample_interval_s=1.0, clock=clock, sleep=fake_sleep
)
pending.cancel()
lines = [r for r in caplog.records if r.name == LOGGER]
assert len(lines) == 1
assert lines[0].levelno == logging.INFO
message = lines[0].getMessage()
assert message.startswith("runtime diagnostics role=combined ")
assert f" pid={os.getpid()} window_s=1.0 " in message
if busy:
assert " inflight=1 " in message
assert " started_1m=2 finished_1m=1 ok=1 failed=0 cancelled=0 " in message
else:
assert " inflight=0 " in message
assert " started_1m=0 finished_1m=0 " in message
assert " gate_held_s=0.0 " in message
@pytest.mark.asyncio
@pytest.mark.parametrize(
("outcome", "bucket"),
[
("ok", "ok"),
("raised", "failed"),
("flagged", "failed"),
("cancelled", "cancelled"),
("flagged_and_cancelled", "cancelled"),
("flag_raises", "failed"),
],
)
async def test_finish_outcome_bucket(manager, caplog, outcome, bucket):
caplog.set_level(logging.ERROR, logger=MANAGER_LOGGER)
handled_failure = outcome.startswith("flagged")
release = asyncio.Event()
async def run() -> None:
await release.wait()
if outcome == "raised":
raise RuntimeError("boom")
def flag() -> bool:
if outcome == "flag_raises":
raise RuntimeError("flag broke")
return handled_failure
task = asyncio.create_task(run())
manager.register_task(1, task, failure_flag=flag)
await asyncio.sleep(0)
if outcome.endswith("cancelled"):
task.cancel()
release.set()
await asyncio.gather(task, return_exceptions=True)
await _settle()
started, ok, failed, cancelled = _counts(manager)
assert started == 1
assert {"ok": ok, "failed": failed, "cancelled": cancelled} == {
name: int(name == bucket) for name in ("ok", "failed", "cancelled")
}
raised_reports = [
r
for r in caplog.records
if r.getMessage() == "Background execution for task 1 raised"
]
assert len(raised_reports) == int(outcome == "raised")
@pytest.mark.asyncio
@pytest.mark.parametrize("entry", ["register", "promote"])
async def test_unawaited_execution_error_is_logged_once_with_traceback(
manager, caplog, entry
):
failure = RuntimeError("boom")
async def run() -> None:
raise failure
task = asyncio.create_task(run())
if entry == "register":
manager.register_task(7, task)
else:
assert manager.reserve_resume(7)
manager.register_reserved_resume(7, task, run_id="run-7")
manager.promote_resume_task(7, task)
caplog.set_level(logging.ERROR)
# asyncio.wait does not retrieve the exception: nothing awaits the handle.
await asyncio.wait({task})
await _settle()
del task
gc.collect()
# Every record carrying this exception, from any logger: asyncio's own
# "never retrieved" report would be a second one.
reports = [
(r.name, r.levelno, r.getMessage())
for r in caplog.records
if r.exc_info is not None and r.exc_info[1] is failure
]
assert reports == [
(MANAGER_LOGGER, logging.ERROR, "Background execution for task 7 raised")
]
assert _counts(manager) == (1, 0, 1, 0)
@pytest.mark.asyncio
async def test_drain_resets_and_shutdown_keeps_counts(manager):
task = asyncio.create_task(asyncio.sleep(3600))
manager.register_task(1, task)
await manager.shutdown()
await _settle()
assert _counts(manager) == (1, 0, 0, 1)
assert _counts(manager) == (0, 0, 0, 0)
@pytest.mark.asyncio
async def test_new_lifespan_starts_from_zero(manager):
task = asyncio.create_task(asyncio.sleep(0))
manager.register_task(1, task)
await task
await _settle()
manager.start_accepting()
assert _counts(manager) == (0, 0, 0, 0)
@pytest.mark.asyncio
async def test_same_task_registered_twice_counted_once(manager):
task = asyncio.create_task(asyncio.sleep(0))
manager.register_task(1, task)
manager.register_task(1, task)
await task
await _settle()
assert _counts(manager) == (1, 1, 0, 0)
@pytest.mark.asyncio
async def test_replaced_registration_still_counts_inflight(manager):
older = asyncio.create_task(asyncio.sleep(3600))
newer = asyncio.create_task(asyncio.sleep(3600))
manager.register_task(1, older)
manager.register_task(1, newer)
assert manager.inflight_counts() == (2, 0)
for task in (older, newer):
task.cancel()
await asyncio.gather(older, newer, return_exceptions=True)
@pytest.mark.asyncio
async def test_promoted_resume_counted_and_fields_disjoint(manager):
coordinator = asyncio.create_task(asyncio.sleep(3600))
assert manager.reserve_resume(1)
manager.register_reserved_resume(1, coordinator, run_id="run-1")
assert manager.inflight_counts() == (0, 1)
assert _counts(manager) == (0, 0, 0, 0)
manager.promote_resume_task(1, coordinator)
assert manager.inflight_counts() == (1, 0)
coordinator.cancel()
await asyncio.gather(coordinator, return_exceptions=True)
await _settle()
assert _counts(manager) == (1, 0, 0, 1)
@pytest.mark.asyncio
async def test_registration_rejected_while_shutting_down_is_not_counted(manager):
await manager.shutdown()
task = asyncio.create_task(asyncio.sleep(3600))
with pytest.raises(RuntimeError, match="shutting down"):
manager.register_task(1, task)
await asyncio.gather(task, return_exceptions=True)
await _settle()
assert _counts(manager) == (0, 0, 0, 0)
@pytest.mark.asyncio
async def test_lag_sampler_counts_overrun(manager, monkeypatch):
clock = _FakeClock()
sleeps = 0
async def fake_sleep(seconds: float) -> None:
nonlocal sleeps
sleeps += 1
if sleeps == 3:
raise asyncio.CancelledError
# The second sleep overruns by 0.8 s.
clock.now += seconds + (0.8 if sleeps == 2 else 0.0)
monkeypatch.setattr(process_diagnostics, "_window", process_diagnostics._Window())
with pytest.raises(asyncio.CancelledError):
await process_diagnostics.run_process_diagnostics(
flush_interval_s=3600.0,
sample_interval_s=1.0,
clock=clock,
sleep=fake_sleep,
)
counters = process_diagnostics.drain_execution_counters(now=clock.now)
assert counters.loop_lag_samples == 1
assert counters.loop_lag_max_ms == pytest.approx(800.0)
@pytest.mark.asyncio
async def test_flush_failure_keeps_loop_running(manager, monkeypatch, caplog):
clock = _FakeClock()
real_drain = process_diagnostics.drain_execution_counters
calls = 0
def flaky_drain(now=None):
nonlocal calls
calls += 1
if calls == 1:
raise RuntimeError("flush broke")
return real_drain(now)
async def fake_sleep(seconds: float) -> None:
if calls == 2:
raise asyncio.CancelledError
clock.now += seconds
monkeypatch.setattr(process_diagnostics, "drain_execution_counters", flaky_drain)
caplog.set_level(logging.INFO, logger=LOGGER)
with pytest.raises(asyncio.CancelledError):
await process_diagnostics.run_process_diagnostics(
flush_interval_s=1.0, sample_interval_s=1.0, clock=clock, sleep=fake_sleep
)
records = [r for r in caplog.records if r.name == LOGGER]
assert [r.levelno for r in records] == [logging.ERROR, logging.INFO]
assert records[1].getMessage().startswith("runtime diagnostics ")
@pytest.mark.asyncio
async def test_bad_role_still_starts_and_stop_logs_final_line(
manager, monkeypatch, caplog
):
monkeypatch.setenv("XAGENT_TASK_EXECUTION_ROLE", "nonsense")
caplog.set_level(logging.INFO, logger=LOGGER)
task = process_diagnostics.start_process_diagnostics()
assert task is not None and not task.done()
execution = asyncio.create_task(asyncio.sleep(3600))
manager.register_task(1, execution)
await manager.shutdown()
await process_diagnostics.stop_process_diagnostics(task)
assert task.cancelled()
lines = [r.getMessage() for r in caplog.records if r.name == LOGGER]
assert len(lines) == 1
assert lines[0].startswith("runtime diagnostics role=unknown ")
assert " started_1m=1 finished_1m=1 ok=0 failed=0 cancelled=1 " in lines[0]
@pytest.mark.asyncio
async def test_stop_keeps_the_callers_own_cancellation(manager):
task = process_diagnostics.start_process_diagnostics()
assert task is not None
gate = asyncio.Event()
async def caller() -> None:
await gate.wait()
await process_diagnostics.stop_process_diagnostics(task)
outer = asyncio.create_task(caller())
await asyncio.sleep(0)
gate.set()
await asyncio.sleep(0)
outer.cancel()
with pytest.raises(asyncio.CancelledError):
await outer
@pytest.mark.asyncio
async def test_app_lifespan_starts_one_diagnostics_task_and_stops_it(manager):
from fastapi import FastAPI
from xagent.web import app as app_module
test_app = FastAPI()
app_module.start_app_process_diagnostics(test_app)
first = test_app.state.process_diagnostics_task
app_module.start_app_process_diagnostics(test_app)
assert first is not None and not first.done()
assert test_app.state.process_diagnostics_task is first
await app_module.stop_app_process_diagnostics(test_app)
assert test_app.state.process_diagnostics_task is None
assert first.cancelled()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failing_step", [None, "drain"], ids=["drain_returns", "drain_raises"]
)
async def test_app_shutdown_stops_diagnostics_after_the_drain(
monkeypatch, failing_step
):
from xagent.web import app as app_module
order: list[str] = []
async def record(name: str) -> None:
order.append(name)
if name == failing_step:
raise RuntimeError("drain failed")
class _Channel:
enabled = False
async def stop(self) -> None:
return None
class _Sandboxes:
async def cleanup(self) -> None:
return None
for module_name, attribute in (
("xagent.web.channels.telegram.bot", "get_telegram_channel"),
("xagent.web.channels.feishu.bot", "get_feishu_channel"),
("xagent.web.channels.slack.bot", "get_slack_channel"),
):
fake = ModuleType(module_name)
setattr(fake, attribute, lambda: _Channel())
monkeypatch.setitem(sys.modules, module_name, fake)
monkeypatch.setattr(app_module, "flush_langfuse", lambda: None)
for name in (
"stop_runtime_performance_monitor",
"stop_orphan_upload_gc_task",
"stop_retention_purge_task",
"stop_task_cleanup_retry_task",
"stop_uploaded_file_recovery_task",
"stop_task_auto_resume_task",
"stop_task_lease_recovery_task",
):
monkeypatch.setattr(app_module, name, lambda _app: record("other"))
monkeypatch.setattr(
app_module, "stop_app_process_diagnostics", lambda _app: record("diagnostics")
)
monkeypatch.setattr(
"xagent.web.services.task_execution.background_task_manager.shutdown",
lambda: record("drain"),
)
monkeypatch.setattr(
"xagent.web.services.task_lease_service.wait_for_heartbeat_manager_idle",
lambda: record("other"),
)
for name in (
"_task_command_dispatcher_task",
"_sandbox_idle_sweep_task",
"_file_storage_startup_sync_task",
"_trigger_dispatcher_task",
"_migration_task",
):
monkeypatch.setattr(app_module, name, None)
monkeypatch.setattr(
"xagent.web.sandbox_manager.get_sandbox_manager", lambda: _Sandboxes()
)
app_module.app.state.metadata_rebuild_task = None
for attribute in ("telegram_task", "slack_task"):
if hasattr(app_module.app.state, attribute):
monkeypatch.delattr(app_module.app.state, attribute)
if failing_step is None:
await app_module.shutdown_event()
else:
with pytest.raises(RuntimeError, match="drain failed"):
await app_module.shutdown_event()
stops = [name for name in order if name != "other"]
assert stops == ["drain", "diagnostics"]
def _gate_probe(manager, monkeypatch, *, window_started_at: float = 0.0):
"""Return (clock, close, open_, drain) helpers around one manager.
``close`` / ``open_`` make the process full / empty at a clock reading and
build a cap snapshot with limit 1; ``drain`` is the diagnostics drain.
"""
monkeypatch.setenv("XAGENT_TASK_EXECUTION_ROLE", "combined")
monkeypatch.setattr(process_diagnostics._window, "started_at", window_started_at)
clock = _FakeClock()
manager._cap_clock = clock
def close(at: float) -> None:
clock.now = at
manager._live_executions[1] = {object()} # type: ignore[assignment]
manager.dispatch_cap_snapshot(ONE_SLOT)
def open_(at: float) -> None:
clock.now = at
manager._live_executions.pop(1, None)
manager.dispatch_cap_snapshot(ONE_SLOT)
def drain(at: float) -> float:
return process_diagnostics.drain_execution_counters(now=at).gate_held_s
return clock, close, open_, drain
@pytest.mark.asyncio
@pytest.mark.parametrize(
"row",
["never_closed", "closed_within_window", "split_across_drain"]
+ ["bounded_by_window", "drain_observes", "zeroed_after_drain"],
)
async def test_gate_held_seconds_measures_closed_time(manager, monkeypatch, row):
started_at = 100.0 if row == "bounded_by_window" else 0.0
_, close, open_, drain = _gate_probe(
manager, monkeypatch, window_started_at=started_at
)
if row == "never_closed":
open_(1)
open_(5)
assert drain(10) == 0.0
elif row == "closed_within_window":
open_(1)
close(3)
open_(5)
assert drain(10) == 2.0
elif row == "split_across_drain":
close(8)
assert drain(10) == 2.0
open_(11)
assert drain(13) == 1.0
elif row == "bounded_by_window":
close(90)
assert drain(110) == 10.0
elif row == "drain_observes":
open_(0)
manager._live_executions[1] = {object()} # type: ignore[assignment]
assert drain(10) == 0.0
assert drain(15) == 5.0
else:
close(3)
open_(5)
assert drain(10) == 2.0
assert drain(20) == 0.0
@pytest.mark.asyncio
async def test_gate_held_counts_a_full_process_with_nothing_queued(
manager, monkeypatch
):
clock, _, _, drain = _gate_probe(manager, monkeypatch)
running = asyncio.create_task(asyncio.sleep(3600))
try:
manager.register_task(1, running)
clock.now = 0.0
manager.dispatch_cap_snapshot(ONE_SLOT)
assert drain(60) == 60.0
finally:
running.cancel()
await asyncio.gather(running, return_exceptions=True)
@pytest.mark.asyncio
@pytest.mark.parametrize("row", ["shutdown_closes_interval", "new_lifespan_resets"])
async def test_gate_held_after_shutdown_and_new_lifespan(manager, monkeypatch, row):
clock, _, _, drain = _gate_probe(manager, monkeypatch)
clock.now = 0.0
# A reservation outlives cancellation, so only the observation that
# follows shutdown's clearing of the tables can end the interval at 4.0.
assert (
manager.reserve_dispatch_slot(1, policy=ONE_SLOT, interactive=False) is not None
)
manager.dispatch_cap_snapshot(ONE_SLOT)
clock.now = 4.0
await manager.shutdown()
if row == "shutdown_closes_interval":
assert drain(10) == 4.0
else:
clock.now = 6.0
manager.start_accepting()
assert drain(10) == 0.0
def _counters(**overrides) -> process_diagnostics.ExecutionCounters:
fields = {
"role": "combined",
"pid": 1,
"window_s": 60.0,
"inflight": 0,
"resume_inflight": 0,
"started": 0,
"finished_ok": 0,
"finished_failed": 0,
"finished_cancelled": 0,
"gate_held_s": 0.0,
"slot_tenants": 0,
"slot_top": 0,
"loop_lag_max_ms": 0.0,
"loop_lag_samples": 0,
"fair_misses": 0,
"fair_fallbacks": 0,
"fair_all_missed": 0,
}
fields.update(overrides)
return process_diagnostics.ExecutionCounters(**fields)
def test_the_line_reports_slot_tenants_between_the_gate_and_the_loop_lag():
line = _counters(slot_tenants=2, slot_top=3).format_line()
assert " gate_held_s=0.0 slot_tenants=2 slot_top=3 loop_lag_max_ms=0.0 " in line
@pytest.mark.asyncio
async def test_drain_reads_the_slot_tenant_counts_from_the_manager(
manager, monkeypatch
):
monkeypatch.setenv("XAGENT_TASK_EXECUTION_ROLE", "combined")
monkeypatch.setattr(process_diagnostics._window, "started_at", 0.0)
policy = DispatchPolicy(limit=10, interactive_reserve=0)
for task_id, key in ((1, "team:1"), (2, "team:1"), (3, "team:2")):
assert manager.reserve_dispatch_slot(
task_id, policy=policy, interactive=False, fairness_key=key
)
counters = process_diagnostics.drain_execution_counters(now=10.0)
assert (counters.slot_tenants, counters.slot_top) == (2, 2)
assert " slot_tenants=2 slot_top=2 " in counters.format_line()
@pytest.mark.asyncio
async def test_a_process_that_dispatches_nothing_reports_no_slot_tenants(
manager, monkeypatch
):
monkeypatch.setenv("XAGENT_TASK_EXECUTION_ROLE", "combined")
monkeypatch.setattr(process_diagnostics._window, "started_at", 0.0)
counters = process_diagnostics.drain_execution_counters(now=10.0)
assert (counters.slot_tenants, counters.slot_top) == (0, 0)
assert " slot_tenants=0 slot_top=0 " in counters.format_line()
def test_the_line_carries_the_active_degradations_before_the_fair_counts():
idle = _counters().format_line()
assert idle.endswith(
" loop_lag_samples=0 degradations=- fair_misses=0 fair_fallbacks=0"
" fair_all_missed=0"
)
line = _counters(degradations=("a_signal", "b_signal")).format_line()
assert " degradations=a_signal,b_signal fair_misses=0 " in line
@pytest.mark.asyncio
async def test_drain_reports_this_processes_degradations_sorted(manager, monkeypatch):
from xagent.web.services import ops_signals
monkeypatch.setattr(ops_signals, "_signals", {})
ops_signals.register_degradation("z_signal", "detail stays out of the line")
ops_signals.register_degradation("a_signal", "detail")
monkeypatch.setattr(process_diagnostics._window, "started_at", 0.0)
line = process_diagnostics.drain_execution_counters(now=10.0).format_line()
assert " degradations=a_signal,z_signal fair_misses=" in line
assert "detail" not in line
def test_the_line_ends_with_the_fair_counts_of_the_window():
line = _counters(fair_misses=2, fair_fallbacks=3, fair_all_missed=4).format_line()
assert line.endswith(
" degradations=- fair_misses=2 fair_fallbacks=3 fair_all_missed=4"
)
@pytest.mark.asyncio
async def test_a_window_reports_the_fair_misses_since_the_last_one(
manager, monkeypatch
):
monkeypatch.setenv("XAGENT_TASK_EXECUTION_ROLE", "combined")
monkeypatch.setattr(process_diagnostics._window, "started_at", 0.0)
monkeypatch.setattr(transport, "_fair_state", transport._FairState())
transport._fair_record_miss("team:1", "batch")
transport._fair_record_miss("team:1", "interactive")
first = process_diagnostics.drain_execution_counters(now=10.0)
second = process_diagnostics.drain_execution_counters(now=20.0)
assert (first.fair_misses, second.fair_misses) == (2, 0)
assert " fair_misses=2 " in first.format_line()
assert " fair_misses=0 " in second.format_line()
@pytest.mark.asyncio
async def test_a_window_reports_the_fallback_rounds_and_all_missed_scans_since_the_last_one(
manager, monkeypatch
):
monkeypatch.setenv("XAGENT_TASK_EXECUTION_ROLE", "combined")
monkeypatch.setattr(process_diagnostics._window, "started_at", 0.0)
state = transport._FairState()
state.fallbacks, state.all_missed = 3, 1
monkeypatch.setattr(transport, "_fair_state", state)
first = process_diagnostics.drain_execution_counters(now=10.0)
second = process_diagnostics.drain_execution_counters(now=20.0)
assert (first.fair_fallbacks, first.fair_all_missed) == (3, 1)
assert (second.fair_fallbacks, second.fair_all_missed) == (0, 0)
assert first.format_line().endswith(" fair_fallbacks=3 fair_all_missed=1")
assert second.format_line().endswith(" fair_fallbacks=0 fair_all_missed=0")