Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import itertools
import json
import logging
from dataclasses import FrozenInstanceError
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from auth.authentication import get_current_subject
from core.inference.api_monitor import ApiMonitor, _trim
import routes.inference as inference_route
def _start(monitor, **overrides):
"""Begin a monitored request, defaulting the fields a test does not vary."""
return monitor.start(
**{
"endpoint": "/v1/chat/completions",
"method": "POST",
"model": "m",
"prompt": "hi",
**overrides,
}
)
def test_terminal_api_usage_receipt_is_immutable_and_emitted_once():
receipts = []
monitor = ApiMonitor(terminal_callback = receipts.append)
entry_id = _start(
monitor,
model = "org/model",
prompt = "do not persist this",
subject = "alice",
via_api_key = True,
)
monitor.set_usage(entry_id, prompt_tokens = 40, completion_tokens = 10, total_tokens = 50)
monitor.finish(entry_id)
monitor.finish(entry_id)
monitor.fail(entry_id, "late error")
monitor.set_usage(entry_id, total_tokens = 999)
assert len(receipts) == 1
receipt = receipts[0]
assert (receipt.prompt_tokens, receipt.completion_tokens, receipt.total_tokens) == (40, 10, 50)
assert receipt.status == "completed"
assert not hasattr(receipt, "prompt")
assert not hasattr(receipt, "reply")
with pytest.raises(FrozenInstanceError):
receipt.total_tokens = 999
@pytest.mark.parametrize(
"terminal,status",
[
(lambda monitor, entry_id: monitor.finish(entry_id, "cancelled"), "cancelled"),
(lambda monitor, entry_id: monitor.fail(entry_id, "boom"), "error"),
(lambda monitor, entry_id: monitor.fail_open(entry_id, "boom"), "error"),
],
)
def test_partial_terminal_api_usage_is_emitted_once(terminal, status):
receipts = []
monitor = ApiMonitor(terminal_callback = receipts.append)
entry_id = _start(
monitor,
endpoint = "/v1/responses",
prompt = "private",
subject = "alice",
via_api_key = True,
)
monitor.set_usage(entry_id, completion_tokens = 7)
terminal(monitor, entry_id)
terminal(monitor, entry_id)
assert len(receipts) == 1
assert receipts[0].status == status
assert receipts[0].total_tokens == 7
def test_terminal_callback_runs_outside_monitor_lock():
lock_was_free = []
monitor = ApiMonitor()
def callback(_receipt):
acquired = monitor._lock.acquire(blocking = False)
lock_was_free.append(acquired)
if acquired:
monitor._lock.release()
assert monitor.snapshot()
monitor.set_terminal_callback(callback)
entry_id = _start(
monitor,
endpoint = "/v1/messages",
prompt = "private",
subject = "alice",
via_api_key = True,
)
monitor.set_usage(entry_id, total_tokens = 1)
monitor.finish(entry_id)
assert lock_was_free == [True]
def test_terminal_callback_failure_is_logged_and_swallowed(caplog):
def broken(_receipt):
raise RuntimeError("storage unavailable")
monitor = ApiMonitor(terminal_callback = broken)
entry_id = _start(
monitor,
endpoint = "/v1/completions",
prompt = "private",
subject = "alice",
via_api_key = True,
)
monitor.set_usage(entry_id, total_tokens = 1)
with caplog.at_level(logging.WARNING):
monitor.finish(entry_id)
assert monitor.get(entry_id)["status"] == "completed"
assert "api_monitor.terminal_callback_failed" in caplog.text
def test_overlapping_callback_leases_do_not_disable_the_live_owner():
monitor = ApiMonitor()
older_receipts = []
newer_receipts = []
older = monitor.acquire_terminal_callback(older_receipts.append)
newer = monitor.acquire_terminal_callback(newer_receipts.append)
monitor.release_terminal_callback(older)
entry_id = _start(
monitor,
endpoint = "/v1/responses",
prompt = "private",
subject = "alice",
via_api_key = True,
)
monitor.set_usage(entry_id, total_tokens = 1)
monitor.finish(entry_id)
assert older_receipts == []
assert len(newer_receipts) == 1
monitor.release_terminal_callback(newer)
def test_request_ids_keep_the_full_uuid(monkeypatch):
full_hex = "0123456789ab" + "cdef" * 5
monkeypatch.setattr(
"core.inference.api_monitor.uuid.uuid4",
lambda: type("FakeUuid", (), {"hex": full_hex})(),
)
monitor = ApiMonitor()
entry_id = monitor.start(endpoint = "/v1", method = "POST", model = "m", prompt = "")
lifecycle_id = monitor.record_lifecycle(event = "unload", model = "m")
assert entry_id == f"apireq_{full_hex}"
assert lifecycle_id == f"apievt_{full_hex[:12]}"
def test_non_external_and_non_request_rows_never_emit_usage_receipts():
receipts = []
monitor = ApiMonitor(terminal_callback = receipts.append)
studio = _start(monitor, prompt = "studio", subject = "alice", via_api_key = False)
monitor.set_usage(studio, total_tokens = 10)
monitor.finish(studio)
anonymous = _start(monitor, prompt = "anonymous", via_api_key = True)
monitor.set_usage(anonymous, total_tokens = 10)
monitor.finish(anonymous)
lifecycle = monitor.record_lifecycle(
event = "load", model = "m", running = True, subject = "alice", via_api_key = True
)
monitor.finish(lifecycle)
monitor.record_lifecycle(event = "unload", model = "m", subject = "alice", via_api_key = True)
discarded = _start(
monitor,
endpoint = "/v1/responses",
prompt = "private",
subject = "alice",
via_api_key = True,
)
monitor.discard(discarded)
monitor.finish(discarded)
disabled = ApiMonitor(enabled = False, terminal_callback = receipts.append)
assert disabled.start(endpoint = "/v1", method = "POST", model = "m", prompt = "") == ""
assert receipts == []
def _get_monitor(monkeypatch, *, enabled: bool):
"""Swaps in a fresh monitor, since snapshot() ignores _enabled and a shared one leaks earlier rows."""
monkeypatch.setattr(inference_route, "api_monitor", ApiMonitor(enabled = enabled))
app = FastAPI()
app.include_router(inference_route.studio_router)
# Dict literal: verify_import_hoist.py misses names in assignment targets.
app.dependency_overrides = {get_current_subject: lambda: "test-user"}
return TestClient(app).get("/monitor")
def test_api_monitor_tracks_reply_usage_and_context():
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello", context_length = 100)
monitor.append_reply(entry_id, "hi")
monitor.append_reply(entry_id, " there")
monitor.set_usage(
entry_id,
prompt_tokens = 4,
completion_tokens = 6,
)
monitor.finish(entry_id)
[entry] = monitor.snapshot()
assert entry["status"] == "completed"
assert entry["reply"] == "hi there"
assert entry["total_tokens"] == 10
assert entry["context_usage"] == 0.1
assert entry["duration_ms"] is not None
def test_api_monitor_summary_omits_full_prompt_and_reply():
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, model = "local-model", prompt = "p" * 500)
monitor.set_reply(entry_id, "r" * 500)
[summary] = monitor.snapshot(include_details = False)
assert "prompt" not in summary
assert "reply" not in summary
assert summary["prompt_preview"].endswith("...")
assert summary["reply_preview"].endswith("...")
assert summary["prompt_truncated"] is True
assert summary["reply_truncated"] is True
detail = monitor.get(entry_id)
assert detail is not None
assert detail["prompt"] == "p" * 500
assert detail["reply"] == "r" * 500
def test_api_monitor_filters_entries_by_subject():
monitor = ApiMonitor(max_entries = 3)
alice = _start(monitor, prompt = "alice prompt", subject = "alice")
bob = _start(monitor, prompt = "bob prompt", subject = "bob")
monitor.finish(bob)
alice_entries = monitor.snapshot(subject = "alice")
assert [entry["id"] for entry in alice_entries] == [alice]
assert monitor.get(bob, subject = "alice") is None
assert monitor.get(bob, subject = "bob")["id"] == bob
assert monitor.active_count(subject = "alice") == 1
assert monitor.active_count(subject = "bob") == 0
def test_api_monitor_keeps_bounded_recent_history():
monitor = ApiMonitor(max_entries = 2)
first = _start(monitor, prompt = "first")
second = _start(monitor, prompt = "second")
third = _start(monitor, prompt = "third")
monitor.finish(first)
monitor.finish(second)
monitor.finish(third)
entries = monitor.snapshot()
ids = [entry["id"] for entry in entries]
assert ids[0] == third
assert [entry["prompt"] for entry in entries] == ["third", "second"]
assert first not in ids
assert monitor.active_count() == 0
def test_api_monitor_keeps_running_entries_beyond_history_limit():
monitor = ApiMonitor(max_entries = 1)
running = _start(monitor, prompt = "running")
for prompt in ("done-1", "done-2", "done-3"):
entry_id = _start(monitor, prompt = prompt)
monitor.finish(entry_id)
entries = monitor.snapshot()
ids = [entry["id"] for entry in entries]
assert running in ids
assert monitor.active_count() == 1
monitor.finish(running)
[entry] = monitor.snapshot()
assert entry["id"] == running
assert entry["status"] == "completed"
assert monitor.active_count() == 0
def test_api_monitor_finish_is_idempotent():
monitor = ApiMonitor(max_entries = 2)
entry_id = _start(monitor)
monitor.finish(entry_id)
first = monitor.snapshot()[0]
monitor.finish(entry_id)
second = monitor.snapshot()[0]
assert first["finished_at"] == second["finished_at"]
assert first["duration_ms"] == second["duration_ms"]
def test_api_monitor_preserves_authoritative_total_tokens():
monitor = ApiMonitor(max_entries = 2)
entry_id = _start(monitor)
monitor.set_usage(
entry_id,
prompt_tokens = 10,
completion_tokens = 20,
total_tokens = 33,
)
monitor.set_usage(entry_id, prompt_tokens = 11)
assert monitor.snapshot()[0]["total_tokens"] == 33
def test_api_monitor_recomputes_derived_total_tokens():
monitor = ApiMonitor(max_entries = 2)
entry_id = _start(monitor)
monitor.set_usage(entry_id, prompt_tokens = 10)
assert monitor.snapshot()[0]["total_tokens"] == 10
monitor.set_usage(entry_id, completion_tokens = 20)
entry = monitor.snapshot()[0]
assert entry["prompt_tokens"] == 10
assert entry["completion_tokens"] == 20
assert entry["total_tokens"] == 30
def test_api_monitor_duration_non_negative_under_clock_step(monkeypatch):
import core.inference.api_monitor as m
fake_now = [1000.0]
monkeypatch.setattr(m.time, "time", lambda: fake_now[0])
monitor = ApiMonitor(max_entries = 1)
entry_id = _start(monitor, endpoint = "/x")
fake_now[0] = 500.0
monitor.finish(entry_id)
assert monitor.snapshot()[0]["duration_ms"] >= 0
def test_api_monitor_trim_guards_tiny_limit():
assert _trim("abcdefgh", 2) == ".."
assert _trim("abcdefgh", 0) == ""
assert _trim("abcdefgh", 3) == "..."
assert _trim("abcdefgh", 4) == "a..."
assert _trim("abcdefgh", 100) == "abcdefgh"
def test_api_monitor_append_reply_caps_without_regrowing():
import core.inference.api_monitor as m
monitor = ApiMonitor(max_entries = 1)
entry_id = _start(monitor, prompt = "go")
monitor.append_reply(entry_id, "x" * (m._MAX_REPLY_CHARS + 500))
capped = monitor.snapshot()[0]["reply"]
assert len(capped) == m._MAX_REPLY_CHARS and capped.endswith("...")
monitor.append_reply(entry_id, "y" * 1000)
assert monitor.snapshot()[0]["reply"] == capped
def test_api_monitor_append_reply_exact_cap_then_more_marks_truncated():
import core.inference.api_monitor as m
monitor = ApiMonitor(max_entries = 1)
entry_id = _start(monitor, prompt = "go")
monitor.append_reply(entry_id, "x" * m._MAX_REPLY_CHARS)
assert not monitor.snapshot()[0]["reply"].endswith("...")
monitor.append_reply(entry_id, "y")
reply = monitor.snapshot()[0]["reply"]
assert len(reply) == m._MAX_REPLY_CHARS and reply.endswith("...")
def test_clear_keeps_the_callers_own_request_that_is_still_running():
"""Clear keeps an in-flight request, since dropping it zeroes the active count and the later
finish then has no entry to land on."""
monitor = ApiMonitor(max_entries = 4)
done = _start(monitor, prompt = "finished", subject = "alice")
monitor.finish(done)
live = _start(monitor, prompt = "in flight", subject = "alice")
monitor.clear(subject = "alice")
assert [entry["id"] for entry in monitor.snapshot(subject = "alice")] == [live]
assert monitor.active_count(subject = "alice") == 1
monitor.finish(live)
assert monitor.get(live, subject = "alice")["status"] == "completed"
def test_api_monitor_clear_is_scoped_to_one_subject():
monitor = ApiMonitor(max_entries = 4)
alice = _start(monitor, prompt = "alice prompt", subject = "alice")
bob = _start(monitor, prompt = "bob prompt", subject = "bob")
# Clear keeps running rows, so finish first.
monitor.finish(alice)
monitor.clear(subject = "alice")
assert monitor.snapshot(subject = "alice") == []
assert [entry["id"] for entry in monitor.snapshot(subject = "bob")] == [bob]
assert monitor.active_count(subject = "bob") == 1
assert monitor.get(alice, subject = "alice") is None
monitor.clear()
assert monitor.snapshot(subject = "bob") == []
def test_api_monitor_records_whether_the_caller_used_an_api_key():
monitor = ApiMonitor(max_entries = 4)
ui = _start(monitor, endpoint = "/api/inference/chat", subject = "u")
api = _start(monitor, subject = "u", via_api_key = True)
by_id = {entry["id"]: entry for entry in monitor.snapshot(subject = "u")}
assert by_id[ui]["via_api_key"] is False
assert by_id[api]["via_api_key"] is True
def test_api_monitor_disabled_is_noop():
monitor = ApiMonitor(max_entries = 3, enabled = False)
request_id = _start(monitor, model = "local-model", prompt = "user: hello", context_length = 100)
load_id = monitor.record_lifecycle(
event = "load",
model = "local-model",
running = True,
)
unload_id = monitor.record_lifecycle(
event = "unload",
model = "local-model",
)
assert request_id == load_id == unload_id == ""
monitor.append_reply(request_id, "hi")
monitor.set_reply(request_id, "hi")
monitor.set_usage(request_id, prompt_tokens = 4, completion_tokens = 6)
monitor.relabel(load_id, "renamed-model")
monitor.set_progress(load_id, 50)
monitor.finish(load_id)
monitor.fail_open(load_id, "boom")
monitor.fail(request_id, "boom")
monitor.discard(unload_id)
assert monitor.snapshot() == []
assert monitor.active_count() == 0
assert monitor.get(request_id) is None
def test_api_monitor_disable_env_var_truthy(monkeypatch):
import core.inference.api_monitor as m
for value in ("1", "true", "yes", "on", "TRUE", "On", " yes "):
monkeypatch.setenv(m._DISABLE_ENV, value)
assert m._api_monitor_disabled() is True, value
def test_api_monitor_disable_env_var_falsy(monkeypatch):
import core.inference.api_monitor as m
for value in ("", "0", "false", "no", "off", "disabled"):
monkeypatch.setenv(m._DISABLE_ENV, value)
assert m._api_monitor_disabled() is False, value
def test_api_monitor_disable_env_var_unset(monkeypatch):
import core.inference.api_monitor as m
monkeypatch.delenv(m._DISABLE_ENV, raising = False)
assert m._api_monitor_disabled() is False
def test_lifecycle_load_row_opens_running_then_closes():
monitor = ApiMonitor(max_entries = 5)
event_id = monitor.record_lifecycle(event = "load", model = "org/A-GGUF", running = True)
row = monitor.snapshot()[0]
assert row["kind"] == "lifecycle" and row["event"] == "load"
assert row["status"] == "running" and row["duration_ms"] is None
assert monitor.active_count() == 0
monitor.relabel(event_id, "org/A-GGUF:Q4_K_M")
monitor.finish(event_id)
row = monitor.snapshot()[0]
assert row["status"] == "completed"
assert row["model"] == "org/A-GGUF:Q4_K_M"
assert row["duration_ms"] is not None
def test_lifecycle_unload_row_is_terminal_on_arrival():
monitor = ApiMonitor(max_entries = 5)
monitor.record_lifecycle(event = "unload", model = "org/A-GGUF", reason = "idle")
row = monitor.snapshot()[0]
assert row["status"] == "completed"
assert (row["event"], row["reason"]) == ("unload", "idle")
assert monitor.active_count() == 0
def test_lifecycle_rows_are_visible_to_every_subject():
monitor = ApiMonitor(max_entries = 5)
_start(monitor, subject = "alice")
event_id = monitor.record_lifecycle(event = "unload", model = "org/A-GGUF")
bob = monitor.snapshot(subject = "bob")
assert [r["kind"] for r in bob] == ["lifecycle"]
assert monitor.get(event_id, subject = "bob") is not None
assert len(monitor.snapshot(subject = "alice")) == 2
def test_request_rows_stay_private_to_their_subject():
monitor = ApiMonitor(max_entries = 5)
rid = _start(monitor, subject = "alice")
assert monitor.snapshot(subject = "bob") == []
assert monitor.get(rid, subject = "bob") is None
def test_discard_drops_a_row_that_never_happened():
monitor = ApiMonitor(max_entries = 5)
event_id = monitor.record_lifecycle(event = "load", model = "org/A-GGUF", running = True)
monitor.discard(event_id)
assert monitor.snapshot() == []
monitor.discard(event_id)
def test_fail_open_never_touches_a_finished_row():
monitor = ApiMonitor(max_entries = 5)
event_id = monitor.record_lifecycle(event = "load", model = "org/A-GGUF", running = True)
monitor.finish(event_id)
monitor.fail_open(event_id, "Load did not complete")
row = monitor.snapshot()[0]
assert row["status"] == "completed" and row["error"] is None
still_open = monitor.record_lifecycle(event = "load", model = "org/B-GGUF", running = True)
monitor.fail_open(still_open, "Load did not complete")
assert monitor.snapshot()[0]["status"] == "error"
def test_lifecycle_rows_share_the_retention_budget():
monitor = ApiMonitor(max_entries = 2)
for i in range(4):
monitor.record_lifecycle(event = "unload", model = f"org/M{i}")
models = [r["model"] for r in monitor.snapshot()]
assert models == ["org/M3", "org/M2"]
def test_request_rows_report_kind_request():
monitor = ApiMonitor(max_entries = 2)
monitor.start(endpoint = "/v1/chat/completions", method = "POST", model = "m", prompt = "hi")
assert monitor.snapshot()[0]["kind"] == "request"
def test_clear_hides_shared_lifecycle_rows_for_that_caller_only():
"""Clear hides shared lifecycle rows for that caller only; deleting them would erase others' history."""
monitor = ApiMonitor(max_entries = 10)
mine = _start(monitor, model = "org/A", prompt = "user: hi", subject = "alice")
monitor.finish(mine)
shared = monitor.record_lifecycle(event = "unload", model = "org/A")
assert {e["id"] for e in monitor.snapshot(subject = "alice")} == {mine, shared}
assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {shared}
monitor.clear(subject = "alice")
assert monitor.snapshot(subject = "alice") == []
assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {shared}
assert monitor.get(shared, subject = "alice") is None
assert monitor.get(shared, subject = "bob") is not None
def test_clear_leaves_a_running_shared_row_visible():
"""A load still running is live state, so clearing the log must not hide its row."""
monitor = ApiMonitor(max_entries = 10)
running = monitor.record_lifecycle(event = "load", model = "org/A", running = True)
monitor.clear(subject = "alice")
assert {e["id"] for e in monitor.snapshot(subject = "alice")} == {running}
def test_hidden_shared_ids_do_not_outlive_their_entries():
"""The hidden set names only rows that exist, so it stays bounded by the ring buffer."""
monitor = ApiMonitor(max_entries = 2)
monitor.record_lifecycle(event = "unload", model = "org/A")
monitor.clear(subject = "alice")
from utils.account_context import current_account_id
key = (current_account_id(), "alice")
assert monitor._hidden_shared.get(key)
for i in range(5):
monitor.record_lifecycle(event = "unload", model = f"org/M{i}")
assert not monitor._hidden_shared.get(key)
def test_an_api_triggered_lifecycle_row_carries_the_attribution():
"""A refused auto-switch never reaches start, so its lifecycle row must carry via_api_key."""
monitor = ApiMonitor(max_entries = 5)
api_load = monitor.record_lifecycle(
event = "load", model = "org/Repo-GGUF", running = True, via_api_key = True
)
monitor.record_lifecycle(event = "unload", model = "org/Repo-GGUF", reason = "idle")
rows = {e["id"]: e for e in monitor.snapshot()}
assert rows[api_load]["via_api_key"] is True
idle = [e for e in rows.values() if e["event"] == "unload"]
assert idle and all(e["via_api_key"] is False for e in idle)
monitor.fail(api_load, error = "auto-switch refused")
after = {e["id"]: e for e in monitor.snapshot()}
assert after[api_load]["via_api_key"] is True
assert after[api_load]["status"] == "error"
def test_an_api_lifecycle_row_pops_the_overlay_only_for_its_own_caller():
"""The row is shared, but via_api_key only opens the overlay for the caller that made the request."""
monitor = ApiMonitor(max_entries = 5)
row = monitor.record_lifecycle(
event = "load",
model = "org/Repo-GGUF",
running = True,
via_api_key = True,
subject = "alice",
)
mine = {e["id"]: e for e in monitor.snapshot(subject = "alice")}
theirs = {e["id"]: e for e in monitor.snapshot(subject = "bob")}
assert row in mine and row in theirs
assert mine[row]["via_api_key"] is True
assert theirs[row]["via_api_key"] is False
assert monitor.get(row, subject = "alice")["via_api_key"] is True
assert monitor.get(row, subject = "bob")["via_api_key"] is False
assert monitor.get(row)["via_api_key"] is True
def test_clearing_hides_a_shared_row_this_caller_owns_rather_than_deleting_it():
"""An API-key load now owns its shared row. A subject-scoped clear drops that
subject's rows, so without this the owner's Clear would delete a row every
other caller can still see and wipe it out of their history too."""
monitor = ApiMonitor(max_entries = 10)
row = monitor.record_lifecycle(
event = "unload", model = "org/Repo-GGUF", via_api_key = True, subject = "alice"
)
monitor.clear(subject = "alice")
assert monitor.snapshot(subject = "alice") == []
assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {row}
assert monitor.get(row, subject = "bob") is not None
def test_sse_done_detection_accepts_both_spacings():
done = inference_route._is_openai_sse_done
assert done("data: [DONE]") is True
assert done("data:[DONE]") is True
assert done('data: {"choices": []}') is False
assert done("event: ping") is False
assert done("") is False
def test_sse_done_detection_is_independent_of_the_monitor():
"""The proxy sets sent_done from the line itself, so a logging flag cannot add a second [DONE]."""
line = "data: [DONE]"
assert inference_route._monitor_openai_sse_line(None, line) is None
assert inference_route._is_openai_sse_done(line) is True
def test_monitor_route_reports_enabled(monkeypatch):
response = _get_monitor(monkeypatch, enabled = True)
assert response.status_code == 200
assert response.json()["logging_enabled"] is True
def test_monitor_route_reports_disabled(monkeypatch):
response = _get_monitor(monkeypatch, enabled = False)
assert response.status_code == 200
payload = response.json()
assert payload["logging_enabled"] is False
assert payload["entries"] == []
def test_monitor_route_disabled_still_hides_recorded_rows(monkeypatch):
"""A disabled monitor records nothing and the route hides already-recorded rows too."""
monkeypatch.setattr(inference_route, "api_monitor", ApiMonitor(enabled = False))
_start(inference_route.api_monitor, subject = "test-user")
app = FastAPI()
app.include_router(inference_route.studio_router)
app.dependency_overrides = {get_current_subject: lambda: "test-user"}
payload = TestClient(app).get("/monitor").json()
assert payload["logging_enabled"] is False
assert payload["entries"] == []
def test_set_perf_records_stats_and_snapshot_reports_them():
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
monitor.set_reply(entry_id, "hi")
monitor.set_perf(
entry_id,
tok_per_sec = 42.5,
prompt_tok_per_sec = 81.25,
prompt_ms = 123.4,
stop_reason = "length",
)
monitor.finish(entry_id)
[entry] = monitor.snapshot()
assert entry["tok_per_sec"] == 42.5
assert entry["prompt_tok_per_sec"] == 81.25
assert entry["ttft_ms"] == 123
assert entry["stop_reason"] == "length"
def test_measured_ttft_wins_over_engine_prefill():
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
entry = next(e for e in monitor._entries if e.id == entry_id)
entry.started_monotonic -= 2.0
monitor.append_reply(entry_id, "hi")
monitor.set_perf(entry_id, tok_per_sec = 42.5, prompt_ms = 120.0)
monitor.finish(entry_id)
[snapshot] = monitor.snapshot()
assert snapshot["ttft_ms"] >= 2000
def test_set_perf_rejects_non_finite_values():
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
monitor.set_perf(
entry_id,
tok_per_sec = float("nan"),
prompt_tok_per_sec = float("inf"),
prompt_ms = float("inf"),
)
monitor.set_perf(entry_id, tok_per_sec = "bogus", prompt_tok_per_sec = -1, prompt_ms = None)
monitor.finish(entry_id)
[entry] = monitor.snapshot()
assert entry["tok_per_sec"] is None
assert entry["prompt_tok_per_sec"] is None
assert entry["ttft_ms"] is None
def test_full_response_reply_does_not_stamp_ttft():
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
monitor.set_reply(entry_id, "full response")
monitor.append_reply(entry_id, " tail", stamp_first_token = False)
monitor.finish(entry_id)
[entry] = monitor.snapshot()
assert entry["ttft_ms"] is None
assert entry["tok_per_sec"] is None
def test_set_usage_rejects_malformed_token_counts():
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
monitor.append_reply(entry_id, "hi")
monitor.set_usage(
entry_id,
prompt_tokens = -3,
completion_tokens = "bogus",
total_tokens = 12,
)
monitor.finish(entry_id)
[entry] = monitor.snapshot()
assert entry["prompt_tokens"] is None
assert entry["completion_tokens"] is None
assert entry["total_tokens"] == 12
assert entry["tok_per_sec"] is None
def test_mark_first_token_stamps_ttft_for_reasoning_only_streams():
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
monitor.mark_first_token(entry_id)
monitor.finish(entry_id)
[entry] = monitor.snapshot()
assert entry["ttft_ms"] is not None
entry_id2 = _start(monitor, model = "local-model", prompt = "user: hello")
monitor.mark_first_token(entry_id2)
first = monitor._find_locked(entry_id2).first_token_monotonic
monitor.append_reply(entry_id2, "visible")
assert monitor._find_locked(entry_id2).first_token_monotonic == first
def test_queue_state_counts_direct_overflow_as_queued(monkeypatch):
# Direct calls hold no lease, so overflow must show as queued.
from types import SimpleNamespace
import routes.inference as inf
monkeypatch.setattr(
inf,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_diffusion = False,
base_url = "http://llama.test",
effective_parallel_slots = 1,
),
)
monkeypatch.setattr(inf, "peek_llama_admission_snapshot", lambda _base: None)
monkeypatch.setattr(inf, "_direct_llama_inflight", 2)
state = inf._monitor_queue_state()
assert state == {"capacity": 1, "active": 1, "queued": 1, "free": 0}
def test_non_streaming_responses_reports_its_finish_reason(monkeypatch):
import routes.inference as inf
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf, "api_monitor", monitor)
entry_id = _start(
monitor,
endpoint = "/v1/responses",
model = "local-model",
prompt = "user: hello",
)
body = {
"choices": [{"message": {"content": "hi"}, "finish_reason": "length"}],
"timings": {"predicted_per_second": 12.5, "prompt_ms": 80.0},
}
choices = body.get("choices", [])
inf._monitor_usage(
entry_id,
{"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
None,
timings = body.get("timings"),
stop_reason = (choices[0].get("finish_reason") if choices else None),
)
monitor.finish(entry_id)
[entry] = monitor.snapshot()
assert entry["stop_reason"] == "length"
assert entry["tok_per_sec"] == 12.5
assert entry["ttft_ms"] == 80
def test_parked_tool_resume_counts_as_queued():
from core.inference.llama_admission import LlamaAdmissionQueue
queue = LlamaAdmissionQueue("http://llama.test")
queue._unpark_tickets.append(1)
assert queue.snapshot().queued == 1
def test_free_never_reports_a_slot_admission_would_refuse():
"""free must match _can_admit_locked, since a resume ticket holds a slot back from new arrivals."""
from core.inference.llama_admission import LlamaAdmissionQueue
for capacity, held, tickets in itertools.product(range(1, 5), range(0, 5), range(0, 3)):
if held > capacity:
continue
queue = LlamaAdmissionQueue("http://llama.test")
queue._resize_pool_locked(capacity)
if any(queue._take_slot_locked(0) is None for _ in range(held)):
continue
for _ in range(tickets):
queue._unpark_seq += 1
queue._unpark_tickets.append(queue._unpark_seq)
snapshot = queue.snapshot()
admittable = queue._can_admit_locked(len(queue._unpark_tickets))
assert (snapshot.free > 0) == admittable, (
f"capacity={capacity} held={held} tickets={tickets}: "
f"free={snapshot.free} but _can_admit_locked={admittable}"
)
def test_queue_panel_never_shows_a_free_slot_next_to_a_resume(monkeypatch):
"""The panel derives free from the snapshot, so it inherits that invariant."""
from types import SimpleNamespace
import routes.inference as inf
from core.inference.llama_admission import LlamaAdmissionQueue
queue = LlamaAdmissionQueue("http://llama.test")
queue._unpark_tickets.append(1)
monkeypatch.setattr(
inf,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_diffusion = False,
base_url = "http://llama.test",
effective_parallel_slots = 1,
),
)
monkeypatch.setattr(inf, "peek_llama_admission_snapshot", lambda _base: queue.snapshot())
monkeypatch.setattr(inf, "_direct_llama_inflight", 0)
assert inf._monitor_queue_state() == {"capacity": 1, "active": 0, "queued": 1, "free": 0}
def test_set_perf_survives_an_out_of_range_engine_number():
"""float() of a huge int raises OverflowError, which a ValueError handler misses, inside a stream."""
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
huge = int("9" * 400)
monitor.set_perf(entry_id, tok_per_sec = huge, prompt_ms = huge, stop_reason = "stop")
monitor.finish(entry_id)
[entry] = monitor.snapshot()
assert entry["tok_per_sec"] is None
assert entry["ttft_ms"] is None
assert entry["stop_reason"] == "stop"
@pytest.mark.parametrize(
"chunk",
[
{"choices": [{"delta": {"content": "x"}}], "timings": {"predicted_per_second": 10**400}},
{"choices": [{"delta": {"content": "x"}}], "timings": {"prompt_ms": 10**400}},
{"choices": [{"delta": {"content": "x"}}], "usage": {"total_tokens": 10**400}},
{"choices": [{"delta": {"content": "x"}}], "usage": "not-a-dict"},
{"choices": [{"delta": {"content": "x"}}], "timings": "not-a-dict"},
{"choices": "not-a-list"},
{"choices": [{"delta": "not-a-dict"}]},
{},
],
)
def test_monitor_chunk_never_raises_on_a_malformed_upstream_chunk(monkeypatch, chunk):
"""A raise here escapes into the SSE generator and truncates the response."""
import routes.inference as inf
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf, "api_monitor", monitor)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
inf._monitor_openai_chunk(entry_id, chunk, 4096, streaming = True)
assert monitor.snapshot()
def test_direct_llama_counter_is_started_last_before_its_guarding_try():
"""started() must come last before the try, since anything raised between them leaks a busy slot."""
import ast
import inspect
import textwrap
for func_name in ("openai_completions", "_monitored_llama_call", "_direct_llama_request"):
tree = ast.parse(textwrap.dedent(inspect.getsource(getattr(inference_route, func_name))))
started = [
node
for node in ast.walk(tree)
if isinstance(node, ast.Expr)
and isinstance(node.value, ast.Call)
and isinstance(node.value.func, ast.Name)
and node.value.func.id == "_direct_llama_request_started"
]
assert started, f"{func_name}: no _direct_llama_request_started() call found"
for node in ast.walk(tree):
block = getattr(node, "body", None)
if not isinstance(block, list):
continue
for index, stmt in enumerate(block):
if not any(stmt is call for call in started):
continue
assert index + 1 < len(block) and isinstance(block[index + 1], ast.Try), (
f"{func_name}: _direct_llama_request_started() is not immediately "
f"followed by the try whose finally decrements it:\n"
+ "\n".join(ast.unparse(s) for s in block[index : index + 3])
)
def _llama_slot_readout(
monkeypatch,
*,
is_audio = False,
slots = 4,
):
"""Point the slot readout at a loaded llama-server with ``slots`` free slots."""
from types import SimpleNamespace
import routes.inference as inf
backend = SimpleNamespace(
is_loaded = True,
is_diffusion = False,
is_vision = True,
base_url = "http://llama.test",
effective_parallel_slots = slots,
context_length = 4096,
model_identifier = "some/tts-GGUF",
_is_audio = is_audio,
_audio_type = "snac",
_auth_headers = None,
)
monkeypatch.setattr(inf, "get_llama_cpp_backend", lambda: backend)
monkeypatch.setattr(inf, "peek_llama_admission_snapshot", lambda _base: None)
monkeypatch.setattr(inf, "_direct_llama_inflight", 0)
return backend
def test_queue_state_counts_rag_vision_captioning(monkeypatch):
"""RAG captioning reaches llama-server with no lease, so the panel must count it directly."""
import routes.inference as inf
from core.rag import captioner
_llama_slot_readout(monkeypatch)
seen = {}
class _Resp:
status_code = 200
def raise_for_status(self):
pass
def json(self):
return {"choices": [{"message": {"content": "a caption"}}]}
def _fake_post(*_a, **_k):
seen["state"] = inf._monitor_queue_state()
return _Resp()
monkeypatch.setattr("httpx.post", _fake_post)
assert captioner._caption_one("http://llama.test", "local", b"png", 5.0) == "a caption"
assert seen["state"] == {"capacity": 4, "active": 1, "queued": 0, "free": 3}
assert inf._monitor_queue_state() == {"capacity": 4, "active": 0, "queued": 0, "free": 4}
@pytest.mark.parametrize("fails", [False, True])
def test_rag_vision_call_balances_the_direct_counter(monkeypatch, fails):
"""Both outcomes must return the active count to zero, since a failed vision call is swallowed
as non-fatal and a leak would pin the panel at busy for the whole session."""
import routes.inference as inf
from core.rag import captioner
_llama_slot_readout(monkeypatch)
class _Resp:
status_code = 500
def raise_for_status(self):
raise RuntimeError("llama-server said no")
def json(self):
return {"choices": [{"message": {"content": "ok"}}]}
def _fake_post(*_a, **_k):
if fails:
raise RuntimeError("connection refused")
return _Resp()
monkeypatch.setattr("httpx.post", _fake_post)
assert captioner._ocr_one("http://llama.test", "local", b"png", 5.0) is None
assert inf._direct_llama_inflight == 0
def _run_gguf_tts(monkeypatch, backend, generate):
"""Drive POST /audio/generate onto the GGUF (llama-server) branch."""
import asyncio
import routes.inference as inf
from models.inference import ChatCompletionRequest
backend.generate_audio_response = generate
monkeypatch.setattr(inf, "_llama_public_model_id", lambda *_a, **_k: "some/tts-GGUF")
monkeypatch.setattr(inf, "_fill_recommended_sampling_openai", lambda *_a, **_k: None)
async def _noop_switch(*_a, **_k):
return None
monkeypatch.setattr(inf, "_maybe_auto_switch_model", _noop_switch)
payload = ChatCompletionRequest(
model = "some/tts-GGUF", messages = [{"role": "user", "content": "hi"}]
)
return asyncio.run(inf.generate_audio(payload, request = None, current_subject = "t"))
def test_queue_state_counts_gguf_tts(monkeypatch):
"""GGUF TTS holds a llama-server slot for the whole request without a lease."""
import routes.inference as inf
backend = _llama_slot_readout(monkeypatch, is_audio = True)
seen = {}
def _generate(**_kwargs):
seen["state"] = inf._monitor_queue_state()
return (b"RIFFfake", 24000)
_run_gguf_tts(monkeypatch, backend, _generate)
assert seen["state"] == {"capacity": 4, "active": 1, "queued": 0, "free": 3}
assert inf._monitor_queue_state() == {"capacity": 4, "active": 0, "queued": 0, "free": 4}
@pytest.mark.parametrize("outcome", ["completed", "raised", "disconnected"])
def test_gguf_tts_balances_the_direct_counter(monkeypatch, outcome):
"""Every exit from the TTS branch gives the slot back; the counter has no reset hook."""
from fastapi import HTTPException
import routes.inference as inf
backend = _llama_slot_readout(monkeypatch, is_audio = True)
def _generate(**kwargs):
if outcome == "raised":
raise RuntimeError("llama-server died")
if outcome == "disconnected":
kwargs["cancel_event"].set()
raise RuntimeError("stream closed")
return (b"RIFFfake", 24000)
if outcome == "completed":
_run_gguf_tts(monkeypatch, backend, _generate)
else:
with pytest.raises(HTTPException) as excinfo:
_run_gguf_tts(monkeypatch, backend, _generate)
assert excinfo.value.status_code == (499 if outcome == "disconnected" else 500)
assert inf._direct_llama_inflight == 0
def test_top_level_provider_tool_event_stamps_first_token(monkeypatch):
import routes.inference as inf
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf, "api_monitor", monitor)
entry_id = _start(monitor, model = "provider/model")
inf._monitor_openai_chunk(
entry_id,
{
"object": "chat.completion.chunk",
"choices": [{"index": 0, "delta": {}, "finish_reason": None}],
"_toolEvent": {"type": "web_search"},
},
streaming = True,
)
monitor.finish(entry_id)
[entry] = monitor.snapshot()
assert entry["ttft_ms"] is not None
def test_disagreeing_choice_finish_reasons_report_no_stop_reason(monkeypatch):
import routes.inference as inf
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf, "api_monitor", monitor)
def row_for(reasons):
entry_id = _start(monitor)
inf._monitor_openai_chunk(
entry_id,
{
"choices": [
{"index": i, "message": {"content": "x"}, "finish_reason": r}
for i, r in enumerate(reasons)
]
},
)
monitor.finish(entry_id)
return next(r for r in monitor.snapshot() if r["id"] == entry_id)["stop_reason"]
assert row_for(["stop"]) == "stop"
assert row_for(["stop", "stop"]) == "stop"
assert row_for(["stop", "length"]) is None
def test_streamed_choice_finish_reasons_are_compared_across_chunks(monkeypatch):
# n > 1 streams one choice per chunk; judge agreement across the stream.
import routes.inference as inf
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf, "api_monitor", monitor)
def row_for(reasons):
entry_id = _start(monitor, endpoint = "/v1/completions")
for i, reason in enumerate(reasons):
inf._monitor_openai_sse_line(
entry_id,
"data: "
+ json.dumps({"choices": [{"index": i, "text": "x", "finish_reason": reason}]}),
)
monitor.finish(entry_id)
return next(r for r in monitor.snapshot() if r["id"] == entry_id)["stop_reason"]
assert row_for(["stop"]) == "stop"
assert row_for(["stop", "stop"]) == "stop"
assert row_for(["stop", "length"]) is None
def test_streamed_stop_reason_is_withheld_until_the_request_finishes(monkeypatch):
# Publishing the first choice's reason would later be retracted.
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, endpoint = "/v1/completions")
def row():
return next(r for r in monitor.snapshot() if r["id"] == entry_id)
monitor.note_stop_reason(entry_id, "stop")
assert row()["finished_at"] is None
assert row()["stop_reason"] is None
monitor.note_stop_reason(entry_id, "stop")
assert row()["stop_reason"] is None
monitor.finish(entry_id)
assert row()["stop_reason"] == "stop"
@pytest.mark.parametrize("writer", ["note_stop_reason", "set_perf"])
@pytest.mark.parametrize(
"status, expected",
[("completed", "stop"), ("cancelled", None), ("failed", None), ("error", None)],
)
def test_stop_reason_is_kept_only_by_completed_requests(writer, status, expected):
# Cancelled streams may record 'stop' before cancellation is stamped.
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, endpoint = "/v1/completions")
if writer == "set_perf":
monitor.set_perf(entry_id, stop_reason = "stop")
else:
monitor.note_stop_reason(entry_id, "stop")
if status == "error":
monitor.fail(entry_id, "boom")
else:
monitor.finish(entry_id, status)
row = next(r for r in monitor.snapshot() if r["id"] == entry_id)
assert row["status"] == status
assert row["stop_reason"] == expected
def test_non_streaming_stop_reason_survives_the_finish(monkeypatch):
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, endpoint = "/v1/completions")
monitor.set_perf(entry_id, stop_reason = "stop")
monitor.finish(entry_id)
assert next(r for r in monitor.snapshot() if r["id"] == entry_id)["stop_reason"] == "stop"
@pytest.mark.parametrize(
"queue, expected",
[
({"capacity": 4, "active": 1, "queued": 0, "free": 3}, "generating"),
({"capacity": 4, "active": 0, "queued": 2, "free": 0}, "generating"),
({"capacity": 4, "active": 0, "queued": 0, "free": 4}, "ready"),
(None, "ready"),
],
)
def test_monitor_status_counts_slots_no_row_can_see(monkeypatch, queue, expected):
# Direct llama calls open no row, so rows alone would report Ready.
import routes.inference as inf
monkeypatch.setattr(inf, "api_monitor", ApiMonitor(max_entries = 3))
monkeypatch.setattr(inf, "_monitor_active_model", lambda: "org/M-GGUF")
monkeypatch.setattr(inf, "_monitor_context_length", lambda: 4096)
monkeypatch.setattr(inf, "_monitor_queue_state", lambda: queue)
app = FastAPI()
app.include_router(inf.studio_router)
app.dependency_overrides = {get_current_subject: lambda: "alice"}
body = TestClient(app).get("/monitor").json()
assert body["active_requests"] == 0
assert body["status"] == expected
def test_monitor_status_is_idle_without_a_model(monkeypatch):
import routes.inference as inf
monkeypatch.setattr(inf, "api_monitor", ApiMonitor(max_entries = 3))
monkeypatch.setattr(inf, "_monitor_active_model", lambda: None)
monkeypatch.setattr(inf, "_monitor_context_length", lambda: None)
monkeypatch.setattr(inf, "_monitor_queue_state", lambda: None)
app = FastAPI()
app.include_router(inf.studio_router)
app.dependency_overrides = {get_current_subject: lambda: "alice"}
assert TestClient(app).get("/monitor").json()["status"] == "idle"
def test_tool_card_starts_ttft_but_not_the_token_rate_clock(monkeypatch):
# Tool run / confirmation time must not count as decoding time.
import core.inference.api_monitor as m
clock = [100.0]
monkeypatch.setattr(m.time, "monotonic", lambda: clock[0])
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor)
monitor.mark_first_token(entry_id, decoded = False)
clock[0] = 160.0
monitor.append_reply(entry_id, "first real token")
clock[0] = 162.0
monitor.set_usage(entry_id, completion_tokens = 21)
monitor.finish(entry_id)
row = next(r for r in monitor.snapshot() if r["id"] == entry_id)
assert row["ttft_ms"] == 0
assert row["tok_per_sec"] == 10.0
def test_a_decoded_first_token_starts_both_clocks(monkeypatch):
import core.inference.api_monitor as m
clock = [100.0]
monkeypatch.setattr(m.time, "monotonic", lambda: clock[0])
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor)
monitor.mark_first_token(entry_id)
clock[0] = 102.0
monitor.set_usage(entry_id, completion_tokens = 21)
monitor.finish(entry_id)
assert next(r for r in monitor.snapshot() if r["id"] == entry_id)["tok_per_sec"] == 10.0
def test_a_stop_reason_written_after_finish_escapes_the_clearing():
# The settle runs once at finish, so later writes would undo it.
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor)
monitor.set_perf(entry_id, stop_reason = "stop")
monitor.finish(entry_id, "cancelled")
assert next(r for r in monitor.snapshot() if r["id"] == entry_id)["stop_reason"] is None
monitor.set_perf(entry_id, stop_reason = "stop")
assert next(r for r in monitor.snapshot() if r["id"] == entry_id)["stop_reason"] == "stop"
@pytest.mark.parametrize(
"line, dropped",
[
('data: {"choices":[],"usage":{"completion_tokens":9}}', True),
('data: {"choices":[{"delta":{"content":"x"}}],"usage":{"completion_tokens":9}}', False),
('data: {"choices":[{"delta":{"content":"x"}}]}', False),
# Quoted key passes the prefilter; only the parse can tell.
('data: {"choices":[{"delta":{"content":"\\"usage\\":1"}}]}', False),
("data: [DONE]", False),
("data: not json", False),
(": keepalive comment", False),
],
)
def test_usage_only_sse_is_recognized_for_relay_filtering(line, dropped):
# A client that did not opt in would index choices[0] on a usage chunk.
import routes.inference as inf
assert inf._is_openai_usage_only_sse(line) is dropped
@pytest.mark.parametrize("include_usage, expected", [(True, True), (False, False)])
def test_wants_stream_usage_reads_the_callers_opt_in(include_usage, expected):
import routes.inference as inf
from types import SimpleNamespace
payload = SimpleNamespace(stream_options = {"include_usage": include_usage})
assert inf._wants_stream_usage(payload) is expected
assert inf._wants_stream_usage(SimpleNamespace(stream_options = None)) is False
def test_direct_llama_work_is_busy_without_the_admission_snapshot(monkeypatch):
# With admission off the readout is None, so track direct calls.
import routes.inference as inf
monkeypatch.setattr(inf, "api_monitor", ApiMonitor(max_entries = 3))
monkeypatch.setattr(inf, "_monitor_active_model", lambda: "org/M-GGUF")
monkeypatch.setattr(inf, "_monitor_context_length", lambda: 4096)
monkeypatch.setattr(inf, "_monitor_queue_state", lambda: None)
monkeypatch.setattr(inf, "_direct_llama_inflight", 1)
app = FastAPI()
app.include_router(inf.studio_router)
app.dependency_overrides = {get_current_subject: lambda: "alice"}
body = TestClient(app).get("/monitor").json()
assert body["active_requests"] == 0
assert body["queue"] is None
assert body["status"] == "generating"
monkeypatch.setattr(inf, "_direct_llama_inflight", 0)
assert TestClient(app).get("/monitor").json()["status"] == "ready"
def test_direct_busy_reads_the_live_counter(monkeypatch):
import routes.inference as inf
monkeypatch.setattr(inf, "_direct_llama_inflight", 0)
assert inf._direct_llama_is_busy() is False
monkeypatch.setattr(inf, "_direct_llama_inflight", 2)
assert inf._direct_llama_is_busy() is True
def test_the_decode_span_comes_only_from_engine_timings(monkeypatch):
"""duration_ms carries the queue wait, which read a 50 tok/s model as 5. decode_ms must not."""
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inference_route, "api_monitor", monitor)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
inference_route._monitor_usage(
entry_id,
{"prompt_tokens": 11, "completion_tokens": 50},
4096,
timings = {
"prompt_ms": 9000.0,
"prompt_per_second": 11.0,
"predicted_ms": 1000.0,
"predicted_per_second": 50.0,
},
)
monitor.finish(entry_id)
[row] = monitor.snapshot()
assert row["decode_ms"] == 1000
assert row["completion_tokens"] / (row["decode_ms"] / 1000) == 50.0
assert row["prompt_tok_per_sec"] == 11.0
def test_prompt_progress_updates_for_followup_prefill_and_decode_phase(monkeypatch):
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inference_route, "api_monitor", monitor)
entry_id = _start(monitor)
monitor.set_prompt_progress(
entry_id,
total = 1000,
processed = 400,
cached = 100,
time_ms = 250.25,
)
row = monitor.get(entry_id)
assert row["running_phase"] == "prompt_processing"
assert row["prompt_progress"] == {
"total": 1000,
"processed": 400,
"cached": 100,
"time_ms": 250.25,
"percent": 40.0,
}
monitor.set_running_phase(entry_id, "token_generation")
monitor.set_prompt_progress(entry_id, total = 1000, processed = 500)
row = monitor.get(entry_id)
assert row["running_phase"] == "prompt_processing"
assert row["prompt_progress"]["processed"] == 500
inference_route._monitor_usage(
entry_id,
None,
timings = {
"prompt_progress": {"total": 1000, "processed": 600},
"running_phase": "token_generation",
},
)
row = monitor.get(entry_id)
assert row["running_phase"] == "token_generation"
assert row["prompt_progress"]["processed"] == 600
def test_monitor_usage_relays_live_progress_and_decode_phase(monkeypatch):
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inference_route, "api_monitor", monitor)
entry_id = _start(monitor)
inference_route._monitor_usage(
entry_id,
None,
timings = {
"prompt_progress": {
"total": 800,
"processed": 200,
"cache": 0,
"time_ms": 10,
}
},
)
inference_route._monitor_usage(
entry_id,
None,
timings = {"running_phase": "token_generation"},
)
row = monitor.get(entry_id)
assert row["running_phase"] == "token_generation"
assert row["prompt_progress"]["percent"] == 25.0
@pytest.mark.parametrize(
"delta",
[
{
"tool_calls": [
{
"index": 0,
"id": "call_1",
"type": "function",
"function": {"name": "search", "arguments": '{"query":'},
}
]
},
{"function_call": {"name": "search", "arguments": '{"query":'}},
],
)
def test_tool_only_stream_leaves_prefill_phase(monkeypatch, delta):
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inference_route, "api_monitor", monitor)
entry_id = _start(monitor)
inference_route._monitor_openai_chunk(
entry_id,
{"prompt_progress": {"total": 1000, "processed": 1000}, "choices": []},
streaming = True,
)
inference_route._monitor_openai_chunk(
entry_id,
{"choices": [{"index": 0, "delta": delta}]},
streaming = True,
)
assert monitor.get(entry_id)["running_phase"] == "token_generation"
def test_streaming_response_exposes_monitor_id():
response = inference_route._sse_streaming_response(
iter(()),
monitor_id = "apireq_test",
)
assert response.headers["X-Unsloth-Monitor-ID"] == "apireq_test"
assert "X-Unsloth-Monitor-ID" not in inference_route._sse_streaming_response(iter(())).headers
def test_monitor_id_reads_the_matching_concurrent_request(monkeypatch):
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inference_route, "api_monitor", monitor)
first_id = _start(monitor, subject = "alice", model = "same-model", prompt = "same")
second_id = _start(monitor, subject = "alice", model = "same-model", prompt = "same")
monitor.set_prompt_progress(first_id, total = 1000, processed = 200)
monitor.set_prompt_progress(second_id, total = 1000, processed = 800)
app = FastAPI()
app.include_router(inference_route.studio_router)
app.dependency_overrides = {get_current_subject: lambda: "alice"}
client = TestClient(app)
first = client.get(f"/monitor/{first_id}").json()
second = client.get(f"/monitor/{second_id}").json()
assert first["id"] != second["id"]
assert first["prompt_progress"]["processed"] == 200
assert second["prompt_progress"]["processed"] == 800
def test_a_timings_only_final_chunk_still_sets_the_decode_span(monkeypatch):
"""llama-server can end a stream with timings and no usage."""
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inference_route, "api_monitor", monitor)
entry_id = _start(
monitor,
endpoint = "/v1/completions",
model = "local-model",
prompt = "user: hello",
)
inference_route._monitor_usage(entry_id, None, None, timings = {"predicted_ms": 1000})
monitor.finish(entry_id)
assert monitor.snapshot()[0]["decode_ms"] == 1000
def test_a_streamed_reply_alone_reports_no_decode_span():
"""Timing the stream misses the first chunk and reasoning tokens, so report nothing."""
monitor = ApiMonitor(max_entries = 3)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
monitor.append_reply(entry_id, "hi")
monitor.append_reply(entry_id, " there")
monitor.finish(entry_id)
row = monitor.snapshot()[0]
assert row["decode_ms"] is None
assert row["duration_ms"] is not None
@pytest.mark.parametrize(
# "1000" is absent: _finite_float_or_none coerces numeric strings.
"predicted_ms",
[float("inf"), float("nan"), -1, "abc", None, 1e308, 10**400, {}, []],
)
def test_a_bad_predicted_ms_is_dropped_rather_than_raising(monkeypatch, predicted_ms):
"""json.loads accepts a bare Infinity; a raise in a streaming generator truncates the reply."""
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inference_route, "api_monitor", monitor)
entry_id = _start(monitor, model = "local-model", prompt = "user: hello")
inference_route._monitor_usage(
entry_id,
{"completion_tokens": 50},
None,
timings = {"predicted_ms": predicted_ms},
)
monitor.finish(entry_id)
assert monitor.snapshot()[0]["decode_ms"] is None
def test_perf_callback_reports_decode_phase_once_per_prefill_round(monkeypatch):
import routes.inference as inf_mod
from core.inference.llama_cpp import _report_live_llama_timings
seen = []
monkeypatch.setattr(
inf_mod,
"_monitor_usage",
lambda *_a, timings, **_k: seen.append(timings.get("running_phase")),
)
callback = inf_mod._monitor_perf_callback("apireq_x", 4096)
progress = {
"choices": [{"index": 0, "delta": {"role": "assistant", "content": None}}],
"prompt_progress": {"total": 10, "processed": 10, "cache": 0, "time_ms": 1},
}
token = {"choices": [{"index": 0, "delta": {"content": "a"}}], "timings": {"predicted_n": 1}}
for chunk in (progress, token, token, token, progress, token, token):
_report_live_llama_timings(callback, chunk)
assert seen == [None, "token_generation", None, None, None, "token_generation", None]
def test_progress_updates_advance_updated_at_on_a_frozen_clock(monkeypatch):
import core.inference.api_monitor as monitor_mod
monitor = ApiMonitor(max_entries = 3)
entry_id = monitor.start(endpoint = "/v1/chat/completions", method = "POST", model = "m", prompt = "p")
monkeypatch.setattr(monitor_mod.time, "time", lambda: 1_000.0)
seen = []
for processed in (10, 20):
monitor.set_prompt_progress(entry_id, total = 100, processed = processed)
seen.append(monitor.snapshot()[0]["updated_at"])
monitor.set_running_phase(entry_id, "token_generation")
seen.append(monitor.snapshot()[0]["updated_at"])
assert seen[0] < seen[1] < seen[2]
def test_mistral_reasoning_stream_reaches_the_client_and_the_monitor(monkeypatch):
import asyncio
from types import SimpleNamespace
import httpx
from core.inference import external_provider as ep
from models.inference import ChatCompletionRequest
thinking = [{"type": "thinking", "thinking": [{"type": "text", "text": "Multiply 17 by 23."}]}]
deltas = [
{"role": "assistant", "content": thinking},
{
"content": [
{"type": "thinking", "thinking": [{"type": "text", "text": ""}]},
{"type": "text", "text": "17 * 23"},
]
},
{"content": " = 391"},
]
chunks = [
{"choices": [{"index": 0, "delta": delta, "finish_reason": None}]} for delta in deltas
]
chunks.append({"choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}]})
sse = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n"
def handle(_request):
return httpx.Response(
200, headers = {"content-type": "text/event-stream"}, content = sse.encode()
)
async def run():
async with httpx.AsyncClient(transport = httpx.MockTransport(handle)) as client:
monkeypatch.setattr(ep, "_http_client", client)
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inference_route, "api_monitor", monitor)
payload = ChatCompletionRequest(
messages = [{"role": "user", "content": "What is 17 * 23?"}],
stream = True,
provider_type = "mistral",
external_model = "mistral-medium-latest",
)
async def disconnected():
return False
request = SimpleNamespace(
headers = {},
state = SimpleNamespace(skip_api_monitor = False),
url = SimpleNamespace(path = "/v1/chat/completions"),
method = "POST",
is_disconnected = disconnected,
)
response = await inference_route._proxy_to_external_provider(payload, request)
body = [part async for part in response.body_iterator]
return body, monitor
body, monitor = asyncio.run(run())
sent = []
for part in body:
part = part.decode() if isinstance(part, bytes) else part
for line in part.splitlines():
if line.startswith("data: {"):
sent.append(json.loads(line[6:]))
assert not [chunk for chunk in sent if "error" in chunk]
contents = [
choice["delta"]["content"]
for chunk in sent
for choice in chunk.get("choices") or []
if choice.get("delta", {}).get("content")
]
assert contents == [delta["content"] for delta in deltas]
row = monitor.snapshot()[0]
assert row["status"] == "completed"
assert row["reply"] == "17 * 23 = 391"
def test_a_monitor_failure_does_not_abort_the_stream(monkeypatch):
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inference_route, "api_monitor", monitor)
entry_id = _start(monitor)
def boom(*_args, **_kwargs):
raise RuntimeError("monitor bug")
monkeypatch.setattr(monitor, "append_reply", boom)
line = "data: " + json.dumps({"choices": [{"index": 0, "delta": {"content": "hi"}}]})
assert inference_route._monitor_openai_sse_line(entry_id, line) is None
def test_mistral_thinking_part_starts_token_generation(monkeypatch):
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inference_route, "api_monitor", monitor)
entry_id = _start(monitor)
thinking = [{"type": "thinking", "thinking": [{"type": "text", "text": "Hmm."}]}]
line = "data: " + json.dumps({"choices": [{"index": 0, "delta": {"content": thinking}}]})
inference_route._monitor_openai_sse_line(entry_id, line)
row = monitor.snapshot()[0]
assert row["running_phase"] == "token_generation"
assert row["reply"] == ""