Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: report a throughput the engine could have produced (#10384)

Daniel Han46792093a213
diff --git a/studio/backend/core/inference/llama_stats.py b/studio/backend/core/inference/llama_stats.py
index d63e38801ee..95241275c9c 100644
--- a/studio/backend/core/inference/llama_stats.py
+++ b/studio/backend/core/inference/llama_stats.py
@@ -66,11 +66,30 @@ class LlamaServerStatsLogger:
out = {}
for k, v in _METRIC_RE.findall(body):
try: # a malformed value must not kill the daemon thread
- out[k] = float(v)
+ value = float(v)
except ValueError:
continue
+ # float() overflows a long digit string to inf without raising; _env_float
+ # refuses the same text. No printed double reaches 1.8e308, so nothing is lost.
+ if math.isfinite(value):
+ out[k] = value
return out
+ @staticmethod
+ def _prompt_rate(base, tokens, seconds):
+ """Prompt tokens per second over the engine's OWN measure of the time they took.
+
+ Prompt only: add_prompt(n, n, t_us) pairs those totals, while metrics_on_prediction()
+ passes n_gen and n_gen - 1. The baseline is held until both move, since /metrics
+ renders six significant digits and one can cross a boundary a scrape before the other.
+ """
+ if base is None or tokens < base[0] or seconds < base[1]:
+ return 0.0, (tokens, seconds) # first reading, or counters that went backwards
+ d_tokens, d_seconds = tokens - base[0], seconds - base[1]
+ if d_tokens <= 0.0 or d_seconds <= 0.0:
+ return 0.0, base
+ return d_tokens / d_seconds, (tokens, seconds)
+
def _stalled_for(self, now, running, decode_calls):
"""Seconds the engine has held a slot without calling llama_decode().
@@ -121,7 +140,8 @@ class LlamaServerStatsLogger:
def _run(self):
misses = 0
- prev = None # (monotonic_t, tokens_predicted_total, prompt_tokens_total)
+ prev = None # (monotonic_t, n_decode_total)
+ gen_base = prompt_base = None # (tokens, seconds) at the last tick that carried both
while not self._stop.wait(self._interval):
m = self._scrape()
if not m:
@@ -130,30 +150,35 @@ class LlamaServerStatsLogger:
self._log.debug("engine_stats: /metrics scrape failing, still retrying")
continue # real shutdown is driven by stop() from _kill_process
misses = 0
- # Generation tokens come from tokens_predicted_total (counter) and predicted_tokens_seconds (gauge);
- # n_decode_total counts llama_decode() calls, not tokens, so it must not feed tok/s.
now = time.monotonic()
predicted = m.get("tokens_predicted_total", 0.0)
prompt = m.get("prompt_tokens_total", 0.0)
- gen_delta = prompt_delta = 0.0
- if prev is not None and now > prev[0]:
- dt = now - prev[0]
- gen_delta = max(0.0, (predicted - prev[1]) / dt)
- prompt_delta = max(0.0, (prompt - prev[2]) / dt)
- prev = (now, predicted, prompt)
- # Prefer llama.cpp's own throughput gauges; fall back to the counter delta for binaries that expose only the
- # counters.
- gen_tps = m.get("predicted_tokens_seconds") or gen_delta
- prompt_tps = m.get("prompt_tokens_seconds") or prompt_delta
+ predicted_s = m.get("tokens_predicted_seconds_total", 0.0)
+ prompt_s = m.get("prompt_seconds_total", 0.0)
+ # A build without n_decode_total reads None and never "changes", so the wedge
+ # message is chosen at report time.
+ decode_calls = m.get("n_decode_total")
running, waiting = (
int(m.get("requests_processing", 0)),
int(m.get("requests_deferred", 0)),
)
- # a build without n_decode_total reads None and never "changes", accumulating the same way
- # A held slot not calling llama_decode() is a wedge, and its only symptom is an endless run of identical
- # info lines. A build without n_decode_total reads None and never "changes", accumulating the same way, so
- # the message is chosen at report time.
- decode_calls = m.get("n_decode_total")
+ prompt_delta, prompt_base = self._prompt_rate(prompt_base, prompt, prompt_s)
+ gen_moved = gen_base is not None and (predicted, predicted_s) != gen_base
+ gen_base = (predicted, predicted_s)
+ # Calls, not tokens, and never fed into tok/s: the only counter that moves
+ # within a tick, so the only sign of progress while a generation runs.
+ decode_rate = None
+ if prev is not None and now > prev[0] and None not in (decode_calls, prev[1]):
+ decode_rate = max(0.0, (decode_calls - prev[1]) / (now - prev[0]))
+ prev = (now, decode_calls)
+ # The bucket empties on every /metrics read, ours or another client's, so a zero
+ # gauge is a reading. The counters read a free first token against a millisecond.
+ gen_tps = m.get("predicted_tokens_seconds")
+ # /metrics renders one table, so a scrape with no prompt metric measured none.
+ prompt_measured = "prompt_tokens_seconds" in m or (
+ "prompt_tokens_total" in m and "prompt_seconds_total" in m
+ )
+ prompt_tps = m.get("prompt_tokens_seconds") or prompt_delta
stalled_for = self._stalled_for(now, running, decode_calls)
if self._stall_timeout and stalled_for >= self._stall_timeout:
if decode_calls is None:
@@ -168,15 +193,19 @@ class LlamaServerStatsLogger:
)
elif not self._stall_reported:
self._report_stall(running, waiting, stalled_for, decode_calls)
- # Gate on real activity this tick so a stale gauge never logs at idle.
- if running or waiting or gen_delta or prompt_delta:
- self._log.info(
- "engine_stats",
- gen_tok_s = round(float(gen_tps), 1),
- prompt_tok_s = round(float(prompt_tps), 1),
- running = running,
- waiting = waiting,
- )
+ # Gate on real activity this tick, so an idle engine stays quiet.
+ if running or waiting or gen_tps or gen_moved or prompt_tps:
+ # Absent, not 0.0: a build with no gauge, no n_decode_total or no prompt
+ # metric was never measured, and 0.0 states it was.
+ fields = {}
+ if gen_tps is not None:
+ fields["gen_tok_s"] = round(float(gen_tps), 1)
+ if prompt_measured:
+ fields["prompt_tok_s"] = round(float(prompt_tps), 1)
+ fields["running"], fields["waiting"] = running, waiting
+ if decode_rate is not None:
+ fields["decode_calls_s"] = round(float(decode_rate), 1)
+ self._log.info("engine_stats", **fields)
# bounded by threading.TIMEOUT_MAX as well
diff --git a/studio/backend/tests/test_engine_stats_live_rate.py b/studio/backend/tests/test_engine_stats_live_rate.py
new file mode 100644
index 00000000000..f61509b3113
--- /dev/null
+++ b/studio/backend/tests/test_engine_stats_live_rate.py
@@ -0,0 +1,282 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""engine_stats must not attribute work to the tick its counter moved on.
+
+Neither token counter moves while the work happens, so the poll interval prices a window
+the work did not run in. The sides are not symmetric: add_prompt(n, n, t) makes the prompt
+counters a pair, metrics_on_prediction() passes n_gen and n_gen - 1, so generation
+throughput comes from llama.cpp's gauge or from nowhere. The clock is faked because the
+shared _drive helper polls at 1 ms on the real one.
+"""
+
+from __future__ import annotations
+
+import core.inference.llama_stats as ls
+from core.inference.llama_stats import LlamaServerStatsLogger
+
+_TICK_S = 10.0
+
+
+class _Capture:
+ def __init__(self):
+ self.events = []
+
+ def info(self, event, **kw):
+ self.events.append((event, dict(kw)))
+
+ def debug(self, *a, **k):
+ pass
+
+ def warning(self, *a, **k):
+ pass
+
+
+def _drive(
+ snaps,
+ monkeypatch,
+ tick_s = _TICK_S,
+):
+ """Run _run() over `snaps` on a clock that advances tick_s per scrape."""
+ clock = {"t": 1000.0}
+ monkeypatch.setattr(ls.time, "monotonic", lambda: clock["t"])
+
+ cap = _Capture()
+ lg = LlamaServerStatsLogger("http://127.0.0.1:0", cap)
+ lg._interval = 0.001 # the real sleep between ticks, not the faked elapsed time
+ state = {"i": 0}
+
+ def fake_scrape():
+ i = state["i"]
+ state["i"] += 1
+ if i >= len(snaps):
+ lg.stop()
+ return None
+ clock["t"] += tick_s
+ return snaps[i]
+
+ lg._scrape = fake_scrape
+ lg._run()
+ return [kw for ev, kw in cap.events if ev == "engine_stats"]
+
+
+def _busy(
+ predicted = 0.0,
+ predicted_s = 0.0,
+ prompt = 0.0,
+ prompt_s = 0.0,
+ decode = 0.0,
+ running = 1.0,
+ waiting = 0.0,
+ gen_gauge = None,
+):
+ snap = {
+ "tokens_predicted_total": predicted,
+ "tokens_predicted_seconds_total": predicted_s,
+ "prompt_tokens_total": prompt,
+ "prompt_seconds_total": prompt_s,
+ "n_decode_total": decode,
+ "requests_processing": running,
+ "requests_deferred": waiting,
+ }
+ if gen_gauge is not None:
+ snap["predicted_tokens_seconds"] = gen_gauge
+ return snap
+
+
+def test_a_prefill_is_priced_by_the_seconds_it_reports_with_it(monkeypatch):
+ snaps = [_busy(decode = float(i)) for i in range(8)] + [
+ _busy(prompt = 1837.0, prompt_s = 80.0, decode = 8.0)
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert max(s["prompt_tok_s"] for s in stats) == 23.0
+ # What the old arithmetic (count / poll interval) would have said.
+ assert 1837.0 / _TICK_S == 183.7
+
+
+def test_an_idle_gap_is_not_charged_to_the_prefill_after_it(monkeypatch):
+ idle = [_busy(running = 0.0) for _ in range(6)]
+ working = [_busy(decode = 1.0), _busy(prompt = 100.0, prompt_s = 20.0, decode = 2.0)]
+ stats = _drive(idle + working, monkeypatch)
+
+ assert max(s["prompt_tok_s"] for s in stats) == 5.0
+
+
+def test_deferred_requests_do_not_stretch_the_denominator(monkeypatch):
+ """A queued request is not running, so charging its wait understates instead."""
+ snaps = [
+ _busy(decode = 1.0),
+ *[_busy(running = 0.0, waiting = 1.0, decode = 1.0) for _ in range(6)],
+ _busy(prompt = 100.0, prompt_s = 20.0, decode = 2.0),
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert max(s["prompt_tok_s"] for s in stats) == 5.0
+ # What charging the queued ticks would have said.
+ assert round(100.0 / 80.0, 1) == 1.2
+
+
+def test_work_already_running_at_the_first_poll_is_priced_whole(monkeypatch):
+ """Starting mid-prefill, elapsed poll time omits up to an interval; the engine's does not."""
+ snaps = [
+ _busy(prompt = 200.0, prompt_s = 40.0, decode = 4.0),
+ _busy(prompt = 1837.0, prompt_s = 80.0, decode = 8.0),
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ # 1637 tokens in the 40 seconds between the two readings.
+ assert max(s["prompt_tok_s"] for s in stats) == 40.9
+ assert 1637.0 / _TICK_S == 163.7
+
+
+def test_a_long_prefill_is_not_attributed_to_the_tick_it_flushed_on(monkeypatch):
+ """The prompt counter and its seconds flush together, on a decode with output."""
+ snaps = (
+ [_busy(prompt = 0.0)]
+ + [_busy() for _ in range(64)]
+ + [_busy(prompt = 130000.0, prompt_s = 650.0)]
+ )
+ stats = _drive(snaps, monkeypatch)
+
+ assert max(s["prompt_tok_s"] for s in stats) == 200.0
+ assert 130000.0 / _TICK_S == 13000.0
+
+
+def test_the_decode_counter_reports_while_the_token_counters_are_still(monkeypatch):
+ """Why the line read 0 for 98.8% of the time: only n_decode_total moves here."""
+ snaps = [_busy(decode = float(i * 20), gen_gauge = 0.0) for i in range(4)]
+ stats = _drive(snaps, monkeypatch)
+
+ assert all(s["gen_tok_s"] == 0.0 for s in stats)
+ # Nothing on the first line: one sample is not a rate.
+ assert [s.get("decode_calls_s") for s in stats] == [None, 2.0, 2.0, 2.0]
+
+
+def test_a_build_without_the_decode_counter_omits_the_field(monkeypatch):
+ """0.0 would state the engine was measured making no calls."""
+ snaps = [
+ {"tokens_predicted_total": 0.0, "prompt_tokens_total": 0.0, "requests_processing": 1.0},
+ {"tokens_predicted_total": 50.0, "prompt_tokens_total": 0.0, "requests_processing": 1.0},
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert stats and all("decode_calls_s" not in s for s in stats)
+
+
+def test_a_zero_gauge_is_a_reading_and_not_a_missing_one(monkeypatch):
+ """One-token completion: the free prompt-batch token is no decode step, so 0."""
+ snaps = [
+ _busy(gen_gauge = 0.0),
+ _busy(predicted = 1.0, predicted_s = 0.0001, gen_gauge = 0.0),
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert stats and all(s["gen_tok_s"] == 0.0 for s in stats)
+ assert 1.0 / 0.0001 == 10000.0
+
+
+def test_a_zero_gauge_beside_moved_counters_is_still_a_reading(monkeypatch):
+ """Another client's scrape empties the bucket, and understating beats inventing."""
+ snaps = [
+ _busy(predicted = 100.0, predicted_s = 5.0, gen_gauge = 20.0),
+ _busy(predicted = 300.0, predicted_s = 15.0, gen_gauge = 0.0),
+ _busy(predicted = 400.0, predicted_s = 20.0, gen_gauge = 20.0),
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert [s["gen_tok_s"] for s in stats] == [20.0, 0.0, 20.0]
+ # What filling the gap from the counters would have said.
+ assert 200.0 / 10.0 == 20.0
+
+
+def test_tokens_with_no_seconds_yet_are_kept_for_the_tick_that_brings_them(monkeypatch):
+ """Six significant digits can move one total a scrape before the other, so the
+ baseline is held until both have moved rather than charged to the next seconds."""
+ snaps = [
+ _busy(prompt = 100.0, prompt_s = 1000.0),
+ _busy(prompt = 200.0, prompt_s = 1000.0),
+ _busy(prompt = 300.0, prompt_s = 1020.0),
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert [s["prompt_tok_s"] for s in stats] == [0.0, 0.0, 10.0]
+
+
+def test_seconds_that_resolve_before_their_tokens_are_kept_too(monkeypatch):
+ snaps = [
+ _busy(prompt = 100.0, prompt_s = 1000.0),
+ _busy(prompt = 100.0, prompt_s = 1020.0),
+ _busy(prompt = 200.0, prompt_s = 1020.0),
+ _busy(prompt = 300.0, prompt_s = 1040.0),
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert [s["prompt_tok_s"] for s in stats] == [0.0, 0.0, 5.0, 5.0]
+ # What discarding the held seconds would have said for the third tick.
+ assert 100.0 / 20.0 == 5.0
+
+
+def test_a_build_without_the_generation_gauge_omits_the_field(monkeypatch):
+ """The seconds time n_gen - 1 steps against n_gen tokens, so the ratio is not a rate."""
+ snaps = [
+ _busy(predicted = 0.0),
+ _busy(predicted = 1.0, predicted_s = 0.0001),
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert stats and all("gen_tok_s" not in s for s in stats)
+ assert 1.0 / 0.0001 == 10000.0
+
+
+def test_a_build_with_no_prompt_metric_omits_the_field(monkeypatch):
+ """The third unmeasurable case: no prompt gauge and no prompt counters."""
+ snaps = [
+ {"tokens_predicted_total": 0.0, "requests_processing": 1.0},
+ {"tokens_predicted_total": 50.0, "requests_processing": 1.0},
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert stats and all("prompt_tok_s" not in s for s in stats)
+ assert all(s["running"] == 1 for s in stats)
+
+
+def test_a_non_finite_metric_value_never_reaches_the_line(monkeypatch):
+ """float() turns a long digit string into inf; _env_float refuses the same text."""
+
+ class _Resp:
+ status = 200
+
+ def __init__(self, body):
+ self._body = body
+
+ def read(self):
+ return self._body
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *a):
+ return False
+
+ body = (
+ "llamacpp:prompt_tokens_seconds " + "9" * 400 + "\n"
+ "llamacpp:prompt_tokens_total 10\n"
+ "llamacpp:prompt_seconds_total 1\n"
+ ).encode()
+ monkeypatch.setattr(ls.urllib.request, "urlopen", lambda *a, **k: _Resp(body))
+
+ m = LlamaServerStatsLogger("http://127.0.0.1:0", _Capture())._scrape()
+
+ assert "prompt_tokens_seconds" not in m
+ assert m["prompt_tokens_total"] == 10.0
+
+
+def test_the_llama_cpp_gauge_still_wins_when_it_reports(monkeypatch):
+ snaps = [
+ _busy(gen_gauge = 24.6),
+ _busy(predicted = 1837.0, predicted_s = 80.0, gen_gauge = 24.6),
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert stats and all(s["gen_tok_s"] == 24.6 for s in stats)
diff --git a/studio/backend/tests/test_llama_stats.py b/studio/backend/tests/test_llama_stats.py
index 5e43d80f256..c46ac40ecf8 100644
--- a/studio/backend/tests/test_llama_stats.py
+++ b/studio/backend/tests/test_llama_stats.py
@@ -117,12 +117,25 @@ def test_scrape_parses_labelled_and_bare_metrics(monkeypatch):
assert m["requests_processing"] == 1.0
-def test_counter_delta_fallback_without_gauges():
- # Older binaries expose only the counters; throughput falls back to deltas.
+def test_counters_without_gauges_still_report_what_is_measurable():
+ # Older binaries expose only the counters. The generation pair is not a rate (the
+ # seconds time n_gen - 1 steps while the tokens count n_gen), so no gen_tok_s is
+ # claimed; running=1 keeps the line going out with the fields that are measured.
+ # /metrics renders one table, so a build with prompt_tokens_total has the seconds too.
snaps = [
- {"tokens_predicted_total": 100.0, "prompt_tokens_total": 0.0, "requests_processing": 1.0},
- {"tokens_predicted_total": 100.0, "prompt_tokens_total": 0.0, "requests_processing": 1.0},
+ {
+ "tokens_predicted_total": 100.0,
+ "prompt_tokens_total": 0.0,
+ "prompt_seconds_total": 0.0,
+ "requests_processing": 1.0,
+ },
+ {
+ "tokens_predicted_total": 100.0,
+ "prompt_tokens_total": 0.0,
+ "prompt_seconds_total": 0.0,
+ "requests_processing": 1.0,
+ },
]
stats = _drive(snaps)
- # running=1 keeps it emitting; gen_tok_s falls back to the (here zero) delta.
- assert stats and all(s["gen_tok_s"] >= 0.0 for s in stats)
+ assert stats and all("gen_tok_s" not in s for s in stats)
+ assert all(s["prompt_tok_s"] == 0.0 and s["running"] == 1 for s in stats)
diff --git a/studio/backend/tests/test_llama_stats_stall.py b/studio/backend/tests/test_llama_stats_stall.py
index c5aba864a1c..9cb11fca3a3 100644
--- a/studio/backend/tests/test_llama_stats_stall.py
+++ b/studio/backend/tests/test_llama_stats_stall.py
@@ -78,11 +78,15 @@ def _stalls(cap):
def _wedged(n, *, decode = 8192.0):
- """The user's signature: a held slot that never calls llama_decode()."""
+ """The user's signature: a held slot that never calls llama_decode().
+
+ The gauges are in the snapshot because /metrics renders them from the same table."""
return [
{
"tokens_predicted_total": 4096.0,
"prompt_tokens_total": 512.0,
+ "predicted_tokens_seconds": 0.0,
+ "prompt_tokens_seconds": 0.0,
"n_decode_total": decode,
"requests_processing": 1.0,
"requests_deferred": 0.0,