Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: log a live generation rate while a reply is still streaming (#15390)

Nilay940a42ab568b
diff --git a/studio/backend/core/inference/llama_stats.py b/studio/backend/core/inference/llama_stats.py
index 3615f19135f..5d2413b363d 100644
--- a/studio/backend/core/inference/llama_stats.py
+++ b/studio/backend/core/inference/llama_stats.py
@@ -3,6 +3,7 @@
"""Turns llama-server /metrics into periodic vLLM-style stats lines, only while busy; needs --metrics."""
+import json
import math
import os
import re
@@ -31,6 +32,7 @@ class LlamaServerStatsLogger:
headers = None,
):
self._url = f"{base_url.rstrip('/')}/metrics"
+ self._slots_url = f"{base_url.rstrip('/')}/slots"
self._headers = dict(headers or {})
# No HTTPSHandler: it builds an SSL context this loopback scrape never needs.
self._opener = urllib.request.OpenerDirector()
@@ -51,6 +53,7 @@ class LlamaServerStatsLogger:
self._stall_since = None
self._stall_reported = False
self._unmeasurable_reported = False
+ self._slot_base = None
def start(self):
if self._thread is None:
@@ -60,17 +63,22 @@ class LlamaServerStatsLogger:
def stop(self):
self._stop.set()
- def _scrape(self):
+ def _get(self, url):
try:
# No proxy: the bearer must only ever go to the loopback server.
with self._opener.open(
- urllib.request.Request(self._url, headers = self._headers), timeout = 3
+ urllib.request.Request(url, headers = self._headers), timeout = 3
) as r:
if r.status != 200:
return None
- body = r.read().decode("utf-8", "replace")
+ return r.read().decode("utf-8", "replace")
except Exception:
return None
+
+ def _scrape(self):
+ body = self._get(self._url)
+ if body is None:
+ return None
out = {}
for k, v in _METRIC_RE.findall(body):
try: # a malformed value must not kill the daemon thread
@@ -82,6 +90,23 @@ class LlamaServerStatsLogger:
out[k] = value
return out
+ def _live_gen_rate(self, now, running):
+ slots = {}
+ if running:
+ try:
+ slots = {
+ (s["id"], s["id_task"]): float(s["next_token"][0]["n_decoded"])
+ for s in json.loads(self._get(self._slots_url))
+ if s.get("is_processing")
+ }
+ except Exception:
+ slots = None
+ base, self._slot_base = self._slot_base, None if slots is None else (now, slots)
+ if slots is None or base is None or now <= base[0]:
+ return None
+ tokens = sum(max(0.0, n - base[1].get(k, 0.0)) for k, n in slots.items())
+ return tokens / (now - base[0])
+
@staticmethod
def _prompt_rate(base, tokens, seconds):
"""Prompt tokens/s over the prompt time the engine measures; baseline held until both totals move, as /metrics rounds to six digits."""
@@ -150,6 +175,13 @@ class LlamaServerStatsLogger:
prev = (now, decode_calls)
# The bucket empties on every /metrics read, from any client, so a zero gauge is a reading; counters read a free first token against a millisecond.
gen_tps = m.get("predicted_tokens_seconds")
+ # The gauge only counts a reply once its slot is released, so a long one reads 0 until it ends.
+ live_gen = self._live_gen_rate(now, running)
+ if running and not gen_tps:
+ if live_gen is not None:
+ gen_tps = live_gen
+ elif decode_rate:
+ gen_tps = None
# /metrics renders one table, so a scrape with no prompt metric measured none.
prompt_measured = "prompt_tokens_seconds" in m or (
"prompt_tokens_total" in m and "prompt_seconds_total" in m
diff --git a/studio/backend/tests/test_engine_stats_live_rate.py b/studio/backend/tests/test_engine_stats_live_rate.py
index 1e025b18eb2..26140a19815 100644
--- a/studio/backend/tests/test_engine_stats_live_rate.py
+++ b/studio/backend/tests/test_engine_stats_live_rate.py
@@ -5,12 +5,30 @@
from __future__ import annotations
+import json
+
import core.inference.llama_stats as ls
from core.inference.llama_stats import LlamaServerStatsLogger
_TICK_S = 10.0
+class _Resp:
+ status = 200
+
+ def __init__(self, body):
+ self._body = body
+
+ def read(self):
+ return self._body
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *a):
+ return False
+
+
class _Capture:
def __init__(self):
self.events = []
@@ -139,7 +157,7 @@ def test_the_decode_counter_reports_while_the_token_counters_are_still(monkeypat
snaps = [_busy(decode = float(i * 20), gen_gauge = 0.0) for i in range(4)]
stats = _drive(snaps, monkeypatch)
- assert all(s["gen_tok_s"] == 0.0 for s in stats)
+ assert [s.get("gen_tok_s") for s in stats] == [0.0, None, None, None]
assert [s.get("decode_calls_s") for s in stats] == [None, 2.0, 2.0, 2.0]
@@ -232,21 +250,6 @@ def test_a_build_with_no_prompt_metric_omits_the_field(monkeypatch):
def test_a_non_finite_metric_value_never_reaches_the_line(monkeypatch):
"""float() turns a long digit string into inf; _env_float refuses the same text."""
- class _Resp:
- status = 200
-
- def __init__(self, body):
- self._body = body
-
- def read(self):
- return self._body
-
- def __enter__(self):
- return self
-
- def __exit__(self, *a):
- return False
-
body = (
"llamacpp:prompt_tokens_seconds " + "9" * 400 + "\n"
"llamacpp:prompt_tokens_total 10\n"
@@ -268,3 +271,45 @@ def test_the_llama_cpp_gauge_still_wins_when_it_reports(monkeypatch):
stats = _drive(snaps, monkeypatch)
assert stats and all(s["gen_tok_s"] == 24.6 for s in stats)
+
+
+def _slots(monkeypatch, *ticks):
+ bodies = iter(
+ json.dumps(
+ [
+ {
+ "id": slot,
+ "is_processing": True,
+ "id_task": task,
+ "next_token": [{"n_decoded": n}],
+ }
+ for slot, task, n in tick
+ ]
+ ).encode()
+ for tick in ticks
+ )
+ monkeypatch.setattr(
+ ls.urllib.request.OpenerDirector, "open", lambda *a, **k: _Resp(next(bodies))
+ )
+
+
+def test_a_long_reply_reports_its_rate_while_it_is_still_generating(monkeypatch):
+ _slots(monkeypatch, [(0, 7, 250)], [(0, 7, 500)], [(0, 7, 750)])
+ snaps = [_busy(running = 0.0)] + [
+ _busy(decode = float(i * 250), gen_gauge = 0.0) for i in range(1, 4)
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert [s["gen_tok_s"] for s in stats] == [25.0, 25.0, 25.0]
+
+
+def test_a_new_task_on_a_slot_is_counted_from_zero(monkeypatch):
+ _slots(monkeypatch, [(0, 7, 900)], [(0, 8, 100)])
+ snaps = [
+ _busy(running = 0.0),
+ _busy(decode = 900.0, gen_gauge = 0.0),
+ _busy(decode = 1000.0, gen_gauge = 0.0),
+ ]
+ stats = _drive(snaps, monkeypatch)
+
+ assert [s["gen_tok_s"] for s in stats] == [90.0, 10.0]