Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: log a live generation rate while a reply is still streaming (#15390)
Nilay940a42ab568b
diff --git a/studio/backend/core/inference/llama_stats.py b/studio/backend/core/inference/llama_stats.pyindex 3615f19135f..5d2413b363d 100644--- a/studio/backend/core/inference/llama_stats.py+++ b/studio/backend/core/inference/llama_stats.py@@ -3,6 +3,7 @@"""Turns llama-server /metrics into periodic vLLM-style stats lines, only while busy; needs --metrics."""+import jsonimport mathimport osimport re@@ -31,6 +32,7 @@ class LlamaServerStatsLogger:headers = None,):self._url = f"{base_url.rstrip('/')}/metrics"+ self._slots_url = f"{base_url.rstrip('/')}/slots"self._headers = dict(headers or {})# No HTTPSHandler: it builds an SSL context this loopback scrape never needs.self._opener = urllib.request.OpenerDirector()@@ -51,6 +53,7 @@ class LlamaServerStatsLogger:self._stall_since = Noneself._stall_reported = Falseself._unmeasurable_reported = False+ self._slot_base = Nonedef start(self):if self._thread is None:@@ -60,17 +63,22 @@ class LlamaServerStatsLogger:def stop(self):self._stop.set()- def _scrape(self):+ def _get(self, url):try:# No proxy: the bearer must only ever go to the loopback server.with self._opener.open(- urllib.request.Request(self._url, headers = self._headers), timeout = 3+ urllib.request.Request(url, headers = self._headers), timeout = 3) as r:if r.status != 200:return None- body = r.read().decode("utf-8", "replace")+ return r.read().decode("utf-8", "replace")except Exception:return None++ def _scrape(self):+ body = self._get(self._url)+ if body is None:+ return Noneout = {}for k, v in _METRIC_RE.findall(body):try: # a malformed value must not kill the daemon thread@@ -82,6 +90,23 @@ class LlamaServerStatsLogger:out[k] = valuereturn out+ def _live_gen_rate(self, now, running):+ slots = {}+ if running:+ try:+ slots = {+ (s["id"], s["id_task"]): float(s["next_token"][0]["n_decoded"])+ for s in json.loads(self._get(self._slots_url))+ if s.get("is_processing")+ }+ except Exception:+ slots = None+ base, self._slot_base = self._slot_base, None if slots is None else (now, slots)+ if slots is None or base is None or now <= base[0]:+ return None+ tokens = sum(max(0.0, n - base[1].get(k, 0.0)) for k, n in slots.items())+ return tokens / (now - base[0])+@staticmethoddef _prompt_rate(base, tokens, seconds):"""Prompt tokens/s over the prompt time the engine measures; baseline held until both totals move, as /metrics rounds to six digits."""@@ -150,6 +175,13 @@ class LlamaServerStatsLogger:prev = (now, decode_calls)# The bucket empties on every /metrics read, from any client, so a zero gauge is a reading; counters read a free first token against a millisecond.gen_tps = m.get("predicted_tokens_seconds")+ # The gauge only counts a reply once its slot is released, so a long one reads 0 until it ends.+ live_gen = self._live_gen_rate(now, running)+ if running and not gen_tps:+ if live_gen is not None:+ gen_tps = live_gen+ elif decode_rate:+ gen_tps = None# /metrics renders one table, so a scrape with no prompt metric measured none.prompt_measured = "prompt_tokens_seconds" in m or ("prompt_tokens_total" in m and "prompt_seconds_total" in mdiff --git a/studio/backend/tests/test_engine_stats_live_rate.py b/studio/backend/tests/test_engine_stats_live_rate.pyindex 1e025b18eb2..26140a19815 100644--- a/studio/backend/tests/test_engine_stats_live_rate.py+++ b/studio/backend/tests/test_engine_stats_live_rate.py@@ -5,12 +5,30 @@from __future__ import annotations+import json+import core.inference.llama_stats as lsfrom core.inference.llama_stats import LlamaServerStatsLogger_TICK_S = 10.0+class _Resp:+ status = 200++ def __init__(self, body):+ self._body = body++ def read(self):+ return self._body++ def __enter__(self):+ return self++ def __exit__(self, *a):+ return False++class _Capture:def __init__(self):self.events = []@@ -139,7 +157,7 @@ def test_the_decode_counter_reports_while_the_token_counters_are_still(monkeypatsnaps = [_busy(decode = float(i * 20), gen_gauge = 0.0) for i in range(4)]stats = _drive(snaps, monkeypatch)- assert all(s["gen_tok_s"] == 0.0 for s in stats)+ assert [s.get("gen_tok_s") for s in stats] == [0.0, None, None, None]assert [s.get("decode_calls_s") for s in stats] == [None, 2.0, 2.0, 2.0]@@ -232,21 +250,6 @@ def test_a_build_with_no_prompt_metric_omits_the_field(monkeypatch):def test_a_non_finite_metric_value_never_reaches_the_line(monkeypatch):"""float() turns a long digit string into inf; _env_float refuses the same text."""- class _Resp:- status = 200-- def __init__(self, body):- self._body = body-- def read(self):- return self._body-- def __enter__(self):- return self-- def __exit__(self, *a):- return False-body = ("llamacpp:prompt_tokens_seconds " + "9" * 400 + "\n""llamacpp:prompt_tokens_total 10\n"@@ -268,3 +271,45 @@ def test_the_llama_cpp_gauge_still_wins_when_it_reports(monkeypatch):stats = _drive(snaps, monkeypatch)assert stats and all(s["gen_tok_s"] == 24.6 for s in stats)+++def _slots(monkeypatch, *ticks):+ bodies = iter(+ json.dumps(+ [+ {+ "id": slot,+ "is_processing": True,+ "id_task": task,+ "next_token": [{"n_decoded": n}],+ }+ for slot, task, n in tick+ ]+ ).encode()+ for tick in ticks+ )+ monkeypatch.setattr(+ ls.urllib.request.OpenerDirector, "open", lambda *a, **k: _Resp(next(bodies))+ )+++def test_a_long_reply_reports_its_rate_while_it_is_still_generating(monkeypatch):+ _slots(monkeypatch, [(0, 7, 250)], [(0, 7, 500)], [(0, 7, 750)])+ snaps = [_busy(running = 0.0)] + [+ _busy(decode = float(i * 250), gen_gauge = 0.0) for i in range(1, 4)+ ]+ stats = _drive(snaps, monkeypatch)++ assert [s["gen_tok_s"] for s in stats] == [25.0, 25.0, 25.0]+++def test_a_new_task_on_a_slot_is_counted_from_zero(monkeypatch):+ _slots(monkeypatch, [(0, 7, 900)], [(0, 8, 100)])+ snaps = [+ _busy(running = 0.0),+ _busy(decode = 900.0, gen_gauge = 0.0),+ _busy(decode = 1000.0, gen_gauge = 0.0),+ ]+ stats = _drive(snaps, monkeypatch)++ assert [s["gen_tok_s"] for s in stats] == [90.0, 10.0]