Skip to content

Distribute and run LLMs with a single file.

feat(chat): unify /stats across both chat backends (#1076)

Davide Eynard879941ff301d
diff --git a/llamafile/chatbot.h b/llamafile/chatbot.h
index c37bee45..7d976c77 100644
--- a/llamafile/chatbot.h
+++ b/llamafile/chatbot.h
@@ -124,6 +124,16 @@ void on_undo(const std::vector<std::string> &);
void on_upload(const std::vector<std::string> &);
void print(const std::string_view &);
void print_ephemeral(const std::string_view &);
+
+// Shared /stats rendering for TUI (works both in default combined mode
+// and in standalone `--chat` mode). Information available in both appears
+// first, followed by backend-specific extras.
+// gen_rate_n is how many decode steps gen_ms covers, which is not always
+// gen_n: the server's generation timer starts at the first sampled token, so
+// it spans gen_n - 1 steps there. Dividing by gen_n instead inflates the rate.
+// Returns false when nothing has been processed yet.
+bool print_stats_core(int prompt_n, double prompt_ms, int gen_n, double gen_ms,
+ int gen_rate_n, double total_ms);
void record_undo(void);
void repl(ChatBackend &backend);
void rewind(int);
diff --git a/llamafile/chatbot_api.cpp b/llamafile/chatbot_api.cpp
index 55af684e..54c1d2c5 100644
--- a/llamafile/chatbot_api.cpp
+++ b/llamafile/chatbot_api.cpp
@@ -176,6 +176,29 @@ public:
total_tokens_ = usage["total_tokens"].get<int>();
}
+ // Extract timings from the final chunk. The server
+ // always attaches these once its stats are set, so no
+ // request flag is needed; without them /stats falls
+ // back to token counts alone.
+ if (chunk.contains("timings")) {
+ auto &t = chunk["timings"];
+ if (t.contains("prompt_n"))
+ last_prompt_n_ = t["prompt_n"].get<int>();
+ if (t.contains("prompt_ms"))
+ last_prompt_ms_ = t["prompt_ms"].get<double>();
+ if (t.contains("predicted_n"))
+ last_predicted_n_ = t["predicted_n"].get<int>();
+ if (t.contains("predicted_ms"))
+ last_predicted_ms_ = t["predicted_ms"].get<double>();
+ if (t.contains("cache_n"))
+ last_cache_n_ = t["cache_n"].get<int>();
+ if (t.contains("draft_n"))
+ last_draft_n_ = t["draft_n"].get<int>();
+ if (t.contains("draft_n_accepted"))
+ last_draft_accepted_ = t["draft_n_accepted"].get<int>();
+ have_timings_ = true;
+ }
+
if (!chunk.contains("choices") || chunk["choices"].empty())
continue;
@@ -236,10 +259,29 @@ public:
}
void print_stats() override {
- printf("prompt tokens: %d\n"
- "completion tokens: %d\n"
- "total tokens: %d\n",
- last_prompt_tokens_, last_completion_tokens_, total_tokens_);
+ // Prefer the timings' own token counts: they are what the millisecond
+ // figures were measured over, so the derived rates stay consistent.
+ // usage.prompt_tokens also counts cache hits, which were never decoded.
+ const int prompt_n = have_timings_ ? last_prompt_n_ : last_prompt_tokens_;
+ const int gen_n = have_timings_ ? last_predicted_n_ : last_completion_tokens_;
+
+ // The server's generation timer starts when the first token is
+ // sampled -- that token comes free from the last prompt batch's
+ // logits -- so predicted_ms covers predicted_n - 1 decode steps. This
+ // is what the server's own n_gen_tps() divides by.
+ const int gen_rate_n = gen_n > 0 ? gen_n - 1 : 0;
+
+ if (!print_stats_core(prompt_n, last_prompt_ms_, gen_n,
+ last_predicted_ms_, gen_rate_n,
+ last_prompt_ms_ + last_predicted_ms_))
+ return;
+
+ // server only: the direct backend has no equivalent for these
+ if (last_cache_n_ > 0)
+ printf(" %-10s %6d tokens\n", "cached", last_cache_n_);
+ if (last_draft_n_ > 0)
+ printf(" %-10s %6d/%d accepted\n", "draft", last_draft_accepted_,
+ last_draft_n_);
}
void on_clear() override {
@@ -248,6 +290,14 @@ public:
last_prompt_tokens_ = 0;
last_completion_tokens_ = 0;
total_tokens_ = 0;
+ have_timings_ = false;
+ last_prompt_n_ = 0;
+ last_prompt_ms_ = 0;
+ last_predicted_n_ = 0;
+ last_predicted_ms_ = 0;
+ last_cache_n_ = 0;
+ last_draft_n_ = 0;
+ last_draft_accepted_ = 0;
}
void on_push() override {
@@ -306,6 +356,16 @@ private:
int last_completion_tokens_ = 0;
int total_tokens_ = 0;
+ // Perf tracking (from the SSE timings object on the final chunk)
+ bool have_timings_ = false;
+ int last_prompt_n_ = 0;
+ double last_prompt_ms_ = 0;
+ int last_predicted_n_ = 0;
+ double last_predicted_ms_ = 0;
+ int last_cache_n_ = 0;
+ int last_draft_n_ = 0;
+ int last_draft_accepted_ = 0;
+
// Cached context size from server /props endpoint
int context_max_ = 0;
diff --git a/llamafile/chatbot_comm.cpp b/llamafile/chatbot_comm.cpp
index 9a0d4b90..512c132d 100644
--- a/llamafile/chatbot_comm.cpp
+++ b/llamafile/chatbot_comm.cpp
@@ -29,6 +29,39 @@
namespace lf {
namespace chatbot {
+// one row of the shared /stats block; the rate is derived rather than taken
+// from the backend so both backends round it the same way. rate_n is how many
+// steps ms actually covers, which is not always the token count displayed --
+// see the gen_rate_n note on print_stats_core.
+static void print_stats_row(const char *label, int n, double ms, int rate_n) {
+ if (ms <= 0 || n <= 0) {
+ printf(" %-10s %6d tokens\n", label, n);
+ } else if (rate_n > 0) {
+ printf(" %-10s %6d tokens %10.2f ms %8.1f tok/s\n", label, n, ms,
+ rate_n / ms * 1e3);
+ } else {
+ // duration known but no step to divide by, e.g. a one-token reply
+ printf(" %-10s %6d tokens %10.2f ms\n", label, n, ms);
+ }
+}
+
+bool print_stats_core(int prompt_n, double prompt_ms, int gen_n, double gen_ms,
+ int gen_rate_n, double total_ms) {
+ if (prompt_n <= 0 && gen_n <= 0) {
+ printf(" no tokens processed yet\n");
+ return false;
+ }
+ print_stats_row("prompt", prompt_n, prompt_ms, prompt_n);
+ print_stats_row("generated", gen_n, gen_ms, gen_rate_n);
+ if (total_ms > 0) {
+ printf(" %-10s %6d tokens %10.2f ms\n", "total", prompt_n + gen_n,
+ total_ms);
+ } else {
+ printf(" %-10s %6d tokens\n", "total", prompt_n + gen_n);
+ }
+ return true;
+}
+
// handle irc style commands like: `/arg0 arg1 arg2`
bool handle_command(const char *command) {
if (!strcmp(command, "/?")) {
diff --git a/llamafile/chatbot_direct.cpp b/llamafile/chatbot_direct.cpp
index f947c272..e337898f 100644
--- a/llamafile/chatbot_direct.cpp
+++ b/llamafile/chatbot_direct.cpp
@@ -95,9 +95,33 @@ public:
}
void print_stats() override {
- FLAG_log_disable = false;
- common_perf_print(g_ctx, g_sampler);
- FLAG_log_disable = true;
+ // Format and print directly rather than going through llama.cpp's
+ // common_perf_print(). This keeps /stats consistent across chat and
+ // combined mode and off the logging path entirely.
+ const llama_perf_context_data ctx_perf = llama_perf_context(g_ctx);
+ const llama_perf_sampler_data smpl_perf =
+ llama_perf_sampler(common_sampler_get(g_sampler));
+ const double total_ms = 1e-3 * ggml_time_us() - ctx_perf.t_start_ms;
+
+ // llama_perf_context() clamps both token counts with std::max(1, ...)
+ // (llama-context.cpp, perf_get_data) so that its own printer can divide
+ // by them. Before anything is decoded the real counters are 0 and that
+ // guard would show up here as a phantom token on each row. The
+ // durations are not clamped, so use them to tell the cases apart.
+ const int prompt_n = ctx_perf.t_p_eval_ms > 0 ? ctx_perf.n_p_eval : 0;
+ const int gen_n = ctx_perf.t_eval_ms > 0 ? ctx_perf.n_eval : 0;
+
+ // n_eval and t_eval_us accumulate together, one per decoded token, so
+ // here the rate basis is simply the token count
+ if (!print_stats_core(prompt_n, ctx_perf.t_p_eval_ms, gen_n,
+ ctx_perf.t_eval_ms, gen_n, total_ms))
+ return; // nothing decoded yet; load time reappears after the first turn
+
+ // --chat only: the server backend has no equivalent for these
+ printf(" %-10s %17.2f ms\n", "load", ctx_perf.t_load_ms);
+ if (smpl_perf.n_sample > 0)
+ printf(" %-10s %17.2f ms\n", "samplers", smpl_perf.t_sample_ms);
+ printf(" %-10s %17d\n", "graphs", ctx_perf.n_reused);
}
void on_clear() override {
diff --git a/llamafile/chatbot_help.cpp b/llamafile/chatbot_help.cpp
index 1aa47f26..eb7118a3 100644
--- a/llamafile/chatbot_help.cpp
+++ b/llamafile/chatbot_help.cpp
@@ -89,11 +89,13 @@ the usage of that specific command will be printed.\n\
} else if (args[1] == "stats") {
fprintf(stderr, "\
" BOLD "usage: /stats" RESET "\n\
-prints performance statistics for current session. this includes prompt\n\
-evaluation time in tokens per second, which indicates prefill speed, or\n\
-how quickly llamafile is able to read text. the 'eval time' statistic\n\
-gives you prediction or token generation speed, in tokens per second,\n\
-which tells you how quickly llamafile is able to write text.\n\
+prints performance statistics for current session. the 'prompt' row is\n\
+prefill speed, in tokens per second: how quickly llamafile reads text.\n\
+the 'generated' row is prediction speed, in tokens per second: how\n\
+quickly llamafile writes text. both rows are reported the same way in\n\
+--chat and server mode. further rows depend on the mode: --chat adds\n\
+model load time, sampler time and compute graph reuse, while server mode\n\
+adds prompt cache hits and, with speculative decoding, draft acceptance.\n\
");
} else if (args[1] == "clear") {
fprintf(stderr, "\
diff --git a/llamafile/chatbot_main.cpp b/llamafile/chatbot_main.cpp
index d061e2ad..080b0cad 100644
--- a/llamafile/chatbot_main.cpp
+++ b/llamafile/chatbot_main.cpp
@@ -330,6 +330,15 @@ int main(int argc, char **argv) {
clear_ephemeral();
}
+ // Startup decoding (the BOS token above, and a base model's system prompt)
+ // is synchronized, so llama_context books it as generation. Left in place
+ // it is charged to the user's first /stats as a token they never asked
+ // for, and on add_bos models an idle /stats would report a decode before
+ // the conversation has begun. perf_reset() keeps t_load_us, so the load
+ // row survives; it also restarts the total-time clock here, which excludes
+ // model loading from the session total.
+ llama_perf_context_reset(g_ctx);
+
repl(*backend);
// Synchronize before cleanup to ensure all GPU operations complete
diff --git a/tests/integration/tests/test_combined.py b/tests/integration/tests/test_combined.py
index 5ffff04f..bc373866 100644
--- a/tests/integration/tests/test_combined.py
+++ b/tests/integration/tests/test_combined.py
@@ -2,7 +2,15 @@
import pytest
-from utils.llamafile import LlamafileRunner, read_until_idle, stop_tui
+from utils.llamafile import (
+ LlamafileRunner,
+ STATS_IDLE_LINE,
+ assert_generated_rate_basis,
+ assert_stats_core,
+ read_until_idle,
+ stop_tui,
+ strip_ansi,
+)
from utils.prompts import ADD_1_1, ADD_2_2, ADD_3_3, GREETING_PROMPT
@@ -82,3 +90,95 @@ class TestCombinedMode:
finally:
stop_tui(proc)
+
+
+class TestCombinedStats:
+ """/stats in combined mode, served by ApiBackend over HTTP."""
+
+ def _stats_output(self, llamafile, server_port, timeouts, generate_first):
+ """Drive the combined-mode TUI and return its output after /stats."""
+ proc = llamafile.start_combined(port=server_port)
+ try:
+ ready = LlamafileRunner.wait_for_server(
+ server_port, timeout=timeouts.server_ready, proc=proc
+ )
+ assert ready, "Server did not become ready in combined mode"
+
+ # discard startup banner and model info block
+ _ = read_until_idle(proc.stdout, idle_timeout=0.5, max_timeout=5.0)
+
+ if generate_first:
+ proc.stdin.write(f"{ADD_2_2.prompt}\n")
+ proc.stdin.flush()
+ read_until_idle(
+ proc.stdout,
+ idle_timeout=2.0 * timeouts.multiplier,
+ max_timeout=timeouts.cli,
+ )
+
+ proc.stdin.write("/stats\n")
+ proc.stdin.flush()
+ return read_until_idle(
+ proc.stdout,
+ idle_timeout=2.0 * timeouts.multiplier,
+ max_timeout=timeouts.cli,
+ )
+ finally:
+ stop_tui(proc)
+
+ def test_stats_reports_perf_metrics(self, llamafile, server_port, timeouts):
+ """/stats prints the same core block the --chat backend prints.
+
+ ApiBackend reads these from the timings object the server attaches
+ to the final SSE chunk; before unification it reported only token
+ counts, with no durations or rates at all.
+ """
+ output = self._stats_output(
+ llamafile, server_port, timeouts, generate_first=True
+ )
+
+ assert len(output) > 0, "TUI produced no output for /stats"
+ assert_stats_core(output, where="combined mode")
+
+ def test_stats_generated_rate_excludes_the_free_first_token(
+ self, llamafile, server_port, timeouts
+ ):
+ """The server's generation timer skips the first token; the rate must too.
+
+ predicted_ms spans predicted_n - 1 decode steps, so dividing by
+ predicted_n overstates the rate by n/(n-1) and reads ~1e6 tok/s on a
+ one-token reply.
+ """
+ output = self._stats_output(
+ llamafile, server_port, timeouts, generate_first=True
+ )
+
+ assert_generated_rate_basis(
+ output, free_first_token=True, where="combined mode"
+ )
+
+ def test_stats_omits_direct_backend_extras(self, llamafile, server_port, timeouts):
+ """Rows the server cannot source must not appear in combined mode."""
+ output = self._stats_output(
+ llamafile, server_port, timeouts, generate_first=True
+ )
+ # assert the core rendered first, so this cannot pass merely because
+ # /stats printed nothing worth inspecting
+ assert_stats_core(output, where="combined mode")
+ clean = strip_ansi(output)
+
+ for row in ("load", "graphs", "samplers"):
+ assert f" {row}" not in clean, (
+ f"combined-mode /stats printed --chat-only row '{row}': {clean}"
+ )
+
+ def test_stats_when_idle_reports_nothing(self, llamafile, server_port, timeouts):
+ """An idle /stats reads the same here as it does in --chat."""
+ clean = strip_ansi(
+ self._stats_output(llamafile, server_port, timeouts, generate_first=False)
+ )
+
+ assert STATS_IDLE_LINE in clean, f"/stats did not report an idle session: {clean}"
diff --git a/tests/integration/tests/test_tui.py b/tests/integration/tests/test_tui.py
index f1d7de6c..8bb3f33a 100644
--- a/tests/integration/tests/test_tui.py
+++ b/tests/integration/tests/test_tui.py
@@ -2,6 +2,12 @@
import pytest
+from utils.llamafile import (
+ STATS_IDLE_LINE,
+ assert_generated_rate_basis,
+ assert_stats_core,
+ strip_ansi,
+)
from utils.prompts import ADD_2_2
@@ -53,6 +59,67 @@ class TestTUIBasic:
assert result.returncode == 0
+class TestTUIStats:
+ """/stats in --chat mode, served by DirectBackend."""
+
+ def test_stats_reports_perf_metrics(self, llamafile, tmp_path, timeouts):
+ """/stats prints the shared core block after generating.
+
+ Regression test for #1047, where this printed nothing at all: the
+ metrics went through LOG_INF(), which chat mode silences.
+ """
+ input_file = tmp_path / "input.txt"
+ input_file.write_text(f"{ADD_2_2.prompt}\n/stats\n/exit\n")
+
+ result = llamafile.run_tui(str(input_file), timeout=timeouts.tui)
+
+ assert result.returncode == 0, f"TUI failed: {result.stderr}"
+ assert_stats_core(result.stdout, where="--chat")
+
+ def test_stats_generated_rate_counts_every_token(self, llamafile, tmp_path, timeouts):
+ """--chat decodes every generated token, so the rate divides by all of them."""
+ input_file = tmp_path / "input.txt"
+ input_file.write_text(f"{ADD_2_2.prompt}\n/stats\n/exit\n")
+
+ result = llamafile.run_tui(str(input_file), timeout=timeouts.tui)
+
+ assert result.returncode == 0
+ assert_generated_rate_basis(
+ result.stdout, free_first_token=False, where="--chat"
+ )
+
+ def test_stats_reports_direct_backend_extras(self, llamafile, tmp_path, timeouts):
+ """--chat adds load time and graph reuse, which the server has no equivalent for."""
+ input_file = tmp_path / "input.txt"
+ input_file.write_text(f"{ADD_2_2.prompt}\n/stats\n/exit\n")
+
+ result = llamafile.run_tui(str(input_file), timeout=timeouts.tui)
+ clean = strip_ansi(result.stdout)
+
+ assert result.returncode == 0
+ for row in ("load", "graphs"):
+ assert f" {row}" in clean, f"/stats missing '{row}' row: {clean}"
+
+ def test_stats_when_idle_reports_nothing(self, llamafile, tmp_path, timeouts):
+ """/stats before generating says so, and prints no rows of its own.
+
+ llama_perf_context() clamps its token counts with std::max(1, ...),
+ so an unguarded implementation shows a phantom token on each row.
+ """
+ input_file = tmp_path / "input.txt"
+ input_file.write_text("/stats\n/exit\n")
+
+ result = llamafile.run_tui(str(input_file), timeout=timeouts.tui)
+ clean = strip_ansi(result.stdout)
+
+ assert result.returncode == 0
+ assert STATS_IDLE_LINE in clean, f"/stats did not report an idle session: {clean}"
+ # no per-mode extras belong under an idle report
+ for row in ("load", "graphs", "samplers"):
+ assert f" {row}" not in clean, f"idle /stats printed '{row}' row: {clean}"
+
+
@pytest.mark.tui
@pytest.mark.thinking
class TestTUIThinking:
diff --git a/tests/integration/utils/llamafile.py b/tests/integration/utils/llamafile.py
index 28a10942..3e71fd50 100644
--- a/tests/integration/utils/llamafile.py
+++ b/tests/integration/utils/llamafile.py
@@ -5,6 +5,7 @@ import fcntl
import logging
import os
import platform
+import re
import select
import subprocess
import time
@@ -16,6 +17,103 @@ import requests
logger = logging.getLogger(__name__)
+_ANSI_RE = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]")
+
+# Rows of the shared /stats block, in the order both backends print them.
+# "total" carries no rate, so it is checked separately from the other two.
+STATS_CORE_ROWS = ("prompt", "generated", "total")
+STATS_IDLE_LINE = "no tokens processed yet"
+
+# Far above anything local inference reaches, so only an arithmetic fault
+# trips it -- a duration divided by a count of steps it does not cover.
+MAX_PLAUSIBLE_TOK_PER_SEC = 100_000.0
+
+
+def strip_ansi(text: str) -> str:
+ """Remove ANSI escape sequences so TUI output can be matched by line."""
+ return _ANSI_RE.sub("", text)
+
+
+def parse_stats_row(output: str, row: str) -> tuple[int, float, float] | None:
+ """Return (tokens, ms, tok_per_sec) for a /stats row, or None if absent.
+
+ Returns None when the row carries no rate, which is legitimate: a reply
+ short enough that the duration covers no decode step prints tokens and
+ milliseconds only.
+ """
+ m = re.search(
+ rf"^\s+{row}\s+(\d+) tokens\s+([\d.]+) ms\s+([\d.]+) tok/s",
+ strip_ansi(output),
+ re.M,
+ )
+ return (int(m.group(1)), float(m.group(2)), float(m.group(3))) if m else None
+
+
+def assert_generated_rate_basis(output: str, free_first_token: bool, where: str = "") -> None:
+ """Assert the generated row's rate divides by the steps its duration covers.
+
+ The two backends differ here and the difference is easy to get wrong.
+ ``--chat`` accumulates n_eval and t_eval_us together, one per decoded
+ token, so the rate divides by the token count. The server starts its
+ generation timer at the first sampled token -- that one comes free from
+ the last prompt batch's logits -- so its duration spans one step fewer
+ than it reports tokens, and dividing by the count inflates the rate by
+ n/(n-1).
+ """
+ parsed = parse_stats_row(output, "generated")
+ assert parsed is not None, f"/stats has no generated rate{f' in {where}' if where else ''}"
+ n, ms, rate = parsed
+
+ steps = (n - 1) if free_first_token else n
+ expected = steps / ms * 1e3
+ # generous: guards the basis, not float formatting
+ assert abs(rate - expected) < max(1.0, expected * 0.02), (
+ f"/stats generated rate {rate} tok/s{f' in {where}' if where else ''} "
+ f"does not divide {n} tokens over {steps} step(s) of {ms} ms "
+ f"(expected ~{expected:.1f})"
+ )
+
+
+def assert_stats_core(output: str, where: str = "") -> None:
+ """Assert the shared /stats core block is present and well formed.
+
+ Both backends must render this block identically -- that is the whole
+ point of having one, so both modes assert against this same helper. If
+ they ever diverge in column layout or wording, one of the two callers
+ fails here rather than silently drifting.
+ """
+ clean = strip_ansi(output)
+ context = f" in {where}" if where else ""
+
+ assert STATS_IDLE_LINE not in clean, (
+ f"/stats reported an idle session{context} after generating: {clean}"
+ )
+
+ for row in STATS_CORE_ROWS:
+ assert re.search(rf"^\s+{row}\s+\d+ tokens", clean, re.M), (
+ f"/stats missing '{row}' row{context}: {clean}"
+ )
+
+ # prompt and generated carry a duration and a derived rate; total omits
+ # the rate because a combined tokens-per-second figure would be meaningless
+ for row in ("prompt", "generated"):
+ m = re.search(
+ rf"^\s+{row}\s+\d+ tokens\s+[\d.]+ ms\s+([\d.]+) tok/s", clean, re.M
+ )
+ assert m, f"/stats '{row}' row lacks duration and rate{context}: {clean}"
+
+ # A rate this high means the duration did not cover the count it was
+ # divided by, not that the hardware is fast. The server's generation
+ # timer spans one step fewer than it reports tokens, and it clamps a
+ # sub-microsecond span to 1us, so dividing by the token count yields
+ # ~1e6 tok/s on a one-token reply.
+ rate = float(m.group(1))
+ assert rate < MAX_PLAUSIBLE_TOK_PER_SEC, (
+ f"/stats '{row}' rate of {rate} tok/s is not physically plausible"
+ f"{context}; the duration likely covers fewer steps than the token"
+ f" count it was divided by: {clean}"
+ )
+
def skip_if_bundled(executable: str) -> None:
"""Skip the current test when *executable* is a pre-built .llamafile bundle.