Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
feat(chat): unify /stats across both chat backends (#1076)
Davide Eynard879941ff301d
diff --git a/llamafile/chatbot.h b/llamafile/chatbot.hindex c37bee45..7d976c77 100644--- a/llamafile/chatbot.h+++ b/llamafile/chatbot.h@@ -124,6 +124,16 @@ void on_undo(const std::vector<std::string> &);void on_upload(const std::vector<std::string> &);void print(const std::string_view &);void print_ephemeral(const std::string_view &);++// Shared /stats rendering for TUI (works both in default combined mode+// and in standalone `--chat` mode). Information available in both appears+// first, followed by backend-specific extras.+// gen_rate_n is how many decode steps gen_ms covers, which is not always+// gen_n: the server's generation timer starts at the first sampled token, so+// it spans gen_n - 1 steps there. Dividing by gen_n instead inflates the rate.+// Returns false when nothing has been processed yet.+bool print_stats_core(int prompt_n, double prompt_ms, int gen_n, double gen_ms,+ int gen_rate_n, double total_ms);void record_undo(void);void repl(ChatBackend &backend);void rewind(int);diff --git a/llamafile/chatbot_api.cpp b/llamafile/chatbot_api.cppindex 55af684e..54c1d2c5 100644--- a/llamafile/chatbot_api.cpp+++ b/llamafile/chatbot_api.cpp@@ -176,6 +176,29 @@ public:total_tokens_ = usage["total_tokens"].get<int>();}+ // Extract timings from the final chunk. The server+ // always attaches these once its stats are set, so no+ // request flag is needed; without them /stats falls+ // back to token counts alone.+ if (chunk.contains("timings")) {+ auto &t = chunk["timings"];+ if (t.contains("prompt_n"))+ last_prompt_n_ = t["prompt_n"].get<int>();+ if (t.contains("prompt_ms"))+ last_prompt_ms_ = t["prompt_ms"].get<double>();+ if (t.contains("predicted_n"))+ last_predicted_n_ = t["predicted_n"].get<int>();+ if (t.contains("predicted_ms"))+ last_predicted_ms_ = t["predicted_ms"].get<double>();+ if (t.contains("cache_n"))+ last_cache_n_ = t["cache_n"].get<int>();+ if (t.contains("draft_n"))+ last_draft_n_ = t["draft_n"].get<int>();+ if (t.contains("draft_n_accepted"))+ last_draft_accepted_ = t["draft_n_accepted"].get<int>();+ have_timings_ = true;+ }+if (!chunk.contains("choices") || chunk["choices"].empty())continue;@@ -236,10 +259,29 @@ public:}void print_stats() override {- printf("prompt tokens: %d\n"- "completion tokens: %d\n"- "total tokens: %d\n",- last_prompt_tokens_, last_completion_tokens_, total_tokens_);+ // Prefer the timings' own token counts: they are what the millisecond+ // figures were measured over, so the derived rates stay consistent.+ // usage.prompt_tokens also counts cache hits, which were never decoded.+ const int prompt_n = have_timings_ ? last_prompt_n_ : last_prompt_tokens_;+ const int gen_n = have_timings_ ? last_predicted_n_ : last_completion_tokens_;++ // The server's generation timer starts when the first token is+ // sampled -- that token comes free from the last prompt batch's+ // logits -- so predicted_ms covers predicted_n - 1 decode steps. This+ // is what the server's own n_gen_tps() divides by.+ const int gen_rate_n = gen_n > 0 ? gen_n - 1 : 0;++ if (!print_stats_core(prompt_n, last_prompt_ms_, gen_n,+ last_predicted_ms_, gen_rate_n,+ last_prompt_ms_ + last_predicted_ms_))+ return;++ // server only: the direct backend has no equivalent for these+ if (last_cache_n_ > 0)+ printf(" %-10s %6d tokens\n", "cached", last_cache_n_);+ if (last_draft_n_ > 0)+ printf(" %-10s %6d/%d accepted\n", "draft", last_draft_accepted_,+ last_draft_n_);}void on_clear() override {@@ -248,6 +290,14 @@ public:last_prompt_tokens_ = 0;last_completion_tokens_ = 0;total_tokens_ = 0;+ have_timings_ = false;+ last_prompt_n_ = 0;+ last_prompt_ms_ = 0;+ last_predicted_n_ = 0;+ last_predicted_ms_ = 0;+ last_cache_n_ = 0;+ last_draft_n_ = 0;+ last_draft_accepted_ = 0;}void on_push() override {@@ -306,6 +356,16 @@ private:int last_completion_tokens_ = 0;int total_tokens_ = 0;+ // Perf tracking (from the SSE timings object on the final chunk)+ bool have_timings_ = false;+ int last_prompt_n_ = 0;+ double last_prompt_ms_ = 0;+ int last_predicted_n_ = 0;+ double last_predicted_ms_ = 0;+ int last_cache_n_ = 0;+ int last_draft_n_ = 0;+ int last_draft_accepted_ = 0;+// Cached context size from server /props endpointint context_max_ = 0;diff --git a/llamafile/chatbot_comm.cpp b/llamafile/chatbot_comm.cppindex 9a0d4b90..512c132d 100644--- a/llamafile/chatbot_comm.cpp+++ b/llamafile/chatbot_comm.cpp@@ -29,6 +29,39 @@namespace lf {namespace chatbot {+// one row of the shared /stats block; the rate is derived rather than taken+// from the backend so both backends round it the same way. rate_n is how many+// steps ms actually covers, which is not always the token count displayed --+// see the gen_rate_n note on print_stats_core.+static void print_stats_row(const char *label, int n, double ms, int rate_n) {+ if (ms <= 0 || n <= 0) {+ printf(" %-10s %6d tokens\n", label, n);+ } else if (rate_n > 0) {+ printf(" %-10s %6d tokens %10.2f ms %8.1f tok/s\n", label, n, ms,+ rate_n / ms * 1e3);+ } else {+ // duration known but no step to divide by, e.g. a one-token reply+ printf(" %-10s %6d tokens %10.2f ms\n", label, n, ms);+ }+}++bool print_stats_core(int prompt_n, double prompt_ms, int gen_n, double gen_ms,+ int gen_rate_n, double total_ms) {+ if (prompt_n <= 0 && gen_n <= 0) {+ printf(" no tokens processed yet\n");+ return false;+ }+ print_stats_row("prompt", prompt_n, prompt_ms, prompt_n);+ print_stats_row("generated", gen_n, gen_ms, gen_rate_n);+ if (total_ms > 0) {+ printf(" %-10s %6d tokens %10.2f ms\n", "total", prompt_n + gen_n,+ total_ms);+ } else {+ printf(" %-10s %6d tokens\n", "total", prompt_n + gen_n);+ }+ return true;+}+// handle irc style commands like: `/arg0 arg1 arg2`bool handle_command(const char *command) {if (!strcmp(command, "/?")) {diff --git a/llamafile/chatbot_direct.cpp b/llamafile/chatbot_direct.cppindex f947c272..e337898f 100644--- a/llamafile/chatbot_direct.cpp+++ b/llamafile/chatbot_direct.cpp@@ -95,9 +95,33 @@ public:}void print_stats() override {- FLAG_log_disable = false;- common_perf_print(g_ctx, g_sampler);- FLAG_log_disable = true;+ // Format and print directly rather than going through llama.cpp's+ // common_perf_print(). This keeps /stats consistent across chat and+ // combined mode and off the logging path entirely.+ const llama_perf_context_data ctx_perf = llama_perf_context(g_ctx);+ const llama_perf_sampler_data smpl_perf =+ llama_perf_sampler(common_sampler_get(g_sampler));+ const double total_ms = 1e-3 * ggml_time_us() - ctx_perf.t_start_ms;++ // llama_perf_context() clamps both token counts with std::max(1, ...)+ // (llama-context.cpp, perf_get_data) so that its own printer can divide+ // by them. Before anything is decoded the real counters are 0 and that+ // guard would show up here as a phantom token on each row. The+ // durations are not clamped, so use them to tell the cases apart.+ const int prompt_n = ctx_perf.t_p_eval_ms > 0 ? ctx_perf.n_p_eval : 0;+ const int gen_n = ctx_perf.t_eval_ms > 0 ? ctx_perf.n_eval : 0;++ // n_eval and t_eval_us accumulate together, one per decoded token, so+ // here the rate basis is simply the token count+ if (!print_stats_core(prompt_n, ctx_perf.t_p_eval_ms, gen_n,+ ctx_perf.t_eval_ms, gen_n, total_ms))+ return; // nothing decoded yet; load time reappears after the first turn++ // --chat only: the server backend has no equivalent for these+ printf(" %-10s %17.2f ms\n", "load", ctx_perf.t_load_ms);+ if (smpl_perf.n_sample > 0)+ printf(" %-10s %17.2f ms\n", "samplers", smpl_perf.t_sample_ms);+ printf(" %-10s %17d\n", "graphs", ctx_perf.n_reused);}void on_clear() override {diff --git a/llamafile/chatbot_help.cpp b/llamafile/chatbot_help.cppindex 1aa47f26..eb7118a3 100644--- a/llamafile/chatbot_help.cpp+++ b/llamafile/chatbot_help.cpp@@ -89,11 +89,13 @@ the usage of that specific command will be printed.\n\} else if (args[1] == "stats") {fprintf(stderr, "\" BOLD "usage: /stats" RESET "\n\-prints performance statistics for current session. this includes prompt\n\-evaluation time in tokens per second, which indicates prefill speed, or\n\-how quickly llamafile is able to read text. the 'eval time' statistic\n\-gives you prediction or token generation speed, in tokens per second,\n\-which tells you how quickly llamafile is able to write text.\n\+prints performance statistics for current session. the 'prompt' row is\n\+prefill speed, in tokens per second: how quickly llamafile reads text.\n\+the 'generated' row is prediction speed, in tokens per second: how\n\+quickly llamafile writes text. both rows are reported the same way in\n\+--chat and server mode. further rows depend on the mode: --chat adds\n\+model load time, sampler time and compute graph reuse, while server mode\n\+adds prompt cache hits and, with speculative decoding, draft acceptance.\n\");} else if (args[1] == "clear") {fprintf(stderr, "\diff --git a/llamafile/chatbot_main.cpp b/llamafile/chatbot_main.cppindex d061e2ad..080b0cad 100644--- a/llamafile/chatbot_main.cpp+++ b/llamafile/chatbot_main.cpp@@ -330,6 +330,15 @@ int main(int argc, char **argv) {clear_ephemeral();}+ // Startup decoding (the BOS token above, and a base model's system prompt)+ // is synchronized, so llama_context books it as generation. Left in place+ // it is charged to the user's first /stats as a token they never asked+ // for, and on add_bos models an idle /stats would report a decode before+ // the conversation has begun. perf_reset() keeps t_load_us, so the load+ // row survives; it also restarts the total-time clock here, which excludes+ // model loading from the session total.+ llama_perf_context_reset(g_ctx);+repl(*backend);// Synchronize before cleanup to ensure all GPU operations completediff --git a/tests/integration/tests/test_combined.py b/tests/integration/tests/test_combined.pyindex 5ffff04f..bc373866 100644--- a/tests/integration/tests/test_combined.py+++ b/tests/integration/tests/test_combined.py@@ -2,7 +2,15 @@import pytest-from utils.llamafile import LlamafileRunner, read_until_idle, stop_tui+from utils.llamafile import (+ LlamafileRunner,+ STATS_IDLE_LINE,+ assert_generated_rate_basis,+ assert_stats_core,+ read_until_idle,+ stop_tui,+ strip_ansi,+)from utils.prompts import ADD_1_1, ADD_2_2, ADD_3_3, GREETING_PROMPT@@ -82,3 +90,95 @@ class TestCombinedMode:finally:stop_tui(proc)+++class TestCombinedStats:+ """/stats in combined mode, served by ApiBackend over HTTP."""++ def _stats_output(self, llamafile, server_port, timeouts, generate_first):+ """Drive the combined-mode TUI and return its output after /stats."""+ proc = llamafile.start_combined(port=server_port)+ try:+ ready = LlamafileRunner.wait_for_server(+ server_port, timeout=timeouts.server_ready, proc=proc+ )+ assert ready, "Server did not become ready in combined mode"++ # discard startup banner and model info block+ _ = read_until_idle(proc.stdout, idle_timeout=0.5, max_timeout=5.0)++ if generate_first:+ proc.stdin.write(f"{ADD_2_2.prompt}\n")+ proc.stdin.flush()+ read_until_idle(+ proc.stdout,+ idle_timeout=2.0 * timeouts.multiplier,+ max_timeout=timeouts.cli,+ )++ proc.stdin.write("/stats\n")+ proc.stdin.flush()+ return read_until_idle(+ proc.stdout,+ idle_timeout=2.0 * timeouts.multiplier,+ max_timeout=timeouts.cli,+ )+ finally:+ stop_tui(proc)++ def test_stats_reports_perf_metrics(self, llamafile, server_port, timeouts):+ """/stats prints the same core block the --chat backend prints.++ ApiBackend reads these from the timings object the server attaches+ to the final SSE chunk; before unification it reported only token+ counts, with no durations or rates at all.+ """+ output = self._stats_output(+ llamafile, server_port, timeouts, generate_first=True+ )++ assert len(output) > 0, "TUI produced no output for /stats"+ assert_stats_core(output, where="combined mode")++ def test_stats_generated_rate_excludes_the_free_first_token(+ self, llamafile, server_port, timeouts+ ):+ """The server's generation timer skips the first token; the rate must too.++ predicted_ms spans predicted_n - 1 decode steps, so dividing by+ predicted_n overstates the rate by n/(n-1) and reads ~1e6 tok/s on a+ one-token reply.+ """+ output = self._stats_output(+ llamafile, server_port, timeouts, generate_first=True+ )++ assert_generated_rate_basis(+ output, free_first_token=True, where="combined mode"+ )++ def test_stats_omits_direct_backend_extras(self, llamafile, server_port, timeouts):+ """Rows the server cannot source must not appear in combined mode."""+ output = self._stats_output(+ llamafile, server_port, timeouts, generate_first=True+ )+ # assert the core rendered first, so this cannot pass merely because+ # /stats printed nothing worth inspecting+ assert_stats_core(output, where="combined mode")+ clean = strip_ansi(output)++ for row in ("load", "graphs", "samplers"):+ assert f" {row}" not in clean, (+ f"combined-mode /stats printed --chat-only row '{row}': {clean}"+ )++ def test_stats_when_idle_reports_nothing(self, llamafile, server_port, timeouts):+ """An idle /stats reads the same here as it does in --chat."""+ clean = strip_ansi(+ self._stats_output(llamafile, server_port, timeouts, generate_first=False)+ )++ assert STATS_IDLE_LINE in clean, f"/stats did not report an idle session: {clean}"diff --git a/tests/integration/tests/test_tui.py b/tests/integration/tests/test_tui.pyindex f1d7de6c..8bb3f33a 100644--- a/tests/integration/tests/test_tui.py+++ b/tests/integration/tests/test_tui.py@@ -2,6 +2,12 @@import pytest+from utils.llamafile import (+ STATS_IDLE_LINE,+ assert_generated_rate_basis,+ assert_stats_core,+ strip_ansi,+)from utils.prompts import ADD_2_2@@ -53,6 +59,67 @@ class TestTUIBasic:assert result.returncode == 0+class TestTUIStats:+ """/stats in --chat mode, served by DirectBackend."""++ def test_stats_reports_perf_metrics(self, llamafile, tmp_path, timeouts):+ """/stats prints the shared core block after generating.++ Regression test for #1047, where this printed nothing at all: the+ metrics went through LOG_INF(), which chat mode silences.+ """+ input_file = tmp_path / "input.txt"+ input_file.write_text(f"{ADD_2_2.prompt}\n/stats\n/exit\n")++ result = llamafile.run_tui(str(input_file), timeout=timeouts.tui)++ assert result.returncode == 0, f"TUI failed: {result.stderr}"+ assert_stats_core(result.stdout, where="--chat")++ def test_stats_generated_rate_counts_every_token(self, llamafile, tmp_path, timeouts):+ """--chat decodes every generated token, so the rate divides by all of them."""+ input_file = tmp_path / "input.txt"+ input_file.write_text(f"{ADD_2_2.prompt}\n/stats\n/exit\n")++ result = llamafile.run_tui(str(input_file), timeout=timeouts.tui)++ assert result.returncode == 0+ assert_generated_rate_basis(+ result.stdout, free_first_token=False, where="--chat"+ )++ def test_stats_reports_direct_backend_extras(self, llamafile, tmp_path, timeouts):+ """--chat adds load time and graph reuse, which the server has no equivalent for."""+ input_file = tmp_path / "input.txt"+ input_file.write_text(f"{ADD_2_2.prompt}\n/stats\n/exit\n")++ result = llamafile.run_tui(str(input_file), timeout=timeouts.tui)+ clean = strip_ansi(result.stdout)++ assert result.returncode == 0+ for row in ("load", "graphs"):+ assert f" {row}" in clean, f"/stats missing '{row}' row: {clean}"++ def test_stats_when_idle_reports_nothing(self, llamafile, tmp_path, timeouts):+ """/stats before generating says so, and prints no rows of its own.++ llama_perf_context() clamps its token counts with std::max(1, ...),+ so an unguarded implementation shows a phantom token on each row.+ """+ input_file = tmp_path / "input.txt"+ input_file.write_text("/stats\n/exit\n")++ result = llamafile.run_tui(str(input_file), timeout=timeouts.tui)+ clean = strip_ansi(result.stdout)++ assert result.returncode == 0+ assert STATS_IDLE_LINE in clean, f"/stats did not report an idle session: {clean}"+ # no per-mode extras belong under an idle report+ for row in ("load", "graphs", "samplers"):+ assert f" {row}" not in clean, f"idle /stats printed '{row}' row: {clean}"++@pytest.mark.tui@pytest.mark.thinkingclass TestTUIThinking:diff --git a/tests/integration/utils/llamafile.py b/tests/integration/utils/llamafile.pyindex 28a10942..3e71fd50 100644--- a/tests/integration/utils/llamafile.py+++ b/tests/integration/utils/llamafile.py@@ -5,6 +5,7 @@ import fcntlimport loggingimport osimport platform+import reimport selectimport subprocessimport time@@ -16,6 +17,103 @@ import requestslogger = logging.getLogger(__name__)+_ANSI_RE = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]")++# Rows of the shared /stats block, in the order both backends print them.+# "total" carries no rate, so it is checked separately from the other two.+STATS_CORE_ROWS = ("prompt", "generated", "total")+STATS_IDLE_LINE = "no tokens processed yet"++# Far above anything local inference reaches, so only an arithmetic fault+# trips it -- a duration divided by a count of steps it does not cover.+MAX_PLAUSIBLE_TOK_PER_SEC = 100_000.0+++def strip_ansi(text: str) -> str:+ """Remove ANSI escape sequences so TUI output can be matched by line."""+ return _ANSI_RE.sub("", text)+++def parse_stats_row(output: str, row: str) -> tuple[int, float, float] | None:+ """Return (tokens, ms, tok_per_sec) for a /stats row, or None if absent.++ Returns None when the row carries no rate, which is legitimate: a reply+ short enough that the duration covers no decode step prints tokens and+ milliseconds only.+ """+ m = re.search(+ rf"^\s+{row}\s+(\d+) tokens\s+([\d.]+) ms\s+([\d.]+) tok/s",+ strip_ansi(output),+ re.M,+ )+ return (int(m.group(1)), float(m.group(2)), float(m.group(3))) if m else None+++def assert_generated_rate_basis(output: str, free_first_token: bool, where: str = "") -> None:+ """Assert the generated row's rate divides by the steps its duration covers.++ The two backends differ here and the difference is easy to get wrong.+ ``--chat`` accumulates n_eval and t_eval_us together, one per decoded+ token, so the rate divides by the token count. The server starts its+ generation timer at the first sampled token -- that one comes free from+ the last prompt batch's logits -- so its duration spans one step fewer+ than it reports tokens, and dividing by the count inflates the rate by+ n/(n-1).+ """+ parsed = parse_stats_row(output, "generated")+ assert parsed is not None, f"/stats has no generated rate{f' in {where}' if where else ''}"+ n, ms, rate = parsed++ steps = (n - 1) if free_first_token else n+ expected = steps / ms * 1e3+ # generous: guards the basis, not float formatting+ assert abs(rate - expected) < max(1.0, expected * 0.02), (+ f"/stats generated rate {rate} tok/s{f' in {where}' if where else ''} "+ f"does not divide {n} tokens over {steps} step(s) of {ms} ms "+ f"(expected ~{expected:.1f})"+ )+++def assert_stats_core(output: str, where: str = "") -> None:+ """Assert the shared /stats core block is present and well formed.++ Both backends must render this block identically -- that is the whole+ point of having one, so both modes assert against this same helper. If+ they ever diverge in column layout or wording, one of the two callers+ fails here rather than silently drifting.+ """+ clean = strip_ansi(output)+ context = f" in {where}" if where else ""++ assert STATS_IDLE_LINE not in clean, (+ f"/stats reported an idle session{context} after generating: {clean}"+ )++ for row in STATS_CORE_ROWS:+ assert re.search(rf"^\s+{row}\s+\d+ tokens", clean, re.M), (+ f"/stats missing '{row}' row{context}: {clean}"+ )++ # prompt and generated carry a duration and a derived rate; total omits+ # the rate because a combined tokens-per-second figure would be meaningless+ for row in ("prompt", "generated"):+ m = re.search(+ rf"^\s+{row}\s+\d+ tokens\s+[\d.]+ ms\s+([\d.]+) tok/s", clean, re.M+ )+ assert m, f"/stats '{row}' row lacks duration and rate{context}: {clean}"++ # A rate this high means the duration did not cover the count it was+ # divided by, not that the hardware is fast. The server's generation+ # timer spans one step fewer than it reports tokens, and it clamps a+ # sub-microsecond span to 1us, so dividing by the token count yields+ # ~1e6 tok/s on a one-token reply.+ rate = float(m.group(1))+ assert rate < MAX_PLAUSIBLE_TOK_PER_SEC, (+ f"/stats '{row}' rate of {rate} tok/s is not physically plausible"+ f"{context}; the duration likely covers fewer steps than the token"+ f" count it was divided by: {clean}"+ )+def skip_if_bundled(executable: str) -> None:"""Skip the current test when *executable* is a pre-built .llamafile bundle.