Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

coli chat: configurable generation statistics with exact token throughput

JustVuggd12d449efc7b
diff --git a/c/coli b/c/coli
index 86786775..06804f1e 100755
--- a/c/coli
+++ b/c/coli
@@ -1328,6 +1328,38 @@ def server_probe(base, api_key=None, timeout=1.5):
except Exception:
return None
+# ---- generation statistics for the chat footer (#1475) ----------------------
+# Measurement and rendering apart: gen_stats() counts (exact completion tokens
+# from the server's usage block when it sends one, the historical chars/4
+# estimate otherwise), render_gen_stats() lays it out for one of three modes.
+# The private-engine chat already prints the engine's own exact counters; the
+# attached path is where the estimate lived and where tok/s was missing.
+CHAT_STATS_MODES = ("full", "compact", "off")
+
+def chat_stats_mode(a):
+ mode = getattr(a, "stats", None) or os.environ.get("COLI_CHAT_STATS", "full")
+ return mode if mode in CHAT_STATS_MODES else "full"
+
+def gen_stats(reply_text, elapsed, exact_tokens=None, interrupted=False):
+ exact = isinstance(exact_tokens, int) and not isinstance(exact_tokens, bool) and exact_tokens >= 0
+ tokens = exact_tokens if exact else len(reply_text) // 4
+ tps = tokens / elapsed if elapsed > 0 else None
+ return {"tokens": tokens, "exact": exact, "elapsed": elapsed, "tps": tps,
+ "interrupted": bool(interrupted)}
+
+def render_gen_stats(st, mode):
+ """'' for off; otherwise the footer body, without the box glyph or colour:
+ full ~370 tok · 313s · ~1.18 tok/s (372 tok · 313s · 1.19 tok/s when exact)
+ compact ~370 tok · ~1.18 tok/s
+ plus ' · ⏹ interrupted' when the answer was cut short."""
+ if mode == "off": return ""
+ approx = "" if st["exact"] else "~"
+ parts = [f"{approx}{st['tokens']} tok"]
+ if mode == "full": parts.append(f"{st['elapsed']:.0f}s")
+ if st["tps"] is not None: parts.append(f"{approx}{st['tps']:.2f} tok/s")
+ if st["interrupted"]: parts.append("⏹ interrupted")
+ return " · ".join(parts)
+
def chat_attached(a, base, model_id):
"""The chat REPL over HTTP against a running `coli serve`.
@@ -1364,7 +1396,8 @@ def chat_attached(a, base, model_id):
# motore che streamma gli esperti da disco, "max" vuol dire centinaia
# di token di riflessione prima della prima parola di risposta.
request = {"model":model_id,"messages":msgs,"stream":True,
- "max_tokens":ngen_for(a,interactive=True)}
+ "max_tokens":ngen_for(a,interactive=True),
+ "stream_options":{"include_usage":True}} # exact completion_tokens in the last chunk
if getattr(a,"effort",None): request["reasoning_effort"]=a.effort
if getattr(a,"think",None) is not None: request["enable_thinking"]=a.think
body=json.dumps(request).encode()
@@ -1374,6 +1407,7 @@ def chat_attached(a, base, model_id):
print(f"\n {C.teal}◆ colibri{C.r}")
sp=Spinner("thinking…"); sp.start()
md=MDStream(" "); reply=[]; first=True; t0=time.time(); interrupted=False; think_open=False
+ exact_tokens=None
try:
with urllib.request.urlopen(req) as r:
for raw in r:
@@ -1383,6 +1417,9 @@ def chat_attached(a, base, model_id):
if data=="[DONE]": break
try: ev=json.loads(data)
except ValueError: continue
+ usage=ev.get("usage")
+ if isinstance(usage,dict) and isinstance(usage.get("completion_tokens"),int):
+ exact_tokens=usage["completion_tokens"]
for ch in ev.get("choices",[]):
d=ch.get("delta",{})
rtxt=d.get("reasoning_content")
@@ -1436,8 +1473,9 @@ def chat_attached(a, base, model_id):
if reply: msgs.append({"role":"assistant","content":"".join(reply)})
else: msgs.pop() # turno vuoto: non sporcare la history
el=time.time()-t0
- note=" · ⏹ interrupted" if interrupted else ""
- print(f"\r {C.dgray}└─ ~{len(''.join(reply))//4} tok · {el:.0f}s{note}{C.r}\n")
+ footer=render_gen_stats(gen_stats("".join(reply), el, exact_tokens, interrupted), chat_stats_mode(a))
+ if footer: print(f"\r {C.dgray}└─ {footer}{C.r}\n")
+ else: print()
print(f" {C.dim}goodbye — the engine keeps running for the next chat 🐦{C.r}")
def kv_resume_notice(model_dir):
@@ -1714,7 +1752,10 @@ def cmd_chat(a):
sp2.stop()
if st is None: engine_diag(p, errlog); break
el=time.time()-t0
- if st.get("tok"):
+ if st.get("tok") and chat_stats_mode(a) == "off":
+ if st.get("interrupted"): print(f" {C.yel}⏹ interrupted; type :more to continue the response{C.r}")
+ print()
+ elif st.get("tok"):
print(f"\r {C.dgray}└─ {st['tok']} tok · {st['tps']:.2f} tok/s · hit {st['hit']:.0f}% · RSS {st['rss']:.1f} GB · {el:.0f}s{C.r}")
if st.get("interrupted"):
print(f" {C.yel}⏹ interrupted; type :more to continue the response{C.r}")
@@ -2199,6 +2240,10 @@ def main():
pc.add_argument("--no-attach", action="store_true",
help="never auto-attach, always spawn a private engine")
pc.add_argument("--api-key", default=os.environ.get("COLI_API_KEY"))
+ pc.add_argument("--stats", choices=list(CHAT_STATS_MODES), default=None,
+ help="generation statistics footer after each answer: full (tokens, seconds, tok/s), "
+ "compact (tokens, tok/s), off. Default from COLI_CHAT_STATS, else full. "
+ "Counts are exact when the server reports completion_tokens, ~estimated otherwise.")
ps=sub.add_parser("serve", parents=[common])
ps.add_argument("--host",default="127.0.0.1"); ps.add_argument("--port",type=int,default=8000)
ps.add_argument("--model-id",default=os.environ.get("COLI_MODEL_ID"))
diff --git a/c/tests/test_chat_stats.py b/c/tests/test_chat_stats.py
new file mode 100644
index 00000000..864567be
--- /dev/null
+++ b/c/tests/test_chat_stats.py
@@ -0,0 +1,79 @@
+"""#1475: the chat footer is a rendering of measured generation statistics,
+not a hard-coded line. gen_stats() takes the server's exact completion_tokens
+when it sent them and falls back to the chars/4 estimate; render_gen_stats()
+lays the result out per --stats mode (COLI_CHAT_STATS sets the default)."""
+import importlib.machinery
+import importlib.util
+import os
+import sys
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+CLI = Path(__file__).resolve().parent.parent / "coli"
+
+
+def load_cli():
+ loader = importlib.machinery.SourceFileLoader("coli_stats_t", str(CLI))
+ spec = importlib.util.spec_from_loader(loader.name, loader)
+ module = importlib.util.module_from_spec(spec)
+ argv = sys.argv
+ sys.argv = ["coli"]
+ try:
+ loader.exec_module(module)
+ finally:
+ sys.argv = argv
+ return module
+
+
+class ChatStatsTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.m = load_cli()
+
+ def test_estimate_full(self):
+ st = self.m.gen_stats("x" * 1480, 313.0)
+ self.assertEqual(st["tokens"], 370)
+ self.assertFalse(st["exact"])
+ self.assertEqual(self.m.render_gen_stats(st, "full"), "~370 tok · 313s · ~1.18 tok/s")
+
+ def test_exact_full(self):
+ st = self.m.gen_stats("x" * 1480, 313.0, exact_tokens=372)
+ self.assertTrue(st["exact"])
+ self.assertEqual(self.m.render_gen_stats(st, "full"), "372 tok · 313s · 1.19 tok/s")
+
+ def test_compact_and_off(self):
+ st = self.m.gen_stats("x" * 1480, 313.0)
+ self.assertEqual(self.m.render_gen_stats(st, "compact"), "~370 tok · ~1.18 tok/s")
+ self.assertEqual(self.m.render_gen_stats(st, "off"), "")
+
+ def test_interrupted_suffix(self):
+ st = self.m.gen_stats("x" * 120, 23.0, interrupted=True)
+ self.assertEqual(self.m.render_gen_stats(st, "full"), "~30 tok · 23s · ~1.30 tok/s · ⏹ interrupted")
+
+ def test_zero_elapsed_has_no_rate(self):
+ st = self.m.gen_stats("abcd", 0.0)
+ self.assertIsNone(st["tps"])
+ self.assertEqual(self.m.render_gen_stats(st, "full"), "~1 tok · 0s")
+
+ def test_bool_is_not_an_exact_count(self):
+ self.assertFalse(self.m.gen_stats("abcdefgh", 1.0, exact_tokens=True)["exact"])
+ self.assertFalse(self.m.gen_stats("abcdefgh", 1.0, exact_tokens=-1)["exact"])
+
+ def test_mode_resolution(self):
+ m = self.m
+ self.assertEqual(m.chat_stats_mode(SimpleNamespace(stats="compact")), "compact")
+ old = os.environ.get("COLI_CHAT_STATS")
+ try:
+ os.environ["COLI_CHAT_STATS"] = "off"
+ self.assertEqual(m.chat_stats_mode(SimpleNamespace(stats=None)), "off")
+ self.assertEqual(m.chat_stats_mode(SimpleNamespace(stats="full")), "full") # the flag wins
+ os.environ["COLI_CHAT_STATS"] = "bogus"
+ self.assertEqual(m.chat_stats_mode(SimpleNamespace()), "full")
+ finally:
+ if old is None: os.environ.pop("COLI_CHAT_STATS", None)
+ else: os.environ["COLI_CHAT_STATS"] = old
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index e5cb6147..aa0584f1 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -467,6 +467,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe
| `COLI_QUEUE_TIMEOUT` | `300` | Seconds a request may wait in the queue. |
| `COLI_KV_SLOTS` | `1` | Independent KV conversation slots (→ engine `KV_SLOTS`). |
| `COLI_POLICY` | `quality` | Resource policy (shared with the engine): `quality` \| `balanced` \| `experimental-fast`. |
+| `COLI_CHAT_STATS` | `full` | Default for `coli chat --stats`: the footer after each answer. `full` = tokens, seconds, tok/s; `compact` = tokens, tok/s; `off` = no footer. Counts are exact (no `~`) when the server reports `completion_tokens` in the streamed usage block, the chars/4 estimate otherwise. The flag wins over the variable. |
| `COLI_COLOR` | auto (TTY) | `COLI_COLOR=1` forces colored `coli` output when not a TTY. |
| `COLI_RAW` | `0` | `coli` raw output mode. |