Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
coli chat: configurable generation statistics with exact token throughput
JustVuggd12d449efc7b
diff --git a/c/coli b/c/coliindex 86786775..06804f1e 100755--- a/c/coli+++ b/c/coli@@ -1328,6 +1328,38 @@ def server_probe(base, api_key=None, timeout=1.5):except Exception:return None+# ---- generation statistics for the chat footer (#1475) ----------------------+# Measurement and rendering apart: gen_stats() counts (exact completion tokens+# from the server's usage block when it sends one, the historical chars/4+# estimate otherwise), render_gen_stats() lays it out for one of three modes.+# The private-engine chat already prints the engine's own exact counters; the+# attached path is where the estimate lived and where tok/s was missing.+CHAT_STATS_MODES = ("full", "compact", "off")++def chat_stats_mode(a):+ mode = getattr(a, "stats", None) or os.environ.get("COLI_CHAT_STATS", "full")+ return mode if mode in CHAT_STATS_MODES else "full"++def gen_stats(reply_text, elapsed, exact_tokens=None, interrupted=False):+ exact = isinstance(exact_tokens, int) and not isinstance(exact_tokens, bool) and exact_tokens >= 0+ tokens = exact_tokens if exact else len(reply_text) // 4+ tps = tokens / elapsed if elapsed > 0 else None+ return {"tokens": tokens, "exact": exact, "elapsed": elapsed, "tps": tps,+ "interrupted": bool(interrupted)}++def render_gen_stats(st, mode):+ """'' for off; otherwise the footer body, without the box glyph or colour:+ full ~370 tok · 313s · ~1.18 tok/s (372 tok · 313s · 1.19 tok/s when exact)+ compact ~370 tok · ~1.18 tok/s+ plus ' · ⏹ interrupted' when the answer was cut short."""+ if mode == "off": return ""+ approx = "" if st["exact"] else "~"+ parts = [f"{approx}{st['tokens']} tok"]+ if mode == "full": parts.append(f"{st['elapsed']:.0f}s")+ if st["tps"] is not None: parts.append(f"{approx}{st['tps']:.2f} tok/s")+ if st["interrupted"]: parts.append("⏹ interrupted")+ return " · ".join(parts)+def chat_attached(a, base, model_id):"""The chat REPL over HTTP against a running `coli serve`.@@ -1364,7 +1396,8 @@ def chat_attached(a, base, model_id):# motore che streamma gli esperti da disco, "max" vuol dire centinaia# di token di riflessione prima della prima parola di risposta.request = {"model":model_id,"messages":msgs,"stream":True,- "max_tokens":ngen_for(a,interactive=True)}+ "max_tokens":ngen_for(a,interactive=True),+ "stream_options":{"include_usage":True}} # exact completion_tokens in the last chunkif getattr(a,"effort",None): request["reasoning_effort"]=a.effortif getattr(a,"think",None) is not None: request["enable_thinking"]=a.thinkbody=json.dumps(request).encode()@@ -1374,6 +1407,7 @@ def chat_attached(a, base, model_id):print(f"\n {C.teal}◆ colibri{C.r}")sp=Spinner("thinking…"); sp.start()md=MDStream(" "); reply=[]; first=True; t0=time.time(); interrupted=False; think_open=False+ exact_tokens=Nonetry:with urllib.request.urlopen(req) as r:for raw in r:@@ -1383,6 +1417,9 @@ def chat_attached(a, base, model_id):if data=="[DONE]": breaktry: ev=json.loads(data)except ValueError: continue+ usage=ev.get("usage")+ if isinstance(usage,dict) and isinstance(usage.get("completion_tokens"),int):+ exact_tokens=usage["completion_tokens"]for ch in ev.get("choices",[]):d=ch.get("delta",{})rtxt=d.get("reasoning_content")@@ -1436,8 +1473,9 @@ def chat_attached(a, base, model_id):if reply: msgs.append({"role":"assistant","content":"".join(reply)})else: msgs.pop() # turno vuoto: non sporcare la historyel=time.time()-t0- note=" · ⏹ interrupted" if interrupted else ""- print(f"\r {C.dgray}└─ ~{len(''.join(reply))//4} tok · {el:.0f}s{note}{C.r}\n")+ footer=render_gen_stats(gen_stats("".join(reply), el, exact_tokens, interrupted), chat_stats_mode(a))+ if footer: print(f"\r {C.dgray}└─ {footer}{C.r}\n")+ else: print()print(f" {C.dim}goodbye — the engine keeps running for the next chat 🐦{C.r}")def kv_resume_notice(model_dir):@@ -1714,7 +1752,10 @@ def cmd_chat(a):sp2.stop()if st is None: engine_diag(p, errlog); breakel=time.time()-t0- if st.get("tok"):+ if st.get("tok") and chat_stats_mode(a) == "off":+ if st.get("interrupted"): print(f" {C.yel}⏹ interrupted; type :more to continue the response{C.r}")+ print()+ elif st.get("tok"):print(f"\r {C.dgray}└─ {st['tok']} tok · {st['tps']:.2f} tok/s · hit {st['hit']:.0f}% · RSS {st['rss']:.1f} GB · {el:.0f}s{C.r}")if st.get("interrupted"):print(f" {C.yel}⏹ interrupted; type :more to continue the response{C.r}")@@ -2199,6 +2240,10 @@ def main():pc.add_argument("--no-attach", action="store_true",help="never auto-attach, always spawn a private engine")pc.add_argument("--api-key", default=os.environ.get("COLI_API_KEY"))+ pc.add_argument("--stats", choices=list(CHAT_STATS_MODES), default=None,+ help="generation statistics footer after each answer: full (tokens, seconds, tok/s), "+ "compact (tokens, tok/s), off. Default from COLI_CHAT_STATS, else full. "+ "Counts are exact when the server reports completion_tokens, ~estimated otherwise.")ps=sub.add_parser("serve", parents=[common])ps.add_argument("--host",default="127.0.0.1"); ps.add_argument("--port",type=int,default=8000)ps.add_argument("--model-id",default=os.environ.get("COLI_MODEL_ID"))diff --git a/c/tests/test_chat_stats.py b/c/tests/test_chat_stats.pynew file mode 100644index 00000000..864567be--- /dev/null+++ b/c/tests/test_chat_stats.py@@ -0,0 +1,79 @@+"""#1475: the chat footer is a rendering of measured generation statistics,+not a hard-coded line. gen_stats() takes the server's exact completion_tokens+when it sent them and falls back to the chars/4 estimate; render_gen_stats()+lays the result out per --stats mode (COLI_CHAT_STATS sets the default)."""+import importlib.machinery+import importlib.util+import os+import sys+import unittest+from pathlib import Path+from types import SimpleNamespace++CLI = Path(__file__).resolve().parent.parent / "coli"+++def load_cli():+ loader = importlib.machinery.SourceFileLoader("coli_stats_t", str(CLI))+ spec = importlib.util.spec_from_loader(loader.name, loader)+ module = importlib.util.module_from_spec(spec)+ argv = sys.argv+ sys.argv = ["coli"]+ try:+ loader.exec_module(module)+ finally:+ sys.argv = argv+ return module+++class ChatStatsTest(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.m = load_cli()++ def test_estimate_full(self):+ st = self.m.gen_stats("x" * 1480, 313.0)+ self.assertEqual(st["tokens"], 370)+ self.assertFalse(st["exact"])+ self.assertEqual(self.m.render_gen_stats(st, "full"), "~370 tok · 313s · ~1.18 tok/s")++ def test_exact_full(self):+ st = self.m.gen_stats("x" * 1480, 313.0, exact_tokens=372)+ self.assertTrue(st["exact"])+ self.assertEqual(self.m.render_gen_stats(st, "full"), "372 tok · 313s · 1.19 tok/s")++ def test_compact_and_off(self):+ st = self.m.gen_stats("x" * 1480, 313.0)+ self.assertEqual(self.m.render_gen_stats(st, "compact"), "~370 tok · ~1.18 tok/s")+ self.assertEqual(self.m.render_gen_stats(st, "off"), "")++ def test_interrupted_suffix(self):+ st = self.m.gen_stats("x" * 120, 23.0, interrupted=True)+ self.assertEqual(self.m.render_gen_stats(st, "full"), "~30 tok · 23s · ~1.30 tok/s · ⏹ interrupted")++ def test_zero_elapsed_has_no_rate(self):+ st = self.m.gen_stats("abcd", 0.0)+ self.assertIsNone(st["tps"])+ self.assertEqual(self.m.render_gen_stats(st, "full"), "~1 tok · 0s")++ def test_bool_is_not_an_exact_count(self):+ self.assertFalse(self.m.gen_stats("abcdefgh", 1.0, exact_tokens=True)["exact"])+ self.assertFalse(self.m.gen_stats("abcdefgh", 1.0, exact_tokens=-1)["exact"])++ def test_mode_resolution(self):+ m = self.m+ self.assertEqual(m.chat_stats_mode(SimpleNamespace(stats="compact")), "compact")+ old = os.environ.get("COLI_CHAT_STATS")+ try:+ os.environ["COLI_CHAT_STATS"] = "off"+ self.assertEqual(m.chat_stats_mode(SimpleNamespace(stats=None)), "off")+ self.assertEqual(m.chat_stats_mode(SimpleNamespace(stats="full")), "full") # the flag wins+ os.environ["COLI_CHAT_STATS"] = "bogus"+ self.assertEqual(m.chat_stats_mode(SimpleNamespace()), "full")+ finally:+ if old is None: os.environ.pop("COLI_CHAT_STATS", None)+ else: os.environ["COLI_CHAT_STATS"] = old+++if __name__ == "__main__":+ unittest.main()diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex e5cb6147..aa0584f1 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -467,6 +467,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe| `COLI_QUEUE_TIMEOUT` | `300` | Seconds a request may wait in the queue. || `COLI_KV_SLOTS` | `1` | Independent KV conversation slots (→ engine `KV_SLOTS`). || `COLI_POLICY` | `quality` | Resource policy (shared with the engine): `quality` \| `balanced` \| `experimental-fast`. |+| `COLI_CHAT_STATS` | `full` | Default for `coli chat --stats`: the footer after each answer. `full` = tokens, seconds, tok/s; `compact` = tokens, tok/s; `off` = no footer. Counts are exact (no `~`) when the server reports `completion_tokens` in the streamed usage block, the chars/4 estimate otherwise. The flag wins over the variable. || `COLI_COLOR` | auto (TTY) | `COLI_COLOR=1` forces colored `coli` output when not a TTY. || `COLI_RAW` | `0` | `coli` raw output mode. |