Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(coli): truthful splash + visible engine status lines in chat
Andrey Aristov657606923419
diff --git a/c/coli b/c/coliindex 44a09179..12d746d9 100755--- a/c/coli+++ b/c/coli@@ -137,12 +137,37 @@ def sprite_lines():out.append(row+"\033[0m")return out-def banner(sub=""):+def backend_tag():+ # Requested backend, from the same env the engine child will read. The engine+ # still prints its own [METAL]/[CUDA] line once live (and falls back per-block+ # on faults) — this only stops the splash claiming "CPU" under COLI_METAL=1.+ e=os.environ+ def on(k): v=e.get(k); return v is not None and v!="0"+ if on("COLI_METAL"): return "Metal"+ if on("COLI_CUDA"): return "CUDA"+ if on("COLI_VULKAN"): return "Vulkan"+ return "CPU"++def model_tag(model=None):+ # Splash strapline from the model actually being loaded (the old line was a+ # hardcoded "GLM-5.2 · 744B MoE · int4" from the single-model days). Falls back+ # to the historic strapline for commands that have no model dir at hand.+ if model and os.path.isdir(model):+ name=os.path.basename(os.path.normpath(model))+ try:+ with open(os.path.join(model,"config.json"),encoding="utf-8") as f: cfg=json.load(f)+ except (OSError,ValueError): cfg={}+ ne=cfg.get("n_routed_experts") or cfg.get("num_experts")+ kind=f"{ne}-expert MoE" if ne else (cfg.get("model_type") or "transformer")+ return f"{name} · {kind} · streaming {backend_tag()}"+ return f"GLM-5.2 · 744B MoE · int4 · streaming {backend_tag()}"++def banner(sub="", model=None):sp=sprite_lines()txt=[f"{C.teal}{C.b}colibri{C.r} {C.dim}v{_version}{C.r}",f"{C.dim}tiny engine, immense model{C.r}",- f"{C.gray}GLM-5.2 · 744B MoE · int4 · streaming CPU{C.r}",+ f"{C.gray}{model_tag(model)}{C.r}",f"{C.dgray}{sub}{C.r}" if sub else "","",]@@ -592,7 +617,7 @@ def cmd_plan(a):if a.json:print(json.dumps(plan,indent=2))return- banner("plan · Disk / RAM / VRAM")+ banner("plan · Disk / RAM / VRAM", model=a.model)print(textwrap.indent(format_plan(plan)," "))print()@@ -640,7 +665,7 @@ def cmd_tune(a):a.no_tune_profile=Truebase_env=env_for(a)prompt=f"[gMASK]<sop><|user|>{a.prompt}<|assistant|><think></think>"- banner("tune · measured execution profile")+ banner("tune · measured execution profile", model=a.model)print(f" fixed replay: {a.tokens} tokens · {a.repeats} run(s) per candidate")print(" only quality-preserving scheduling knobs are eligible\n")try:@@ -667,7 +692,7 @@ def cmd_tune(a):def cmd_run(a):need_model(a.model)prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"')- banner("run")+ banner("run", model=a.model)# template ufficiale GLM-5.2: niente \n dopo i ruoli; <think></think> = risposta diretta (nothink).# THINK=1 lascia <think> aperto, stessa convenzione del serve mode (glm.c). EN: THINK=1 leaves# <think> open so the engine emits its reasoning block; the default stays nothink.@@ -835,7 +860,7 @@ def cmd_chat(a):try: p.wait(timeout=10)except subprocess.TimeoutExpired: p.kill()need_model(a.model)- banner(f"chat · {os.path.basename(a.model)} · ram {a.ram or '-'}GB · topp {a.topp or 'off'}")+ banner(f"chat · ram {a.ram or '-'}GB · topp {a.topp or 'off'}", model=a.model)kv_resume_notice(a.model)errlog=tempfile.NamedTemporaryFile(mode="w+", suffix=".log", delete=False)e=env_for(a); e["SERVE"]="1"@@ -851,7 +876,13 @@ def cmd_chat(a):p=subprocess.Popen([GLM,str(a.cap if a.cap is not None else 0)], env=e, stdin=subprocess.PIPE,stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0)sp=Spinner("waking the giant (744B)…"); sp.start()- st=stream_turn(p, READY, lambda b: None)+ # Keep the pre-READY stdout preamble instead of discarding it: the engine's+ # "loaded in Xs | resident dense: Y MB" line is a printf, i.e. STDOUT+ # (colibri.c:9209 — same in inkling.c/olmoe.c), so it arrives HERE and never+ # in errlog. The ready-line regex below used to scan stderr for it and could+ # never match. A few hundred bytes of banner; nothing else reads them.+ _preamble=bytearray()+ st=stream_turn(p, READY, _preamble.extend)sp.stop()if st is None:try: errlog.write(p.stderr.read().decode("utf-8","replace"))@@ -867,7 +898,16 @@ def cmd_chat(a):# of load-time status ([RAM_GB], [MTP], ...) that were already emitted._drain_box={"done":False}def _drain():- try: errlog.write(p.stderr.read().decode("utf-8","replace"))+ # readline(), NOT read(): read() returns only at EOF, and the engine is alive+ # on stdin, so it never returned and errlog.write() below it never ran — the+ # bounded wait always timed out on an EMPTY file and the whole status block+ # (including "ready in Xs") silently displayed nothing. Per-line write+flush+ # makes each status line visible to the reader below as soon as it arrives.+ try:+ while True:+ raw=p.stderr.readline()+ if not raw: break+ errlog.write(raw.decode("utf-8","replace")); errlog.flush()except (OSError, ValueError): pass_drain_box["done"]=Truethreading.Thread(target=_drain, daemon=True).start()@@ -876,12 +916,17 @@ def cmd_chat(a):if _drain_box["done"]: breaktime.sleep(0.05)errlog.flush()+ mload=re.search(r"loaded in ([0-9.]+)s \| resident dense: ([0-9.]+) MB",+ _preamble.decode("utf-8","replace"))+ if mload: print(f" {C.grn}✓{C.r} ready in {mload.group(1)}s {C.dim}· resident {float(mload.group(2))/1000:.1f} GB · RSS {st.get('rss','?')} GB{C.r}")try:elog=open(errlog.name).read()- mload=re.search(r"loaded in ([0-9.]+)s \| resident dense: ([0-9.]+) MB", elog)- if mload: print(f" {C.grn}✓{C.r} ready in {mload.group(1)}s {C.dim}· resident {float(mload.group(2))/1000:.1f} GB · RSS {st.get('rss','?')} GB{C.r}")for l in elog.splitlines(): # una riga di stato per riga, senza path- if l.startswith(("[RAM_GB","[PIN]","[MTP]","[USAGE]","[DSA]","[KV]")):+ # [METAL]/[CUDA] first: the splash tagline (backend_tag) only reflects the+ # REQUESTED backend from the environment — this engine line is the actual+ # confirmation the GPU tier engaged (it can still fall back per-block). The+ # engine prints it to stderr, we drain it into errlog; keep it whitelisted.+ if l.startswith(("[METAL]","[CUDA]","[RAM_GB","[PIN]","[MTP]","[USAGE]","[DSA]","[KV]")):l=re.sub(r" ?\(?/[^ )]+\)?","",l.strip()) # via i percorsi lunghil=re.sub(r" from$","",l)for chunk in textwrap.wrap(l, term_w()-4) or [l]:@@ -1064,7 +1109,7 @@ def cmd_web(a):def cmd_bench(a):need_model(a.model)- banner("bench")+ banner("bench", model=a.model)# python con `tokenizers`: l'ambiente del progetto se c'e', altrimenti quello correntevenv_py = os.path.join(HERE, "mio_env", "Scripts" if sys.platform == "win32" else "bin", "python3")py = venv_py if os.path.exists(venv_py) else sys.executable