Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(coli): truthful splash + visible engine status lines in chat

Andrey Aristov657606923419
diff --git a/c/coli b/c/coli
index 44a09179..12d746d9 100755
--- a/c/coli
+++ b/c/coli
@@ -137,12 +137,37 @@ def sprite_lines():
out.append(row+"\033[0m")
return out
-def banner(sub=""):
+def backend_tag():
+ # Requested backend, from the same env the engine child will read. The engine
+ # still prints its own [METAL]/[CUDA] line once live (and falls back per-block
+ # on faults) — this only stops the splash claiming "CPU" under COLI_METAL=1.
+ e=os.environ
+ def on(k): v=e.get(k); return v is not None and v!="0"
+ if on("COLI_METAL"): return "Metal"
+ if on("COLI_CUDA"): return "CUDA"
+ if on("COLI_VULKAN"): return "Vulkan"
+ return "CPU"
+
+def model_tag(model=None):
+ # Splash strapline from the model actually being loaded (the old line was a
+ # hardcoded "GLM-5.2 · 744B MoE · int4" from the single-model days). Falls back
+ # to the historic strapline for commands that have no model dir at hand.
+ if model and os.path.isdir(model):
+ name=os.path.basename(os.path.normpath(model))
+ try:
+ with open(os.path.join(model,"config.json"),encoding="utf-8") as f: cfg=json.load(f)
+ except (OSError,ValueError): cfg={}
+ ne=cfg.get("n_routed_experts") or cfg.get("num_experts")
+ kind=f"{ne}-expert MoE" if ne else (cfg.get("model_type") or "transformer")
+ return f"{name} · {kind} · streaming {backend_tag()}"
+ return f"GLM-5.2 · 744B MoE · int4 · streaming {backend_tag()}"
+
+def banner(sub="", model=None):
sp=sprite_lines()
txt=[
f"{C.teal}{C.b}colibri{C.r} {C.dim}v{_version}{C.r}",
f"{C.dim}tiny engine, immense model{C.r}",
- f"{C.gray}GLM-5.2 · 744B MoE · int4 · streaming CPU{C.r}",
+ f"{C.gray}{model_tag(model)}{C.r}",
f"{C.dgray}{sub}{C.r}" if sub else "",
"",
]
@@ -592,7 +617,7 @@ def cmd_plan(a):
if a.json:
print(json.dumps(plan,indent=2))
return
- banner("plan · Disk / RAM / VRAM")
+ banner("plan · Disk / RAM / VRAM", model=a.model)
print(textwrap.indent(format_plan(plan)," "))
print()
@@ -640,7 +665,7 @@ def cmd_tune(a):
a.no_tune_profile=True
base_env=env_for(a)
prompt=f"[gMASK]<sop><|user|>{a.prompt}<|assistant|><think></think>"
- banner("tune · measured execution profile")
+ banner("tune · measured execution profile", model=a.model)
print(f" fixed replay: {a.tokens} tokens · {a.repeats} run(s) per candidate")
print(" only quality-preserving scheduling knobs are eligible\n")
try:
@@ -667,7 +692,7 @@ def cmd_tune(a):
def cmd_run(a):
need_model(a.model)
prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"')
- banner("run")
+ banner("run", model=a.model)
# template ufficiale GLM-5.2: niente \n dopo i ruoli; <think></think> = risposta diretta (nothink).
# THINK=1 lascia <think> aperto, stessa convenzione del serve mode (glm.c). EN: THINK=1 leaves
# <think> open so the engine emits its reasoning block; the default stays nothink.
@@ -835,7 +860,7 @@ def cmd_chat(a):
try: p.wait(timeout=10)
except subprocess.TimeoutExpired: p.kill()
need_model(a.model)
- banner(f"chat · {os.path.basename(a.model)} · ram {a.ram or '-'}GB · topp {a.topp or 'off'}")
+ banner(f"chat · ram {a.ram or '-'}GB · topp {a.topp or 'off'}", model=a.model)
kv_resume_notice(a.model)
errlog=tempfile.NamedTemporaryFile(mode="w+", suffix=".log", delete=False)
e=env_for(a); e["SERVE"]="1"
@@ -851,7 +876,13 @@ def cmd_chat(a):
p=subprocess.Popen([GLM,str(a.cap if a.cap is not None else 0)], env=e, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0)
sp=Spinner("waking the giant (744B)…"); sp.start()
- st=stream_turn(p, READY, lambda b: None)
+ # Keep the pre-READY stdout preamble instead of discarding it: the engine's
+ # "loaded in Xs | resident dense: Y MB" line is a printf, i.e. STDOUT
+ # (colibri.c:9209 — same in inkling.c/olmoe.c), so it arrives HERE and never
+ # in errlog. The ready-line regex below used to scan stderr for it and could
+ # never match. A few hundred bytes of banner; nothing else reads them.
+ _preamble=bytearray()
+ st=stream_turn(p, READY, _preamble.extend)
sp.stop()
if st is None:
try: errlog.write(p.stderr.read().decode("utf-8","replace"))
@@ -867,7 +898,16 @@ def cmd_chat(a):
# of load-time status ([RAM_GB], [MTP], ...) that were already emitted.
_drain_box={"done":False}
def _drain():
- try: errlog.write(p.stderr.read().decode("utf-8","replace"))
+ # readline(), NOT read(): read() returns only at EOF, and the engine is alive
+ # on stdin, so it never returned and errlog.write() below it never ran — the
+ # bounded wait always timed out on an EMPTY file and the whole status block
+ # (including "ready in Xs") silently displayed nothing. Per-line write+flush
+ # makes each status line visible to the reader below as soon as it arrives.
+ try:
+ while True:
+ raw=p.stderr.readline()
+ if not raw: break
+ errlog.write(raw.decode("utf-8","replace")); errlog.flush()
except (OSError, ValueError): pass
_drain_box["done"]=True
threading.Thread(target=_drain, daemon=True).start()
@@ -876,12 +916,17 @@ def cmd_chat(a):
if _drain_box["done"]: break
time.sleep(0.05)
errlog.flush()
+ mload=re.search(r"loaded in ([0-9.]+)s \| resident dense: ([0-9.]+) MB",
+ _preamble.decode("utf-8","replace"))
+ if mload: print(f" {C.grn}✓{C.r} ready in {mload.group(1)}s {C.dim}· resident {float(mload.group(2))/1000:.1f} GB · RSS {st.get('rss','?')} GB{C.r}")
try:
elog=open(errlog.name).read()
- mload=re.search(r"loaded in ([0-9.]+)s \| resident dense: ([0-9.]+) MB", elog)
- if mload: print(f" {C.grn}✓{C.r} ready in {mload.group(1)}s {C.dim}· resident {float(mload.group(2))/1000:.1f} GB · RSS {st.get('rss','?')} GB{C.r}")
for l in elog.splitlines(): # una riga di stato per riga, senza path
- if l.startswith(("[RAM_GB","[PIN]","[MTP]","[USAGE]","[DSA]","[KV]")):
+ # [METAL]/[CUDA] first: the splash tagline (backend_tag) only reflects the
+ # REQUESTED backend from the environment — this engine line is the actual
+ # confirmation the GPU tier engaged (it can still fall back per-block). The
+ # engine prints it to stderr, we drain it into errlog; keep it whitelisted.
+ if l.startswith(("[METAL]","[CUDA]","[RAM_GB","[PIN]","[MTP]","[USAGE]","[DSA]","[KV]")):
l=re.sub(r" ?\(?/[^ )]+\)?","",l.strip()) # via i percorsi lunghi
l=re.sub(r" from$","",l)
for chunk in textwrap.wrap(l, term_w()-4) or [l]:
@@ -1064,7 +1109,7 @@ def cmd_web(a):
def cmd_bench(a):
need_model(a.model)
- banner("bench")
+ banner("bench", model=a.model)
# python con `tokenizers`: l'ambiente del progetto se c'e', altrimenti quello corrente
venv_py = os.path.join(HERE, "mio_env", "Scripts" if sys.platform == "win32" else "bin", "python3")
py = venv_py if os.path.exists(venv_py) else sys.executable