Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Say why the engine died instead of dying mute (#305)
JustVuggd57955e95a6d
diff --git a/c/coli b/c/coliindex efc8526f..ad2f13ef 100755--- a/c/coli+++ b/c/coli@@ -344,6 +344,34 @@ class Spinner:if self.th: self.th.join(timeout=0.4)if TTY: sys.stdout.write("\r\033[K"); sys.stdout.flush()+def engine_diag(p, errlog=None):+ """Lo stdout del motore ha chiuso: il processo e' morto. Dire PERCHE'.+ Un SIGKILL dell'OOM-killer del kernel e' altrimenti invisibile — niente errore,+ niente exit code, il motore muore muto e sembra un bug nostro (issue #305).+ Quasi sempre e' memoria: il picco reale ha sforato la RAM della macchina."""+ try: rc=p.wait(timeout=5)+ except Exception: rc=p.poll()+ if rc is None: why="its output closed but the process is still alive"+ elif rc<0: # POSIX: morte per segnale+ try: why=f"killed by {signal.Signals(-rc).name}"+ except Exception: why=f"killed by signal {-rc}"+ elif rc>0: why=f"exit code {rc}"+ else: why="exited cleanly"+ print(f"\n {C.yel}[engine terminated: {why}]{C.r}")+ if rc is not None and rc<0 and -rc==getattr(signal,"SIGKILL",-1):+ print(f" {C.yel}nothing in the engine sends SIGKILL to itself: this is the kernel's\n"+ f" OOM-killer. The peak RSS exceeded the machine's free memory.\n"+ f" Lower --ram, lower PIN_GB, or shorten the context.{C.r}")+ if errlog is not None:+ try:+ errlog.flush()+ for _ in range(20): # il drain thread scrive su errlog quando il child chiude+ tail=open(errlog.name).read().strip()+ if tail: break+ time.sleep(0.05)+ if tail: print(f" {C.dgray}" + "\n ".join(tail.splitlines()[-6:]) + f"{C.r}")+ except Exception: pass+def stream_turn(p, sentinel, on_bytes):"""legge fino alla sentinella; on_bytes riceve i chunk della risposta. Poi legge la riga STAT.Il PRIMO Ctrl-C durante lo stream non chiude la sessione: il motore (handler SIGINT)@@ -478,7 +506,9 @@ def cmd_chat(a):if st is None:try: errlog.write(p.stderr.read().decode("utf-8","replace"))except (OSError, ValueError): pass- errlog.seek(0); print(errlog.read()[-1500:]); sys.exit("the engine exited while loading")+ errlog.flush(); errlog.seek(0); print(errlog.read()[-1500:])+ engine_diag(p) # perche' e' morto (OOM-kill compreso); errlog e' gia' stampato sopra+ sys.exit("the engine exited while loading")p.stdout.readline() # TIERS line (web-dashboard protocol): emitted once right after STAT;# left unread it leaks into the first answer's text# READY received. Drain the child's stderr into errlog without blocking:@@ -568,7 +598,7 @@ def cmd_chat(a):st=stream_turn(p, END, echo)if not raw: md.close()sp2.stop()- if st is None: print(f"\n {C.yel}[engine terminated]{C.r}"); break+ if st is None: engine_diag(p, errlog); breakel=time.time()-t0if st.get("tok"):print(f"\r {C.dgray}└─ {st['tok']} tok · {st['tps']:.2f} tok/s · hit {st['hit']:.0f}% · RSS {st['rss']:.1f} GB · {el:.0f}s{C.r}")diff --git a/c/glm.c b/c/glm.cindex 7e05ad94..ae18c554 100644--- a/c/glm.c+++ b/c/glm.c@@ -5632,7 +5632,30 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){double slack = 1.2e9 + pc_b + ws_b + kv_b + kvb_b;double avail = ram_gb*1e9 - (double)m->resident_bytes - slack;int capmax = (avail>0 && nsp>0) ? (int)(avail/((double)nsp*eb)) : 0;+ int floored = capmax<1; /* il budget non regge nemmeno UNO slot per layer */if(capmax<1) capmax=1;+ /* Il floor a 1 e' una bugia comoda: con avail negativo capmax sarebbe 0, cioe'+ * "non ci sto nel tuo budget". Alzarlo a 1 e proseguire trasforma "non ci sto"+ * in "sforo" -- ed e' esattamente l'OOM-kill a meta' generazione che questa+ * funzione esiste per evitare. Il kernel uccide con SIGKILL: nessun errore,+ * nessun log, il motore muore muto (issue #305). Dirlo, e fermarsi se il picco+ * non entra nemmeno nella RAM realmente disponibile misurata all'avvio. */+ if(floored){+ double peak = (double)m->resident_bytes + (double)capmax*nsp*eb + slack;+ fprintf(stderr,"[RAM_GB=%.1f%s] WARNING: cap=1 is the floor, projected peak %.1f GB is "+ "%.1f GB OVER the budget (resident %.1f GB + reserve %.1f GB).%s\n",+ ram_gb,auto_b?" auto":"",peak/1e9,(peak-ram_gb*1e9)/1e9,+ m->resident_bytes/1e9,slack/1e9,+ getenv("PIN_GB")?" PIN_GB is inflating the resident set: lower it or drop it.":"");+ if(g_mem_avail_boot>0 && peak > g_mem_avail_boot*1e9 &&+ !(getenv("COLI_RAM_OVERCOMMIT") && atoi(getenv("COLI_RAM_OVERCOMMIT")))){+ fprintf(stderr,"[RAM] refusing to start: that peak also exceeds the %.1f GB actually "+ "available on this machine, so the kernel would OOM-kill this run mid-generation.\n"+ "[RAM] lower PIN_GB, lower the context, or raise the RAM budget if the box really has it "+ "(COLI_RAM_OVERCOMMIT=1 overrides this check).\n", g_mem_avail_boot);+ exit(2);+ }+ }if(capmax < m->ecap){fprintf(stderr,"[RAM_GB=%.1f%s] resident %.1f GB + reserve %.1f GB (ws %.1f, KV %dx%d %.1f, kvb %.1f), ""experts %.1f MB x %d layers -> cap lowered %d->%d (projected peak %.1f GB)\n",