Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Say why the engine died instead of dying mute (#305)

JustVuggd57955e95a6d
diff --git a/c/coli b/c/coli
index efc8526f..ad2f13ef 100755
--- a/c/coli
+++ b/c/coli
@@ -344,6 +344,34 @@ class Spinner:
if self.th: self.th.join(timeout=0.4)
if TTY: sys.stdout.write("\r\033[K"); sys.stdout.flush()
+def engine_diag(p, errlog=None):
+ """Lo stdout del motore ha chiuso: il processo e' morto. Dire PERCHE'.
+ Un SIGKILL dell'OOM-killer del kernel e' altrimenti invisibile — niente errore,
+ niente exit code, il motore muore muto e sembra un bug nostro (issue #305).
+ Quasi sempre e' memoria: il picco reale ha sforato la RAM della macchina."""
+ try: rc=p.wait(timeout=5)
+ except Exception: rc=p.poll()
+ if rc is None: why="its output closed but the process is still alive"
+ elif rc<0: # POSIX: morte per segnale
+ try: why=f"killed by {signal.Signals(-rc).name}"
+ except Exception: why=f"killed by signal {-rc}"
+ elif rc>0: why=f"exit code {rc}"
+ else: why="exited cleanly"
+ print(f"\n {C.yel}[engine terminated: {why}]{C.r}")
+ if rc is not None and rc<0 and -rc==getattr(signal,"SIGKILL",-1):
+ print(f" {C.yel}nothing in the engine sends SIGKILL to itself: this is the kernel's\n"
+ f" OOM-killer. The peak RSS exceeded the machine's free memory.\n"
+ f" Lower --ram, lower PIN_GB, or shorten the context.{C.r}")
+ if errlog is not None:
+ try:
+ errlog.flush()
+ for _ in range(20): # il drain thread scrive su errlog quando il child chiude
+ tail=open(errlog.name).read().strip()
+ if tail: break
+ time.sleep(0.05)
+ if tail: print(f" {C.dgray}" + "\n ".join(tail.splitlines()[-6:]) + f"{C.r}")
+ except Exception: pass
+
def stream_turn(p, sentinel, on_bytes):
"""legge fino alla sentinella; on_bytes riceve i chunk della risposta. Poi legge la riga STAT.
Il PRIMO Ctrl-C durante lo stream non chiude la sessione: il motore (handler SIGINT)
@@ -478,7 +506,9 @@ def cmd_chat(a):
if st is None:
try: errlog.write(p.stderr.read().decode("utf-8","replace"))
except (OSError, ValueError): pass
- errlog.seek(0); print(errlog.read()[-1500:]); sys.exit("the engine exited while loading")
+ errlog.flush(); errlog.seek(0); print(errlog.read()[-1500:])
+ engine_diag(p) # perche' e' morto (OOM-kill compreso); errlog e' gia' stampato sopra
+ sys.exit("the engine exited while loading")
p.stdout.readline() # TIERS line (web-dashboard protocol): emitted once right after STAT;
# left unread it leaks into the first answer's text
# READY received. Drain the child's stderr into errlog without blocking:
@@ -568,7 +598,7 @@ def cmd_chat(a):
st=stream_turn(p, END, echo)
if not raw: md.close()
sp2.stop()
- if st is None: print(f"\n {C.yel}[engine terminated]{C.r}"); break
+ if st is None: engine_diag(p, errlog); break
el=time.time()-t0
if st.get("tok"):
print(f"\r {C.dgray}└─ {st['tok']} tok · {st['tps']:.2f} tok/s · hit {st['hit']:.0f}% · RSS {st['rss']:.1f} GB · {el:.0f}s{C.r}")
diff --git a/c/glm.c b/c/glm.c
index 7e05ad94..ae18c554 100644
--- a/c/glm.c
+++ b/c/glm.c
@@ -5632,7 +5632,30 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){
double slack = 1.2e9 + pc_b + ws_b + kv_b + kvb_b;
double avail = ram_gb*1e9 - (double)m->resident_bytes - slack;
int capmax = (avail>0 && nsp>0) ? (int)(avail/((double)nsp*eb)) : 0;
+ int floored = capmax<1; /* il budget non regge nemmeno UNO slot per layer */
if(capmax<1) capmax=1;
+ /* Il floor a 1 e' una bugia comoda: con avail negativo capmax sarebbe 0, cioe'
+ * "non ci sto nel tuo budget". Alzarlo a 1 e proseguire trasforma "non ci sto"
+ * in "sforo" -- ed e' esattamente l'OOM-kill a meta' generazione che questa
+ * funzione esiste per evitare. Il kernel uccide con SIGKILL: nessun errore,
+ * nessun log, il motore muore muto (issue #305). Dirlo, e fermarsi se il picco
+ * non entra nemmeno nella RAM realmente disponibile misurata all'avvio. */
+ if(floored){
+ double peak = (double)m->resident_bytes + (double)capmax*nsp*eb + slack;
+ fprintf(stderr,"[RAM_GB=%.1f%s] WARNING: cap=1 is the floor, projected peak %.1f GB is "
+ "%.1f GB OVER the budget (resident %.1f GB + reserve %.1f GB).%s\n",
+ ram_gb,auto_b?" auto":"",peak/1e9,(peak-ram_gb*1e9)/1e9,
+ m->resident_bytes/1e9,slack/1e9,
+ getenv("PIN_GB")?" PIN_GB is inflating the resident set: lower it or drop it.":"");
+ if(g_mem_avail_boot>0 && peak > g_mem_avail_boot*1e9 &&
+ !(getenv("COLI_RAM_OVERCOMMIT") && atoi(getenv("COLI_RAM_OVERCOMMIT")))){
+ fprintf(stderr,"[RAM] refusing to start: that peak also exceeds the %.1f GB actually "
+ "available on this machine, so the kernel would OOM-kill this run mid-generation.\n"
+ "[RAM] lower PIN_GB, lower the context, or raise the RAM budget if the box really has it "
+ "(COLI_RAM_OVERCOMMIT=1 overrides this check).\n", g_mem_avail_boot);
+ exit(2);
+ }
+ }
if(capmax < m->ecap){
fprintf(stderr,"[RAM_GB=%.1f%s] resident %.1f GB + reserve %.1f GB (ws %.1f, KV %dx%d %.1f, kvb %.1f), "
"experts %.1f MB x %d layers -> cap lowered %d->%d (projected peak %.1f GB)\n",