Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

prefill progress: engine reports layer N/78 on stderr, coli spinner shows it live

JustVugg1a8d2bcff38b
diff --git a/c/coli b/c/coli
index f10225dd..9dd0ff24 100644
--- a/c/coli
+++ b/c/coli
@@ -106,14 +106,20 @@ def env_for(a):
class Spinner:
FRAMES=["⠋","⠙","⠹","⠸","⠼","⠴","⠦","⠧","⠇","⠏"]
- def __init__(self,label): self.label=label; self.stop_evt=threading.Event(); self.t0=time.time(); self.th=None
+ def __init__(self,label,tick=None):
+ self.label=label; self.tick=tick; self.suffix=""
+ self.stop_evt=threading.Event(); self.t0=time.time(); self.th=None
def start(self):
if not TTY: return
def run():
i=0
while not self.stop_evt.is_set():
el=time.time()-self.t0
- sys.stdout.write(f"\r {C.teal}{self.FRAMES[i%10]}{C.r} {C.dim}{self.label} {el:.0f}s{C.r}\033[K")
+ if self.tick and i%8==0: # ~1 Hz: legge il progresso dal log
+ try: self.suffix=self.tick() or self.suffix
+ except Exception: pass
+ suf=f" {C.dgray}· {self.suffix}{C.r}" if self.suffix else ""
+ sys.stdout.write(f"\r {C.teal}{self.FRAMES[i%10]}{C.r} {C.dim}{self.label} {el:.0f}s{C.r}{suf}\033[K")
sys.stdout.flush(); i+=1; time.sleep(0.12)
self.th=threading.Thread(target=run,daemon=True); self.th.start()
def stop(self):
@@ -244,7 +250,14 @@ def cmd_chat(a):
print(f"\n {C.teal}◆ colibrì{C.r}")
dec=codecs.getincrementaldecoder("utf-8")("replace")
state={"first":True}
- sp2=Spinner("pensa…"); sp2.start()
+ def prefill_tick(path=errlog.name):
+ try:
+ with open(path) as f:
+ f.seek(max(0, os.path.getsize(path)-1500)); tail=f.read()
+ pl=[l for l in tail.splitlines() if l.startswith("[prefill]")]
+ return pl[-1].replace("[prefill] ","prefill ") if pl else ""
+ except Exception: return ""
+ sp2=Spinner("pensa…", tick=prefill_tick); sp2.start()
def echo(bs, _dec=dec, _st=state):
if _st["first"]:
sp2.stop(); _st["first"]=False
diff --git a/c/glm.c b/c/glm.c
index 21abbc90..23c2da91 100644
--- a/c/glm.c
+++ b/c/glm.c
@@ -725,7 +725,13 @@ static void layer_forward(Model *m, Layer *l, int li, float *x, int S, int pos_b
static void layers_forward(Model *m, float *x, int S, int pos_base){
Cfg *c=&m->c; int D=c->hidden;
float *nrm=falloc((int64_t)S*D), *tmp=falloc((int64_t)S*D);
- for(int i=0;i<c->n_layers;i++) layer_forward(m,&m->L[i],i,x,S,pos_base,nrm,tmp);
+ for(int i=0;i<c->n_layers;i++){
+ /* progresso su stderr per i batch grossi (prefill): il primo byte di risposta
+ * puo' arrivare dopo MINUTI di streaming — al buio sembra un blocco. */
+ if(S>=8 && (i%4==0 || i==c->n_layers-1))
+ fprintf(stderr,"[prefill] layer %d/%d · %d token\n", i+1, c->n_layers, S);
+ layer_forward(m,&m->L[i],i,x,S,pos_base,nrm,tmp);
+ }
free(nrm); free(tmp);
}