Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
feat(engine): per-token logprob channel, ECHO prefill read-out, token-ID intake
monotophic075ca6ba40c6
diff --git a/c/colibri.c b/c/colibri.cindex 9668be55..2762d4f9 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -6865,8 +6865,13 @@ static void intr_install(void){}* a mux spec turn is running and are reset at each turn boundary, so every other* spec_decode caller (chat, run, oracle) sees them permanently 0. */static volatile sig_atomic_t g_mux_stop=0, g_mux_cancel=0;+/* emit callback contract (U7a): `lo` is the vocab-sized logit row the token+ * was picked or verified from -- live only for the duration of the call.+ * Accepted DRAFT tokens get their verification row, so the per-token numeric+ * channel has no gap on the speculative path (they bypass every pick_tok+ * call site in the mux loop). Callers that don't need it ignore it. */static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *logit,- void (*emit)(int,void*), void *ud, int *kv_out, float **logit_out){+ void (*emit)(int,const float*,void*), void *ud, int *kv_out, float **logit_out){Cfg *c=&m->c; int V=c->vocab; int emitted=0, done=0;int draft[64]; if(g_draft>63) g_draft=63;int carry_ban=-1; /* token rifiutato dalla verifica: escluso dal resample */@@ -6889,9 +6894,9 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *louint64_t cp_prop0=g_corp_prop, cp_acc0=g_corp_acc; int cp_pause=0;while(emitted<n_new && !done && !g_intr && !g_mux_stop && !g_mux_cancel){/* g_intr / g_mux_*: stessa uscita del tetto n_new (#678) */- int next=pick_tok(logit,V,carry_ban); carry_ban=-1; free(logit); logit=NULL;- if((eos>=0 && next==eos) || is_stop(next)) break;- emit(next,ud); all[kv]=next; emitted++; m->n_emit++;+ int next=pick_tok(logit,V,carry_ban); carry_ban=-1;+ if((eos>=0 && next==eos) || is_stop(next)){ free(logit); logit=NULL; break; }+ emit(next,logit,ud); free(logit); logit=NULL; all[kv]=next; emitted++; m->n_emit++;gr_feed(&g_grd,next); /* il walker segue l'output emesso *//* One-shot generation does not need logits or KV for the last token.* Stateful callers do: their kv_out becomes chat history, feeds MORE,@@ -6965,7 +6970,7 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *loaccept = (rndu() < g_pbuf[draft[k]]); }if(!accept){ if(g_temp>0) carry_ban=draft[k]; break; }if((eos>=0 && draft[k]==eos) || is_stop(draft[k])){ done=1; break; }- emit(draft[k],ud); all[kv+1+k]=draft[k]; emitted++; m->n_emit++;+ emit(draft[k],lo+(int64_t)k*V,ud); all[kv+1+k]=draft[k]; emitted++; m->n_emit++;gr_feed(&g_grd,draft[k]); k++;}if(gsrc==1) g_grd.acc+=(uint64_t)k;@@ -6991,10 +6996,11 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo/* emit callback: accumula in un array (validazione) */typedef struct { int *dst; int n; } EmitStore;-static void emit_store(int t, void *ud){ EmitStore *e=(EmitStore*)ud; e->dst[e->n++]=t; }+static void emit_store(int t, const float *lo, void *ud){ (void)lo; EmitStore *e=(EmitStore*)ud; e->dst[e->n++]=t; }/* emit callback: detokenizza e stampa in streaming (chat/run), con heartbeat */typedef struct { Tok *T; Model *m; double t0; int count; int quiet; } EmitStream;-static void emit_stream(int t, void *ud){+static void emit_stream(int t, const float *lo, void *ud){+ (void)lo;EmitStream *e=(EmitStream*)ud; char dec[64];int dn=tok_decode(e->T,&t,1,dec,63); dec[dn]=0; fputs(dec,stdout); fflush(stdout);if(!e->quiet && ++e->count%16==0){ double tt=e->m->hits+e->m->miss;@@ -7781,6 +7787,8 @@ typedef struct {float *spec_logit; /* continuation logits between chunks; NULL = the lastemitted token sits at hist[len], not yet forwarded */unsigned long long id;+ int logprobs; /* per-token numeric channel (U7a): requested+ top-k count from SUBMIT logprobs=k; 0 = off */float temp, top_p;double started;uint64_t hits0, miss0;@@ -7788,12 +7796,99 @@ typedef struct {feeds the PROF protocol line and the PROF=1 report */} ServeReq;-static void mux_data(Tok *T, unsigned long long id, int token){+/* Numeric tail shared by opted-in DATA and ECHO frames (U7a): writes+ * " <lp> <k> [<tid> <tlp>]*k" into dst. lp = log-softmax of `token` over the+ * logit row; the top-k table selects by logit (identical order to selecting+ * by log-probability) and is UNSORTED, same as run_ablate_score's tk output.+ * The token's own entry, when it appears in the table, is the SAME double as+ * lp printed through the same format -- the bit-identity the harness's+ * is_greedy check depends on. lo==NULL (an echo's position 0: nothing to+ * condition on) writes " nan 0"; the server maps that to OpenAI's null.+ * k is capped by COLI_SUBMIT_TOPK_MAX (=32, run_ablate_score's ceiling). */+static int logprob_tail(char *dst, size_t cap, const float *lo, int V, int token, int topk){+ int tk_id[COLI_SUBMIT_TOPK_MAX]; float tk_val[COLI_SUBMIT_TOPK_MAX];+ int w;+ if(!lo || token<0 || token>=V) return snprintf(dst,cap," nan 0");+ if(topk>COLI_SUBMIT_TOPK_MAX) topk=COLI_SUBMIT_TOPK_MAX;+ if(topk>V) topk=V;+ float mx=lo[0]; for(int i=1;i<V;i++) if(lo[i]>mx) mx=lo[i];+ double se=0; for(int i=0;i<V;i++) se+=exp((double)lo[i]-mx);+ double logZ=(double)mx+log(se);+ for(int k=0;k<topk;k++){ tk_id[k]=-1; tk_val[k]=-1e30f; }+ for(int i=0;i<V;i++){ float v=lo[i];+ int mn=0; for(int k=1;k<topk;k++) if(tk_val[k]<tk_val[mn]) mn=k;+ if(v>tk_val[mn]){ tk_val[mn]=v; tk_id[mn]=i; } }+ w=snprintf(dst,cap," %.6f %d",(double)lo[token]-logZ,topk);+ for(int k=0;k<topk && w>0 && (size_t)w<cap;k++)+ w+=snprintf(dst+w,cap-(size_t)w," %d %.6f",tk_id[k],(double)tk_val[k]-logZ);+ return w;+}++/* One generated token -> one DATA frame. Opted-in requests (SUBMIT logprobs=k)+ * carry the numeric channel in the header: "DATA <id> <n> <lp> <k> [tid tlp]*k";+ * opt-out requests keep the exact legacy 3-field frame, byte for byte -- an old+ * server (whose dispatcher hard-fails on unknown framing) can only ever be+ * paired with requests that never opt in, so it never sees the extended form. */+static void mux_data(Tok *T, unsigned long long id, int token,+ const float *lo, int V, int topk){+ char out[256]; int n=tok_decode(T,&token,1,out,sizeof(out));+ if(topk>0 && lo){+ char tail[1024]; logprob_tail(tail,sizeof(tail),lo,V,token,topk);+ printf("DATA %llu %d%s\n",id,n,tail);+ } else+ printf("DATA %llu %d\n",id,n);+ if(n>0) fwrite(out,1,(size_t)n,stdout); putchar('\n');+ fflush(stdout);+}++/* Echoed-prompt read-out frame (U7a, opt-in only): "ECHO <id> <n> <pos> <lp>+ * <k> [tid tlp]*k" + payload framed exactly like DATA (n bytes + '\n').+ * One frame per prompt position, in position order, BEFORE any DATA frame;+ * position 0 carries " nan 0" (nothing to condition on). Never emitted unless+ * the request set SUBMIT logprobs=k, so an old server can never receive one. */+static void mux_echo(Tok *T, unsigned long long id, int pos, int token,+ const float *lo, int V, int topk){char out[256]; int n=tok_decode(T,&token,1,out,sizeof(out));- printf("DATA %llu %d\n",id,n); if(n>0) fwrite(out,1,(size_t)n,stdout); putchar('\n');+ char tail[1024]; logprob_tail(tail,sizeof(tail),lo,V,token,topk);+ printf("ECHO %llu %d %d%s\n",id,n,pos,tail);+ if(n>0) fwrite(out,1,(size_t)n,stdout); putchar('\n');fflush(stdout);}+/* Opt-in prefill read-out (U7a, SUBMIT logprobs=k): one full-prompt forward+ * whose per-position hidden states are read out through lm_head -- one ECHO+ * frame per prompt position. Structurally run_score's loop, but live inside+ * the serve path and gated per request instead of a launch-env-gated+ * exit-early mode. Cost: one vocab-sized lm_head matmul per prompt position+ * (P x vocab, P = prompt length), paid ONLY by requests that opted in; the+ * opt-out path keeps step()'s single last-position lm_head untouched.+ * Returns the last position's logits (step()'s contract) as the generation+ * continuation. The caller resets the slot to len 0 first: the read-out+ * needs logits at EVERY position, so this path takes no cached-prefix skip+ * and no cross-slot KV adoption (KV rows [0,nt) are rewritten in full). */+static float *mux_prefill_echo(Model *m, Tok *T, unsigned long long id,+ const int *ids, int nt, int topk){+ Cfg *c=&m->c; int D=c->hidden, V=c->vocab;+ float *x=falloc((int64_t)nt*D);+ for(int s=0;s<nt;s++) embed_row(m, ids[s], x+(int64_t)s*D);+ layers_forward(m,x,nt,0);+ if(m->hlast) memcpy(m->hlast, x+(int64_t)(nt-1)*D, D*sizeof(float));+ if(m->has_mtp && nt>=2 && g_draft>0) mtp_absorb(m, ids+1, x, nt-1, 0); /* same as step() */+ float *lo=falloc(V), *row=falloc(D);+ mux_echo(T,id,0,ids[0],NULL,V,0);+ double th0=now_s();+ for(int pos=1; pos<nt; pos++){+ rmsnorm(row, x+(int64_t)(pos-1)*D, m->final_norm, D, c->eps);+ matmul_qt(lo, row, &m->lm_head, 1);+ mux_echo(T,id,pos,ids[pos],lo,V,topk);+ }+ rmsnorm(row, x+(int64_t)(nt-1)*D, m->final_norm, D, c->eps);+ matmul_qt(lo, row, &m->lm_head, 1);+ m->t_head += now_s()-th0;+ free(x); free(row);+ return lo; /* last position: the generation continuation */+}+/* #678: non-blocking stdin poll while a single-slot spec turn is running. Consumes* pending STOP/CANCEL lines (raising g_mux_stop/g_mux_cancel for the active id) so* the server's stop takes effect within ~1 token instead of after max_tokens. The@@ -7844,10 +7939,13 @@ static void mux_ctl_poll(unsigned long long id){}}-/* emit callback for the single-slot speculative path: stream straight to the mux protocol */-typedef struct { Tok *T; unsigned long long id; } MuxEmit;-static void mux_spec_emit(int t, void *ud){- MuxEmit *e=(MuxEmit*)ud; mux_data(e->T,e->id,t);+/* emit callback for the single-slot speculative path: stream straight to the mux+ * protocol. `lo` is the row the token was picked/verified from (spec_decode's+ * emit contract) -- accepted draft tokens included, so opted-in requests get a+ * numeric value on EVERY generated token, with no speculative-path gap. */+typedef struct { Tok *T; unsigned long long id; int V, logprobs; } MuxEmit;+static void mux_spec_emit(int t, const float *lo, void *ud){+ MuxEmit *e=(MuxEmit*)ud; mux_data(e->T,e->id,t,lo,e->V,e->logprobs);mux_ctl_poll(e->id); /* #678: honor STOP/CANCEL within ~1 token */}@@ -7973,7 +8071,20 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g* mid-markup and emitted a bare "<". Retries were identical because the surviving *head** never changes when a client appends to the end (hence "prefill 0" on every retry).* Refuse loudly instead; the gateway turns this into a 400 context_length_exceeded. */- int nt=tok_encode(T,raw,(int)sub.bytes,tmp,maxctx-1);+ int nt;+ if(sub.tok_ids){+ /* Pre-tokenized intake (SUBMIT ids=1, U7a): the payload is ASCII token+ * ids fed straight into the same tmp[] buffer the text path fills --+ * identical embedding/position machinery downstream, no tok_encode, no+ * detokenize/re-encode round trip. Same one-token headroom contract as+ * the text arm above (coli_ids_parse reports the cap on overflow). */+ nt=coli_ids_parse(raw,(size_t)sub.bytes,tmp,maxctx-1,m->c.vocab);+ if(nt<0){+ free(tmp); free(raw); free(line);+ printf("ERROR %llu BAD_REQUEST\n",sub.id); fflush(stdout); return 0;+ }+ } else+ nt=tok_encode(T,raw,(int)sub.bytes,tmp,maxctx-1);free(raw); free(line);if(nt<1){ free(tmp); printf("ERROR %llu EMPTY_PROMPT\n",sub.id); fflush(stdout); return 0; }if(nt>maxctx-2){@@ -7989,7 +8100,12 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g* first with an ERROR, so a request yields exactly one of ACCEPT or an early ERROR -- which* lets a CONTEXT_EXCEEDED become a clean HTTP 400 instead of a broken already-200 stream. */printf("ACCEPT %llu %d\n",sub.id,nt); fflush(stdout);- int prefix=0; while(prefix<sc->len && prefix<nt && sc->hist[prefix]==tmp[prefix]) prefix++;+ /* Echo read-out (U7a): needs logits at EVERY prompt position, so the whole+ * prompt re-prefills from position 0 -- no cached-prefix skip and no+ * cross-slot adoption below (either would leave positions with no logits). */+ int echo = sub.logprobs>0;+ int prefix=0;+ if(!echo) while(prefix<sc->len && prefix<nt && sc->hist[prefix]==tmp[prefix]) prefix++;if(prefix<sc->len){ sc->len=prefix; if(m->has_mtp) m->kv_start[m->c.n_layers]=-1;kv_disk_truncate(m,sc->len); }/* Cross-slot prefix adoption (COLI_KV_SHARE=1) β RadixAttention's benefit@@ -8004,7 +8120,7 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g* generated tokens identical to the full-prefill run. */static int kvshare=-1;if(kvshare<0) kvshare=getenv("COLI_KV_SHARE")?atoi(getenv("COLI_KV_SHARE")):0;- if(kvshare && nctx>1 && prefix>=sc->len){+ if(kvshare && !echo && nctx>1 && prefix>=sc->len){int best=-1, blen=sc->len;for(int i=0;i<nctx;i++){if(i==sub.slot) continue;@@ -8041,11 +8157,13 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *gif(add>0) memcpy(sc->hist+sc->len,tmp+sc->len,(size_t)add*sizeof(int));fprintf(stderr,"[API] KV slot %d prefix %d/%d token, prefill %d\n",sub.slot,sc->len,nt,add);free(tmp);- float *logit = add>0 ? step(m,sc->hist+sc->len,add,sc->len)- : step(m,sc->hist+sc->len-1,1,sc->len-1);+ float *logit = echo ? mux_prefill_echo(m,T,sub.id,sc->hist,nt,sub.logprobs)+ : add>0 ? step(m,sc->hist+sc->len,add,sc->len)+ : step(m,sc->hist+sc->len-1,1,sc->len-1);sc->len+=add; sc->first=0;ServeReq *r=&req[sub.slot]; memset(r,0,sizeof(*r));r->id=sub.id; r->maximum=sub.max_tokens; r->temp=sub.temperature; r->top_p=sub.top_p;+ r->logprobs=sub.logprobs;r->prompt_tokens=nt; r->started=now_s(); r->hits0=m->hits; r->miss0=m->miss;prof_base(m,&r->pb); /* a few loads: cheap enough to always track *//* Clamp to the KV room WITHOUT flagging: length_limited must mean "the@@ -8067,12 +8185,13 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *gr->spec=1; r->spec_logit=logit; r->active=1;return 1;}- int next=pick_tok(logit,m->c.vocab,-1); free(logit);- if(r->maximum<=0){ r->length_limited=1; mux_done(m,sc,r); return 1; } /* no room at all */- if(next==eos || is_stop(next)){ mux_done(m,sc,r); return 1; }+ int next=pick_tok(logit,m->c.vocab,-1);+ if(r->maximum<=0){ free(logit); r->length_limited=1; mux_done(m,sc,r); return 1; } /* no room at all */+ if(next==eos || is_stop(next)){ free(logit); mux_done(m,sc,r); return 1; }r->pending=next; r->emitted=1; r->active=1; sc->hist[sc->len]=next; m->n_emit++;if(grd[sub.slot].on){ grammar_reset(&grd[sub.slot]); gr_feed(&grd[sub.slot],next); }- mux_data(T,r->id,next);+ mux_data(T,r->id,next,logit,m->c.vocab,r->logprobs);+ free(logit);if(r->emitted>=r->maximum){ r->length_limited=1; mux_done(m,sc,r); }return 1;}@@ -8168,7 +8287,7 @@ static void run_serve_mux(Model *m, const char *snap){kv_bind(m,&sc->kv);g_temp=r->temp; g_nuc=r->top_p;float *lg=r->spec_logit; r->spec_logit=NULL; /* spec_decode takes ownership */- MuxEmit ud={&T,r->id};+ MuxEmit ud={&T,r->id,m->c.vocab,r->logprobs};g_mux_stop=0; g_mux_cancel=0; /* fresh per turn (#678) */int prod=spec_decode(m,sc->hist,sc->len,r->maximum-r->emitted,eos,lg,mux_spec_emit,&ud,&sc->len,NULL);@@ -8209,7 +8328,7 @@ static void run_serve_mux(Model *m, const char *snap){if(next==eos || is_stop(next)){ mux_done(m,sc,r); done=1; break; }r->pending=next; sc->hist[sc->len]=next; r->emitted++; m->n_emit++;if(gd->on) gr_feed(gd,next);- mux_data(&T,r->id,next);+ mux_data(&T,r->id,next,lo+(int64_t)j*m->c.vocab,m->c.vocab,r->logprobs);if(r->emitted>=r->maximum){ r->length_limited=1; mux_done(m,sc,r); done=1; break; }if(j<k){if(next!=draft[j]) break; /* rejected: seq[j+1..] stale, overwritten next forward */@@ -8233,7 +8352,7 @@ static void run_serve_mux(Model *m, const char *snap){if(next==eos || is_stop(next)){mux_done(m,sc,r);continue;}r->pending=next; sc->hist[sc->len]=next; r->emitted++; m->n_emit++;if(grd[i].on) gr_feed(&grd[i],next); /* walker stays in sync when not drafting */- mux_data(&T,r->id,next);+ mux_data(&T,r->id,next,lo+(int64_t)s*m->c.vocab,m->c.vocab,r->logprobs);if(r->emitted>=r->maximum){ r->length_limited=1; mux_done(m,sc,r); }}free(lo);diff --git a/c/decode_batch.h b/c/decode_batch.hindex 9f4dd343..12e3367e 100644--- a/c/decode_batch.h+++ b/c/decode_batch.h@@ -3,6 +3,7 @@#include <stddef.h>#include <stdio.h>+#include <string.h>#include <math.h>/* `base` belongs to one sequence's KV state. Keeping this arithmetic in a@@ -12,31 +13,86 @@ static inline float *coli_kv_row(float *base, int position, int width)return base + (size_t)position * (size_t)width;}+/* Per-token top-k emission cap: run_ablate_score's existing top-32 read-out+ * ceiling, adopted as the wire cap too (no named consumer asks for more). */+#define COLI_SUBMIT_TOPK_MAX 32+typedef struct {unsigned long long id, bytes, gbytes;int slot, max_tokens;float temperature, top_p;+ int logprobs; /* requested per-token top-k count; 0 = channel off (opt-in) */+ int tok_ids; /* 1 = payload is ASCII token ids, not raw prompt text */} ColiSubmit;+/* Extension fields, 8th whitespace field onward: "key=value" tokens after the+ * full 7-field header, e.g. "SUBMIT 1 0 12 16 0 1 0 logprobs=5 ids=1".+ * A namespace instead of more positional fields: the next per-request knob+ * (a future seed, say) claims a key here rather than forcing another+ * field-count migration of this scarce wire budget. Unknown keys reject the+ * whole frame -- framing stays as unambiguous as the legacy arms'+ * trailing-field guard. An OLD engine rejects ANY extended header cleanly:+ * both legacy sscanf arms fail on the extra field (their trailing %c matches+ * the separating space) and the engine answers ERROR 0 BAD_REQUEST. */+static inline int coli_submit_ext(const char *p, ColiSubmit *s)+{+ int seen = 0;+ while (*p == ' ' || *p == '\t') p++;+ while (*p) {+ char key[16];+ unsigned long long val;+ int n = 0;+ if (sscanf(p, "%15[a-z_]=%llu%n", key, &val, &n) != 2 || n <= 0)+ return 0;+ if (p[n] && p[n] != ' ' && p[n] != '\t') return 0;+ if (!strcmp(key, "logprobs")) { /* per-token top-k emission */+ if (val > COLI_SUBMIT_TOPK_MAX) return 0;+ s->logprobs = (int)val;+ } else if (!strcmp(key, "ids")) { /* pre-tokenized prompt intake */+ if (val > 1) return 0;+ s->tok_ids = (int)val;+ } else {+ return 0;+ }+ seen = 1;+ p += n;+ while (*p == ' ' || *p == '\t') p++;+ }+ return seen;+}+/* Parse the textual header. The payload is read separately using `bytes`, so* it may contain newlines. Reject trailing fields to keep framing unambiguous.* Optional 7th field `gbytes`: length of a per-request grammar (raw GBNF, or a* JSON-Schema compiled engine-side) appended to the payload AFTER the prompt- * bytes. 6-field headers remain valid (gbytes = 0). */+ * bytes. 6-field headers remain valid (gbytes = 0). Fields past the 7th are+ * key=value extension tokens (coli_submit_ext above), only ever sent by a+ * server that knows this engine understands them. */static inline int coli_submit_parse(const char *line, ColiSubmit *s){char tail;+ int ok = 0, base = 0;if (!line || !s) return 0;s->gbytes = 0;+ s->logprobs = 0;+ s->tok_ids = 0;if (sscanf(line, "SUBMIT %llu %d %llu %d %f %f %llu %c", &s->id, &s->slot,&s->bytes, &s->max_tokens, &s->temperature, &s->top_p,- &s->gbytes, &tail) != 7) {+ &s->gbytes, &tail) == 7) ok = 1;+ if (!ok) {s->gbytes = 0;if (sscanf(line, "SUBMIT %llu %d %llu %d %f %f %c", &s->id, &s->slot,&s->bytes, &s->max_tokens, &s->temperature, &s->top_p,- &tail) != 6)- return 0;+ &tail) == 6) ok = 1;+ }+ if (!ok) {+ s->gbytes = 0;+ if (sscanf(line, "SUBMIT %llu %d %llu %d %f %f %llu%n", &s->id,+ &s->slot, &s->bytes, &s->max_tokens, &s->temperature,+ &s->top_p, &s->gbytes, &base) == 7 && base > 0 &&+ coli_submit_ext(line + base, s)) ok = 1;}+ if (!ok) return 0;return s->id > 0 && s->bytes <= (16u << 20) && s->gbytes <= (1u << 20) &&s->slot >= 0 && s->max_tokens >= 1 &&isfinite(s->temperature) && isfinite(s->top_p) &&@@ -44,4 +100,34 @@ static inline int coli_submit_parse(const char *line, ColiSubmit *s)s->top_p > 0 && s->top_p <= 1;}+/* SUBMIT ids=1 payload: ASCII decimal token ids separated by whitespace,+ * parsed straight into the prompt buffer -- the pre-tokenized intake path+ * that bypasses tok_encode entirely (no detokenize/re-encode round trip, so+ * the ids the caller sent are exactly the ids the engine scores). Returns+ * the id count, or -1 on any malformed or out-of-range id. More than `cap`+ * ids reports `cap`: the caller parses with one spare slot and treats a full+ * buffer as overflow, mirroring tok_encode's stop-at-cap contract (#401). */+static inline int coli_ids_parse(const char *buf, size_t len, int *out,+ int cap, int vocab)+{+ size_t i = 0;+ int n = 0;+ if (!buf || !out || cap < 1 || vocab < 1) return -1;+ while (i < len) {+ while (i < len && (unsigned char)buf[i] <= ' ') i++;+ if (i >= len) break;+ if (n >= cap) return cap; /* overflow: caller refuses loudly */+ {+ long v = 0;+ size_t d = i;+ while (i < len && buf[i] >= '0' && buf[i] <= '9' && v < vocab)+ v = v * 10 + (buf[i++] - '0');+ if (i == d || v >= vocab || (i < len && (unsigned char)buf[i] > ' '))+ return -1;+ out[n++] = (int)v;+ }+ }+ return n;+}+#endifdiff --git a/c/openai_server.py b/c/openai_server.pyindex 19108c4b..12c83718 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1889,7 +1889,14 @@ class Engine:if not fields:continuekind = fields[0]- if kind == "DATA" and len(fields) == 3:+ if kind == "DATA" and len(fields) >= 3:+ # 3 fields: the legacy frame. More: the U7a per-token+ # numeric channel ("DATA <id> <n> <lp> <k> [tid tlp]*k"),+ # emitted only for requests that opted in via the SUBMIT+ # logprobs field. The payload framing is identical; the+ # numeric fields are consumed by the server feature half+ # (U7b) -- accepted here so the frame never kills the+ # dispatcher (and with it every in-flight request).request_id = fields[1]size = int(fields[2])if not 0 <= size <= 65536:@@ -1901,6 +1908,19 @@ class Engine:events = self.pending.get(request_id)if events is not None:events.put(("data", data))+ elif kind == "ECHO" and len(fields) >= 6:+ # U7a prefill read-out: "ECHO <id> <n> <pos> <lp> <k>+ # [tid tlp]*k" plus a DATA-framed payload (n bytes + LF).+ # Emitted only for opted-in requests; no current request+ # path opts in, so the frame is read (to keep the stream+ # in sync) and dropped -- U7b delivers it to the response+ # assembly when it wires the opt-in.+ size = int(fields[2])+ if not 0 <= size <= 65536:+ raise RuntimeError("invalid engine DATA size")+ self._read_exact(size)+ if self._read_exact(1) != b"\n":+ raise RuntimeError("invalid engine DATA terminator")elif kind == "ACCEPT" and len(fields) >= 3:# #597: the engine validated the submission (fits context) before prefill.# Keep it pending β DATA/DONE still follow β and let generate() commit thediff --git a/c/tests/test_decode_batch.c b/c/tests/test_decode_batch.cindex f5b216e2..851182a6 100644--- a/c/tests/test_decode_batch.c+++ b/c/tests/test_decode_batch.c@@ -52,11 +52,86 @@ static void test_submit_header(void)assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 512 extra", &sub));}+/* U7a extension fields: key=value tokens from the 8th field onward. The two+ * legacy arms stay byte-identical in behavior (asserted above); the third arm+ * accepts only known keys with in-range values, so an unknown or malformed+ * extension rejects the whole frame exactly like a trailing garbage field. */+static void test_submit_extension_fields(void)+{+ ColiSubmit sub;+ /* legacy headers leave the new capabilities off */+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95", &sub));+ assert(sub.logprobs == 0 && sub.tok_ids == 0);+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 512", &sub));+ assert(sub.logprobs == 0 && sub.tok_ids == 0);+ /* the 8-field opt-in forms */+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=5", &sub));+ assert(sub.logprobs == 5 && sub.tok_ids == 0 && sub.gbytes == 0);+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 ids=1", &sub));+ assert(sub.tok_ids == 1 && sub.logprobs == 0);+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 512 logprobs=32 ids=1", &sub));+ assert(sub.logprobs == 32 && sub.tok_ids == 1 && sub.gbytes == 512);+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=0 ids=0", &sub));+ assert(sub.logprobs == 0 && sub.tok_ids == 0);+ /* rejects: over-cap, out-of-range, unknown key, malformed token */+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=33", &sub));+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 ids=2", &sub));+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 seed=7", &sub));+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=", &sub));+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 =5", &sub));+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=5x", &sub));+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=5 junk", &sub));+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs", &sub));+ /* an extended header still validates the base fields */+ assert(!coli_submit_parse("SUBMIT 0 3 17 64 0.7 0.95 0 logprobs=5", &sub));+ assert(!coli_submit_parse("SUBMIT 42 3 17 0 0.7 0.95 0 logprobs=5", &sub));+}++/* U7a token-ID intake: the ids the caller formatted are exactly the ids that+ * come back out (the zero-round-trip property) -- and malformed input never+ * parses partially. */+static void test_ids_parse_round_trip(void)+{+ /* fixed known-good example: format then parse must reproduce the array */+ const int ids[6] = {0, 154822, 154824, 12, 7, 99999};+ char buf[128];+ int out[8], n, k;+ n = snprintf(buf, sizeof(buf), "%d %d %d %d %d %d",+ ids[0], ids[1], ids[2], ids[3], ids[4], ids[5]);+ assert(n > 0);+ k = coli_ids_parse(buf, (size_t)n, out, 8, 160000);+ assert(k == 6);+ for (int i = 0; i < 6; i++) assert(out[i] == ids[i]);++ /* whitespace forms: leading/trailing/newlines are all separators */+ k = coli_ids_parse(" 5\n7\t9 ", 9, out, 8, 100);+ assert(k == 3 && out[0] == 5 && out[1] == 7 && out[2] == 9);++ /* rejects: sign, non-digit, vocab bound (exact and above), garbage tail */+ assert(coli_ids_parse("-1", 2, out, 8, 100) == -1);+ assert(coli_ids_parse("5x", 2, out, 8, 100) == -1);+ assert(coli_ids_parse("1 two 3", 7, out, 8, 100) == -1);+ assert(coli_ids_parse("100", 3, out, 8, 100) == -1); /* id == vocab */+ assert(coli_ids_parse("99", 2, out, 8, 100) == 1 && out[0] == 99);+ assert(coli_ids_parse("101", 3, out, 8, 100) == -1);+ assert(coli_ids_parse("123456789012345", 15, out, 8, 100) == -1);++ /* empty payload parses to zero ids (the caller's EMPTY_PROMPT arm) */+ assert(coli_ids_parse(" ", 3, out, 8, 100) == 0);++ /* overflow reports the cap so the caller's context check refuses loudly,+ * mirroring tok_encode's stop-at-cap contract (#401) */+ assert(coli_ids_parse("1 2 3 4 5", 9, out, 4, 100) == 4);+ assert(coli_ids_parse("1 2 3 4", 7, out, 4, 100) == 4);+}+int main(void){test_rows_use_their_own_sequence_storage();test_const_reader_selects_the_same_row();test_submit_header();+ test_submit_extension_fields();+ test_ids_parse_round_trip();puts("decode batch helper tests: ok");return 0;}diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex c7127cb0..adf01c91 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -660,6 +660,57 @@ class DispatcherTest(unittest.TestCase):"attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15,}])+ def test_accepts_u7a_echo_and_extended_data_frames(self):+ # U7a forward-compat: the engine's opt-in per-token numeric channel --+ # ECHO frames for echoed prompt positions and DATA frames extended+ # with "<lp> <k> [tid tlp]*k" -- must NOT trip the dispatcher's+ # catch-all (which kills every in-flight request). Text delivery and+ # the DONE stats stay exactly as for legacy frames; the numeric+ # fields are consumed by the server feature half (U7b).+ def respond(process, frame):+ request_id = frame.split()[1]+ process.stdout.feed(b"ACCEPT " + request_id + b" 3\n")+ process.stdout.feed(b"ECHO " + request_id + b" 2 0 nan 0\nHi\n")+ process.stdout.feed(+ b"ECHO " + request_id ++ b" 6 1 -0.105361 2 7 -0.105361 9 -2.302585\n world\n")+ process.stdout.feed(+ b"ECHO " + request_id ++ b" 1 2 -1.203973 2 4 -0.803973 6 -1.203973\n!\n")+ process.stdout.feed(+ b"DATA " + request_id ++ b" 2 -0.223144 2 3 -0.223144 8 -1.723144\nok\n")+ process.stdout.feed(+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 3 0\n")++ process = FakeProcess(respond)+ with patch("openai_server.subprocess.Popen", return_value=process):+ engine = Engine("glm", "model")+ chunks = []+ stats = engine.generate("Hi world!", 4, 0.0, 1.0, chunks.append)+ self.assertEqual(chunks, ["ok"])+ self.assertEqual(stats["completion_tokens"], 1)+ self.assertEqual(stats["prompt_tokens"], 3)+ self.assertIsNone(engine.dispatcher_error)+ engine.close()++ def test_unknown_frame_still_stops_dispatcher(self):+ # The catch-all that makes an unrecognized frame a hard failure is+ # load-bearing for the U7a compatibility asymmetry (a new engine's+ # frame reaching an OLD server kills the dispatcher -- the reason the+ # engine half ships first and stays opt-in). Accepting ECHO/extended+ # DATA must not have widened acceptance beyond those frames.+ def respond(process, frame):+ request_id = frame.split()[1]+ process.stdout.feed(b"LOGPROB " + request_id + b" 0.5\n")++ process = FakeProcess(respond)+ with patch("openai_server.subprocess.Popen", return_value=process):+ engine = Engine("glm", "model")+ with self.assertRaisesRegex(RuntimeError, "invalid engine response: LOGPROB"):+ engine.generate("hello", 4, 0.7, 0.9, lambda _: None)+ engine.close()+def test_cancels_generation_after_consumer_disconnects(self):request_id = Nonediff --git a/c/tests/test_spec_decode_state.c b/c/tests/test_spec_decode_state.cindex e23783b7..1360628d 100644--- a/c/tests/test_spec_decode_state.c+++ b/c/tests/test_spec_decode_state.c@@ -162,6 +162,40 @@ static int test_persisted_tail(void) {return 0;}+/* U7a: spec_decode's emit callback receives the logit row its token was+ * picked/verified from -- including ACCEPTED DRAFT tokens, which bypass every+ * pick_tok call site in the mux loop (the speculative-gap hazard the packet's+ * Fork 5 flags). Greedy contract: no row may be NULL, and every row's argmax+ * must be the emitted token -- proving the row really is that token's scoring+ * distribution, not a stale or shifted one. */+typedef struct { int n, null_rows, mismatches; } EmitProbe;+static void emit_probe(int t, const float *lo, void *ud) {+ EmitProbe *e = (EmitProbe *)ud;+ e->n++;+ if (!lo) { e->null_rows++; return; }+ if (argmax_v(lo, V) != t) e->mismatches++;+}++static int test_emit_carries_scoring_logits_for_accepted_drafts(void) {+ const int next[V] = {0, 2, 3, 1, 4, 5, 6, 7};+ Toy t;+ toy_init(&t, next);+ reset_decode_globals(2);++ /* Same shape as test_accepted_speculative_tokens: the repeated [1,2]+ * bigram proposes [3,1]; one picked token + two accepted drafts. */+ int all[12] = {1, 2, 3, 1};+ EmitProbe probe = {0, 0, 0};+ int kv = -1;+ CHECK(spec_decode(&t.m, all, 4, 3, -1, choose(2), emit_probe, &probe, &kv, NULL) == 3);+ CHECK(probe.n == 3); /* 1 pick_tok token + 2 accepted draft tokens */+ CHECK(probe.null_rows == 0); /* the draft-accept arm has no numeric gap */+ CHECK(probe.mismatches == 0); /* each row scores exactly its own token */++ toy_free(&t);+ return 0;+}+static int test_accepted_speculative_tokens(void) {const int next[V] = {0, 2, 3, 1, 4, 5, 6, 7};Toy t;@@ -217,6 +251,7 @@ int main(void) {CHECK(test_ngen_zero_and_one_shot() == 0);CHECK(test_stateful_ngen_and_more() == 0);CHECK(test_persisted_tail() == 0);+ CHECK(test_emit_carries_scoring_logits_for_accepted_drafts() == 0);CHECK(test_accepted_speculative_tokens() == 0);CHECK(test_eos_and_non_limit_stop() == 0);puts("spec_decode state tests: ok");