Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(engine): per-token logprob channel, ECHO prefill read-out, token-ID intake

monotophic075ca6ba40c6
diff --git a/c/colibri.c b/c/colibri.c
index 9668be55..2762d4f9 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -6865,8 +6865,13 @@ static void intr_install(void){}
* a mux spec turn is running and are reset at each turn boundary, so every other
* spec_decode caller (chat, run, oracle) sees them permanently 0. */
static volatile sig_atomic_t g_mux_stop=0, g_mux_cancel=0;
+/* emit callback contract (U7a): `lo` is the vocab-sized logit row the token
+ * was picked or verified from -- live only for the duration of the call.
+ * Accepted DRAFT tokens get their verification row, so the per-token numeric
+ * channel has no gap on the speculative path (they bypass every pick_tok
+ * call site in the mux loop). Callers that don't need it ignore it. */
static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *logit,
- void (*emit)(int,void*), void *ud, int *kv_out, float **logit_out){
+ void (*emit)(int,const float*,void*), void *ud, int *kv_out, float **logit_out){
Cfg *c=&m->c; int V=c->vocab; int emitted=0, done=0;
int draft[64]; if(g_draft>63) g_draft=63;
int carry_ban=-1; /* token rifiutato dalla verifica: escluso dal resample */
@@ -6889,9 +6894,9 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo
uint64_t cp_prop0=g_corp_prop, cp_acc0=g_corp_acc; int cp_pause=0;
while(emitted<n_new && !done && !g_intr && !g_mux_stop && !g_mux_cancel){
/* g_intr / g_mux_*: stessa uscita del tetto n_new (#678) */
- int next=pick_tok(logit,V,carry_ban); carry_ban=-1; free(logit); logit=NULL;
- if((eos>=0 && next==eos) || is_stop(next)) break;
- emit(next,ud); all[kv]=next; emitted++; m->n_emit++;
+ int next=pick_tok(logit,V,carry_ban); carry_ban=-1;
+ if((eos>=0 && next==eos) || is_stop(next)){ free(logit); logit=NULL; break; }
+ emit(next,logit,ud); free(logit); logit=NULL; all[kv]=next; emitted++; m->n_emit++;
gr_feed(&g_grd,next); /* il walker segue l'output emesso */
/* One-shot generation does not need logits or KV for the last token.
* Stateful callers do: their kv_out becomes chat history, feeds MORE,
@@ -6965,7 +6970,7 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo
accept = (rndu() < g_pbuf[draft[k]]); }
if(!accept){ if(g_temp>0) carry_ban=draft[k]; break; }
if((eos>=0 && draft[k]==eos) || is_stop(draft[k])){ done=1; break; }
- emit(draft[k],ud); all[kv+1+k]=draft[k]; emitted++; m->n_emit++;
+ emit(draft[k],lo+(int64_t)k*V,ud); all[kv+1+k]=draft[k]; emitted++; m->n_emit++;
gr_feed(&g_grd,draft[k]); k++;
}
if(gsrc==1) g_grd.acc+=(uint64_t)k;
@@ -6991,10 +6996,11 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo
/* emit callback: accumula in un array (validazione) */
typedef struct { int *dst; int n; } EmitStore;
-static void emit_store(int t, void *ud){ EmitStore *e=(EmitStore*)ud; e->dst[e->n++]=t; }
+static void emit_store(int t, const float *lo, void *ud){ (void)lo; EmitStore *e=(EmitStore*)ud; e->dst[e->n++]=t; }
/* emit callback: detokenizza e stampa in streaming (chat/run), con heartbeat */
typedef struct { Tok *T; Model *m; double t0; int count; int quiet; } EmitStream;
-static void emit_stream(int t, void *ud){
+static void emit_stream(int t, const float *lo, void *ud){
+ (void)lo;
EmitStream *e=(EmitStream*)ud; char dec[64];
int dn=tok_decode(e->T,&t,1,dec,63); dec[dn]=0; fputs(dec,stdout); fflush(stdout);
if(!e->quiet && ++e->count%16==0){ double tt=e->m->hits+e->m->miss;
@@ -7781,6 +7787,8 @@ typedef struct {
float *spec_logit; /* continuation logits between chunks; NULL = the last
emitted token sits at hist[len], not yet forwarded */
unsigned long long id;
+ int logprobs; /* per-token numeric channel (U7a): requested
+ top-k count from SUBMIT logprobs=k; 0 = off */
float temp, top_p;
double started;
uint64_t hits0, miss0;
@@ -7788,12 +7796,99 @@ typedef struct {
feeds the PROF protocol line and the PROF=1 report */
} ServeReq;
-static void mux_data(Tok *T, unsigned long long id, int token){
+/* Numeric tail shared by opted-in DATA and ECHO frames (U7a): writes
+ * " <lp> <k> [<tid> <tlp>]*k" into dst. lp = log-softmax of `token` over the
+ * logit row; the top-k table selects by logit (identical order to selecting
+ * by log-probability) and is UNSORTED, same as run_ablate_score's tk output.
+ * The token's own entry, when it appears in the table, is the SAME double as
+ * lp printed through the same format -- the bit-identity the harness's
+ * is_greedy check depends on. lo==NULL (an echo's position 0: nothing to
+ * condition on) writes " nan 0"; the server maps that to OpenAI's null.
+ * k is capped by COLI_SUBMIT_TOPK_MAX (=32, run_ablate_score's ceiling). */
+static int logprob_tail(char *dst, size_t cap, const float *lo, int V, int token, int topk){
+ int tk_id[COLI_SUBMIT_TOPK_MAX]; float tk_val[COLI_SUBMIT_TOPK_MAX];
+ int w;
+ if(!lo || token<0 || token>=V) return snprintf(dst,cap," nan 0");
+ if(topk>COLI_SUBMIT_TOPK_MAX) topk=COLI_SUBMIT_TOPK_MAX;
+ if(topk>V) topk=V;
+ float mx=lo[0]; for(int i=1;i<V;i++) if(lo[i]>mx) mx=lo[i];
+ double se=0; for(int i=0;i<V;i++) se+=exp((double)lo[i]-mx);
+ double logZ=(double)mx+log(se);
+ for(int k=0;k<topk;k++){ tk_id[k]=-1; tk_val[k]=-1e30f; }
+ for(int i=0;i<V;i++){ float v=lo[i];
+ int mn=0; for(int k=1;k<topk;k++) if(tk_val[k]<tk_val[mn]) mn=k;
+ if(v>tk_val[mn]){ tk_val[mn]=v; tk_id[mn]=i; } }
+ w=snprintf(dst,cap," %.6f %d",(double)lo[token]-logZ,topk);
+ for(int k=0;k<topk && w>0 && (size_t)w<cap;k++)
+ w+=snprintf(dst+w,cap-(size_t)w," %d %.6f",tk_id[k],(double)tk_val[k]-logZ);
+ return w;
+}
+
+/* One generated token -> one DATA frame. Opted-in requests (SUBMIT logprobs=k)
+ * carry the numeric channel in the header: "DATA <id> <n> <lp> <k> [tid tlp]*k";
+ * opt-out requests keep the exact legacy 3-field frame, byte for byte -- an old
+ * server (whose dispatcher hard-fails on unknown framing) can only ever be
+ * paired with requests that never opt in, so it never sees the extended form. */
+static void mux_data(Tok *T, unsigned long long id, int token,
+ const float *lo, int V, int topk){
+ char out[256]; int n=tok_decode(T,&token,1,out,sizeof(out));
+ if(topk>0 && lo){
+ char tail[1024]; logprob_tail(tail,sizeof(tail),lo,V,token,topk);
+ printf("DATA %llu %d%s\n",id,n,tail);
+ } else
+ printf("DATA %llu %d\n",id,n);
+ if(n>0) fwrite(out,1,(size_t)n,stdout); putchar('\n');
+ fflush(stdout);
+}
+
+/* Echoed-prompt read-out frame (U7a, opt-in only): "ECHO <id> <n> <pos> <lp>
+ * <k> [tid tlp]*k" + payload framed exactly like DATA (n bytes + '\n').
+ * One frame per prompt position, in position order, BEFORE any DATA frame;
+ * position 0 carries " nan 0" (nothing to condition on). Never emitted unless
+ * the request set SUBMIT logprobs=k, so an old server can never receive one. */
+static void mux_echo(Tok *T, unsigned long long id, int pos, int token,
+ const float *lo, int V, int topk){
char out[256]; int n=tok_decode(T,&token,1,out,sizeof(out));
- printf("DATA %llu %d\n",id,n); if(n>0) fwrite(out,1,(size_t)n,stdout); putchar('\n');
+ char tail[1024]; logprob_tail(tail,sizeof(tail),lo,V,token,topk);
+ printf("ECHO %llu %d %d%s\n",id,n,pos,tail);
+ if(n>0) fwrite(out,1,(size_t)n,stdout); putchar('\n');
fflush(stdout);
}
+/* Opt-in prefill read-out (U7a, SUBMIT logprobs=k): one full-prompt forward
+ * whose per-position hidden states are read out through lm_head -- one ECHO
+ * frame per prompt position. Structurally run_score's loop, but live inside
+ * the serve path and gated per request instead of a launch-env-gated
+ * exit-early mode. Cost: one vocab-sized lm_head matmul per prompt position
+ * (P x vocab, P = prompt length), paid ONLY by requests that opted in; the
+ * opt-out path keeps step()'s single last-position lm_head untouched.
+ * Returns the last position's logits (step()'s contract) as the generation
+ * continuation. The caller resets the slot to len 0 first: the read-out
+ * needs logits at EVERY position, so this path takes no cached-prefix skip
+ * and no cross-slot KV adoption (KV rows [0,nt) are rewritten in full). */
+static float *mux_prefill_echo(Model *m, Tok *T, unsigned long long id,
+ const int *ids, int nt, int topk){
+ Cfg *c=&m->c; int D=c->hidden, V=c->vocab;
+ float *x=falloc((int64_t)nt*D);
+ for(int s=0;s<nt;s++) embed_row(m, ids[s], x+(int64_t)s*D);
+ layers_forward(m,x,nt,0);
+ if(m->hlast) memcpy(m->hlast, x+(int64_t)(nt-1)*D, D*sizeof(float));
+ if(m->has_mtp && nt>=2 && g_draft>0) mtp_absorb(m, ids+1, x, nt-1, 0); /* same as step() */
+ float *lo=falloc(V), *row=falloc(D);
+ mux_echo(T,id,0,ids[0],NULL,V,0);
+ double th0=now_s();
+ for(int pos=1; pos<nt; pos++){
+ rmsnorm(row, x+(int64_t)(pos-1)*D, m->final_norm, D, c->eps);
+ matmul_qt(lo, row, &m->lm_head, 1);
+ mux_echo(T,id,pos,ids[pos],lo,V,topk);
+ }
+ rmsnorm(row, x+(int64_t)(nt-1)*D, m->final_norm, D, c->eps);
+ matmul_qt(lo, row, &m->lm_head, 1);
+ m->t_head += now_s()-th0;
+ free(x); free(row);
+ return lo; /* last position: the generation continuation */
+}
+
/* #678: non-blocking stdin poll while a single-slot spec turn is running. Consumes
* pending STOP/CANCEL lines (raising g_mux_stop/g_mux_cancel for the active id) so
* the server's stop takes effect within ~1 token instead of after max_tokens. The
@@ -7844,10 +7939,13 @@ static void mux_ctl_poll(unsigned long long id){
}
}
-/* emit callback for the single-slot speculative path: stream straight to the mux protocol */
-typedef struct { Tok *T; unsigned long long id; } MuxEmit;
-static void mux_spec_emit(int t, void *ud){
- MuxEmit *e=(MuxEmit*)ud; mux_data(e->T,e->id,t);
+/* emit callback for the single-slot speculative path: stream straight to the mux
+ * protocol. `lo` is the row the token was picked/verified from (spec_decode's
+ * emit contract) -- accepted draft tokens included, so opted-in requests get a
+ * numeric value on EVERY generated token, with no speculative-path gap. */
+typedef struct { Tok *T; unsigned long long id; int V, logprobs; } MuxEmit;
+static void mux_spec_emit(int t, const float *lo, void *ud){
+ MuxEmit *e=(MuxEmit*)ud; mux_data(e->T,e->id,t,lo,e->V,e->logprobs);
mux_ctl_poll(e->id); /* #678: honor STOP/CANCEL within ~1 token */
}
@@ -7973,7 +8071,20 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g
* mid-markup and emitted a bare "<". Retries were identical because the surviving *head*
* never changes when a client appends to the end (hence "prefill 0" on every retry).
* Refuse loudly instead; the gateway turns this into a 400 context_length_exceeded. */
- int nt=tok_encode(T,raw,(int)sub.bytes,tmp,maxctx-1);
+ int nt;
+ if(sub.tok_ids){
+ /* Pre-tokenized intake (SUBMIT ids=1, U7a): the payload is ASCII token
+ * ids fed straight into the same tmp[] buffer the text path fills --
+ * identical embedding/position machinery downstream, no tok_encode, no
+ * detokenize/re-encode round trip. Same one-token headroom contract as
+ * the text arm above (coli_ids_parse reports the cap on overflow). */
+ nt=coli_ids_parse(raw,(size_t)sub.bytes,tmp,maxctx-1,m->c.vocab);
+ if(nt<0){
+ free(tmp); free(raw); free(line);
+ printf("ERROR %llu BAD_REQUEST\n",sub.id); fflush(stdout); return 0;
+ }
+ } else
+ nt=tok_encode(T,raw,(int)sub.bytes,tmp,maxctx-1);
free(raw); free(line);
if(nt<1){ free(tmp); printf("ERROR %llu EMPTY_PROMPT\n",sub.id); fflush(stdout); return 0; }
if(nt>maxctx-2){
@@ -7989,7 +8100,12 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g
* first with an ERROR, so a request yields exactly one of ACCEPT or an early ERROR -- which
* lets a CONTEXT_EXCEEDED become a clean HTTP 400 instead of a broken already-200 stream. */
printf("ACCEPT %llu %d\n",sub.id,nt); fflush(stdout);
- int prefix=0; while(prefix<sc->len && prefix<nt && sc->hist[prefix]==tmp[prefix]) prefix++;
+ /* Echo read-out (U7a): needs logits at EVERY prompt position, so the whole
+ * prompt re-prefills from position 0 -- no cached-prefix skip and no
+ * cross-slot adoption below (either would leave positions with no logits). */
+ int echo = sub.logprobs>0;
+ int prefix=0;
+ if(!echo) while(prefix<sc->len && prefix<nt && sc->hist[prefix]==tmp[prefix]) prefix++;
if(prefix<sc->len){ sc->len=prefix; if(m->has_mtp) m->kv_start[m->c.n_layers]=-1;
kv_disk_truncate(m,sc->len); }
/* Cross-slot prefix adoption (COLI_KV_SHARE=1) β€” RadixAttention's benefit
@@ -8004,7 +8120,7 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g
* generated tokens identical to the full-prefill run. */
static int kvshare=-1;
if(kvshare<0) kvshare=getenv("COLI_KV_SHARE")?atoi(getenv("COLI_KV_SHARE")):0;
- if(kvshare && nctx>1 && prefix>=sc->len){
+ if(kvshare && !echo && nctx>1 && prefix>=sc->len){
int best=-1, blen=sc->len;
for(int i=0;i<nctx;i++){
if(i==sub.slot) continue;
@@ -8041,11 +8157,13 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g
if(add>0) memcpy(sc->hist+sc->len,tmp+sc->len,(size_t)add*sizeof(int));
fprintf(stderr,"[API] KV slot %d prefix %d/%d token, prefill %d\n",sub.slot,sc->len,nt,add);
free(tmp);
- float *logit = add>0 ? step(m,sc->hist+sc->len,add,sc->len)
- : step(m,sc->hist+sc->len-1,1,sc->len-1);
+ float *logit = echo ? mux_prefill_echo(m,T,sub.id,sc->hist,nt,sub.logprobs)
+ : add>0 ? step(m,sc->hist+sc->len,add,sc->len)
+ : step(m,sc->hist+sc->len-1,1,sc->len-1);
sc->len+=add; sc->first=0;
ServeReq *r=&req[sub.slot]; memset(r,0,sizeof(*r));
r->id=sub.id; r->maximum=sub.max_tokens; r->temp=sub.temperature; r->top_p=sub.top_p;
+ r->logprobs=sub.logprobs;
r->prompt_tokens=nt; r->started=now_s(); r->hits0=m->hits; r->miss0=m->miss;
prof_base(m,&r->pb); /* a few loads: cheap enough to always track */
/* Clamp to the KV room WITHOUT flagging: length_limited must mean "the
@@ -8067,12 +8185,13 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g
r->spec=1; r->spec_logit=logit; r->active=1;
return 1;
}
- int next=pick_tok(logit,m->c.vocab,-1); free(logit);
- if(r->maximum<=0){ r->length_limited=1; mux_done(m,sc,r); return 1; } /* no room at all */
- if(next==eos || is_stop(next)){ mux_done(m,sc,r); return 1; }
+ int next=pick_tok(logit,m->c.vocab,-1);
+ if(r->maximum<=0){ free(logit); r->length_limited=1; mux_done(m,sc,r); return 1; } /* no room at all */
+ if(next==eos || is_stop(next)){ free(logit); mux_done(m,sc,r); return 1; }
r->pending=next; r->emitted=1; r->active=1; sc->hist[sc->len]=next; m->n_emit++;
if(grd[sub.slot].on){ grammar_reset(&grd[sub.slot]); gr_feed(&grd[sub.slot],next); }
- mux_data(T,r->id,next);
+ mux_data(T,r->id,next,logit,m->c.vocab,r->logprobs);
+ free(logit);
if(r->emitted>=r->maximum){ r->length_limited=1; mux_done(m,sc,r); }
return 1;
}
@@ -8168,7 +8287,7 @@ static void run_serve_mux(Model *m, const char *snap){
kv_bind(m,&sc->kv);
g_temp=r->temp; g_nuc=r->top_p;
float *lg=r->spec_logit; r->spec_logit=NULL; /* spec_decode takes ownership */
- MuxEmit ud={&T,r->id};
+ MuxEmit ud={&T,r->id,m->c.vocab,r->logprobs};
g_mux_stop=0; g_mux_cancel=0; /* fresh per turn (#678) */
int prod=spec_decode(m,sc->hist,sc->len,r->maximum-r->emitted,eos,lg,
mux_spec_emit,&ud,&sc->len,NULL);
@@ -8209,7 +8328,7 @@ static void run_serve_mux(Model *m, const char *snap){
if(next==eos || is_stop(next)){ mux_done(m,sc,r); done=1; break; }
r->pending=next; sc->hist[sc->len]=next; r->emitted++; m->n_emit++;
if(gd->on) gr_feed(gd,next);
- mux_data(&T,r->id,next);
+ mux_data(&T,r->id,next,lo+(int64_t)j*m->c.vocab,m->c.vocab,r->logprobs);
if(r->emitted>=r->maximum){ r->length_limited=1; mux_done(m,sc,r); done=1; break; }
if(j<k){
if(next!=draft[j]) break; /* rejected: seq[j+1..] stale, overwritten next forward */
@@ -8233,7 +8352,7 @@ static void run_serve_mux(Model *m, const char *snap){
if(next==eos || is_stop(next)){mux_done(m,sc,r);continue;}
r->pending=next; sc->hist[sc->len]=next; r->emitted++; m->n_emit++;
if(grd[i].on) gr_feed(&grd[i],next); /* walker stays in sync when not drafting */
- mux_data(&T,r->id,next);
+ mux_data(&T,r->id,next,lo+(int64_t)s*m->c.vocab,m->c.vocab,r->logprobs);
if(r->emitted>=r->maximum){ r->length_limited=1; mux_done(m,sc,r); }
}
free(lo);
diff --git a/c/decode_batch.h b/c/decode_batch.h
index 9f4dd343..12e3367e 100644
--- a/c/decode_batch.h
+++ b/c/decode_batch.h
@@ -3,6 +3,7 @@
#include <stddef.h>
#include <stdio.h>
+#include <string.h>
#include <math.h>
/* `base` belongs to one sequence's KV state. Keeping this arithmetic in a
@@ -12,31 +13,86 @@ static inline float *coli_kv_row(float *base, int position, int width)
return base + (size_t)position * (size_t)width;
}
+/* Per-token top-k emission cap: run_ablate_score's existing top-32 read-out
+ * ceiling, adopted as the wire cap too (no named consumer asks for more). */
+#define COLI_SUBMIT_TOPK_MAX 32
+
typedef struct {
unsigned long long id, bytes, gbytes;
int slot, max_tokens;
float temperature, top_p;
+ int logprobs; /* requested per-token top-k count; 0 = channel off (opt-in) */
+ int tok_ids; /* 1 = payload is ASCII token ids, not raw prompt text */
} ColiSubmit;
+/* Extension fields, 8th whitespace field onward: "key=value" tokens after the
+ * full 7-field header, e.g. "SUBMIT 1 0 12 16 0 1 0 logprobs=5 ids=1".
+ * A namespace instead of more positional fields: the next per-request knob
+ * (a future seed, say) claims a key here rather than forcing another
+ * field-count migration of this scarce wire budget. Unknown keys reject the
+ * whole frame -- framing stays as unambiguous as the legacy arms'
+ * trailing-field guard. An OLD engine rejects ANY extended header cleanly:
+ * both legacy sscanf arms fail on the extra field (their trailing %c matches
+ * the separating space) and the engine answers ERROR 0 BAD_REQUEST. */
+static inline int coli_submit_ext(const char *p, ColiSubmit *s)
+{
+ int seen = 0;
+ while (*p == ' ' || *p == '\t') p++;
+ while (*p) {
+ char key[16];
+ unsigned long long val;
+ int n = 0;
+ if (sscanf(p, "%15[a-z_]=%llu%n", key, &val, &n) != 2 || n <= 0)
+ return 0;
+ if (p[n] && p[n] != ' ' && p[n] != '\t') return 0;
+ if (!strcmp(key, "logprobs")) { /* per-token top-k emission */
+ if (val > COLI_SUBMIT_TOPK_MAX) return 0;
+ s->logprobs = (int)val;
+ } else if (!strcmp(key, "ids")) { /* pre-tokenized prompt intake */
+ if (val > 1) return 0;
+ s->tok_ids = (int)val;
+ } else {
+ return 0;
+ }
+ seen = 1;
+ p += n;
+ while (*p == ' ' || *p == '\t') p++;
+ }
+ return seen;
+}
+
/* Parse the textual header. The payload is read separately using `bytes`, so
* it may contain newlines. Reject trailing fields to keep framing unambiguous.
* Optional 7th field `gbytes`: length of a per-request grammar (raw GBNF, or a
* JSON-Schema compiled engine-side) appended to the payload AFTER the prompt
- * bytes. 6-field headers remain valid (gbytes = 0). */
+ * bytes. 6-field headers remain valid (gbytes = 0). Fields past the 7th are
+ * key=value extension tokens (coli_submit_ext above), only ever sent by a
+ * server that knows this engine understands them. */
static inline int coli_submit_parse(const char *line, ColiSubmit *s)
{
char tail;
+ int ok = 0, base = 0;
if (!line || !s) return 0;
s->gbytes = 0;
+ s->logprobs = 0;
+ s->tok_ids = 0;
if (sscanf(line, "SUBMIT %llu %d %llu %d %f %f %llu %c", &s->id, &s->slot,
&s->bytes, &s->max_tokens, &s->temperature, &s->top_p,
- &s->gbytes, &tail) != 7) {
+ &s->gbytes, &tail) == 7) ok = 1;
+ if (!ok) {
s->gbytes = 0;
if (sscanf(line, "SUBMIT %llu %d %llu %d %f %f %c", &s->id, &s->slot,
&s->bytes, &s->max_tokens, &s->temperature, &s->top_p,
- &tail) != 6)
- return 0;
+ &tail) == 6) ok = 1;
+ }
+ if (!ok) {
+ s->gbytes = 0;
+ if (sscanf(line, "SUBMIT %llu %d %llu %d %f %f %llu%n", &s->id,
+ &s->slot, &s->bytes, &s->max_tokens, &s->temperature,
+ &s->top_p, &s->gbytes, &base) == 7 && base > 0 &&
+ coli_submit_ext(line + base, s)) ok = 1;
}
+ if (!ok) return 0;
return s->id > 0 && s->bytes <= (16u << 20) && s->gbytes <= (1u << 20) &&
s->slot >= 0 && s->max_tokens >= 1 &&
isfinite(s->temperature) && isfinite(s->top_p) &&
@@ -44,4 +100,34 @@ static inline int coli_submit_parse(const char *line, ColiSubmit *s)
s->top_p > 0 && s->top_p <= 1;
}
+/* SUBMIT ids=1 payload: ASCII decimal token ids separated by whitespace,
+ * parsed straight into the prompt buffer -- the pre-tokenized intake path
+ * that bypasses tok_encode entirely (no detokenize/re-encode round trip, so
+ * the ids the caller sent are exactly the ids the engine scores). Returns
+ * the id count, or -1 on any malformed or out-of-range id. More than `cap`
+ * ids reports `cap`: the caller parses with one spare slot and treats a full
+ * buffer as overflow, mirroring tok_encode's stop-at-cap contract (#401). */
+static inline int coli_ids_parse(const char *buf, size_t len, int *out,
+ int cap, int vocab)
+{
+ size_t i = 0;
+ int n = 0;
+ if (!buf || !out || cap < 1 || vocab < 1) return -1;
+ while (i < len) {
+ while (i < len && (unsigned char)buf[i] <= ' ') i++;
+ if (i >= len) break;
+ if (n >= cap) return cap; /* overflow: caller refuses loudly */
+ {
+ long v = 0;
+ size_t d = i;
+ while (i < len && buf[i] >= '0' && buf[i] <= '9' && v < vocab)
+ v = v * 10 + (buf[i++] - '0');
+ if (i == d || v >= vocab || (i < len && (unsigned char)buf[i] > ' '))
+ return -1;
+ out[n++] = (int)v;
+ }
+ }
+ return n;
+}
+
#endif
diff --git a/c/openai_server.py b/c/openai_server.py
index 19108c4b..12c83718 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -1889,7 +1889,14 @@ class Engine:
if not fields:
continue
kind = fields[0]
- if kind == "DATA" and len(fields) == 3:
+ if kind == "DATA" and len(fields) >= 3:
+ # 3 fields: the legacy frame. More: the U7a per-token
+ # numeric channel ("DATA <id> <n> <lp> <k> [tid tlp]*k"),
+ # emitted only for requests that opted in via the SUBMIT
+ # logprobs field. The payload framing is identical; the
+ # numeric fields are consumed by the server feature half
+ # (U7b) -- accepted here so the frame never kills the
+ # dispatcher (and with it every in-flight request).
request_id = fields[1]
size = int(fields[2])
if not 0 <= size <= 65536:
@@ -1901,6 +1908,19 @@ class Engine:
events = self.pending.get(request_id)
if events is not None:
events.put(("data", data))
+ elif kind == "ECHO" and len(fields) >= 6:
+ # U7a prefill read-out: "ECHO <id> <n> <pos> <lp> <k>
+ # [tid tlp]*k" plus a DATA-framed payload (n bytes + LF).
+ # Emitted only for opted-in requests; no current request
+ # path opts in, so the frame is read (to keep the stream
+ # in sync) and dropped -- U7b delivers it to the response
+ # assembly when it wires the opt-in.
+ size = int(fields[2])
+ if not 0 <= size <= 65536:
+ raise RuntimeError("invalid engine DATA size")
+ self._read_exact(size)
+ if self._read_exact(1) != b"\n":
+ raise RuntimeError("invalid engine DATA terminator")
elif kind == "ACCEPT" and len(fields) >= 3:
# #597: the engine validated the submission (fits context) before prefill.
# Keep it pending β€” DATA/DONE still follow β€” and let generate() commit the
diff --git a/c/tests/test_decode_batch.c b/c/tests/test_decode_batch.c
index f5b216e2..851182a6 100644
--- a/c/tests/test_decode_batch.c
+++ b/c/tests/test_decode_batch.c
@@ -52,11 +52,86 @@ static void test_submit_header(void)
assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 512 extra", &sub));
}
+/* U7a extension fields: key=value tokens from the 8th field onward. The two
+ * legacy arms stay byte-identical in behavior (asserted above); the third arm
+ * accepts only known keys with in-range values, so an unknown or malformed
+ * extension rejects the whole frame exactly like a trailing garbage field. */
+static void test_submit_extension_fields(void)
+{
+ ColiSubmit sub;
+ /* legacy headers leave the new capabilities off */
+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95", &sub));
+ assert(sub.logprobs == 0 && sub.tok_ids == 0);
+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 512", &sub));
+ assert(sub.logprobs == 0 && sub.tok_ids == 0);
+ /* the 8-field opt-in forms */
+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=5", &sub));
+ assert(sub.logprobs == 5 && sub.tok_ids == 0 && sub.gbytes == 0);
+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 ids=1", &sub));
+ assert(sub.tok_ids == 1 && sub.logprobs == 0);
+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 512 logprobs=32 ids=1", &sub));
+ assert(sub.logprobs == 32 && sub.tok_ids == 1 && sub.gbytes == 512);
+ assert(coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=0 ids=0", &sub));
+ assert(sub.logprobs == 0 && sub.tok_ids == 0);
+ /* rejects: over-cap, out-of-range, unknown key, malformed token */
+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=33", &sub));
+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 ids=2", &sub));
+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 seed=7", &sub));
+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=", &sub));
+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 =5", &sub));
+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=5x", &sub));
+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs=5 junk", &sub));
+ assert(!coli_submit_parse("SUBMIT 42 3 17 64 0.7 0.95 0 logprobs", &sub));
+ /* an extended header still validates the base fields */
+ assert(!coli_submit_parse("SUBMIT 0 3 17 64 0.7 0.95 0 logprobs=5", &sub));
+ assert(!coli_submit_parse("SUBMIT 42 3 17 0 0.7 0.95 0 logprobs=5", &sub));
+}
+
+/* U7a token-ID intake: the ids the caller formatted are exactly the ids that
+ * come back out (the zero-round-trip property) -- and malformed input never
+ * parses partially. */
+static void test_ids_parse_round_trip(void)
+{
+ /* fixed known-good example: format then parse must reproduce the array */
+ const int ids[6] = {0, 154822, 154824, 12, 7, 99999};
+ char buf[128];
+ int out[8], n, k;
+ n = snprintf(buf, sizeof(buf), "%d %d %d %d %d %d",
+ ids[0], ids[1], ids[2], ids[3], ids[4], ids[5]);
+ assert(n > 0);
+ k = coli_ids_parse(buf, (size_t)n, out, 8, 160000);
+ assert(k == 6);
+ for (int i = 0; i < 6; i++) assert(out[i] == ids[i]);
+
+ /* whitespace forms: leading/trailing/newlines are all separators */
+ k = coli_ids_parse(" 5\n7\t9 ", 9, out, 8, 100);
+ assert(k == 3 && out[0] == 5 && out[1] == 7 && out[2] == 9);
+
+ /* rejects: sign, non-digit, vocab bound (exact and above), garbage tail */
+ assert(coli_ids_parse("-1", 2, out, 8, 100) == -1);
+ assert(coli_ids_parse("5x", 2, out, 8, 100) == -1);
+ assert(coli_ids_parse("1 two 3", 7, out, 8, 100) == -1);
+ assert(coli_ids_parse("100", 3, out, 8, 100) == -1); /* id == vocab */
+ assert(coli_ids_parse("99", 2, out, 8, 100) == 1 && out[0] == 99);
+ assert(coli_ids_parse("101", 3, out, 8, 100) == -1);
+ assert(coli_ids_parse("123456789012345", 15, out, 8, 100) == -1);
+
+ /* empty payload parses to zero ids (the caller's EMPTY_PROMPT arm) */
+ assert(coli_ids_parse(" ", 3, out, 8, 100) == 0);
+
+ /* overflow reports the cap so the caller's context check refuses loudly,
+ * mirroring tok_encode's stop-at-cap contract (#401) */
+ assert(coli_ids_parse("1 2 3 4 5", 9, out, 4, 100) == 4);
+ assert(coli_ids_parse("1 2 3 4", 7, out, 4, 100) == 4);
+}
+
int main(void)
{
test_rows_use_their_own_sequence_storage();
test_const_reader_selects_the_same_row();
test_submit_header();
+ test_submit_extension_fields();
+ test_ids_parse_round_trip();
puts("decode batch helper tests: ok");
return 0;
}
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index c7127cb0..adf01c91 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -660,6 +660,57 @@ class DispatcherTest(unittest.TestCase):
"attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15,
}])
+ def test_accepts_u7a_echo_and_extended_data_frames(self):
+ # U7a forward-compat: the engine's opt-in per-token numeric channel --
+ # ECHO frames for echoed prompt positions and DATA frames extended
+ # with "<lp> <k> [tid tlp]*k" -- must NOT trip the dispatcher's
+ # catch-all (which kills every in-flight request). Text delivery and
+ # the DONE stats stay exactly as for legacy frames; the numeric
+ # fields are consumed by the server feature half (U7b).
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ process.stdout.feed(b"ACCEPT " + request_id + b" 3\n")
+ process.stdout.feed(b"ECHO " + request_id + b" 2 0 nan 0\nHi\n")
+ process.stdout.feed(
+ b"ECHO " + request_id +
+ b" 6 1 -0.105361 2 7 -0.105361 9 -2.302585\n world\n")
+ process.stdout.feed(
+ b"ECHO " + request_id +
+ b" 1 2 -1.203973 2 4 -0.803973 6 -1.203973\n!\n")
+ process.stdout.feed(
+ b"DATA " + request_id +
+ b" 2 -0.223144 2 3 -0.223144 8 -1.723144\nok\n")
+ process.stdout.feed(
+ b"DONE " + request_id + b" STAT 1 2.5 0 1.0 3 0\n")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ chunks = []
+ stats = engine.generate("Hi world!", 4, 0.0, 1.0, chunks.append)
+ self.assertEqual(chunks, ["ok"])
+ self.assertEqual(stats["completion_tokens"], 1)
+ self.assertEqual(stats["prompt_tokens"], 3)
+ self.assertIsNone(engine.dispatcher_error)
+ engine.close()
+
+ def test_unknown_frame_still_stops_dispatcher(self):
+ # The catch-all that makes an unrecognized frame a hard failure is
+ # load-bearing for the U7a compatibility asymmetry (a new engine's
+ # frame reaching an OLD server kills the dispatcher -- the reason the
+ # engine half ships first and stays opt-in). Accepting ECHO/extended
+ # DATA must not have widened acceptance beyond those frames.
+ def respond(process, frame):
+ request_id = frame.split()[1]
+ process.stdout.feed(b"LOGPROB " + request_id + b" 0.5\n")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ with self.assertRaisesRegex(RuntimeError, "invalid engine response: LOGPROB"):
+ engine.generate("hello", 4, 0.7, 0.9, lambda _: None)
+ engine.close()
+
def test_cancels_generation_after_consumer_disconnects(self):
request_id = None
diff --git a/c/tests/test_spec_decode_state.c b/c/tests/test_spec_decode_state.c
index e23783b7..1360628d 100644
--- a/c/tests/test_spec_decode_state.c
+++ b/c/tests/test_spec_decode_state.c
@@ -162,6 +162,40 @@ static int test_persisted_tail(void) {
return 0;
}
+/* U7a: spec_decode's emit callback receives the logit row its token was
+ * picked/verified from -- including ACCEPTED DRAFT tokens, which bypass every
+ * pick_tok call site in the mux loop (the speculative-gap hazard the packet's
+ * Fork 5 flags). Greedy contract: no row may be NULL, and every row's argmax
+ * must be the emitted token -- proving the row really is that token's scoring
+ * distribution, not a stale or shifted one. */
+typedef struct { int n, null_rows, mismatches; } EmitProbe;
+static void emit_probe(int t, const float *lo, void *ud) {
+ EmitProbe *e = (EmitProbe *)ud;
+ e->n++;
+ if (!lo) { e->null_rows++; return; }
+ if (argmax_v(lo, V) != t) e->mismatches++;
+}
+
+static int test_emit_carries_scoring_logits_for_accepted_drafts(void) {
+ const int next[V] = {0, 2, 3, 1, 4, 5, 6, 7};
+ Toy t;
+ toy_init(&t, next);
+ reset_decode_globals(2);
+
+ /* Same shape as test_accepted_speculative_tokens: the repeated [1,2]
+ * bigram proposes [3,1]; one picked token + two accepted drafts. */
+ int all[12] = {1, 2, 3, 1};
+ EmitProbe probe = {0, 0, 0};
+ int kv = -1;
+ CHECK(spec_decode(&t.m, all, 4, 3, -1, choose(2), emit_probe, &probe, &kv, NULL) == 3);
+ CHECK(probe.n == 3); /* 1 pick_tok token + 2 accepted draft tokens */
+ CHECK(probe.null_rows == 0); /* the draft-accept arm has no numeric gap */
+ CHECK(probe.mismatches == 0); /* each row scores exactly its own token */
+
+ toy_free(&t);
+ return 0;
+}
+
static int test_accepted_speculative_tokens(void) {
const int next[V] = {0, 2, 3, 1, 4, 5, 6, 7};
Toy t;
@@ -217,6 +251,7 @@ int main(void) {
CHECK(test_ngen_zero_and_one_shot() == 0);
CHECK(test_stateful_ngen_and_more() == 0);
CHECK(test_persisted_tail() == 0);
+ CHECK(test_emit_carries_scoring_logits_for_accepted_drafts() == 0);
CHECK(test_accepted_speculative_tokens() == 0);
CHECK(test_eos_and_non_limit_stop() == 0);
puts("spec_decode state tests: ok");