Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(qwen38): bound serve cache and validate tokens

themorlock85637993c84e
diff --git a/c/Makefile b/c/Makefile
index 09efea07..9d7c165b 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1106,8 +1106,8 @@ tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h
tests/test_st_slice$(EXE): tests/test_st_slice.c st.h json.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_qwen38_tokenizer$(EXE): tests/test_qwen38_tokenizer.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h route_trace.h tok_unicode.h tok_unicode_o200k.h
- $(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)
+tests/test_qwen38_tokenizer$(EXE): tests/test_qwen38_tokenizer.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h edge_runtime.c edge_runtime.h edge_adapters.h edge_adapter_internal.h st.h json.h compat.h quant.h route_trace.h tok_unicode.h tok_unicode_o200k.h
+ $(CC) $(NOCUDA_CFLAGS) $< edge_runtime.c -o $@ $(NOCUDA_LDFLAGS)
tests/test_qwen38_config$(EXE): tests/test_qwen38_config.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h route_trace.h tok_unicode.h tok_unicode_o200k.h
$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)
diff --git a/c/family_registry.py b/c/family_registry.py
index 0733eeb0..c24d1a43 100644
--- a/c/family_registry.py
+++ b/c/family_registry.py
@@ -224,6 +224,9 @@ def _qwen38_geometry(config, context, _model_dir):
index_ratio = _required_int(config, "indexer_compress_ratio", family)
if index_kv != 1 or index_dim < rotary_dim or index_budget % index_ratio:
raise ValueError(f"{family}: invalid sparse indexer geometry")
+ # Serve allocates this complete QSA bank once before publishing READY and
+ # never grows it in place. One context-sized bank is therefore both the
+ # steady allocation and the allocation peak; no old/new growth pair exists.
context_state = full * context * (2 * n_kv * head_dim + index_kv * index_dim) * 4
value_heads = _required_int(config, "linear_num_value_heads", family)
diff --git a/c/qwen38.c b/c/qwen38.c
index 6571b43c..599a353f 100644
--- a/c/qwen38.c
+++ b/c/qwen38.c
@@ -1400,56 +1400,15 @@ static void q38_prefix_cache_release(Model *m){
q38_prefix_cache_dispose(&g_q38_prefix);
}
-/* Grow QSA storage without disturbing rows already described by the prefix.
- * The core's ordinary allocator intentionally frees on growth for cold
- * inference; serve needs this transactional variant for a cached extension. */
-static int q38_serve_ensure_kv(Model *m,int required){
- if(!m||required<1)return 0;
- if(m->K&&required<=m->kv_cap){m->max_t=required;return 1;}
- int old_cap=m->kv_cap,old_max=m->max_t,keep=m->kv_len;
- if(!m->K||old_cap<1||keep<1){m->max_t=required;ensure_kv(m);return 1;}
- if(keep>old_cap)keep=old_cap;if(keep>required)keep=required;
- Cfg *c=&m->c;float **new_k=(float**)calloc((size_t)c->layers,sizeof(float*));
- float **new_v=(float**)calloc((size_t)c->layers,sizeof(float*));
- float **new_ik=(float**)calloc((size_t)c->layers,sizeof(float*));
- if(!new_k||!new_v||!new_ik)goto oom;
- size_t cells=0;
- for(int i=0;i<c->layers;i++)if(c->is_attn[i]){
- if(c->kv_heads<=0||c->head_dim<=0||c->idx_dim<=0||
- !m->K[i]||!m->V[i]||!m->IK[i]||
- !q38_size_mul((size_t)c->kv_heads,(size_t)required,&cells)||
- !q38_size_mul(cells,(size_t)c->head_dim,&cells)||
- cells>SIZE_MAX/sizeof(float))goto oom;
- new_k[i]=(float*)malloc(cells*sizeof(float));new_v[i]=(float*)malloc(cells*sizeof(float));
- if(!new_k[i]||!new_v[i])goto oom;
- size_t row_cells=0,row_bytes=0;
- if(!q38_size_mul((size_t)keep,(size_t)c->head_dim,&row_cells)||
- !q38_size_mul(row_cells,sizeof(float),&row_bytes))goto oom;
- for(int h=0;h<c->kv_heads;h++){
- size_t old_off=0,new_off=0;
- if(!q38_size_mul((size_t)h,(size_t)old_cap,&old_off)||
- !q38_size_mul(old_off,(size_t)c->head_dim,&old_off)||
- !q38_size_mul((size_t)h,(size_t)required,&new_off)||
- !q38_size_mul(new_off,(size_t)c->head_dim,&new_off))goto oom;
- memcpy(new_k[i]+new_off,m->K[i]+old_off,row_bytes);
- memcpy(new_v[i]+new_off,m->V[i]+old_off,row_bytes);
- }
- if(!q38_size_mul((size_t)required,(size_t)c->idx_dim,&cells)||
- cells>SIZE_MAX/sizeof(float))goto oom;
- new_ik[i]=(float*)malloc(cells*sizeof(float));if(!new_ik[i])goto oom;
- size_t index_cells=0,index_bytes=0;
- if(!q38_size_mul((size_t)keep,(size_t)c->idx_dim,&index_cells)||
- !q38_size_mul(index_cells,sizeof(float),&index_bytes))goto oom;
- memcpy(new_ik[i],m->IK[i],index_bytes);
- }
- for(int i=0;i<c->layers;i++){
- free(m->K[i]);free(m->V[i]);free(m->IK[i]);
- }
- free(m->K);free(m->V);free(m->IK);m->K=new_k;m->V=new_v;m->IK=new_ik;
- m->kv_cap=required;m->max_t=required;return 1;
-oom:
- if(new_k)for(int i=0;i<c->layers;i++){free(new_k[i]);free(new_v?new_v[i]:NULL);free(new_ik?new_ik[i]:NULL);}
- free(new_k);free(new_v);free(new_ik);m->max_t=old_max;return 0;
+/* Allocate the complete served QSA bank once, before READY. Prefix reuse
+ * relies on append-only position rows; one planned allocation both preserves
+ * those rows and avoids a growth peak containing the old and new banks at the
+ * same time. An initialized serve model is therefore never grown in place. */
+static int q38_serve_ensure_kv(Model *m,int capacity){
+ if(!m||capacity<1)return 0;
+ if(m->kv_cap>0)return m->K&&m->kv_cap>=capacity;
+ m->max_t=capacity;ensure_kv(m);
+ return m->K&&m->kv_cap>=capacity;
}
static double q38_cache_hit_percent(uint64_t hits,uint64_t misses){
@@ -1478,8 +1437,7 @@ static int q38_format_prof(char *out,size_t capacity,double wall_s,int prompt_to
static int serve_one(Model *m, ServeReq *q){
int *ids=NULL, np=0;
encode_text_n(q->payload,(size_t)q->plen,&ids,&np); /* byte-counted prompt; qwen38 adds no BOS */
- int max_ctx = qwen38_max_ctx();
- if(max_ctx > m->c.max_positions) max_ctx = m->c.max_positions;
+ int max_ctx=m->kv_cap;
if(np<1 || np>max_ctx || q->max_tok<1 || q->max_tok>max_ctx-np){
printf("ERROR %s CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d\n",q->id,np,q->max_tok,max_ctx);
fflush(stdout); free(ids); return 0;
@@ -1488,11 +1446,6 @@ static int serve_one(Model *m, ServeReq *q){
double request_started=now_s();
uint64_t hits_before=m->hits, misses_before=m->miss;
Q38Timers timers_before=m->timers;
- int request_capacity=np+q->max_tok;
- if(!q38_serve_ensure_kv(m,request_capacity)){
- coli_serve_write_error(stdout,q->id,"unable to grow QSA state");
- free(ids);return 0;
- }
int reuse=q38_prefix_restore(m,ids,np);
float *lo=NULL;
if(reuse==np){
@@ -1589,6 +1542,11 @@ static int serve_one(Model *m, ServeReq *q){
static void serve_loop(Model *m){
coli_serve_binary_mode();
setvbuf(stdin,NULL,_IONBF,0);
+ int max_ctx=qwen38_max_ctx();
+ if(max_ctx>m->c.max_positions)max_ctx=m->c.max_positions;
+ if(!q38_serve_ensure_kv(m,max_ctx)){
+ fprintf(stderr,"[serve] unable to allocate QSA state\n");return;
+ }
fputs("\x01\x01READY\x01\x01\n",stdout);
printf("STAT 0 0.00 0.0 %.2f\n",rss_gb());
fflush(stdout);
@@ -2483,8 +2441,8 @@ static int qwen38_edge_detokenize(
void *engine_impl,const int32_t *token_ids,size_t token_count,
char *text,size_t text_capacity,size_t *text_bytes,
char *error,size_t error_size) {
- (void)engine_impl;
- if(!token_ids||!token_count||!text_bytes||token_count>INT_MAX||
+ Qwen38EdgeEngine *engine=(Qwen38EdgeEngine*)engine_impl;
+ if(!engine||!token_ids||!token_count||!text_bytes||token_count>INT_MAX||
token_count>SIZE_MAX/sizeof(int))
return coli_edge_adapter_error(error,error_size,
"invalid Qwen3.8 detokenizer input");
@@ -2492,7 +2450,16 @@ static int qwen38_edge_detokenize(
if(!ids)
return coli_edge_adapter_error(error,error_size,
"out of memory detokenizing Qwen3.8 tokens");
- for(size_t item=0;item<token_count;item++)ids[item]=token_ids[item];
+ for(size_t item=0;item<token_count;item++){
+ int token=token_ids[item];
+ if(token<0||token>=engine->model.c.vocab||token>=g_tok_n||
+ !g_tok||!g_tok[token]){
+ free(ids);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 token ID is not decodable");
+ }
+ ids[item]=token;
+ }
unsigned char *decoded=NULL;size_t decoded_bytes=0;
int result=decode_range_alloc(ids,0,(int)token_count,
&decoded,&decoded_bytes);
diff --git a/c/tests/test_edge_adapters_real.c b/c/tests/test_edge_adapters_real.c
index af777edd..8a3fe6c1 100644
--- a/c/tests/test_edge_adapters_real.c
+++ b/c/tests/test_edge_adapters_real.c
@@ -150,6 +150,18 @@ int main(int argc, char **argv) {
"tokenizer round-trip differs");
free(probe_text); probe_text = NULL;
free(probe_ids); probe_ids = NULL;
+ if (!strcmp(family, "qwen38")) {
+ int32_t invalid_ids[] = {-1, (int32_t)edge_cap.vocab_size};
+ for (size_t invalid = 0;
+ invalid < sizeof(invalid_ids) / sizeof(invalid_ids[0]);
+ invalid++) {
+ size_t rejected_bytes = 0;
+ REQUIRE(coli_edge_detokenize(edge, &invalid_ids[invalid], 1,
+ NULL, 0, &rejected_bytes,
+ error, sizeof(error)) != 0,
+ "Qwen3.8 detokenizer accepted an invalid token ID");
+ }
+ }
uint32_t context = (uint32_t)(prompt_count + max_new + 2);
ColiSegmentEngineOptions segment_options = {
diff --git a/c/tests/test_qwen38_prefix.c b/c/tests/test_qwen38_prefix.c
index 0753956e..91e00e4e 100644
--- a/c/tests/test_qwen38_prefix.c
+++ b/c/tests/test_qwen38_prefix.c
@@ -52,10 +52,11 @@ int main(void){
CHECK(m.DN_rec[0]&&m.DN_conv[0]&&m.PLE_conv_state&&m.ple_history);
CHECK(q38_prefix_cache_layout(&m));
- /* QSA rows are position-indexed. Verify growth keeps the old prefix. */
- m.c.kv_heads=1;m.c.head_dim=2;m.c.idx_dim=2;m.kv_cap=3;m.kv_len=3;
- m.K[1]=malloc(6*sizeof(float));m.V[1]=malloc(6*sizeof(float));m.IK[1]=malloc(6*sizeof(float));
- CHECK(m.K[1]&&m.V[1]&&m.IK[1]);
+ /* Serve allocates its planned QSA ceiling once. Reuse must preserve the
+ * position-indexed prefix without a second allocation or growth peak. */
+ m.c.kv_heads=1;m.c.head_dim=2;m.c.idx_dim=2;
+ CHECK(q38_serve_ensure_kv(&m,6));CHECK(m.kv_cap==6);m.kv_len=3;
+ float *initial_k=m.K[1],*initial_v=m.V[1],*initial_ik=m.IK[1];
for(int i=0;i<6;i++){m.K[1][i]=(float)i;m.V[1][i]=(float)(10+i);m.IK[1][i]=(float)(20+i);}
const int first[]={1,2,3};float first_logits[8];
@@ -65,6 +66,8 @@ int main(void){
const int extension[]={1,2,3,4};
CHECK(q38_prefix_restore(&m,extension,4)==3);CHECK(check_state(&m,7.f));
CHECK(q38_serve_ensure_kv(&m,6));CHECK(m.kv_cap==6&&m.kv_len==3);
+ CHECK(m.K[1]==initial_k&&m.V[1]==initial_v&&m.IK[1]==initial_ik);
+ CHECK(!q38_serve_ensure_kv(&m,7));
for(int i=0;i<6;i++){CHECK(m.K[1][i]==(float)i);CHECK(m.V[1][i]==(float)(10+i));CHECK(m.IK[1][i]==(float)(20+i));}
/* An exact prompt restores the recurrent state and its saved logits. */
@@ -84,6 +87,6 @@ int main(void){
for(size_t i=0;i<g_q38_prefix.conv_cells;i++)CHECK(m.DN_conv[0][i]==0.f);
for(size_t i=0;i<g_q38_prefix.ple_cells;i++)CHECK(m.PLE_conv_state[i]==0.f);
free_fake(&m);
- puts("qwen38 prefix: exact reuse, extension, mismatch reset, state restore: ok");
+ puts("qwen38 prefix: single QSA allocation, exact/extension reuse, reset: ok");
return 0;
}
diff --git a/c/tests/test_qwen38_tokenizer.c b/c/tests/test_qwen38_tokenizer.c
index c851cf41..56e73ca7 100644
--- a/c/tests/test_qwen38_tokenizer.c
+++ b/c/tests/test_qwen38_tokenizer.c
@@ -2,6 +2,7 @@
#define _GNU_SOURCE
#define QWEN38_NO_MAIN
#define QWEN38_TEST_TOKENIZER
+#define COLI_EDGE_ADAPTER
#include "../qwen38.c"
#define CHECK(x) do { if(!(x)){ \
@@ -137,11 +138,20 @@ int main(void) {
invalid_output,sizeof invalid_output,&invalid_n)==12);
for(int i=0;i<12;i+=3)CHECK(!memcmp(invalid_output+i,replacement,3));
- char **grown=realloc(g_tok,15u*sizeof(char*));CHECK(grown!=NULL);g_tok=grown;
+ char **grown=realloc(g_tok,16u*sizeof(char*));CHECK(grown!=NULL);g_tok=grown;
g_tok[12]=malloc(257);CHECK(g_tok[12]!=NULL);
for(int i=0;i<128;i++){g_tok[12][2*i]=(char)0xc4;g_tok[12][2*i+1]=(char)0xa0;}
g_tok[12][256]='\0';g_tok[13]=strdup("<0x00>");CHECK(g_tok[13]!=NULL);
- g_tok[14]=strdup("<0xFF>");CHECK(g_tok[14]!=NULL);g_tok_n=15;
+ g_tok[14]=strdup("<0xFF>");CHECK(g_tok[14]!=NULL);g_tok[15]=NULL;g_tok_n=16;
+ Qwen38EdgeEngine edge={0};edge.model.c.vocab=g_tok_n;
+ const int32_t invalid_edge_ids[]={-1,15,g_tok_n};char edge_error[128];
+ for(size_t index=0;
+ index<sizeof(invalid_edge_ids)/sizeof(invalid_edge_ids[0]);index++){
+ size_t text_bytes=0;
+ CHECK(qwen38_edge_detokenize(&edge,&invalid_edge_ids[index],1,
+ NULL,0,&text_bytes,
+ edge_error,sizeof edge_error)<0);
+ }
const int range_ids[]={12,13,12};unsigned char *range=NULL;size_t range_n=0;
CHECK(decode_range_alloc(range_ids,0,3,&range,&range_n)==0);
CHECK(range_n==257);CHECK(range[128]=='\0');
diff --git a/docs/qwen38.md b/docs/qwen38.md
index 54414039..144bbe3b 100644
--- a/docs/qwen38.md
+++ b/docs/qwen38.md
@@ -65,7 +65,9 @@ saved DeltaNet/PLE recurrent state is restored before evaluating only the
extension. An identical prompt also reuses its saved final logits. Mismatched
or shorter prompts reset every hybrid component rather than guessing at cache
identity. Decode and cancellation mutate only live state, not the published
-prompt snapshot.
+prompt snapshot. The complete configured QSA context bank is allocated once
+before the engine publishes `READY`; serve never grows it alongside a still-live
+old bank, so the planner's single context-bank reservation is also the peak.
QSA caches the two K/V heads and the indexer's raw key. Complete four-token
blocks are pooled and scored, the best 512 blocks are retained, and a causal