Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

refactor(telemetry): one expert history for every engine (#700)

Mario GutiΓ©rrezb9564726cd6f
diff --git a/c/Makefile b/c/Makefile
index 4cd545fb..b0c05cbd 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -358,7 +358,7 @@ $(file >.build-config,$(BUILD_CONFIG))
endif
.build-config: ;
-colibri$(EXE): colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) .build-config
+colibri$(EXE): colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h route_trace.h $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) .build-config
$(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) -o colibri$(EXE) $(LDFLAGS)
# Vulkan backend object (plain C + vulkan headers) and its SPIR-V shaders.
@@ -515,30 +515,30 @@ tests/test_schema_gbnf$(EXE): tests/test_schema_gbnf.c schema_gbnf.h grammar.h j
tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_stops$(EXE): tests/test_stops.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_stops$(EXE): tests/test_stops.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_topp$(EXE): tests/test_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_topp$(EXE): tests/test_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test
# gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp
-tests/bench_topp$(EXE): tests/bench_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/bench_topp$(EXE): tests/bench_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_temp_env$(EXE): tests/test_temp_env.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_temp_env$(EXE): tests/test_temp_env.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c colibri.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c colibri.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# fmt=6 kernel oracle: needs the generated grid table, and its fixture comes from
@@ -561,10 +561,10 @@ fuzz-rans: tests/fuzz_rans.c rans.h
tests/fuzz_rans.c -o tests/fuzz_rans -lm
./tests/fuzz_rans
-tests/test_int3$(EXE): tests/test_int3.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_int3$(EXE): tests/test_int3.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_int3_load$(EXE): tests/test_int3_load.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_int3_load$(EXE): tests/test_int3_load.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_logit_nan$(EXE): tests/test_logit_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
@@ -579,10 +579,13 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c
tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_route_trace$(EXE): tests/test_route_trace.c route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_corpus_draft$(EXE): tests/test_corpus_draft.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
+tests/test_corpus_draft$(EXE): tests/test_corpus_draft.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_cap_precedence$(EXE): tests/test_cap_precedence.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
@@ -593,20 +596,20 @@ tests/test_ssd_probe$(EXE): tests/test_ssd_probe.c colibri.c st.h uring.h json.h
# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356),
# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select
-tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# bench_idot: microbenchmark (single-acc vs independent-acc AVX-VNNI idot), NOT a test gate.
# Build on demand on an AVX-VNNI CPU: make tests/bench_idot ARCH=native
-tests/bench_idot$(EXE): tests/bench_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/bench_idot$(EXE): tests/bench_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# bench_mla_simd: microbenchmark (scalar vs AVX2/NEON MLA-absorb reductions, #442),
# NOT a test gate. Build on demand: make tests/bench_mla_simd
-tests/bench_mla_simd$(EXE): tests/bench_mla_simd.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/bench_mla_simd$(EXE): tests/bench_mla_simd.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_pipe_block$(EXE): tests/test_pipe_block.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
diff --git a/c/colibri.c b/c/colibri.c
index 8bf7cc4b..c3b7ad31 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -61,6 +61,7 @@
#include "grammar.h" /* metodo F: draft grammaticali (#48) */
#include "schema_gbnf.h" /* SCHEMA=: JSON-Schema -> GBNF for method F */
#include "decode_batch.h"
+#include "route_trace.h" /* ROUTE_TRACE + .coli_usage, engine-agnostic (#700) */
#ifdef _OPENMP
#include <omp.h> /* scratch per-thread nell'attention */
#else
@@ -786,10 +787,9 @@ static int g_draft=0; /* metodo E: DRAFT=n token auto-speculati per forward v
* parte (#8). MAI un vincolo sul sampling: solo proposte, la verifica batch-union
* decide β€” grammatica sbagliata = draft rifiutati, output identico.
* GRAMMAR_DRAFT=n (default 24) limita i token forzati per forward. */
-static FILE *g_route_fp=NULL; /* ROUTE_TRACE=<path>: dump per-position top-K routing (ids:gates)
- * per layer β€” offline co-activation / coupling analysis. Zero
- * effect on computation; measurement only. */
-static int g_route_call=0;
+/* ROUTE_TRACE=<path> (per-position top-K routing, for offline co-activation analysis)
+ * and the .coli_usage history both live in route_trace.h now, so every engine writes
+ * the same bytes. Measurement only; zero effect on computation. */
/* COUPLE=<.coli_pairs>: coupling-scored cross-layer prefetch. The routing of layer L
* strongly constrains the routing of L+1/L+2 (measured: median co-activation lift 1.8x
* over independence, p99 40x, and the structure TRANSFERS across workloads β€” it is a
@@ -1296,7 +1296,9 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
#endif
m->eroute=calloc(NR,sizeof(int*)); m->enr=calloc(NR,sizeof(int));
m->pin=calloc(NR,sizeof(ESlot*)); m->npin=calloc(NR,sizeof(int));
- m->eusage=calloc(NR,sizeof(uint32_t*)); m->eheat=calloc(NR,sizeof(uint32_t*));
+ rt_init("glm_moe_dsa",c->n_layers,c->n_experts); /* owns the counters + the format */
+ m->eusage=rt_counts_all(); /* alias: bump sites unchanged */
+ m->eheat=calloc(NR,sizeof(uint32_t*));
m->elast=calloc(NR,sizeof(uint32_t*));
m->elast_dc=calloc(NR,sizeof(uint32_t*)); m->elast_pre=calloc(NR,sizeof(uint32_t*));
m->kv=calloc(1,sizeof(KVState));
@@ -1340,7 +1342,6 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
#endif
m->ecache[i]=calloc(cap,sizeof(ESlot));
m->eroute[i]=calloc(c->topk,sizeof(int)); /* metodo C: ultimo routing del layer */
- m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t));
m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t));
m->elast[i]=calloc(c->n_experts,sizeof(uint32_t));
m->elast_dc[i]=calloc(c->n_experts,sizeof(uint32_t));
@@ -1393,7 +1394,6 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
m->mtp_norm=ld(m,PM("shared_head.norm.weight"));
m->ecache[i]=calloc(cap,sizeof(ESlot));
m->eroute[i]=calloc(c->topk,sizeof(int));
- m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t));
m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t));
m->elast[i]=calloc(c->n_experts,sizeof(uint32_t));
m->elast_dc[i]=calloc(c->n_experts,sizeof(uint32_t));
@@ -1445,6 +1445,13 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
}
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(c->idx_type[i])
rb+=qt_bytes(&m->ix_wq[i])+qt_bytes(&m->ix_wk[i])+qt_bytes(&m->ix_wp[i]);
+ /* Dense layers, and the MTP row when the model has none, do not route and so get no
+ * counter row -- the exact shape eusage had before route_trace.h owned it. rt_init
+ * cannot know this: sparsity is settled while the layers are built, above. A row is
+ * the load admission rule, so dropping these is what keeps a history record naming a
+ * dense layer from being absorbed and written back out. */
+ for(int i=0;i<c->n_layers;i++) if(!m->L[i].sparse) rt_drop_row(i);
+ if(!m->has_mtp) rt_drop_row(c->n_layers);
m->resident_bytes=rb;
}
@@ -3399,16 +3406,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
}
if(c->norm_topk){ float sm=0; for(int kk=0;kk<Ke;kk++) sm+=w[kk]; sm+=1e-20f; for(int kk=0;kk<Ke;kk++) w[kk]/=sm; }
for(int kk=0;kk<Ke;kk++) w[kk]*=c->routed_scale;
- if(g_route_fp){ /* ROUTE_TRACE: one line per (position, layer) */
- fprintf(g_route_fp,"%d %d %d",g_route_call,s,layer);
- for(int kk=0;kk<Ke;kk++) fprintf(g_route_fp," %d:%.4f",idx[kk],w[kk]);
- fputc('\n',g_route_fp);
- }
+ rt_trace(layer,s,idx,w,Ke); /* ROUTE_TRACE: one line per (position, layer) */
for(int d=0;d<D;d++) out[(int64_t)s*D+d]=0;
}
free(rank_buf); free(rank_w);
if(g_prof)m->t_route+=now_s()-route_t0;
- if(g_route_fp) g_route_call++;
+ rt_trace_end();
if(g_couple && cp_pred && S<=8)
for(int s2=0;s2<S;s2++) couple_prefetch(m,layer,idxs+(int64_t)s2*K,keff[s2]);
if(g_looka && S==1 && layer<c->n_layers){
@@ -4758,7 +4761,7 @@ static int pipe_layer_sparse(Model *m, Layer *l, int li, float *x_dev, int S, in
* ROUTE_P / ROUTE_TRACE keep the CPU ranking they need. */
static int lr_idx[64]; static float lr_w[64]; static int lr_keff[1];
int dev_routed=0;
- if(g_cuda_router && S==1 && !g_cache_route && g_route_p<=0.f && !g_route_fp
+ if(g_cuda_router && S==1 && !g_cache_route && g_route_p<=0.f && !rt_tracing()
&& c->n_experts<=4096 && c->topk<=64 && !l->router_cuda_bad){
int E=c->n_experts, K=c->topk;
int Ksel = g_topk>0 ? (g_topk<K?g_topk:K) : K;
@@ -7269,19 +7272,30 @@ static void pin_arena_bind(Model *m, PinRec *r, int *slot_of, int from, int to){
#endif
static double expert_avail(Model *m, double ram_gb, int ebits, int max_ctx); /* def. sotto */
static double g_mem_avail_boot; /* def. sotto (#653: corretta qui su GPU integrate) */
-static void pin_load(Model *m, const char *statspath, double gb){
- FILE *f=fopen(statspath,"r"); if(!f){ perror(statspath); return; }
+/* Admission test unchanged; it just runs from route_trace.h's reader now, so PIN=<file>
+ * accepts every history layout an engine can write instead of only sparse text (#700). */
+typedef struct { Model *m; PinRec *r; int *n, cap; unsigned char *seen; } PinCollect;
+static int pin_collect_cb(int l, int e, uint32_t cnt, void *ud){
+ PinCollect *p=(PinCollect*)ud; Model *m=p->m; Cfg *c=&m->c;
+ if(*p->n>=p->cap) return 0;
+ int ok = l>=0 && e>=0 && e<c->n_experts &&
+ ((l<c->n_layers && m->L[l].sparse) || (l==c->n_layers && m->has_mtp));
+ int64_t key=(int64_t)l*c->n_experts+e;
+ if(ok&&!p->seen[key]){ p->r[(*p->n)++]=(PinRec){l,e,cnt}; p->seen[key]=1; return 1; }
+ return 0;
+}
+/* trusted=1 only when the user typed this path. An auto-discovered history has not been
+ * vouched for by anyone, so it stays subject to the identity check like any other. */
+static void pin_load(Model *m, const char *statspath, double gb, int trusted){
+ { FILE *probe=fopen(statspath,"rb"); /* keep the same message on a bad path */
+ if(!probe){ perror(statspath); return; } fclose(probe); }
Cfg *c=&m->c; int cap=(c->n_layers+1)*c->n_experts;
PinRec *r=malloc((size_t)cap*sizeof(PinRec)); int n=0;
unsigned char *seen=calloc((size_t)(c->n_layers+1)*c->n_experts,1);
- int l,e; uint32_t cnt;
- while(n<cap && fscanf(f,"%d %d %u",&l,&e,&cnt)==3){
- int ok = l>=0 && e>=0 && e<c->n_experts &&
- ((l<c->n_layers && m->L[l].sparse) || (l==c->n_layers && m->has_mtp));
- int64_t key=(int64_t)l*c->n_experts+e;
- if(ok&&!seen[key]){ r[n++]=(PinRec){l,e,cnt}; seen[key]=1; }
- }
- fclose(f);
+ /* A named file is what the identity refusal tells the user to pass, so honouring it
+ * here is what makes that message true. Dimensions and format version still apply:
+ * those refusals never offered a way past them. */
+ { PinCollect pc={m,r,&n,cap,seen}; rt_read_ex(statspath,pin_collect_cb,&pc,trusted); }
int fill=getenv("PIN_FILL")?atoi(getenv("PIN_FILL")):0;
#ifdef COLI_CUDA
if(!getenv("PIN_FILL")&&g_cuda_release_host) fill=1;
@@ -8224,11 +8238,7 @@ int main(int argc, char **argv){
g_idot = getenv("IDOT")?atoi(getenv("IDOT")):1; /* 0 = kernel f32 esatti (A/B) */
g_spec_pin = getenv("SPEC_PIN")?atoi(getenv("SPEC_PIN")):1; /* #163: 0 = gate S-dipendenti storici / legacy S-dependent gates */
corpus_load(); /* COLI_DRAFT_CORPUS: external draft source */
- if(getenv("ROUTE_TRACE")&&*getenv("ROUTE_TRACE")){
- g_route_fp=fopen(getenv("ROUTE_TRACE"),"w");
- if(!g_route_fp) fprintf(stderr,"[ROUTE_TRACE] cannot open %s\n",getenv("ROUTE_TRACE"));
- else fprintf(stderr,"[ROUTE_TRACE] logging routing to %s\n",getenv("ROUTE_TRACE"));
- }
+ rt_trace_open(); /* same place as before, so the log order is identical */
g_repin = getenv("REPIN")?atoi(getenv("REPIN")):0; /* RFC: re-pin ogni n token emessi (0=off) / live re-pin every n emitted tokens (0=off) */
g_absorb = getenv("ABSORB")?atoi(getenv("ABSORB")):-1; /* -1 auto: assorbita per S<=4 */
g_dsa_force = getenv("DSA_FORCE")?atoi(getenv("DSA_FORCE")):0;
@@ -8471,6 +8481,7 @@ int main(int argc, char **argv){
* Va PRIMA di cap_for_ram: i pinnati contano nel residente. */
if(getenv("PIN")){
const char *pin=getenv("PIN"); char pauto[2100];
+ int pin_named=strcmp(pin,"auto")!=0; /* typed by the user, vs auto-discovered */
if(!strcmp(pin,"auto")){
/* PIN=auto: la storia VIVA <SNAP>/.coli_usage (appesa a ogni turno) batte il
* profilo congelato stats.txt β€” il pin di ogni riavvio riflette il carico reale
@@ -8489,7 +8500,7 @@ int main(int argc, char **argv){
}
if(pin){
const char *pin_gb=getenv("PIN_GB");
- pin_load(&m,pin,pin_gb&&!strcmp(pin_gb,"all")?-1.0:pin_gb?atof(pin_gb):10.0); /* PIN_GB=all (#80) */
+ pin_load(&m,pin,pin_gb&&!strcmp(pin_gb,"all")?-1.0:pin_gb?atof(pin_gb):10.0,pin_named); /* PIN_GB=all (#80) */
}
}
#if defined(COLI_CUDA) && defined(COLI_ANS)
@@ -8521,7 +8532,7 @@ int main(int argc, char **argv){
* qualche ora di chat) arriva a meta' del budget expert. */
double conf = (double)hist/200000.0; if(conf>1) conf=1;
double pin_gb = expert_avail(&m,ram_env,ebits,est_ctx)*0.5*conf/1e9;
- if(pin_gb>=0.5) pin_load(&m, g_usage_path, pin_gb);
+ if(pin_gb>=0.5) pin_load(&m, g_usage_path, pin_gb, 0); /* auto-discovered: not trusted */
}
/* SEMPRE: senza clamp la LRU cresce fino a cap*76 layer = decine di GB -> OOM-kill.
* RAM_GB assente o <=0 = budget automatico da MemAvailable. */
diff --git a/c/route_trace.h b/c/route_trace.h
new file mode 100644
index 00000000..a3abaecb
--- /dev/null
+++ b/c/route_trace.h
@@ -0,0 +1,332 @@
+/* route_trace.h β€” engine-agnostic routing telemetry: the ROUTE_TRACE stream and the
+ * .coli_usage expert history, in one place so every engine emits the same bytes.
+ *
+ * Why a separate header and not part of telemetry.h: telemetry.h is bound to GLM's
+ * structures ("include after Model/Cfg/QT/ESlot/shards ... requires qt_bytes(), now_s(),
+ * rss_gb()"), so kimi_k3.c and olmoe.c cannot include it β€” their Model/Cfg are different
+ * types. Routing telemetry does not need shared types: an engine knows its layer index,
+ * the expert ids it selected and their gates, and that is the whole input.
+ *
+ * Dependencies are the C library plus compat.h, and compat.h is NOT optional here: it
+ * redirects rename() to MoveFileEx(MOVEFILE_REPLACE_EXISTING), because the CRT rename
+ * fails with EEXIST when the destination exists and would therefore silently stop
+ * updating the history after its first write on Windows. That shim exists in compat.h
+ * *because of* this very code path β€” see the comment above compat_rename(). Including
+ * only <stdio.h> here compiles fine and passes on Linux; it reintroduces that bug.
+ * compat.h itself is self-contained (system headers only, no engine types), which is why
+ * st.h can include it too.
+ *
+ * ---- history format (.coli_usage) ----
+ * Text, one record per line: "<layer> <expert> <count>", sparse (non-zero only).
+ * Two header records carry the dimensions and the writing engine's identity:
+ *
+ * -1 <n_layers> <n_experts>
+ * -2 <format_version> <engine_id>
+ *
+ * The layer field is NEGATIVE on purpose. Every reader written against the old format is
+ * a `while(fscanf(f,"%d %d %u",&l,&e,&c)==3)` loop guarded by `l>=0`, so it parses these
+ * two records, discards them, and continues into the data β€” which is what lets a file
+ * written here still load in an engine built before this header existed.
+ *
+ * Two constraints that field layout is forced by, and they are easy to get wrong:
+ * - a string cannot go in any slot: a non-numeric field makes fscanf return < 3, which
+ * ends the loop and silently drops every record after it. Hence engine_id is a hash.
+ * - engine_id must sit in the THIRD field, the one old readers parse with "%u". A 32-bit
+ * hash routinely exceeds INT_MAX and the second field is read with "%d", where that
+ * is undefined behaviour. So the version (small) goes second, the hash third.
+ *
+ * Two older layouts are also read, so histories accumulated before this existed keep
+ * working β€” that ranking is the entire value of PIN=auto:
+ * - legacy text: the same triples with no header records. It cannot be validated, so it
+ * is accepted as-is, exactly as before.
+ * - inkling's "IKU1": uint32 {magic, n_layers, n_experts} then a dense
+ * uint32[n_layers][n_experts] block. Only inkling reads it; any other engine refuses
+ * it by name rather than loading another model's ranking.
+ *
+ * ---- trace format (ROUTE_TRACE=<path>) ----
+ * One line per (moe call, batch row): "<call> <row> <layer> <id>:<gate.4f> ...".
+ * Byte-identical to what colibri.c emitted before this header; tools/route_pairs.py and
+ * the .coli_pairs pipeline consume it unchanged.
+ */
+#ifndef ROUTE_TRACE_H
+#define ROUTE_TRACE_H
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include "compat.h" /* rename() -> MoveFileEx on Windows; see above */
+
+#define RT_FORMAT_VERSION 1
+#define RT_IKU1_MAGIC 0x31554B49u /* "IKU1" β€” inkling.c's usage_save/pins_load */
+
+/* engines that write this format; the table exists so a mismatch names both sides */
+static const char *rt_engine_names[] = { "glm_moe_dsa", "inkling", "olmoe", "kimi_k3", NULL };
+
+static uint32_t rt_hash(const char *s){ /* FNV-1a 32, stable across builds */
+ uint32_t h = 2166136261u;
+ for(; s && *s; s++){ h ^= (unsigned char)*s; h *= 16777619u; }
+ return h;
+}
+static const char *rt_engine_of(uint32_t id){
+ for(int i = 0; rt_engine_names[i]; i++)
+ if(rt_hash(rt_engine_names[i]) == id) return rt_engine_names[i];
+ return NULL; /* unknown writer: report the id */
+}
+
+/* ---- state: owned here so a new engine defines no storage and no format ---- */
+static uint32_t **rt_c; /* [n_layers+1][n_experts] counters */
+static int rt_nl = -1, rt_ne; /* rows 0..rt_nl inclusive */
+static uint32_t rt_id; /* this engine's id */
+static const char *rt_engine = "";
+static FILE *rt_fp; /* ROUTE_TRACE stream, NULL = off */
+static int rt_call; /* moe-call counter, first trace field */
+
+/* n_layers is inclusive: rows 0..n_layers exist, matching the extra MTP row GLM keeps at
+ * index n_layers. Engines without one simply never touch it. A failed row allocation
+ * leaves that row NULL and is handled the same way an engine's own failed calloc is. */
+/* Open the trace stream. Separate from rt_init because it needs no dimensions, so an
+ * engine that reads ROUTE_TRACE before it has parsed its config (colibri.c does) keeps
+ * the message exactly where it was. Idempotent; a failed open is not retried. */
+static void rt_trace_open(void){
+ static int tried;
+ if(tried) return;
+ tried = 1;
+ const char *p = getenv("ROUTE_TRACE");
+ if(!p || !*p) return;
+ rt_fp = fopen(p, "w");
+ if(!rt_fp) fprintf(stderr, "[ROUTE_TRACE] cannot open %s\n", p);
+ else fprintf(stderr, "[ROUTE_TRACE] logging routing to %s\n", p);
+}
+
+static void rt_init(const char *engine, int n_layers, int n_experts){
+ if(n_layers < 0 || n_experts < 1) return;
+ rt_engine = engine ? engine : "";
+ rt_id = rt_hash(rt_engine);
+ rt_c = (uint32_t**)calloc((size_t)n_layers + 1, sizeof(uint32_t*));
+ if(!rt_c) return;
+ rt_nl = n_layers; rt_ne = n_experts;
+ for(int i = 0; i <= n_layers; i++)
+ rt_c[i] = (uint32_t*)calloc((size_t)n_experts, sizeof(uint32_t));
+ rt_trace_open(); /* no-op if the engine opened it already */
+}
+
+static uint32_t **rt_counts_all(void){ return rt_c; }
+static uint32_t *rt_counts(int layer){
+ return (rt_c && layer >= 0 && layer <= rt_nl) ? rt_c[layer] : NULL;
+}
+static int rt_tracing(void){ return rt_fp != NULL; }
+
+/* Release a layer's counter row, so rt_counts(layer) is NULL for a layer that does not
+ * route. rt_init cannot know which those are β€” an engine learns its own sparsity while it
+ * builds its layers, after it has told us its dimensions β€” so it drops them afterwards.
+ *
+ * This is not an optimisation, it is the load admission rule. Every reader here treats a
+ * NULL row as "this layer has no experts to attribute a count to", which is precisely what
+ * the code being replaced expressed by leaving eusage[i] NULL on dense layers. Skip the
+ * drop and a history record naming a dense layer is silently absorbed and written back. */
+static void rt_drop_row(int layer){
+ if(rt_c && layer >= 0 && layer <= rt_nl){ free(rt_c[layer]); rt_c[layer] = NULL; }
+}
+
+/* count only β€” for a router that bumps its other per-expert state in the same loop */
+static void rt_count(int layer, const int *ids, int k){
+ uint32_t *row = rt_counts(layer);
+ if(!row || !ids) return;
+ for(int i = 0; i < k; i++)
+ if(ids[i] >= 0 && ids[i] < rt_ne) row[ids[i]]++;
+}
+
+/* trace only β€” once per (row), with the gates the layer will actually apply */
+static void rt_trace(int layer, int row, const int *ids, const float *gates, int k){
+ if(!rt_fp || !ids || !gates) return;
+ fprintf(rt_fp, "%d %d %d", rt_call, row, layer);
+ for(int i = 0; i < k; i++) fprintf(rt_fp, " %d:%.4f", ids[i], gates[i]);
+ fputc('\n', rt_fp);
+}
+/* advance the call counter: once per moe() invocation, after its rows are traced */
+static void rt_trace_end(void){ if(rt_fp) rt_call++; }
+
+/* both, for an engine that does not need them at separate points in its router */
+static void rt_route(int layer, int row, const int *ids, const float *gates, int k){
+ rt_count(layer, ids, k);
+ rt_trace(layer, row, ids, gates, k);
+}
+
+/* A typed path that overrides an identity mismatch says so, loudly and once. A mistyped path
+ * and a deliberate cross-engine experiment are indistinguishable at the call site but not to
+ * the person who typed it, and this line is the only thing separating them. It also travels
+ * with any benchmark pasted into the tracker, so a run whose pins came from another engine
+ * carries its own explanation for looking slow. Requested in #700.
+ *
+ * Only placement is at stake, which is why an override exists here at all and does not for a
+ * mismatched weight container (#529): pins decide which bytes are resident, never what the
+ * model computes. The worst case is slower, self-inflicted, and visible in the hit rate. */
+static void rt_say_override(const char *path, const char *writer){
+ static int said = 0;
+ if(said) return; /* once per process, not once per read */
+ said = 1;
+ fprintf(stderr, "[USAGE] %s: written by %s, this engine is %s\n"
+ " β€” honouring it because you named it explicitly\n"
+ " (placement only; expert ids do not transfer between engines)\n",
+ path, writer, rt_engine);
+}
+
+/* ---- one reader, three layouts. cb() receives every (layer, expert, count) record.
+ * Returns the total read, or -1 if the file was refused (unreadable, or a dimension /
+ * engine mismatch in a file that identifies itself). A file that does not identify
+ * itself is accepted without validation: that is the pre-existing behaviour, and the
+ * reason the header records were added. ---- */
+/* A record callback returns 1 when it accepted the record, and rt_read totals only the
+ * accepted ones. This is not a style choice: both readers this replaces added to their
+ * total INSIDE the same `if` that admitted the record, and their admission tests differ
+ * from each other (usage_load bounds the layer, pin_load bounds it implicitly through its
+ * sparsity test). Totalling every parsed record instead would inflate the
+ * "[USAGE] expert history: N selections" line for any history whose records fall outside
+ * this model's shape β€” which a legacy headerless history from a larger model does, and a
+ * headerless file has no dimension record to be refused by. */
+typedef int (*rt_cb)(int layer, int expert, uint32_t count, void *ud);
+
+/* trusted=1 relaxes the IDENTITY check and nothing else. The identity refusal is the one
+ * that says "pass PIN=<path> to use it anyway", so a path the user typed has to be honoured
+ * or that sentence sends them in a circle; it is also the only way to try one engine's
+ * placement priors on another on purpose. The format version and an IKU1 file's dimensions
+ * are parse geometry rather than identity and are never relaxed β€” see each of them below.
+ *
+ * What earns trust is HOW THE PATH WAS REACHED, not what the file contains. PIN=<file> was
+ * typed by a person; PIN=auto and the AUTOPIN seed are paths the engine found by itself and
+ * nobody vouched for, so they take the full check. The caller has to tell us which, because
+ * one function reads both kinds of path and the file cannot say which it was. */
+static int64_t rt_read_ex(const char *path, rt_cb cb, void *ud, int trusted){
+ FILE *f = fopen(path, "rb"); /* binary: the magic sniff needs raw bytes,
+ * and %d/%u skip \r, so text parses fine */
+ if(!f) return -1;
+ int64_t tot = 0;
+ uint32_t magic = 0;
+ if(fread(&magic, 4, 1, f) == 1 && magic == RT_IKU1_MAGIC){
+ /* no false positive is possible from our own text: it starts with "-1 " */
+ if(strcmp(rt_engine, "inkling") != 0){
+ if(!trusted){
+ fprintf(stderr, "[USAGE] %s: written by inkling, this engine is %s β€” refusing "
+ "(pass PIN=<path> to use it anyway)\n", path, rt_engine);
+ fclose(f); return -1;
+ }
+ rt_say_override(path, "inkling");
+ }
+ uint32_t dim[2];
+ if(fread(dim, 4, 2, f) != 2){ fclose(f); return -1; }
+ /* Not bypassed by trust: here the dimensions ARE the parse geometry, because a
+ * record's layer and expert come from its position in the file rather than from the
+ * record itself. Wrong dimensions mean the bytes are being cut up wrongly, which is
+ * misreading rather than misattributing, and this refusal never promised PIN=. */
+ if((int)dim[0] != rt_nl || (int)dim[1] != rt_ne){
+ fprintf(stderr, "[USAGE] %s: history is %u layers x %u experts, this engine is "
+ "%d x %d β€” refusing rather than misreading it\n",
+ path, dim[0], dim[1], rt_nl, rt_ne);
+ fclose(f); return -1;
+ }
+ uint32_t *rowbuf = (uint32_t*)malloc((size_t)dim[1] * 4);
+ if(!rowbuf){ fclose(f); return -1; }
+ for(uint32_t l = 0; l < dim[0]; l++){
+ if(fread(rowbuf, 4, dim[1], f) != dim[1]) break;
+ for(uint32_t e = 0; e < dim[1]; e++)
+ if(rowbuf[e] && cb((int)l, (int)e, rowbuf[e], ud)) tot += rowbuf[e];
+ }
+ free(rowbuf); fclose(f); return tot;
+ }
+ rewind(f);
+ int l, ver; uint32_t v3;
+ while(fscanf(f, "%d %d %u", &l, &ver, &v3) == 3){
+ if(l == -1){ /* -1 <n_layers> <n_experts> */
+ /* Unconditional, like the version check: a file that declares dimensions this
+ * model does not have is a mistake, not a preference, and this refusal never
+ * offered PIN= as a way past it. Nor is there prior behaviour to keep β€” the old
+ * format carried no dimensions, so no file could previously declare wrong ones.
+ * Only identity, below, bends to trust, because only identity promises PIN=. */
+ if(ver != rt_nl || (int)v3 != rt_ne){
+ fprintf(stderr, "[USAGE] %s: history is %d layers x %u experts, this engine "
+ "is %d x %d β€” refusing rather than misreading it\n",
+ path, ver, v3, rt_nl, rt_ne);
+ fclose(f); return -1;
+ }
+ continue;
+ }
+ if(l == -2){ /* -2 <version> <engine_id> */
+ /* Not bypassed by trust, unlike the identity check below. Trust says whose
+ * history the user accepts, not that this build can parse the bytes: a future
+ * layout read as v1 is misread, which is the failure this format exists to
+ * avoid. There is also no prior behaviour to preserve β€” no v2 file exists. */
+ if(ver > RT_FORMAT_VERSION){
+ fprintf(stderr, "[USAGE] %s: format version %d, this build reads %d β€” "
+ "refusing rather than misreading it\n", path, ver, RT_FORMAT_VERSION);
+ fclose(f); return -1;
+ }
+ if(v3 != rt_id){
+ const char *w = rt_engine_of(v3);
+ if(!trusted){
+ fprintf(stderr, "[USAGE] %s: written by %s, this engine is %s β€” refusing "
+ "(pass PIN=<path> to use it anyway)\n",
+ path, w ? w : "an unknown engine", rt_engine);
+ fclose(f); return -1;
+ }
+ rt_say_override(path, w ? w : "an unknown engine");
+ }
+ continue;
+ }
+ if(l < 0) continue; /* room for future header records */
+ if(cb(l, ver, v3, ud)) tot += v3; /* data row: ver=expert, v3=count */
+ }
+ fclose(f);
+ return tot;
+}
+static int64_t rt_read(const char *path, rt_cb cb, void *ud){
+ return rt_read_ex(path, cb, ud, 0);
+}
+
+/* accumulate a history file into the counters owned here. The admission test is the one
+ * usage_load used: in range, and the layer has a counter row β€” which for an engine that
+ * dropped its dense rows (see rt_drop_row) means the layer actually routes. */
+static int rt_acc_cb(int l, int e, uint32_t c, void *ud){
+ (void)ud;
+ uint32_t *row = rt_counts(l);
+ if(!row || e < 0 || e >= rt_ne) return 0;
+ row[e] += c;
+ return 1;
+}
+static int64_t rt_load(const char *path){
+ int64_t t = rt_read(path, rt_acc_cb, NULL);
+ return t < 0 ? 0 : t; /* callers already treat 0 as "no history" */
+}
+
+/* atomic write of the current counters. quiet=0 prints the one-line summary. */
+static int rt_save(const char *path, int quiet){
+ if(!rt_c || !path || !*path) return 0;
+ int64_t tot = 0, nz = 0;
+ for(int i = 0; i <= rt_nl; i++){
+ if(!rt_c[i]) continue;
+ for(int e = 0; e < rt_ne; e++) if(rt_c[i][e]){ tot += rt_c[i][e]; nz++; }
+ }
+ char tmp[2100];
+ snprintf(tmp, sizeof(tmp), "%s.tmp", path);
+ FILE *f = fopen(tmp, "w");
+ if(!f){ if(!quiet) perror(tmp); return 0; }
+ /* An all-zero history stays a ZERO-BYTE file, the way it was before this header
+ * existed. PIN=auto decides whether a history is usable by testing the file SIZE and
+ * falls back to stats.txt when it is empty, so writing two header lines into an empty
+ * history would silently cost that fallback. */
+ if(nz){
+ fprintf(f, "-1 %d %d\n", rt_nl, rt_ne);
+ fprintf(f, "-2 %d %u\n", RT_FORMAT_VERSION, rt_id);
+ for(int i = 0; i <= rt_nl; i++){
+ if(!rt_c[i]) continue;
+ for(int e = 0; e < rt_ne; e++)
+ if(rt_c[i][e]) fprintf(f, "%d %d %u\n", i, e, rt_c[i][e]);
+ }
+ }
+ fclose(f);
+ if(rename(tmp, path) != 0) return 0;
+ if(!quiet) fprintf(stderr, "[STATS] %lld selections across %lld distinct experts -> %s\n",
+ (long long)tot, (long long)nz, path);
+ return 1;
+}
+
+#endif /* ROUTE_TRACE_H */
diff --git a/c/telemetry.h b/c/telemetry.h
index 074e1625..53725f3f 100644
--- a/c/telemetry.h
+++ b/c/telemetry.h
@@ -180,25 +180,13 @@ static void hits_emit(Model *m){
printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm);
}
-static void stats_dump_q(Model *m, const char *path, int quiet){
- char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path);
- FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; }
- Cfg *c=&m->c; int64_t tot=0, nz=0;
- for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue;
- for(int e=0;e<c->n_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } }
- fclose(f); rename(tmp,path);
- if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path);
-}
+/* The history format lives in route_trace.h so every engine writes the same bytes;
+ * these keep the Model-shaped call sites unchanged. */
+static void stats_dump_q(Model *m, const char *path, int quiet){ (void)m; rt_save(path,quiet); }
static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); }
static char g_usage_path[2100]="";
-static int64_t usage_load(Model *m, const char *path){
- FILE *f=fopen(path,"r"); if(!f) return 0;
- Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0;
- while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3)
- if(l>=0&&l<=c->n_layers&&e>=0&&e<c->n_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; }
- fclose(f); return tot;
-}
+static int64_t usage_load(Model *m, const char *path){ (void)m; return rt_load(path); }
static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); }
#endif /* TELEMETRY_H */
diff --git a/c/tests/test_route_trace.c b/c/tests/test_route_trace.c
new file mode 100644
index 00000000..d1d87fe7
--- /dev/null
+++ b/c/tests/test_route_trace.c
@@ -0,0 +1,305 @@
+/* route_trace.h: the history format, and the property the whole design rests on β€”
+ * that a reader written against the OLD format still recovers every record from a file
+ * written by the new one.
+ *
+ * Case 7 is the important one. The header records are encoded as triples with a negative
+ * layer precisely so that the pre-existing readers (telemetry.h's usage_load and
+ * colibri.c's pin_load, both `while(fscanf(f,"%d %d %u",&l,&e,&c)==3)` guarded by l>=0)
+ * parse them, discard them, and keep going. That contract is invisible in the code and
+ * easy to break by "tidying" the header into a comment line or moving the engine hash
+ * into the second field, so it is asserted here against a literal copy of that loop.
+ *
+ * This test includes route_trace.h and nothing else: no Model, no Cfg, no st.h. That is
+ * the point of the header β€” any engine can use it β€” and compiling this file proves it. */
+#include "../route_trace.h"
+
+static int g_nfails = 0;
+static void check(int cond, const char *what){
+ if(!cond){ printf("FAIL: %s\n", what); g_nfails++; }
+}
+
+#define TMP "test_route_trace.tmp"
+
+static long fsize(const char *p){
+ FILE *f = fopen(p, "rb");
+ if(!f) return -1;
+ fseek(f, 0, SEEK_END);
+ long n = ftell(f);
+ fclose(f);
+ return n;
+}
+static void zero_counts(void){
+ for(int l = 0; l <= rt_nl; l++)
+ if(rt_counts(l)) memset(rt_counts(l), 0, (size_t)rt_ne * sizeof(uint32_t));
+}
+static int64_t g_sum; static int g_recs;
+/* accepts everything, to observe what the reader hands out before any admission rule */
+static int sum_cb(int l, int e, uint32_t c, void *ud){
+ (void)l; (void)e; (void)ud; g_sum += c; g_recs++; return 1;
+}
+
+int main(void){
+ rt_init("glm_moe_dsa", 5, 8); /* 5 layers + the MTP row, 8 experts */
+ check(rt_counts(0) != NULL, "rt_init allocates row 0");
+ check(rt_counts(5) != NULL, "rt_init allocates the inclusive last row");
+ check(rt_counts(6) == NULL, "rt_init does not allocate past n_layers");
+ check(rt_counts(-1) == NULL, "a negative layer has no row");
+
+ /* 1. an all-zero history must stay a ZERO-BYTE file: PIN=auto decides whether a
+ * history is usable by testing the file size, and falls back to stats.txt when it is
+ * empty. Two header lines in an empty history would silently cost that fallback. */
+ remove(TMP);
+ check(rt_save(TMP, 1) == 1, "empty history writes");
+ check(fsize(TMP) == 0, "empty history is a zero-byte file");
+
+ /* 2. with counts, the header appears and the data section is the sparse triples */
+ int ids[3] = {2, 5, 7};
+ rt_count(3, ids, 3);
+ rt_count(3, ids, 1); /* expert 2 twice */
+ check(rt_save(TMP, 1) == 1, "history with counts writes");
+ {
+ char buf[512] = {0};
+ /* text mode on purpose: the writer uses fopen("w"), so on Windows the records are
+ * CRLF-terminated (as they have always been β€” stats_dump_q wrote them the same
+ * way). Reading back in text mode collapses that, which keeps these assertions
+ * about the RECORDS rather than about the platform's line terminator. */
+ FILE *f = fopen(TMP, "r");
+ size_t n = f ? fread(buf, 1, sizeof(buf) - 1, f) : 0;
+ if(f) fclose(f);
+ check(n > 0, "history is non-empty");
+ check(strstr(buf, "-1 5 8\n") == buf, "first record is the dimensions");
+ check(strstr(buf, "\n-2 1 ") != NULL, "second record is version then engine id");
+ check(strstr(buf, "\n3 2 2\n") != NULL, "expert 2 counted twice");
+ check(strstr(buf, "\n3 5 1\n") != NULL, "expert 5 counted once");
+ check(strstr(buf, "\n3 7 1\n") != NULL, "expert 7 counted once");
+ }
+
+ /* 3. round trip through the reader */
+ zero_counts();
+ check(rt_load(TMP) == 4, "round trip recovers every selection");
+ check(rt_counts(3)[2] == 2, "round trip recovers the per-expert count");
+
+ /* 4. a legacy file β€” same triples, no header β€” is accepted as before */
+ {
+ FILE *f = fopen(TMP, "w");
+ fprintf(f, "3 2 2\n3 5 1\n3 7 1\n");
+ fclose(f);
+ zero_counts();
+ check(rt_load(TMP) == 4, "legacy text without a header still loads");
+ }
+
+ /* 4b. a history written on Windows arrives with CRLF endings. The reader opens "rb"
+ * (the magic sniff needs raw bytes), so the \r survives into fscanf β€” which skips it
+ * as whitespace. Asserted rather than assumed, and this runs on the MinGW CI job too. */
+ {
+ FILE *f = fopen(TMP, "wb");
+ fprintf(f, "-1 5 8\r\n-2 1 %u\r\n3 2 2\r\n3 5 1\r\n", rt_hash("glm_moe_dsa"));
+ fclose(f);
+ zero_counts();
+ check(rt_load(TMP) == 3, "a CRLF history loads (Windows-written file)");
+ check(rt_counts(3)[2] == 2, "CRLF: per-expert count intact");
+ }
+
+ /* The cases below exercise the refusal paths, so they print eight [USAGE] lines to stderr
+ * on a PASSING run: seven refusals (5, 6, 7, two in 10, two in 10b) and one three-line
+ * override notice in 10. Say so, or a CI log looks like something went wrong. */
+ printf("route_trace: the [USAGE] refusal and override lines below are expected\n");
+
+ /* 5. a file that identifies itself with the wrong dimensions is refused, not misread */
+ {
+ FILE *f = fopen(TMP, "w");
+ fprintf(f, "-1 99 512\n3 2 2\n");
+ fclose(f);
+ g_sum = 0; g_recs = 0;
+ check(rt_read(TMP, sum_cb, NULL) == -1, "wrong dimensions are refused");
+ check(g_recs == 0, "a refused file yields no records");
+ }
+
+ /* 6. a file from another engine is refused, and the message can name it */
+ {
+ FILE *f = fopen(TMP, "w");
+ fprintf(f, "-1 5 8\n-2 1 %u\n3 2 2\n", rt_hash("kimi_k3"));
+ fclose(f);
+ check(rt_read(TMP, sum_cb, NULL) == -1, "another engine's history is refused");
+ check(rt_engine_of(rt_hash("kimi_k3")) != NULL, "the writer can be named");
+ check(strcmp(rt_engine_of(rt_hash("kimi_k3")), "kimi_k3") == 0, "named correctly");
+ check(rt_engine_of(12345u) == NULL, "an unknown id has no name");
+ }
+
+ /* 7. inkling's IKU1 layout is refused by any engine that is not inkling */
+ {
+ FILE *f = fopen(TMP, "wb");
+ uint32_t hdr[3] = { RT_IKU1_MAGIC, 5, 8 };
+ uint32_t body[5 * 8];
+ memset(body, 0, sizeof(body));
+ fwrite(hdr, 4, 3, f);
+ fwrite(body, 4, 5 * 8, f);
+ fclose(f);
+ check(rt_read(TMP, sum_cb, NULL) == -1, "IKU1 is refused by a non-inkling engine");
+ }
+
+ /* 7b. …and inkling itself still reads it. Same reader, engine identity flipped: without
+ * this the binary path is only ever exercised on its refusal, so a broken IKU1 decode
+ * would look like a pass. Same translation unit, so the identity is directly settable. */
+ {
+ const char *save_engine = rt_engine;
+ uint32_t save_id = rt_id;
+ rt_engine = "inkling"; rt_id = rt_hash("inkling");
+
+ FILE *f = fopen(TMP, "wb");
+ uint32_t hdr[3] = { RT_IKU1_MAGIC, 5, 8 }; /* must match rt_nl / rt_ne */
+ uint32_t body[5 * 8];
+ memset(body, 0, sizeof(body));
+ body[3 * 8 + 2] = 7; /* layer 3, expert 2 -> 7 */
+ body[1 * 8 + 5] = 4; /* layer 1, expert 5 -> 4 */
+ fwrite(hdr, 4, 3, f);
+ fwrite(body, 4, 5 * 8, f);
+ fclose(f);
+
+ zero_counts();
+ check(rt_load(TMP) == 11, "inkling accepts its own IKU1 layout");
+ check(rt_counts(3)[2] == 7, "IKU1 decode places layer 3 / expert 2");
+ check(rt_counts(1)[5] == 4, "IKU1 decode places layer 1 / expert 5");
+ check(rt_counts(0)[0] == 0, "IKU1 decode leaves the zeros alone");
+
+ rt_engine = save_engine; rt_id = save_id;
+ }
+
+ /* 8. THE CONTRACT: a reader written against the old format must recover exactly the
+ * same records from a new-format file. This is a literal copy of the loop in
+ * telemetry.h's usage_load and colibri.c's pin_load. */
+ {
+ zero_counts();
+ rt_count(3, ids, 3);
+ rt_count(3, ids, 1);
+ rt_count(5, ids, 2);
+ check(rt_save(TMP, 1) == 1, "contract fixture writes");
+
+ int64_t legacy_tot = 0; int legacy_recs = 0;
+ FILE *f = fopen(TMP, "r"); /* exactly as the old readers open it */
+ int l, e; uint32_t cnt;
+ while(f && fscanf(f, "%d %d %u", &l, &e, &cnt) == 3)
+ if(l >= 0 && l <= 5 && e >= 0 && e < 8){ legacy_tot += cnt; legacy_recs++; }
+ if(f) fclose(f);
+
+ g_sum = 0; g_recs = 0;
+ rt_read(TMP, sum_cb, NULL);
+ check(legacy_tot == g_sum, "an old reader recovers the same total");
+ check(legacy_recs == g_recs, "an old reader recovers the same record count");
+ check(legacy_tot == 6, "and that total is the one that was written");
+ }
+
+ /* 9. the total counts ACCEPTED records only. Both readers this replaces added to their
+ * total inside the same `if` that admitted the record, so a history holding records
+ * outside this model's shape reported the sum of the usable ones. A legacy headerless
+ * history from a larger model is exactly that, and it carries no dimension record to be
+ * refused by β€” so getting this wrong inflates the count the user is shown. */
+ {
+ FILE *f = fopen(TMP, "w");
+ fprintf(f, "3 2 5\n9 1 100\n1 99 200\n"); /* valid, layer OOR, expert OOR */
+ fclose(f);
+ zero_counts();
+ check(rt_load(TMP) == 5, "out-of-range records are not counted in the total");
+ check(rt_counts(3)[2] == 5, "the in-range record still lands");
+ check(rt_counts(1)[7] == 0, "an out-of-range expert wrote nothing");
+ }
+
+ /* 10. THE ESCAPE HATCH. Every refusal message says "pass PIN=<path> to use it anyway",
+ * and pin_load reads through here, so a trusted read must not re-apply the check the
+ * message is telling the user to get around β€” otherwise that advice cannot work. The
+ * old pin_load had no identity or dimension check at all, so this also keeps it. */
+ {
+ FILE *f = fopen(TMP, "w");
+ fprintf(f, "-1 5 8\n-2 1 %u\n3 2 6\n", rt_hash("kimi_k3"));
+ fclose(f);
+ check(rt_read(TMP, sum_cb, NULL) == -1, "an untrusted read refuses another engine");
+ g_sum = 0; g_recs = 0;
+ check(rt_read_ex(TMP, sum_cb, NULL, 1) == 6, "a trusted read takes it anyway");
+ check(g_recs == 1, "trusted: the data record reached the callback");
+ /* the override announces itself once per process (#700), which must not turn a
+ * repeated trusted read into a different answer β€” the guard is on the message only */
+ g_sum = 0; g_recs = 0;
+ check(rt_read_ex(TMP, sum_cb, NULL, 1) == 6, "a second trusted read still honours it");
+ check(g_recs == 1, "the once-only notice does not gate the read");
+
+ /* but only identity bends to trust. Wrong dimensions are a mistake rather than a
+ * preference, and that refusal never offered PIN= as the way past it. */
+ f = fopen(TMP, "w");
+ fprintf(f, "-1 99 512\n-2 1 %u\n3 2 6\n", rt_hash("glm_moe_dsa"));
+ fclose(f);
+ g_sum = 0; g_recs = 0;
+ check(rt_read_ex(TMP, sum_cb, NULL, 1) == -1, "trust does not excuse wrong dimensions");
+ check(g_recs == 0, "no record from a wrong-dimension file reached the callback");
+ }
+
+ /* 10b. …but trust does not extend to the parse geometry. A format this build cannot read
+ * is refused even on a trusted read: trust says whose history the user accepts, not that
+ * the bytes can be cut up correctly, and a future layout read as v1 is misread silently.
+ * Nothing to preserve here either β€” the refusal for a version never promised PIN=. */
+ {
+ FILE *f = fopen(TMP, "w");
+ fprintf(f, "-1 5 8\n-2 %d %u\n3 2 9\n", RT_FORMAT_VERSION + 1, rt_hash("glm_moe_dsa"));
+ fclose(f);
+ g_sum = 0; g_recs = 0;
+ check(rt_read(TMP, sum_cb, NULL) == -1, "a future format version is refused");
+ check(rt_read_ex(TMP, sum_cb, NULL, 1) == -1, "and a trusted read cannot override it");
+ check(g_recs == 0, "no record from a future version reached the callback");
+ }
+
+ /* 11. every record the writer emits has EXACTLY three fields. Not "at least three": a
+ * fourth field leaves one value unconsumed, which desynchronises every old reader and
+ * makes it fabricate a record out of that leftover plus the start of the next line. The
+ * l>=0 guard does not save anyone, because the invented record has a positive layer β€” so
+ * the count lands on a layer that was never routed and nothing reports it. The writer is
+ * the only place this can be enforced, so it is asserted here. */
+ {
+ zero_counts();
+ rt_count(3, ids, 3);
+ rt_count(1, ids, 2);
+ check(rt_save(TMP, 1) == 1, "three-field fixture writes");
+ FILE *f = fopen(TMP, "r");
+ char line[256]; int bad = 0, lines = 0;
+ while(f && fgets(line, sizeof(line), f)){
+ int n = 0; const char *p = line;
+ for(;;){
+ while(*p == ' ' || *p == '\t') p++;
+ if(*p == '\0' || *p == '\n' || *p == '\r') break;
+ n++;
+ while(*p && *p != ' ' && *p != '\t' && *p != '\n' && *p != '\r') p++;
+ }
+ if(n){ lines++; if(n != 3) bad++; }
+ }
+ if(f) fclose(f);
+ check(lines > 0, "the fixture produced records");
+ check(bad == 0, "every record the writer emits has exactly three fields");
+ }
+
+ /* 12. a dropped row is the load admission rule, not an optimisation: a layer that does
+ * not route must not absorb a count. This is what the replaced code said by leaving
+ * eusage[i] NULL on dense layers, and it runs last because the drop is permanent. */
+ {
+ rt_drop_row(2);
+ check(rt_counts(2) == NULL, "rt_drop_row clears the row");
+ FILE *f = fopen(TMP, "w");
+ fprintf(f, "2 3 50\n3 4 7\n");
+ fclose(f);
+ zero_counts();
+ check(rt_load(TMP) == 7, "a record for a non-routing layer is not counted");
+ check(rt_counts(3)[4] == 7, "the routing layer's record still lands");
+ check(rt_save(TMP, 1) == 1, "saving after a drop works");
+ {
+ char buf[512] = {0};
+ FILE *g = fopen(TMP, "r");
+ size_t n = g ? fread(buf, 1, sizeof(buf) - 1, g) : 0;
+ if(g) fclose(g);
+ check(n > 0, "post-drop history is non-empty");
+ check(strstr(buf, "\n2 3 ") == NULL, "the dropped layer is not written back");
+ }
+ }
+
+ remove(TMP);
+ if(g_nfails){ printf("route_trace: %d FAILED\n", g_nfails); return 1; }
+ printf("route_trace: empty-history size, round trip, legacy read, refusals, "
+ "admitted-only totals, trusted read, dropped rows, old-reader contract ok\n");
+ return 0;
+}
diff --git a/docs/routing-telemetry.md b/docs/routing-telemetry.md
new file mode 100644
index 00000000..ccefad49
--- /dev/null
+++ b/docs/routing-telemetry.md
@@ -0,0 +1,195 @@
+# Routing telemetry: the expert history and the trace stream
+
+*The canonical reference for `.coli_usage` and `ROUTE_TRACE=`, and for `route_trace.h`,
+the header both are implemented in. Background and design discussion in
+[#700](https://github.com/JustVugg/colibri/issues/700).*
+
+## Why one header
+
+colibrì is four engines — `colibri.c` (GLM-5.2), `kimi_k3.c`, `inkling.c`, `olmoe.c` —
+and the learning cache described in [tuning.md](tuning.md) is only as good as the history
+it reads. Before `route_trace.h`, that history was per-engine: `colibri.c` wrote sparse
+text, `inkling.c` wrote a dense binary block with an `IKU1` magic, and the other two wrote
+nothing at all, so `PIN=auto` had nothing to read on them. Both writers defaulted to the
+same filename, so a model directory shared between two engines ended up with one history
+in a format the other refuses.
+
+`route_trace.h` holds the format and nothing else. It depends on the C library plus
+[`compat.h`](../c/compat.h) β€” no `Model`, no `Cfg`, no `st.h` β€” because an engine already
+knows its layer index, the expert ids it selected and their gates, and that is the entire
+input. `compat.h` is not optional and not incidental: on Windows the CRT `rename()` fails
+when the destination exists, so without its shim the history stops updating after the first
+write, silently. Any engine, present or future, gets the history and the trace stream from a
+handful of calls β€” `kimi_k3.c` needs five.
+
+## The history file (`.coli_usage`, `stats.txt`, `PIN=<file>`)
+
+Text, one record per line, sparse β€” only experts with a non-zero count appear:
+
+```
+-1 <n_layers> <n_experts>
+-2 <format_version> <engine_id>
+<layer> <expert> <count>
+<layer> <expert> <count>
+...
+```
+
+The two leading records carry the dimensions and the writing engine's identity so a
+history from another model is **refused by name** rather than misread:
+
+```
+[USAGE] /models/glm/.coli_usage: written by kimi_k3, this engine is glm_moe_dsa
+ β€” refusing (pass PIN=<path> to use it anyway)
+```
+
+### Why the header records look like that
+
+The layer field is negative on purpose, and the field order is not arbitrary. Every reader
+written against the original format is
+
+```c
+while(fscanf(f, "%d %d %u", &l, &e, &cnt) == 3)
+ if(l >= 0 && ...) /* out-of-range records are discarded */
+```
+
+so a record with a negative layer is parsed, dropped, and the loop continues into the
+data. That is what lets a history written today still load in a binary built before the
+header existed β€” which matters, because a `.coli_usage` accumulated over weeks *is* the
+value of `PIN=auto`.
+
+Two consequences worth knowing before editing the format:
+
+- **No field may be non-numeric.** A string makes `fscanf` return < 3, which ends the loop
+ and silently drops every record after it. That is why the engine is identified by a hash
+ rather than by name, with the names kept in `route_trace.h` so a mismatch can still be
+ reported in words.
+- **The hash must stay in the third field.** Old readers parse the second field with `%d`,
+ and a 32-bit hash routinely exceeds `INT_MAX` β€” undefined behaviour there. The version
+ (small) goes second, the hash third. `glm_moe_dsa` hashes to `3815245270`, which is above
+ `INT_MAX`, so this is not hypothetical.
+
+`tests/test_route_trace.c` asserts the contract against a literal copy of the old reader
+loop, so breaking either rule fails CI rather than corrupting a user's history.
+
+### Adding a header record later
+
+The negative layer is a general mechanism, not two special cases. Every reader β€” including
+the two that predate this header, still shipped in older binaries β€” advances through a record
+whose first field is negative and ignores it, because each is a
+`while(fscanf(f,"%d %d %u",...)==3)` guarded by `l>=0`. So a new record type can be added at
+any time and older builds skip it instead of breaking, as long as it obeys the two rules above
+and one more: **exactly three numeric fields, never more.**
+
+That third rule is absolute and the failure if it is broken is silent. A four-field record does
+not get skipped β€” it leaves a field unconsumed, desynchronising the reader, which then
+fabricates a record from that leftover plus the start of the next line:
+
+```
+file: 3 2 5 | -3 7 12 99 | 4 1 8
+an old reader sees: 3 2 5 admitted (correct)
+ -3 7 12 skipped (correct β€” the l>=0 guard works)
+ 99 4 1 ADMITTED <- invented: leftover 99, then two fields
+ of the NEXT line
+```
+
+The real `4 1 8` is consumed into the garbage and a count lands on layer 99. Nothing reports
+it. Measured against a literal copy of the old loop, not inferred from reading it.
+
+What is *not* defined yet is any specific record beyond `-1` and `-2`. If a future policy needs
+data the per-expert counts cannot express β€” a per-*transition* count for an order-1 model over
+`(layer, expert)` pairs, for instance β€” its record number and field meanings should be agreed
+before it is written, so the format grows once rather than twice. Three numeric fields is the
+whole constraint; everything else is open.
+
+### An empty history is a zero-byte file
+
+`PIN=auto` decides whether a history is usable by testing the file size, falling back to
+`stats.txt` when it is empty. A run that routed nothing therefore writes no header at all,
+keeping that fallback intact.
+
+### Older layouts that are still read
+
+- **Legacy text**: the same triples with no header records. Accepted as-is β€” it cannot be
+ validated, which is precisely why the header was added.
+- **`IKU1`**: `inkling`'s dense binary layout (`uint32 {magic, n_layers, n_experts}` then
+ `uint32[n_layers][n_experts]`). Read by `inkling` itself; any other engine refuses it by
+ name rather than loading another model's ranking.
+
+## The trace stream (`ROUTE_TRACE=<path>`)
+
+One line per (moe call, batch row), written as the layer routes:
+
+```
+<call> <row> <layer> <expert>:<gate> <expert>:<gate> ...
+```
+
+`call` increments once per `moe()` invocation, `row` is the position within the forward
+batch, and the gates are the values the layer actually applies (post-normalisation). This
+is the input to `tools/route_pairs.py`, which builds the `.coli_pairs` table used by
+`COUPLE=` cross-layer prefetch. Measurement only: enabling it never changes what the model
+computes, and it does disable the device-side router (which would otherwise bypass the CPU
+ranking the trace records).
+
+## Using it from a new engine
+
+```c
+#include "route_trace.h"
+
+rt_init("my_engine", n_layers, n_experts); /* counters + identity + ROUTE_TRACE */
+...
+for(i) if(!layer_routes(i)) rt_drop_row(i); /* once sparsity is known β€” see below */
+...
+rt_route(layer, row, ids, gates, k); /* per routing decision: traces and counts */
+...
+rt_save(usage_path, 1); /* where the engine already persists */
+```
+
+### A layer with no counter row is a layer that cannot be credited
+
+`rt_init` allocates a row per layer because it is told dimensions, not shape. An engine
+usually learns which layers actually route later, while it builds them, and must then call
+`rt_drop_row(layer)` for the ones that do not β€” dense layers, and the MTP row on a model
+without MTP.
+
+This is the load admission rule, not housekeeping. Every reader treats a NULL row as "there
+is no expert here to attribute a count to". Skip the drop and a history record naming a
+dense layer is silently accumulated, added to the reported total, and written back out on
+the next save β€” a record no engine would ever emit. `colibri.c` drops its rows at the end
+of `model_init`, which is the first point where `L[i].sparse` and `has_mtp` are both final
+and still before any history is read.
+
+### Trusted reads: trust follows the user, not the file
+
+`rt_read(path, cb, ud)` applies every check. `rt_read_ex(path, cb, ud, 1)` marks the read
+trusted, which relaxes exactly one of them. The checks fall into two kinds and only the
+first kind bends:
+
+- **Identity** β€” *whose* history this is. The refusal says *"pass `PIN=<path>` to use it
+ anyway"*, so a trusted read must honour the file, or that sentence sends the user in a
+ circle. It is also the only way to try one engine's placement priors on another, which is
+ a thing worth being able to try on purpose.
+- **Parse geometry** β€” the format version, and the dimensions of an `IKU1` file, where a
+ record's layer and expert come from its *position* rather than from the record. Never
+ relaxed. Trust says which history the user accepts, not that this build can cut the bytes
+ up correctly, and a layout read as the wrong version is misread silently. Declared text
+ dimensions that do not match are refused for the same reason, and neither refusal ever
+ offered a way past it.
+
+What makes a read trusted is **how the path was reached, not what the file contains**.
+`PIN=<file>` was typed by the user, so it is trusted. `PIN=auto` and the `AUTOPIN` history
+are paths the engine found on its own β€” nobody vouched for those, so they get the full
+check. `colibri.c` passes that distinction into `pin_load` as an argument for this reason:
+the same function reads both kinds of path, and the file cannot tell you which it was.
+
+The callback's own bounds always apply, so no read of any kind can write outside the
+counters.
+
+An engine whose router needs the two at different points (`colibri.c` counts before gate
+normalisation and traces after) can call `rt_count()` and `rt_trace()` separately, with
+`rt_trace_end()` once per `moe()` call to advance the call counter. `rt_load(path)` reads a
+history back into the counters, and `rt_read(path, cb, ud)` streams records to a callback for
+consumers that rank rather than accumulate. A callback returns 1 when it accepts a record,
+and `rt_read` totals only accepted ones β€” the readers this replaced counted inside the same
+test that admitted, and a history from a differently-shaped model otherwise inflates the
+number reported to the user. Add the engine's name to `rt_engine_names[]` so mismatches
+involving it can be named.
diff --git a/docs/tuning.md b/docs/tuning.md
index c4a2fb29..bd20341d 100644
--- a/docs/tuning.md
+++ b/docs/tuning.md
@@ -117,6 +117,11 @@ decaying session heat map replaces cold pinned experts with hotter streamed
experts. A 25% hysteresis and a four-swap limit prevent tier thrashing.
Persistent `.coli_usage` remains the long-term signal and is not decayed.
+The history's on-disk format, what happens when one engine is handed another
+engine's history, and how `PIN=<file>` differs from `PIN=auto` in how much it
+trusts a file are documented in
+[routing-telemetry.md](routing-telemetry.md).
+
## Router-lookahead prefetch (`PILOT=1`, experimental)
GLM-5.2's expert routing is measurably predictable *ahead of time* β€” applying