Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen36, olmoe: reuse the previous turn's KV instead of re-prefilling the transcript
JustVugg16bfe58b52fd
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 114e9a0e..d511d5d2 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -472,6 +472,17 @@ jobs:echo "FAIL: sanitizer diagnostic under PILOT"; cat san.log; exit 1fiecho "sanitizers clean:"; grep -E "Matching tokens" san.log || true+ - name: KV prefix reuse is token-identical (serve, several turns)+ run: |+ cd c+ # Reusing a previous turn's state must change the timing and nothing+ # else. A wrong reuse length does not crash: it answers from another+ # conversation's state and still reads plausibly, so this is the only+ # gate that would catch it. The sanitizer step above rebuilt the+ # engine, so rebuild the normal one first.+ make clean >/dev/null 2>&1 || true+ make qwen36+ QWEN36_TINY=qwen36_tiny_c python3 -m unittest -v tests.test_qwen36_prefix_serve- name: Expert kernel A/B on an int4 gs64 fixture (old path vs expert_ffn.h, byte-identical ids)run: |cd c@@ -594,6 +605,14 @@ jobs:run: |cd cCOLI_OLMOE_FIXTURE=olmoe_tiny_c python3 -m unittest -v tests.test_olmoe_context_exceeded+ - name: KV prefix reuse is token-identical (serve, several turns)+ run: |+ cd c+ # Same contract as the Qwen3.6 job runs, from the same harness: a+ # reused prefix must produce exactly what a cold engine produces for+ # the same prompt. Wrong reuse is silent, so the gate is the only+ # thing between it and a user.+ OLMOE_TINY=olmoe_tiny_c python3 -m unittest -v tests.test_olmoe_prefix_servedsv41-tiny-check:name: DeepSeek V4.1 tiny oracle (engram + DSA + fp4 experts + vision)@@ -1136,7 +1155,7 @@ jobs:# rieseguiva un binario non strumentato. Uno di quei test era rosso da# settimane senza che nessuno potesse vederlo.- name: Build the engines those tests exercise- run: make -C c kimi_k3 inkling colibri olmoe qwen38+ run: make -C c kimi_k3 inkling colibri olmoe qwen38 qwen36- name: Python test suiterun: cd c && python3 -m unittest discover -s tests -p 'test_*.py'diff --git a/c/Makefile b/c/Makefileindex 2e66298a..0bf732f0 100644--- a/c/Makefile+++ b/c/Makefile@@ -1046,7 +1046,7 @@ fp8-bench: fp8_bench$(EXE)# the GPU sits idle. Same guard #783 put on kimi_k3, which since gaining an# MXFP4 expert path no longer needs it. tests/test_makefile_cuda_scope.py# asserts this shape.-olmoe$(EXE): olmoe.c cli_args.h st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h serve_codec.h edge_adapters.h edge_runtime.h edge_tok_internal.h fused_simd.h segment_adapter_internal.h segment_adapters.h segment_runtime.h+olmoe$(EXE): olmoe.c cli_args.h st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h kv_prefix.h route_trace.h serve_codec.h edge_adapters.h edge_runtime.h edge_tok_internal.h fused_simd.h segment_adapter_internal.h segment_adapters.h segment_runtime.h$(CC) $(NOCUDA_CFLAGS) olmoe.c -o olmoe$(EXE) $(NOCUDA_LDFLAGS)# Qwen3.6-35B-A3B engine (hybrid Gated Attention + Gated DeltaNet + streaming@@ -1072,7 +1072,7 @@ QWEN36_TIER_SRC =QWEN36_CFLAGS = $(NOCUDA_CFLAGS)QWEN36_LDFLAGS = $(NOCUDA_LDFLAGS)endif-qwen36$(EXE): qwen36.c serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h st.h json.h compat.h omp_tune.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+qwen36$(EXE): qwen36.c serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h st.h json.h compat.h omp_tune.h kv_prefix.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)# DeepSeek V4.1 Flash: one file, like every other portable engine. The fp4 expertsdiff --git a/c/olmoe.c b/c/olmoe.cindex 06360d79..0270f0de 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -38,6 +38,7 @@#endif#include "omp_tune.h"#include "route_trace.h" /* shared routing telemetry (#700) */+#include "kv_prefix.h" /* riuso del prefisso tra turni (shared) */#include "serve_codec.h"#ifdef COLI_SEGMENT_ADAPTER#include "segment_runtime.h"@@ -102,6 +103,10 @@ typedef struct {* plain double: a per-turn delta of it is what the PROF line reports. */uint64_t disk_ns;float **K, **V; int kv_len, max_t;+ /* What the cached keys and values were built from, so a serve turn that+ * resends the transcript prefills only the new tail. Recorded where the+ * tokens are fed (see kv_prefix.h), never derived from a counter. */+ kv_prefix kvp;double dense_load_s;/* IMPROVEMENT 2: expert frequency heatmap */uint32_t **freq; /* per-layer expert counts, owned by route_trace.h */@@ -1009,6 +1014,10 @@ static float *step(Model *m, const int *ids, int S, int pos_base) {if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens)pin_hot_experts(m);m->kv_len = pos_base + S;+ /* Recorded HERE, where the tokens entered the cache: fed[0..len-1] are+ * the ids those positions were built from, and that invariant is the+ * whole safety argument for reusing them next turn. */+ kv_prefix_record(&m->kvp, ids, pos_base, S);float *last = falloc(D);rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps);float *logit = falloc(c->vocab);@@ -1462,6 +1471,10 @@ static void serve_hits(Model *m) {fflush(stdout); free(hex); free(bm);}+/* COLI_KV_PREFIX=0: never reuse a previous turn's cache. The escape hatch,+ * and the B arm of the A/B that shows reuse changes nothing but the time. */+static int kv_prefix_off(void){ const char *e = getenv("COLI_KV_PREFIX"); return e && *e == '0'; }+static int serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {Cfg *c = &m->c;int cap = q->plen + 16;@@ -1478,12 +1491,38 @@ static int serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {coli_serve_write_error(stdout, q->id, message); free(ids); return 0;}g_temp = q->temp; g_nuc = q->top_p;+ /* A chat client resends the whole transcript every turn. If this prompt+ * begins with the ids the cache was built from, those keys and values ARE+ * the state at those positions -- attention is causal, so a row depends on+ * its prefix and nothing later. Prefill only the tail. Reuse is all or+ * nothing: nothing here can rewind a cache. COLI_KV_PREFIX=0 turns it off,+ * COLI_PREFIX_LOG=1 reports the decision and its reason, because "it did+ * not get faster" is otherwise indistinguishable from "it is not wired up".+ *+ * On a miss the record must be CLEARED and not merely overwritten: a+ * shorter prompt writes fewer positions than the last one recorded, and+ * kv_prefix_record only ever grows the length, so the stale tail would+ * claim coverage the cache no longer has. */+ int reuse = kv_prefix_off() ? 0 : kv_prefix_reuse(&m->kvp, ids, np);+ if (!reuse) kv_prefix_clear(&m->kvp);+ if (getenv("COLI_PREFIX_LOG")) {+ if (reuse)+ fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",+ reuse, np, 100.0 * reuse / np);+ else+ fprintf(stderr, "[PREFIX] no reuse: held=%d cap=%d prompt=%d\n",+ m->kvp.len, m->kvp.cap, np);+ fflush(stderr);+ }double t0 = now_s();uint64_t h0 = m->hits, m0 = m->miss;uint64_t disk0 = __atomic_load_n(&m->disk_ns, __ATOMIC_RELAXED);double attn0 = g_prof_attn_s, moe0 = g_prof_moe_s, head0 = g_prof_head_s;long long fwd0 = g_prof_forwards;- float *logit = step(m, ids, np, 0);+ /* `reuse` is the ABSOLUTE position of the first fresh token: attention+ * and the KV rows are position-indexed, so this has to be the real+ * offset, not a count of what is left to do. */+ float *logit = step(m, ids + reuse, np - reuse, reuse);int hist_len = np, gen = 0, limited = 1, cancelled = 0;char buf[512];for (int s = 0; s < q->max_tok && !cancelled; s++) {@@ -1663,6 +1702,12 @@ int main(int argc, char **argv) {m.K[i] = falloc((int64_t)m.c.n_heads * m.max_t * m.c.head_dim);m.V[i] = falloc((int64_t)m.c.n_heads * m.max_t * m.c.head_dim);}+ /* Serve mode only: the record is sized with the cache it describes, and+ * the cache here is allocated once for ctx_cap and never reallocated,+ * so a recorded position stays valid for the life of the process. The+ * CLI paths (generate/run_chat/PPL) leave it unallocated, which makes+ * every kv_prefix call there a no-op. */+ kv_prefix_alloc(&m.kvp, m.max_t);Tok T;char tokpath[2048]; snprintf(tokpath, sizeof(tokpath), "%s/tokenizer.json", snap);tok_load(&T, tokpath);diff --git a/c/qwen36.c b/c/qwen36.cindex 3f793268..11794867 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -61,6 +61,7 @@ static int qwen36_max_ctx(void) {#include "cli_args.h"#include "st.h"#include "omp_tune.h"+#include "kv_prefix.h" /* riuso del prefisso tra turni (shared) */#include "json.h" /* tokenizer.json parsing (reuse minimal parser) */#include "qwen36_tier.h" /* optional CUDA VRAM expert tier */#include "expert_ffn.h" /* routed experts: planar int4 kernel + layer runner */@@ -643,6 +644,11 @@ typedef struct {double t_disk;uint8_t **ehit;float **K, **V; int kv_len, max_t, kv_cap;+ /* What the current attention and DeltaNet state was built from, so a+ * turn that resends the transcript prefills only the new tail. The ids+ * are recorded where they are fed, never derived from a counter: see+ * kv_prefix.h for why that distinction is the whole safety argument. */+ kv_prefix kvp;float *attn_sc; /* [attn_sc_thr * kv_cap] score rows, one per thread */int attn_sc_thr;double dense_load_s;@@ -1073,6 +1079,10 @@ static int g_qdw_n = 0;static uint64_t g_qwen_matmul_d_calls;#endifstatic int dense_i8_on(void){ static int v=-1; if(v<0){ const char *e=getenv("COLI_DENSE_I8"); v=!(e&&*e=='0'); } return v; }+/* COLI_KV_PREFIX=0: never reuse a previous turn's state. Kept as an escape+ * hatch and as the B arm of the A/B that shows reuse changes nothing but+ * the time. */+static int kv_prefix_off(void){ const char *e=getenv("COLI_KV_PREFIX"); return e && *e=='0'; }static int dense_batch_on(void){ const char *e=getenv("QWEN_DENSE_BATCH"); return !(e&&*e=='0'); }static void qdw_register(const float *W, int I, int O){if (!W || !dense_i8_on() || g_qdw_n >= QDW_MAX) return;@@ -2413,6 +2423,10 @@ static float *step(Model *m, const int *ids, int S, int pos_base) {memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float));}layers_forward_range(m, x, S, pos_base, 0, c->n_layers, 1, lf);+ /* Recorded HERE, where the tokens actually entered the state, rather than+ * derived from the caller's bookkeeping: the invariant that fed[0..len-1]+ * are the ids the state was built from is the whole safety argument. */+ kv_prefix_record(&m->kvp, ids, pos_base, S);m->token_count += S; m->freq_token_count += S;if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) pin_hot_experts(m);m->kv_len = pos_base + S;@@ -2552,6 +2566,10 @@ static float *g_last_logit = NULL;* generation (the CLI runs once, so this is also correct there). */static void reset_recurrent(Model *m){Cfg *c = &m->c;+ /* Paired with the record on purpose: whoever zeroes the state must also+ * forget what it was built from, or the two disagree in favour of the one+ * nobody can check. */+ kv_prefix_clear(&m->kvp);for (int i = 0; i < c->n_layers; i++){if (c->is_attn[i]) continue;if (m->DN_rec[i]) memset(m->DN_rec[i], 0, (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim * sizeof(float));@@ -2564,18 +2582,49 @@ static void reset_recurrent(Model *m){* growth so the server doesn't leak KV memory across requests. */static void ensure_kv(Model *m){Cfg *c = &m->c;- if (m->kv_cap >= m->max_t && m->K) return;- if (m->K){- for (int i = 0; i < c->n_layers; i++){ if (m->K[i]) free(m->K[i]); if (m->V[i]) free(m->V[i]); }- free(m->K); free(m->V); m->K = NULL; m->V = NULL;+ if (m->kv_cap >= m->max_t && m->K) {+ /* max_t is the ROW STRIDE of the KV cache, not just a capacity: a row+ * lives at (head*max_t + position)*head_dim. Callers set it per request+ * from prompt+max_tok, so a shorter request used to shrink the stride+ * while the allocation stayed the same size -- harmless only as long as+ * every turn rewrote every row from position 0. Reuse reads rows an+ * earlier turn wrote, so the stride has to stay the one they were+ * written with: the allocation's, which is kv_cap. */+ m->max_t = m->kv_cap;+ return;}+ /* Growth COPIES the rows instead of discarding them. A chat resends a+ * longer transcript every turn, so this reallocation lands on exactly the+ * turn that wants to reuse the previous one's state: freeing the rows here+ * would make kv_prefix_reuse miss in the one case it exists for. The copy+ * is a memcpy at DRAM speed; the prefill it saves is seconds of expert+ * reads on a streaming engine. The row stride IS max_t, so the copy must be+ * per head -- one flat memcpy would land every head but the first at the+ * wrong offset, and that reads as a plausible answer from another+ * conversation rather than as a crash. */+ float **oldK = m->K, **oldV = m->V;+ int old_stride = m->kv_cap, keep = m->K ? m->kvp.len : 0;+ if (keep > old_stride) keep = old_stride;+ if (keep > m->max_t) keep = m->max_t;m->K = calloc((size_t)c->n_layers, sizeof(float*)); m->V = calloc((size_t)c->n_layers, sizeof(float*));for (int i = 0; i < c->n_layers; i++){if (c->is_attn[i]){- m->K[i] = falloc((int64_t)c->kv_heads * m->max_t * c->k_head_dim);- m->V[i] = falloc((int64_t)c->kv_heads * m->max_t * c->k_head_dim);+ int64_t kvd = c->k_head_dim;+ m->K[i] = falloc((int64_t)c->kv_heads * m->max_t * kvd);+ m->V[i] = falloc((int64_t)c->kv_heads * m->max_t * kvd);+ if (keep > 0 && oldK && oldK[i] && oldV[i])+ for (int h = 0; h < c->kv_heads; h++){+ memcpy(m->K[i] + (int64_t)h*m->max_t*kvd,+ oldK[i] + (int64_t)h*old_stride*kvd, (size_t)keep*kvd*sizeof(float));+ memcpy(m->V[i] + (int64_t)h*m->max_t*kvd,+ oldV[i] + (int64_t)h*old_stride*kvd, (size_t)keep*kvd*sizeof(float));+ }} else { m->K[i] = NULL; m->V[i] = NULL; }}+ if (oldK){+ for (int i = 0; i < c->n_layers; i++){ if (oldK[i]) free(oldK[i]); if (oldV[i]) free(oldV[i]); }+ free(oldK); free(oldV);+ }/* Attention scores: one row per thread, indexed by absolute position, so* each row must hold max_t entries. Sized here rather than in attention()* because it grows with the context exactly like the KV cache does, and@@ -2588,6 +2637,15 @@ static void ensure_kv(Model *m){#endifm->attn_sc = falloc((int64_t)m->attn_sc_thr * m->max_t);m->kv_cap = m->max_t;+ /* The record describes those same positions, so it survives with them. If+ * its own allocation fails, reuse simply stops: this is an optimisation and+ * must never be the reason a turn fails. */+ if (keep > 0) {+ if (!kv_prefix_grow(&m->kvp, m->max_t, keep)) { kv_prefix_clear(&m->kvp); m->kv_len = 0; }+ else if (m->kv_len > keep) m->kv_len = keep;+ } else if (!kv_prefix_alloc(&m->kvp, m->max_t)) {+ kv_prefix_clear(&m->kvp);+ }}static void generate(Model *m, const int *prompt, int np, int n_new, int *out) {@@ -2818,7 +2876,29 @@ static void serve_one(Model *m, ServeReq *q){}printf("ACCEPT %s %d\n",q->id,np); fflush(stdout);m->max_t = np + q->max_tok;- reset_recurrent(m); ensure_kv(m); m->kv_len = 0;+ /* Grow the cache BEFORE deciding, so the decision sees the state that will+ * actually be there: ensure_kv preserves both the rows and the record. */+ ensure_kv(m);+ /* A chat client resends the whole transcript every turn. If this prompt+ * begins with the ids the current state was built from, that state already+ * IS the state at those positions: prefill only the tail. Either the reused+ * positions are token-identical or nothing is reused -- there is no partial+ * case, because nothing here can rewind a state. COLI_KV_PREFIX=0 turns it+ * off for an A/B; COLI_PREFIX_LOG=1 reports the decision and its reason,+ * because "it did not get faster" is otherwise indistinguishable from+ * "reuse is not wired up". */+ int reuse = kv_prefix_off() ? 0 : kv_prefix_reuse(&m->kvp, ids, np);+ if (getenv("COLI_PREFIX_LOG")) {+ if (reuse)+ fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",+ reuse, np, 100.0 * reuse / np);+ else+ fprintf(stderr, "[PREFIX] no reuse: held=%d cap=%d prompt=%d%s%s\n",+ m->kvp.len, m->kvp.cap, np, m->kvp.tainted ? " tainted" : "",+ (m->kvp.len > 0 && m->kvp.len < np) ? " (diverged)" : "");+ fflush(stderr);+ }+ if (!reuse) { reset_recurrent(m); m->kv_len = 0; }/* Per-REQUEST state, not per-process: without this the server keeps the* first request's prefill flag and expert-collection set forever, so* COLIBRI_RESIDENT=1 collects on request #1 and never again, and the@@ -2828,7 +2908,9 @@ static void serve_one(Model *m, ServeReq *q){if (m->momentum_logits)memset(m->momentum_logits, 0,(size_t)m->c.n_layers * m->c.n_experts * sizeof(float));- float *lo = step(m, ids, np, 0);+ /* `reuse` is the ABSOLUTE position of the first fresh token: attention and+ * the KV rows are position-indexed, so this has to be the real offset. */+ float *lo = step(m, ids + reuse, np - reuse, reuse);int gen=0, limited=1, forwards=1; /* il prefill e' il primo forward */const double s_disk=m->t_disk, s_attn=tm_sum(0)+tm_sum(1), s_moe=tm_sum(2), s_head=tm_sum(5);int eos_ids[4]; int n_eos=serve_eos_ids(eos_ids,4);diff --git a/c/tests/prefix_serve_harness.py b/c/tests/prefix_serve_harness.pynew file mode 100644index 00000000..fd988228--- /dev/null+++ b/c/tests/prefix_serve_harness.py@@ -0,0 +1,272 @@+"""Shared contract for KV prefix reuse, exercised through the serve protocol.++Reusing a previous turn's attention state must change the timing and nothing+else. If it alters even one token the engine is answering from a state that+belongs to a different conversation, and the reply still reads plausibly --+nothing else in the tree would catch it. So the gate is: the same two turns on+one engine must produce what the second turn alone produces on a cold one.++The scenarios live here rather than in each engine's file because the contract+is the same for every engine, and a contract copied per engine drifts per+engine. Each engine's test module supplies `spawn()` and its skip conditions;+everything below is identical for all of them.++The turns ask for ONE token each unless a scenario says otherwise. That is not+timidity: with max_tok=1 no sampled token is ever fed back, so the recorded+prefix is exactly the prompt and the next turn's prompt can be built as plain+text. Rebuilding it from the reply instead goes through decode-then-encode,+which a fixture whose vocabulary is wider than the byte range cannot round-trip+-- the prompt then diverges and the harness fault looks exactly like an engine+fault. One scenario does replay the reply and asserts the property that holds+either way: reuse or no reuse, the answer must match a cold engine's.++The turn being COMPARED always asks for eight. One token is not a gate: a+deliberately broken reuse that kept the attention rows and zeroed the recurrent+state still produced the same single argmax on the tiny Qwen3.6 fixture, and+only showed up once there were several tokens to disagree about.+"""+import json+import os+import subprocess+import threading+import time++READY = b"\x01\x01READY\x01\x01\n"+++def byte_level_vocab(size):+ """The GPT-2 byte->unicode map that tok.h's tk_build_bytemap builds.++ Bytes 33..126, 161..172 and 174..255 keep their own codepoint; the rest are+ renumbered from 256 upward in byte order. Reproduced here rather than+ imported so a drift between the two shows up as a failing test.+ """+ direct = set(range(33, 127)) | set(range(161, 173)) | set(range(174, 256))+ vocab, spare = {}, 0+ for b in range(size):+ if b in direct:+ cp = b+ else:+ cp, spare = 256 + spare, spare + 1+ vocab[chr(cp)] = b+ return vocab+++def ensure_byte_tokenizer(fixture, size=256):+ """Give a fixture that ships none a one-token-per-byte tokenizer.++ Some tiny fixtures carry weights and a teacher-forcing oracle but no+ tokenizer, because the oracle path feeds token ids directly. Serve mode goes+ through text, so it needs one.+ """+ path = fixture / "tokenizer.json"+ if path.exists():+ return+ path.write_text(json.dumps({+ "version": "1.0",+ "added_tokens": [],+ "model": {"type": "BPE", "vocab": byte_level_vocab(size), "merges": []},+ }), encoding="utf-8")+++class ServeEngine:+ """An engine in serve mode, speaking the protocol in docs/serve_protocol.md."""++ def __init__(self, engine, argv, env, log_prefix=True, reuse=True):+ env = dict(os.environ, **env)+ if log_prefix:+ env["COLI_PREFIX_LOG"] = "1"+ if not reuse:+ env["COLI_KV_PREFIX"] = "0"+ self.p = subprocess.Popen([str(engine)] + [str(a) for a in argv], env=env,+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,+ stderr=subprocess.PIPE, bufsize=0)+ # Drain stderr continuously: reading it only at close() loses whatever+ # the engine wrote after the pipe filled, and an empty capture is+ # indistinguishable from an engine that said nothing -- exactly the+ # ambiguity that makes a first failure here unreadable.+ self._err = []+ self._pump = threading.Thread(target=self._drain, daemon=True)+ self._pump.start()+ deadline = time.time() + 300+ while time.time() < deadline:+ line = self.p.stdout.readline()+ if not line:+ raise RuntimeError("engine exited before READY:\n"+ + "".join(self._err)[-2000:])+ if READY.strip() in line:+ return+ raise RuntimeError("engine never reported READY")++ def ask(self, rid, prompt, max_tok=1):+ """Returns the generated payload as BYTES.++ Never as str: a random-init fixture emits arbitrary byte sequences that+ are mostly not valid UTF-8, so decoding with errors="replace" and+ re-encoding yields entirely different bytes.+ """+ assert isinstance(prompt, bytes), "prompts stay bytes end to end"+ header = f"SUBMIT {rid} 0 {len(prompt)} {max_tok} 0 1\n".encode()+ self.p.stdin.write(header + prompt + b"\n")+ self.p.stdin.flush()+ chunks = []+ while True:+ line = self.p.stdout.readline()+ if not line:+ raise RuntimeError("engine closed mid-request:\n"+ + "".join(self._err)[-2000:])+ text = line.decode("latin-1").rstrip("\n")+ kind = text.split(" ", 1)[0]+ if kind == "DATA":+ chunks.append(self.p.stdout.read(int(text.split()[2])))+ self.p.stdout.readline() # the newline after the payload+ elif kind in ("DONE", "END"):+ return b"".join(chunks)+ elif kind == "ERROR":+ raise RuntimeError(f"engine error: {text}")++ def _drain(self):+ for line in iter(self.p.stderr.readline, b""):+ self._err.append(line.decode(errors="replace"))++ def close(self):+ try:+ self.p.stdin.close()+ self.p.wait(timeout=60)+ except Exception:+ self.p.kill()+ self._pump.join(timeout=10)+ return "".join(self._err)+++class PrefixReuseContract:+ """The five scenarios. Mix into a TestCase that provides spawn(**kwargs).++ Deliberately not a TestCase itself: unittest would collect and run the+ scenarios once more with no engine behind them.+ """++ def spawn(self, log_prefix=True, reuse=True): # pragma: no cover - overridden+ raise NotImplementedError++ def test_reused_prefix_yields_identical_tokens(self):+ warm = self.spawn()+ opening = b"The capital of France is"+ warm.ask("1", opening)+ # Turn 2 EXTENDS what the state already holds -- the shape a chat client+ # produces when it resends the transcript with more appended.+ second = opening + b" a question about geography, and the capital of Spain is"+ reused = warm.ask("2", second, max_tok=8)+ log = warm.close()++ cold = self.spawn(log_prefix=False)+ fresh = cold.ask("1", second, max_tok=8)+ cold.close()++ self.assertIn("[PREFIX] reusing", log,+ "the second turn did not reuse the first turn's state; "+ "this test proves nothing unless it does.\n"+ "engine said:\n" + (log or "(nothing on stderr)"))+ self.assertEqual(reused, fresh,+ "reusing the prefix changed the output -- the engine "+ "answered from a state that is not this conversation")++ def test_three_turns_keep_reusing(self):+ """Each turn holds a longer prefix than the last. On an engine that+ grows its cache this is also the test that covers the growth path: the+ rows and the record must survive the reallocation, per head, or the+ third turn answers from rows that belong to other positions."""+ warm = self.spawn()+ p1 = b"Streaming experts from disk"+ p2 = p1 + b" means the decode rate depends on the drive"+ p3 = p2 + b" and on how many experts stay resident in memory"+ warm.ask("1", p1)+ warm.ask("2", p2)+ third = warm.ask("3", p3, max_tok=8)+ log = warm.close()++ cold = self.spawn(log_prefix=False)+ fresh = cold.ask("1", p3, max_tok=8)+ cold.close()++ self.assertEqual(log.count("[PREFIX] reusing"), 2,+ "expected turns 2 and 3 to reuse:\n" + log)+ self.assertEqual(third, fresh, "reuse across three turns changed the output")++ def test_reuse_matches_the_same_engine_with_reuse_disabled(self):+ """The A/B the kill switch exists for: same process lifetime, same two+ turns, reuse on and off. Stricter than the cold-engine comparison+ because it also holds the expert cache and the router state constant."""+ opening = b"Streaming experts from disk means"+ extended = opening + b" that the decode rate then depends on"++ warm = self.spawn()+ warm.ask("1", opening)+ with_reuse = warm.ask("2", extended, max_tok=8)+ warm.close()++ off = self.spawn(reuse=False)+ off.ask("1", opening)+ without = off.ask("2", extended, max_tok=8)+ log = off.close()++ self.assertNotIn("[PREFIX] reusing", log,+ "COLI_KV_PREFIX=0 did not disable reuse")+ self.assertEqual(with_reuse, without,+ "reuse changed the answer on the same engine")++ def test_a_diverging_prompt_is_not_reused(self):+ """The rejection path matters as much as the reuse path: a prompt that+ shares no prefix must start over, not splice itself onto stale state."""+ other = b"Completely different opening text here"+ eng = self.spawn()+ eng.ask("1", b"The capital of France is")+ diverged = eng.ask("2", other, max_tok=8)+ log = eng.close()++ cold = self.spawn(log_prefix=False)+ fresh = cold.ask("1", other, max_tok=8)+ cold.close()++ self.assertIn("[PREFIX] no reuse", log,+ "a divergent prompt should have reported why it could not "+ "reuse:\n" + (log or "(nothing on stderr)"))+ self.assertEqual(diverged, fresh,+ "a diverging prompt was contaminated by the previous turn")++ def test_a_shorter_prompt_after_a_longer_one(self):+ """The record only ever grows its length, so a miss has to CLEAR it+ rather than overwrite it: a shorter prompt writes fewer positions than+ the last turn recorded, and a stale tail would claim coverage the cache+ no longer has -- which the next turn would then happily reuse."""+ short = b"Short prompt"+ eng = self.spawn()+ eng.ask("1", b"A considerably longer opening than the one that follows it")+ eng.ask("2", short)+ after = eng.ask("3", short + b" extended once more", max_tok=8)+ eng.close()++ cold = self.spawn(log_prefix=False)+ fresh = cold.ask("1", short + b" extended once more", max_tok=8)+ cold.close()++ self.assertEqual(after, fresh,+ "a stale record survived a shorter prompt and was reused")++ def test_a_turn_that_replays_the_reply_is_never_wrong(self):+ """The real chat shape: the client appends the assistant's reply and a+ new question. Whether the reply survives this fixture's decode-encode+ round trip decides whether reuse FIRES, and that is allowed to vary --+ what may never vary is the answer."""+ warm = self.spawn()+ opening = b"The capital of France is"+ reply = warm.ask("1", opening, max_tok=4)+ second = opening + reply + b" and the capital of Spain is"+ got = warm.ask("2", second, max_tok=8)+ warm.close()++ cold = self.spawn(log_prefix=False)+ fresh = cold.ask("1", second, max_tok=8)+ cold.close()++ self.assertEqual(got, fresh, "replaying the reply changed the answer")diff --git a/c/tests/test_olmoe_prefix_serve.py b/c/tests/test_olmoe_prefix_serve.pynew file mode 100644index 00000000..6a1e0969--- /dev/null+++ b/c/tests/test_olmoe_prefix_serve.py@@ -0,0 +1,36 @@+"""KV prefix reuse on OLMoE: the shared contract, against the tiny fixture.++OLMoE is the simple end of the contract -- plain GQA attention, no recurrent+state, and a KV cache allocated once for the context and never reallocated, so+a recorded position stays valid for the life of the process. That makes it the+engine where a reuse bug would be quietest, which is why it is gated too.++Runs against the tiny fixture the oracle job already builds+(tools/make_olmoe_tiny.py + tools/convert_olmoe_merged.py ++tools/make_edge_tiny_tokenizer.py).+"""+import os+import sys+import unittest+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parent))+from prefix_serve_harness import PrefixReuseContract, ServeEngine # noqa: E402++HERE = Path(__file__).resolve().parent.parent+ENGINE = HERE / ("olmoe.exe" if sys.platform == "win32" else "olmoe")+FIXTURE = Path(os.environ.get("OLMOE_TINY", HERE / "olmoe_tiny_c"))++[email protected](ENGINE.exists(), "olmoe is not built")[email protected]((FIXTURE / "tokenizer.json").exists(),+ "tiny olmoe container is absent (tools/convert_olmoe_merged.py)")+class OlmoePrefixServeTest(PrefixReuseContract, unittest.TestCase):+ def spawn(self, log_prefix=True, reuse=True):+ return ServeEngine(ENGINE, ["8", "8"],+ {"SNAP": str(FIXTURE), "SERVE": "1", "CTX": "512"},+ log_prefix=log_prefix, reuse=reuse)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/test_qwen36_prefix_serve.py b/c/tests/test_qwen36_prefix_serve.pynew file mode 100644index 00000000..4d3a9c48--- /dev/null+++ b/c/tests/test_qwen36_prefix_serve.py@@ -0,0 +1,47 @@+"""KV prefix reuse on Qwen3.6: the shared contract, against the tiny fixture.++Qwen3.6 makes the contract stricter than a pure-attention engine would: three of+every four layers are DeltaNet, which carries a RECURRENT state rather than+position-indexed rows. Reuse keeps that state too, so a wrong reuse length+corrupts the recurrence and not just the attention. This engine also GROWS its+KV cache between turns, so tests/prefix_serve_harness.py's three-turn scenario+covers the per-head copy that growth performs.++Runs against the tiny random-init fixture the oracle job already builds+(tools/make_qwen36_tiny.py + tools/convert_qwen36.py): no checkpoint, no disk.+"""+import os+import sys+import unittest+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parent))+from prefix_serve_harness import (PrefixReuseContract, ServeEngine, # noqa: E402+ ensure_byte_tokenizer)++HERE = Path(__file__).resolve().parent.parent+ENGINE = HERE / ("qwen36.exe" if sys.platform == "win32" else "qwen36")+FIXTURE = Path(os.environ.get("QWEN36_TINY", HERE / "qwen36_tiny_c"))++[email protected](ENGINE.exists(), "qwen36 is not built")+# The metadata of this container is TRACKED (config.json, qwen36_meta.json and+# the README), so its presence says nothing about whether the weights were ever+# converted. Gate on a shard the conversion writes, or this skips nothing and+# fails on a checkout that never ran the converter.[email protected]((FIXTURE / "model-globals.safetensors").exists(),+ "tiny qwen36 container is absent (tools/convert_qwen36.py)")+class Qwen36PrefixServeTest(PrefixReuseContract, unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ # The fixture ships no tokenizer: the oracle path feeds ids directly.+ ensure_byte_tokenizer(FIXTURE)++ def spawn(self, log_prefix=True, reuse=True):+ return ServeEngine(ENGINE, ["8", "8"],+ {"SNAP": str(FIXTURE), "SERVE": "1", "COLI_DENSE_I8": "0"},+ log_prefix=log_prefix, reuse=reuse)+++if __name__ == "__main__":+ unittest.main()diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 53664bd0..50fb8ee3 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -374,6 +374,8 @@ Read **only** by `c/inkling.c`.| `INK_SHARED_BATCH` | auto | Prefill rows per shared-expert batch, bounded to 64 MiB of scratch. `=0` restores the scalar per-token path for A/B/debugging; a positive value caps the chunk size. Decode (`S=1`) is unchanged. || `INK_METAL_MIN_S` | `1` | Minimum batch S to send the MoE block to Metal. `=2` restores the prefill-only gate (which mattered when the residency set was absent and per-block `useResource` churn cost ~135 ms). || `INK_PREFIX_LOG` | unset | Log the KV-prefix reuse decision and its reason, as `K3_PREFIX_LOG` does for K3. |+| `COLI_PREFIX_LOG` | unset | Same line for the engines that take the shared record (Qwen3.6, OLMoE): reports how many prompt tokens were reused, or why none were. |+| `COLI_KV_PREFIX` | on | `0` disables KV-prefix reuse on those engines. The escape hatch, and the B arm of the A/B that shows reuse changes nothing but the time. || `GPU_DEV` | `0` | CUDA device index for the inkling CUDA backend. || `NOGPU` | unset | If set, skip GPU init entirely (both CUDA and Metal), regardless of the other GPU variables. |