Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen36, olmoe: reuse the previous turn's KV instead of re-prefilling the transcript

JustVugg16bfe58b52fd
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 114e9a0e..d511d5d2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -472,6 +472,17 @@ jobs:
echo "FAIL: sanitizer diagnostic under PILOT"; cat san.log; exit 1
fi
echo "sanitizers clean:"; grep -E "Matching tokens" san.log || true
+ - name: KV prefix reuse is token-identical (serve, several turns)
+ run: |
+ cd c
+ # Reusing a previous turn's state must change the timing and nothing
+ # else. A wrong reuse length does not crash: it answers from another
+ # conversation's state and still reads plausibly, so this is the only
+ # gate that would catch it. The sanitizer step above rebuilt the
+ # engine, so rebuild the normal one first.
+ make clean >/dev/null 2>&1 || true
+ make qwen36
+ QWEN36_TINY=qwen36_tiny_c python3 -m unittest -v tests.test_qwen36_prefix_serve
- name: Expert kernel A/B on an int4 gs64 fixture (old path vs expert_ffn.h, byte-identical ids)
run: |
cd c
@@ -594,6 +605,14 @@ jobs:
run: |
cd c
COLI_OLMOE_FIXTURE=olmoe_tiny_c python3 -m unittest -v tests.test_olmoe_context_exceeded
+ - name: KV prefix reuse is token-identical (serve, several turns)
+ run: |
+ cd c
+ # Same contract as the Qwen3.6 job runs, from the same harness: a
+ # reused prefix must produce exactly what a cold engine produces for
+ # the same prompt. Wrong reuse is silent, so the gate is the only
+ # thing between it and a user.
+ OLMOE_TINY=olmoe_tiny_c python3 -m unittest -v tests.test_olmoe_prefix_serve
dsv41-tiny-check:
name: DeepSeek V4.1 tiny oracle (engram + DSA + fp4 experts + vision)
@@ -1136,7 +1155,7 @@ jobs:
# rieseguiva un binario non strumentato. Uno di quei test era rosso da
# settimane senza che nessuno potesse vederlo.
- name: Build the engines those tests exercise
- run: make -C c kimi_k3 inkling colibri olmoe qwen38
+ run: make -C c kimi_k3 inkling colibri olmoe qwen38 qwen36
- name: Python test suite
run: cd c && python3 -m unittest discover -s tests -p 'test_*.py'
diff --git a/c/Makefile b/c/Makefile
index 2e66298a..0bf732f0 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1046,7 +1046,7 @@ fp8-bench: fp8_bench$(EXE)
# the GPU sits idle. Same guard #783 put on kimi_k3, which since gaining an
# MXFP4 expert path no longer needs it. tests/test_makefile_cuda_scope.py
# asserts this shape.
-olmoe$(EXE): olmoe.c cli_args.h st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h serve_codec.h edge_adapters.h edge_runtime.h edge_tok_internal.h fused_simd.h segment_adapter_internal.h segment_adapters.h segment_runtime.h
+olmoe$(EXE): olmoe.c cli_args.h st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h kv_prefix.h route_trace.h serve_codec.h edge_adapters.h edge_runtime.h edge_tok_internal.h fused_simd.h segment_adapter_internal.h segment_adapters.h segment_runtime.h
$(CC) $(NOCUDA_CFLAGS) olmoe.c -o olmoe$(EXE) $(NOCUDA_LDFLAGS)
# Qwen3.6-35B-A3B engine (hybrid Gated Attention + Gated DeltaNet + streaming
@@ -1072,7 +1072,7 @@ QWEN36_TIER_SRC =
QWEN36_CFLAGS = $(NOCUDA_CFLAGS)
QWEN36_LDFLAGS = $(NOCUDA_LDFLAGS)
endif
-qwen36$(EXE): qwen36.c serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h st.h json.h compat.h omp_tune.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
+qwen36$(EXE): qwen36.c serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h st.h json.h compat.h omp_tune.h kv_prefix.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)
# DeepSeek V4.1 Flash: one file, like every other portable engine. The fp4 experts
diff --git a/c/olmoe.c b/c/olmoe.c
index 06360d79..0270f0de 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -38,6 +38,7 @@
#endif
#include "omp_tune.h"
#include "route_trace.h" /* shared routing telemetry (#700) */
+#include "kv_prefix.h" /* riuso del prefisso tra turni (shared) */
#include "serve_codec.h"
#ifdef COLI_SEGMENT_ADAPTER
#include "segment_runtime.h"
@@ -102,6 +103,10 @@ typedef struct {
* plain double: a per-turn delta of it is what the PROF line reports. */
uint64_t disk_ns;
float **K, **V; int kv_len, max_t;
+ /* What the cached keys and values were built from, so a serve turn that
+ * resends the transcript prefills only the new tail. Recorded where the
+ * tokens are fed (see kv_prefix.h), never derived from a counter. */
+ kv_prefix kvp;
double dense_load_s;
/* IMPROVEMENT 2: expert frequency heatmap */
uint32_t **freq; /* per-layer expert counts, owned by route_trace.h */
@@ -1009,6 +1014,10 @@ static float *step(Model *m, const int *ids, int S, int pos_base) {
if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens)
pin_hot_experts(m);
m->kv_len = pos_base + S;
+ /* Recorded HERE, where the tokens entered the cache: fed[0..len-1] are
+ * the ids those positions were built from, and that invariant is the
+ * whole safety argument for reusing them next turn. */
+ kv_prefix_record(&m->kvp, ids, pos_base, S);
float *last = falloc(D);
rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps);
float *logit = falloc(c->vocab);
@@ -1462,6 +1471,10 @@ static void serve_hits(Model *m) {
fflush(stdout); free(hex); free(bm);
}
+/* COLI_KV_PREFIX=0: never reuse a previous turn's cache. The escape hatch,
+ * and the B arm of the A/B that shows reuse changes nothing but the time. */
+static int kv_prefix_off(void){ const char *e = getenv("COLI_KV_PREFIX"); return e && *e == '0'; }
+
static int serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {
Cfg *c = &m->c;
int cap = q->plen + 16;
@@ -1478,12 +1491,38 @@ static int serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {
coli_serve_write_error(stdout, q->id, message); free(ids); return 0;
}
g_temp = q->temp; g_nuc = q->top_p;
+ /* A chat client resends the whole transcript every turn. If this prompt
+ * begins with the ids the cache was built from, those keys and values ARE
+ * the state at those positions -- attention is causal, so a row depends on
+ * its prefix and nothing later. Prefill only the tail. Reuse is all or
+ * nothing: nothing here can rewind a cache. COLI_KV_PREFIX=0 turns it off,
+ * COLI_PREFIX_LOG=1 reports the decision and its reason, because "it did
+ * not get faster" is otherwise indistinguishable from "it is not wired up".
+ *
+ * On a miss the record must be CLEARED and not merely overwritten: a
+ * shorter prompt writes fewer positions than the last one recorded, and
+ * kv_prefix_record only ever grows the length, so the stale tail would
+ * claim coverage the cache no longer has. */
+ int reuse = kv_prefix_off() ? 0 : kv_prefix_reuse(&m->kvp, ids, np);
+ if (!reuse) kv_prefix_clear(&m->kvp);
+ if (getenv("COLI_PREFIX_LOG")) {
+ if (reuse)
+ fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",
+ reuse, np, 100.0 * reuse / np);
+ else
+ fprintf(stderr, "[PREFIX] no reuse: held=%d cap=%d prompt=%d\n",
+ m->kvp.len, m->kvp.cap, np);
+ fflush(stderr);
+ }
double t0 = now_s();
uint64_t h0 = m->hits, m0 = m->miss;
uint64_t disk0 = __atomic_load_n(&m->disk_ns, __ATOMIC_RELAXED);
double attn0 = g_prof_attn_s, moe0 = g_prof_moe_s, head0 = g_prof_head_s;
long long fwd0 = g_prof_forwards;
- float *logit = step(m, ids, np, 0);
+ /* `reuse` is the ABSOLUTE position of the first fresh token: attention
+ * and the KV rows are position-indexed, so this has to be the real
+ * offset, not a count of what is left to do. */
+ float *logit = step(m, ids + reuse, np - reuse, reuse);
int hist_len = np, gen = 0, limited = 1, cancelled = 0;
char buf[512];
for (int s = 0; s < q->max_tok && !cancelled; s++) {
@@ -1663,6 +1702,12 @@ int main(int argc, char **argv) {
m.K[i] = falloc((int64_t)m.c.n_heads * m.max_t * m.c.head_dim);
m.V[i] = falloc((int64_t)m.c.n_heads * m.max_t * m.c.head_dim);
}
+ /* Serve mode only: the record is sized with the cache it describes, and
+ * the cache here is allocated once for ctx_cap and never reallocated,
+ * so a recorded position stays valid for the life of the process. The
+ * CLI paths (generate/run_chat/PPL) leave it unallocated, which makes
+ * every kv_prefix call there a no-op. */
+ kv_prefix_alloc(&m.kvp, m.max_t);
Tok T;
char tokpath[2048]; snprintf(tokpath, sizeof(tokpath), "%s/tokenizer.json", snap);
tok_load(&T, tokpath);
diff --git a/c/qwen36.c b/c/qwen36.c
index 3f793268..11794867 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -61,6 +61,7 @@ static int qwen36_max_ctx(void) {
#include "cli_args.h"
#include "st.h"
#include "omp_tune.h"
+#include "kv_prefix.h" /* riuso del prefisso tra turni (shared) */
#include "json.h" /* tokenizer.json parsing (reuse minimal parser) */
#include "qwen36_tier.h" /* optional CUDA VRAM expert tier */
#include "expert_ffn.h" /* routed experts: planar int4 kernel + layer runner */
@@ -643,6 +644,11 @@ typedef struct {
double t_disk;
uint8_t **ehit;
float **K, **V; int kv_len, max_t, kv_cap;
+ /* What the current attention and DeltaNet state was built from, so a
+ * turn that resends the transcript prefills only the new tail. The ids
+ * are recorded where they are fed, never derived from a counter: see
+ * kv_prefix.h for why that distinction is the whole safety argument. */
+ kv_prefix kvp;
float *attn_sc; /* [attn_sc_thr * kv_cap] score rows, one per thread */
int attn_sc_thr;
double dense_load_s;
@@ -1073,6 +1079,10 @@ static int g_qdw_n = 0;
static uint64_t g_qwen_matmul_d_calls;
#endif
static int dense_i8_on(void){ static int v=-1; if(v<0){ const char *e=getenv("COLI_DENSE_I8"); v=!(e&&*e=='0'); } return v; }
+/* COLI_KV_PREFIX=0: never reuse a previous turn's state. Kept as an escape
+ * hatch and as the B arm of the A/B that shows reuse changes nothing but
+ * the time. */
+static int kv_prefix_off(void){ const char *e=getenv("COLI_KV_PREFIX"); return e && *e=='0'; }
static int dense_batch_on(void){ const char *e=getenv("QWEN_DENSE_BATCH"); return !(e&&*e=='0'); }
static void qdw_register(const float *W, int I, int O){
if (!W || !dense_i8_on() || g_qdw_n >= QDW_MAX) return;
@@ -2413,6 +2423,10 @@ static float *step(Model *m, const int *ids, int S, int pos_base) {
memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float));
}
layers_forward_range(m, x, S, pos_base, 0, c->n_layers, 1, lf);
+ /* Recorded HERE, where the tokens actually entered the state, rather than
+ * derived from the caller's bookkeeping: the invariant that fed[0..len-1]
+ * are the ids the state was built from is the whole safety argument. */
+ kv_prefix_record(&m->kvp, ids, pos_base, S);
m->token_count += S; m->freq_token_count += S;
if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) pin_hot_experts(m);
m->kv_len = pos_base + S;
@@ -2552,6 +2566,10 @@ static float *g_last_logit = NULL;
* generation (the CLI runs once, so this is also correct there). */
static void reset_recurrent(Model *m){
Cfg *c = &m->c;
+ /* Paired with the record on purpose: whoever zeroes the state must also
+ * forget what it was built from, or the two disagree in favour of the one
+ * nobody can check. */
+ kv_prefix_clear(&m->kvp);
for (int i = 0; i < c->n_layers; i++){
if (c->is_attn[i]) continue;
if (m->DN_rec[i]) memset(m->DN_rec[i], 0, (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim * sizeof(float));
@@ -2564,18 +2582,49 @@ static void reset_recurrent(Model *m){
* growth so the server doesn't leak KV memory across requests. */
static void ensure_kv(Model *m){
Cfg *c = &m->c;
- if (m->kv_cap >= m->max_t && m->K) return;
- if (m->K){
- for (int i = 0; i < c->n_layers; i++){ if (m->K[i]) free(m->K[i]); if (m->V[i]) free(m->V[i]); }
- free(m->K); free(m->V); m->K = NULL; m->V = NULL;
+ if (m->kv_cap >= m->max_t && m->K) {
+ /* max_t is the ROW STRIDE of the KV cache, not just a capacity: a row
+ * lives at (head*max_t + position)*head_dim. Callers set it per request
+ * from prompt+max_tok, so a shorter request used to shrink the stride
+ * while the allocation stayed the same size -- harmless only as long as
+ * every turn rewrote every row from position 0. Reuse reads rows an
+ * earlier turn wrote, so the stride has to stay the one they were
+ * written with: the allocation's, which is kv_cap. */
+ m->max_t = m->kv_cap;
+ return;
}
+ /* Growth COPIES the rows instead of discarding them. A chat resends a
+ * longer transcript every turn, so this reallocation lands on exactly the
+ * turn that wants to reuse the previous one's state: freeing the rows here
+ * would make kv_prefix_reuse miss in the one case it exists for. The copy
+ * is a memcpy at DRAM speed; the prefill it saves is seconds of expert
+ * reads on a streaming engine. The row stride IS max_t, so the copy must be
+ * per head -- one flat memcpy would land every head but the first at the
+ * wrong offset, and that reads as a plausible answer from another
+ * conversation rather than as a crash. */
+ float **oldK = m->K, **oldV = m->V;
+ int old_stride = m->kv_cap, keep = m->K ? m->kvp.len : 0;
+ if (keep > old_stride) keep = old_stride;
+ if (keep > m->max_t) keep = m->max_t;
m->K = calloc((size_t)c->n_layers, sizeof(float*)); m->V = calloc((size_t)c->n_layers, sizeof(float*));
for (int i = 0; i < c->n_layers; i++){
if (c->is_attn[i]){
- m->K[i] = falloc((int64_t)c->kv_heads * m->max_t * c->k_head_dim);
- m->V[i] = falloc((int64_t)c->kv_heads * m->max_t * c->k_head_dim);
+ int64_t kvd = c->k_head_dim;
+ m->K[i] = falloc((int64_t)c->kv_heads * m->max_t * kvd);
+ m->V[i] = falloc((int64_t)c->kv_heads * m->max_t * kvd);
+ if (keep > 0 && oldK && oldK[i] && oldV[i])
+ for (int h = 0; h < c->kv_heads; h++){
+ memcpy(m->K[i] + (int64_t)h*m->max_t*kvd,
+ oldK[i] + (int64_t)h*old_stride*kvd, (size_t)keep*kvd*sizeof(float));
+ memcpy(m->V[i] + (int64_t)h*m->max_t*kvd,
+ oldV[i] + (int64_t)h*old_stride*kvd, (size_t)keep*kvd*sizeof(float));
+ }
} else { m->K[i] = NULL; m->V[i] = NULL; }
}
+ if (oldK){
+ for (int i = 0; i < c->n_layers; i++){ if (oldK[i]) free(oldK[i]); if (oldV[i]) free(oldV[i]); }
+ free(oldK); free(oldV);
+ }
/* Attention scores: one row per thread, indexed by absolute position, so
* each row must hold max_t entries. Sized here rather than in attention()
* because it grows with the context exactly like the KV cache does, and
@@ -2588,6 +2637,15 @@ static void ensure_kv(Model *m){
#endif
m->attn_sc = falloc((int64_t)m->attn_sc_thr * m->max_t);
m->kv_cap = m->max_t;
+ /* The record describes those same positions, so it survives with them. If
+ * its own allocation fails, reuse simply stops: this is an optimisation and
+ * must never be the reason a turn fails. */
+ if (keep > 0) {
+ if (!kv_prefix_grow(&m->kvp, m->max_t, keep)) { kv_prefix_clear(&m->kvp); m->kv_len = 0; }
+ else if (m->kv_len > keep) m->kv_len = keep;
+ } else if (!kv_prefix_alloc(&m->kvp, m->max_t)) {
+ kv_prefix_clear(&m->kvp);
+ }
}
static void generate(Model *m, const int *prompt, int np, int n_new, int *out) {
@@ -2818,7 +2876,29 @@ static void serve_one(Model *m, ServeReq *q){
}
printf("ACCEPT %s %d\n",q->id,np); fflush(stdout);
m->max_t = np + q->max_tok;
- reset_recurrent(m); ensure_kv(m); m->kv_len = 0;
+ /* Grow the cache BEFORE deciding, so the decision sees the state that will
+ * actually be there: ensure_kv preserves both the rows and the record. */
+ ensure_kv(m);
+ /* A chat client resends the whole transcript every turn. If this prompt
+ * begins with the ids the current state was built from, that state already
+ * IS the state at those positions: prefill only the tail. Either the reused
+ * positions are token-identical or nothing is reused -- there is no partial
+ * case, because nothing here can rewind a state. COLI_KV_PREFIX=0 turns it
+ * off for an A/B; COLI_PREFIX_LOG=1 reports the decision and its reason,
+ * because "it did not get faster" is otherwise indistinguishable from
+ * "reuse is not wired up". */
+ int reuse = kv_prefix_off() ? 0 : kv_prefix_reuse(&m->kvp, ids, np);
+ if (getenv("COLI_PREFIX_LOG")) {
+ if (reuse)
+ fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",
+ reuse, np, 100.0 * reuse / np);
+ else
+ fprintf(stderr, "[PREFIX] no reuse: held=%d cap=%d prompt=%d%s%s\n",
+ m->kvp.len, m->kvp.cap, np, m->kvp.tainted ? " tainted" : "",
+ (m->kvp.len > 0 && m->kvp.len < np) ? " (diverged)" : "");
+ fflush(stderr);
+ }
+ if (!reuse) { reset_recurrent(m); m->kv_len = 0; }
/* Per-REQUEST state, not per-process: without this the server keeps the
* first request's prefill flag and expert-collection set forever, so
* COLIBRI_RESIDENT=1 collects on request #1 and never again, and the
@@ -2828,7 +2908,9 @@ static void serve_one(Model *m, ServeReq *q){
if (m->momentum_logits)
memset(m->momentum_logits, 0,
(size_t)m->c.n_layers * m->c.n_experts * sizeof(float));
- float *lo = step(m, ids, np, 0);
+ /* `reuse` is the ABSOLUTE position of the first fresh token: attention and
+ * the KV rows are position-indexed, so this has to be the real offset. */
+ float *lo = step(m, ids + reuse, np - reuse, reuse);
int gen=0, limited=1, forwards=1; /* il prefill e' il primo forward */
const double s_disk=m->t_disk, s_attn=tm_sum(0)+tm_sum(1), s_moe=tm_sum(2), s_head=tm_sum(5);
int eos_ids[4]; int n_eos=serve_eos_ids(eos_ids,4);
diff --git a/c/tests/prefix_serve_harness.py b/c/tests/prefix_serve_harness.py
new file mode 100644
index 00000000..fd988228
--- /dev/null
+++ b/c/tests/prefix_serve_harness.py
@@ -0,0 +1,272 @@
+"""Shared contract for KV prefix reuse, exercised through the serve protocol.
+
+Reusing a previous turn's attention state must change the timing and nothing
+else. If it alters even one token the engine is answering from a state that
+belongs to a different conversation, and the reply still reads plausibly --
+nothing else in the tree would catch it. So the gate is: the same two turns on
+one engine must produce what the second turn alone produces on a cold one.
+
+The scenarios live here rather than in each engine's file because the contract
+is the same for every engine, and a contract copied per engine drifts per
+engine. Each engine's test module supplies `spawn()` and its skip conditions;
+everything below is identical for all of them.
+
+The turns ask for ONE token each unless a scenario says otherwise. That is not
+timidity: with max_tok=1 no sampled token is ever fed back, so the recorded
+prefix is exactly the prompt and the next turn's prompt can be built as plain
+text. Rebuilding it from the reply instead goes through decode-then-encode,
+which a fixture whose vocabulary is wider than the byte range cannot round-trip
+-- the prompt then diverges and the harness fault looks exactly like an engine
+fault. One scenario does replay the reply and asserts the property that holds
+either way: reuse or no reuse, the answer must match a cold engine's.
+
+The turn being COMPARED always asks for eight. One token is not a gate: a
+deliberately broken reuse that kept the attention rows and zeroed the recurrent
+state still produced the same single argmax on the tiny Qwen3.6 fixture, and
+only showed up once there were several tokens to disagree about.
+"""
+import json
+import os
+import subprocess
+import threading
+import time
+
+READY = b"\x01\x01READY\x01\x01\n"
+
+
+def byte_level_vocab(size):
+ """The GPT-2 byte->unicode map that tok.h's tk_build_bytemap builds.
+
+ Bytes 33..126, 161..172 and 174..255 keep their own codepoint; the rest are
+ renumbered from 256 upward in byte order. Reproduced here rather than
+ imported so a drift between the two shows up as a failing test.
+ """
+ direct = set(range(33, 127)) | set(range(161, 173)) | set(range(174, 256))
+ vocab, spare = {}, 0
+ for b in range(size):
+ if b in direct:
+ cp = b
+ else:
+ cp, spare = 256 + spare, spare + 1
+ vocab[chr(cp)] = b
+ return vocab
+
+
+def ensure_byte_tokenizer(fixture, size=256):
+ """Give a fixture that ships none a one-token-per-byte tokenizer.
+
+ Some tiny fixtures carry weights and a teacher-forcing oracle but no
+ tokenizer, because the oracle path feeds token ids directly. Serve mode goes
+ through text, so it needs one.
+ """
+ path = fixture / "tokenizer.json"
+ if path.exists():
+ return
+ path.write_text(json.dumps({
+ "version": "1.0",
+ "added_tokens": [],
+ "model": {"type": "BPE", "vocab": byte_level_vocab(size), "merges": []},
+ }), encoding="utf-8")
+
+
+class ServeEngine:
+ """An engine in serve mode, speaking the protocol in docs/serve_protocol.md."""
+
+ def __init__(self, engine, argv, env, log_prefix=True, reuse=True):
+ env = dict(os.environ, **env)
+ if log_prefix:
+ env["COLI_PREFIX_LOG"] = "1"
+ if not reuse:
+ env["COLI_KV_PREFIX"] = "0"
+ self.p = subprocess.Popen([str(engine)] + [str(a) for a in argv], env=env,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, bufsize=0)
+ # Drain stderr continuously: reading it only at close() loses whatever
+ # the engine wrote after the pipe filled, and an empty capture is
+ # indistinguishable from an engine that said nothing -- exactly the
+ # ambiguity that makes a first failure here unreadable.
+ self._err = []
+ self._pump = threading.Thread(target=self._drain, daemon=True)
+ self._pump.start()
+ deadline = time.time() + 300
+ while time.time() < deadline:
+ line = self.p.stdout.readline()
+ if not line:
+ raise RuntimeError("engine exited before READY:\n"
+ + "".join(self._err)[-2000:])
+ if READY.strip() in line:
+ return
+ raise RuntimeError("engine never reported READY")
+
+ def ask(self, rid, prompt, max_tok=1):
+ """Returns the generated payload as BYTES.
+
+ Never as str: a random-init fixture emits arbitrary byte sequences that
+ are mostly not valid UTF-8, so decoding with errors="replace" and
+ re-encoding yields entirely different bytes.
+ """
+ assert isinstance(prompt, bytes), "prompts stay bytes end to end"
+ header = f"SUBMIT {rid} 0 {len(prompt)} {max_tok} 0 1\n".encode()
+ self.p.stdin.write(header + prompt + b"\n")
+ self.p.stdin.flush()
+ chunks = []
+ while True:
+ line = self.p.stdout.readline()
+ if not line:
+ raise RuntimeError("engine closed mid-request:\n"
+ + "".join(self._err)[-2000:])
+ text = line.decode("latin-1").rstrip("\n")
+ kind = text.split(" ", 1)[0]
+ if kind == "DATA":
+ chunks.append(self.p.stdout.read(int(text.split()[2])))
+ self.p.stdout.readline() # the newline after the payload
+ elif kind in ("DONE", "END"):
+ return b"".join(chunks)
+ elif kind == "ERROR":
+ raise RuntimeError(f"engine error: {text}")
+
+ def _drain(self):
+ for line in iter(self.p.stderr.readline, b""):
+ self._err.append(line.decode(errors="replace"))
+
+ def close(self):
+ try:
+ self.p.stdin.close()
+ self.p.wait(timeout=60)
+ except Exception:
+ self.p.kill()
+ self._pump.join(timeout=10)
+ return "".join(self._err)
+
+
+class PrefixReuseContract:
+ """The five scenarios. Mix into a TestCase that provides spawn(**kwargs).
+
+ Deliberately not a TestCase itself: unittest would collect and run the
+ scenarios once more with no engine behind them.
+ """
+
+ def spawn(self, log_prefix=True, reuse=True): # pragma: no cover - overridden
+ raise NotImplementedError
+
+ def test_reused_prefix_yields_identical_tokens(self):
+ warm = self.spawn()
+ opening = b"The capital of France is"
+ warm.ask("1", opening)
+ # Turn 2 EXTENDS what the state already holds -- the shape a chat client
+ # produces when it resends the transcript with more appended.
+ second = opening + b" a question about geography, and the capital of Spain is"
+ reused = warm.ask("2", second, max_tok=8)
+ log = warm.close()
+
+ cold = self.spawn(log_prefix=False)
+ fresh = cold.ask("1", second, max_tok=8)
+ cold.close()
+
+ self.assertIn("[PREFIX] reusing", log,
+ "the second turn did not reuse the first turn's state; "
+ "this test proves nothing unless it does.\n"
+ "engine said:\n" + (log or "(nothing on stderr)"))
+ self.assertEqual(reused, fresh,
+ "reusing the prefix changed the output -- the engine "
+ "answered from a state that is not this conversation")
+
+ def test_three_turns_keep_reusing(self):
+ """Each turn holds a longer prefix than the last. On an engine that
+ grows its cache this is also the test that covers the growth path: the
+ rows and the record must survive the reallocation, per head, or the
+ third turn answers from rows that belong to other positions."""
+ warm = self.spawn()
+ p1 = b"Streaming experts from disk"
+ p2 = p1 + b" means the decode rate depends on the drive"
+ p3 = p2 + b" and on how many experts stay resident in memory"
+ warm.ask("1", p1)
+ warm.ask("2", p2)
+ third = warm.ask("3", p3, max_tok=8)
+ log = warm.close()
+
+ cold = self.spawn(log_prefix=False)
+ fresh = cold.ask("1", p3, max_tok=8)
+ cold.close()
+
+ self.assertEqual(log.count("[PREFIX] reusing"), 2,
+ "expected turns 2 and 3 to reuse:\n" + log)
+ self.assertEqual(third, fresh, "reuse across three turns changed the output")
+
+ def test_reuse_matches_the_same_engine_with_reuse_disabled(self):
+ """The A/B the kill switch exists for: same process lifetime, same two
+ turns, reuse on and off. Stricter than the cold-engine comparison
+ because it also holds the expert cache and the router state constant."""
+ opening = b"Streaming experts from disk means"
+ extended = opening + b" that the decode rate then depends on"
+
+ warm = self.spawn()
+ warm.ask("1", opening)
+ with_reuse = warm.ask("2", extended, max_tok=8)
+ warm.close()
+
+ off = self.spawn(reuse=False)
+ off.ask("1", opening)
+ without = off.ask("2", extended, max_tok=8)
+ log = off.close()
+
+ self.assertNotIn("[PREFIX] reusing", log,
+ "COLI_KV_PREFIX=0 did not disable reuse")
+ self.assertEqual(with_reuse, without,
+ "reuse changed the answer on the same engine")
+
+ def test_a_diverging_prompt_is_not_reused(self):
+ """The rejection path matters as much as the reuse path: a prompt that
+ shares no prefix must start over, not splice itself onto stale state."""
+ other = b"Completely different opening text here"
+ eng = self.spawn()
+ eng.ask("1", b"The capital of France is")
+ diverged = eng.ask("2", other, max_tok=8)
+ log = eng.close()
+
+ cold = self.spawn(log_prefix=False)
+ fresh = cold.ask("1", other, max_tok=8)
+ cold.close()
+
+ self.assertIn("[PREFIX] no reuse", log,
+ "a divergent prompt should have reported why it could not "
+ "reuse:\n" + (log or "(nothing on stderr)"))
+ self.assertEqual(diverged, fresh,
+ "a diverging prompt was contaminated by the previous turn")
+
+ def test_a_shorter_prompt_after_a_longer_one(self):
+ """The record only ever grows its length, so a miss has to CLEAR it
+ rather than overwrite it: a shorter prompt writes fewer positions than
+ the last turn recorded, and a stale tail would claim coverage the cache
+ no longer has -- which the next turn would then happily reuse."""
+ short = b"Short prompt"
+ eng = self.spawn()
+ eng.ask("1", b"A considerably longer opening than the one that follows it")
+ eng.ask("2", short)
+ after = eng.ask("3", short + b" extended once more", max_tok=8)
+ eng.close()
+
+ cold = self.spawn(log_prefix=False)
+ fresh = cold.ask("1", short + b" extended once more", max_tok=8)
+ cold.close()
+
+ self.assertEqual(after, fresh,
+ "a stale record survived a shorter prompt and was reused")
+
+ def test_a_turn_that_replays_the_reply_is_never_wrong(self):
+ """The real chat shape: the client appends the assistant's reply and a
+ new question. Whether the reply survives this fixture's decode-encode
+ round trip decides whether reuse FIRES, and that is allowed to vary --
+ what may never vary is the answer."""
+ warm = self.spawn()
+ opening = b"The capital of France is"
+ reply = warm.ask("1", opening, max_tok=4)
+ second = opening + reply + b" and the capital of Spain is"
+ got = warm.ask("2", second, max_tok=8)
+ warm.close()
+
+ cold = self.spawn(log_prefix=False)
+ fresh = cold.ask("1", second, max_tok=8)
+ cold.close()
+
+ self.assertEqual(got, fresh, "replaying the reply changed the answer")
diff --git a/c/tests/test_olmoe_prefix_serve.py b/c/tests/test_olmoe_prefix_serve.py
new file mode 100644
index 00000000..6a1e0969
--- /dev/null
+++ b/c/tests/test_olmoe_prefix_serve.py
@@ -0,0 +1,36 @@
+"""KV prefix reuse on OLMoE: the shared contract, against the tiny fixture.
+
+OLMoE is the simple end of the contract -- plain GQA attention, no recurrent
+state, and a KV cache allocated once for the context and never reallocated, so
+a recorded position stays valid for the life of the process. That makes it the
+engine where a reuse bug would be quietest, which is why it is gated too.
+
+Runs against the tiny fixture the oracle job already builds
+(tools/make_olmoe_tiny.py + tools/convert_olmoe_merged.py +
+tools/make_edge_tiny_tokenizer.py).
+"""
+import os
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from prefix_serve_harness import PrefixReuseContract, ServeEngine # noqa: E402
+
+HERE = Path(__file__).resolve().parent.parent
+ENGINE = HERE / ("olmoe.exe" if sys.platform == "win32" else "olmoe")
+FIXTURE = Path(os.environ.get("OLMOE_TINY", HERE / "olmoe_tiny_c"))
+
+
[email protected](ENGINE.exists(), "olmoe is not built")
[email protected]((FIXTURE / "tokenizer.json").exists(),
+ "tiny olmoe container is absent (tools/convert_olmoe_merged.py)")
+class OlmoePrefixServeTest(PrefixReuseContract, unittest.TestCase):
+ def spawn(self, log_prefix=True, reuse=True):
+ return ServeEngine(ENGINE, ["8", "8"],
+ {"SNAP": str(FIXTURE), "SERVE": "1", "CTX": "512"},
+ log_prefix=log_prefix, reuse=reuse)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_qwen36_prefix_serve.py b/c/tests/test_qwen36_prefix_serve.py
new file mode 100644
index 00000000..4d3a9c48
--- /dev/null
+++ b/c/tests/test_qwen36_prefix_serve.py
@@ -0,0 +1,47 @@
+"""KV prefix reuse on Qwen3.6: the shared contract, against the tiny fixture.
+
+Qwen3.6 makes the contract stricter than a pure-attention engine would: three of
+every four layers are DeltaNet, which carries a RECURRENT state rather than
+position-indexed rows. Reuse keeps that state too, so a wrong reuse length
+corrupts the recurrence and not just the attention. This engine also GROWS its
+KV cache between turns, so tests/prefix_serve_harness.py's three-turn scenario
+covers the per-head copy that growth performs.
+
+Runs against the tiny random-init fixture the oracle job already builds
+(tools/make_qwen36_tiny.py + tools/convert_qwen36.py): no checkpoint, no disk.
+"""
+import os
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from prefix_serve_harness import (PrefixReuseContract, ServeEngine, # noqa: E402
+ ensure_byte_tokenizer)
+
+HERE = Path(__file__).resolve().parent.parent
+ENGINE = HERE / ("qwen36.exe" if sys.platform == "win32" else "qwen36")
+FIXTURE = Path(os.environ.get("QWEN36_TINY", HERE / "qwen36_tiny_c"))
+
+
[email protected](ENGINE.exists(), "qwen36 is not built")
+# The metadata of this container is TRACKED (config.json, qwen36_meta.json and
+# the README), so its presence says nothing about whether the weights were ever
+# converted. Gate on a shard the conversion writes, or this skips nothing and
+# fails on a checkout that never ran the converter.
[email protected]((FIXTURE / "model-globals.safetensors").exists(),
+ "tiny qwen36 container is absent (tools/convert_qwen36.py)")
+class Qwen36PrefixServeTest(PrefixReuseContract, unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ # The fixture ships no tokenizer: the oracle path feeds ids directly.
+ ensure_byte_tokenizer(FIXTURE)
+
+ def spawn(self, log_prefix=True, reuse=True):
+ return ServeEngine(ENGINE, ["8", "8"],
+ {"SNAP": str(FIXTURE), "SERVE": "1", "COLI_DENSE_I8": "0"},
+ log_prefix=log_prefix, reuse=reuse)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 53664bd0..50fb8ee3 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -374,6 +374,8 @@ Read **only** by `c/inkling.c`.
| `INK_SHARED_BATCH` | auto | Prefill rows per shared-expert batch, bounded to 64 MiB of scratch. `=0` restores the scalar per-token path for A/B/debugging; a positive value caps the chunk size. Decode (`S=1`) is unchanged. |
| `INK_METAL_MIN_S` | `1` | Minimum batch S to send the MoE block to Metal. `=2` restores the prefill-only gate (which mattered when the residency set was absent and per-block `useResource` churn cost ~135 ms). |
| `INK_PREFIX_LOG` | unset | Log the KV-prefix reuse decision and its reason, as `K3_PREFIX_LOG` does for K3. |
+| `COLI_PREFIX_LOG` | unset | Same line for the engines that take the shared record (Qwen3.6, OLMoE): reports how many prompt tokens were reused, or why none were. |
+| `COLI_KV_PREFIX` | on | `0` disables KV-prefix reuse on those engines. The escape hatch, and the B arm of the A/B that shows reuse changes nothing but the time. |
| `GPU_DEV` | `0` | CUDA device index for the inkling CUDA backend. |
| `NOGPU` | unset | If set, skip GPU init entirely (both CUDA and Metal), regardless of the other GPU variables. |