Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
diag: say why a turn did not reuse, and capture stderr reliably
JustVuggc7f7f8cefef7
diff --git a/c/inkling.c b/c/inkling.cindex f88b1113..d05e50b6 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -1707,10 +1707,21 @@ static void serve_one(Model *m, Tok *T, SReq *q) {* brings its own audio must not match a text-only state either. */if (naud > 0) kv_prefix_taint(&m->kvp);int reuse = kv_prefix_reuse(&m->kvp, ids, np);+ if (getenv("INK_PREFIX_LOG")) {+ /* Report the decision either way, with the reason when it is no. "It+ * did not get faster" is otherwise indistinguishable from "reuse is not+ * wired up", both for a user and for the CI gate. */+ if (reuse)+ fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",+ reuse, np, 100.0 * reuse / np);+ else+ fprintf(stderr, "[PREFIX] no reuse: held=%d cap=%d prompt=%d%s%s\n",+ m->kvp.len, m->kvp.cap, np,+ m->kvp.tainted ? " tainted" : "",+ (m->kvp.len > 0 && m->kvp.len < np) ? " (diverged)" : "");+ fflush(stderr);+ }if (!reuse) state_reset(m);- else if (getenv("INK_PREFIX_LOG"))- fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",- reuse, np, 100.0 * reuse / np);double t0 = now_s();uint64_t h0 = m->hits, m0 = m->miss;/* per-turn phase snapshot for the PROF line (timers accumulate globally) */diff --git a/c/tests/test_inkling_prefix_serve.py b/c/tests/test_inkling_prefix_serve.pyindex 85d12c19..c4d921ab 100644--- a/c/tests/test_inkling_prefix_serve.py+++ b/c/tests/test_inkling_prefix_serve.py@@ -15,6 +15,7 @@ why this gate lives in CI on a fixture rather than in a benchmark on hardware.import jsonimport osimport subprocess+import threadingimport sysimport timeimport unittest@@ -71,6 +72,13 @@ class Engine:self.p = subprocess.Popen([str(ENGINE), "8"], env=env,stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, bufsize=0)+ # Drain stderr continuously. Reading it only at close() loses whatever+ # the engine wrote after the pipe filled, and an empty capture is+ # indistinguishable from an engine that said nothing -- which is exactly+ # the ambiguity that made the first failure here unreadable.+ self._err = []+ self._pump = threading.Thread(target=self._drain, daemon=True)+ self._pump.start()deadline = time.time() + 300while time.time() < deadline:line = self.p.stdout.readline()@@ -102,13 +110,18 @@ class Engine:elif kind == "ERROR":raise RuntimeError(f"engine error: {text}")+ def _drain(self):+ for line in iter(self.p.stderr.readline, b""):+ self._err.append(line.decode(errors="replace"))+def close(self):try:self.p.stdin.close()self.p.wait(timeout=60)except Exception:self.p.kill()- return self.p.stderr.read().decode(errors="replace")+ self._pump.join(timeout=10)+ return "".join(self._err)@unittest.skipUnless(ENGINE.exists(), "inkling is not built")@@ -145,7 +158,8 @@ class InklingPrefixServeTest(unittest.TestCase):self.assertIn("[PREFIX] reusing", log,"the second turn did not reuse the first turn's state; "- "this test proves nothing unless it does")+ "this test proves nothing unless it does.\n"+ "engine said:\n" + (log or "(nothing on stderr)"))self.assertEqual(reused, fresh,"reusing the prefix changed the output — the engine ""answered from a state that is not this conversation")