Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

diag: say why a turn did not reuse, and capture stderr reliably

JustVuggc7f7f8cefef7
diff --git a/c/inkling.c b/c/inkling.c
index f88b1113..d05e50b6 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -1707,10 +1707,21 @@ static void serve_one(Model *m, Tok *T, SReq *q) {
* brings its own audio must not match a text-only state either. */
if (naud > 0) kv_prefix_taint(&m->kvp);
int reuse = kv_prefix_reuse(&m->kvp, ids, np);
+ if (getenv("INK_PREFIX_LOG")) {
+ /* Report the decision either way, with the reason when it is no. "It
+ * did not get faster" is otherwise indistinguishable from "reuse is not
+ * wired up", both for a user and for the CI gate. */
+ if (reuse)
+ fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",
+ reuse, np, 100.0 * reuse / np);
+ else
+ fprintf(stderr, "[PREFIX] no reuse: held=%d cap=%d prompt=%d%s%s\n",
+ m->kvp.len, m->kvp.cap, np,
+ m->kvp.tainted ? " tainted" : "",
+ (m->kvp.len > 0 && m->kvp.len < np) ? " (diverged)" : "");
+ fflush(stderr);
+ }
if (!reuse) state_reset(m);
- else if (getenv("INK_PREFIX_LOG"))
- fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",
- reuse, np, 100.0 * reuse / np);
double t0 = now_s();
uint64_t h0 = m->hits, m0 = m->miss;
/* per-turn phase snapshot for the PROF line (timers accumulate globally) */
diff --git a/c/tests/test_inkling_prefix_serve.py b/c/tests/test_inkling_prefix_serve.py
index 85d12c19..c4d921ab 100644
--- a/c/tests/test_inkling_prefix_serve.py
+++ b/c/tests/test_inkling_prefix_serve.py
@@ -15,6 +15,7 @@ why this gate lives in CI on a fixture rather than in a benchmark on hardware.
import json
import os
import subprocess
+import threading
import sys
import time
import unittest
@@ -71,6 +72,13 @@ class Engine:
self.p = subprocess.Popen([str(ENGINE), "8"], env=env,
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=0)
+ # Drain stderr continuously. Reading it only at close() loses whatever
+ # the engine wrote after the pipe filled, and an empty capture is
+ # indistinguishable from an engine that said nothing -- which is exactly
+ # the ambiguity that made the first failure here unreadable.
+ self._err = []
+ self._pump = threading.Thread(target=self._drain, daemon=True)
+ self._pump.start()
deadline = time.time() + 300
while time.time() < deadline:
line = self.p.stdout.readline()
@@ -102,13 +110,18 @@ class Engine:
elif kind == "ERROR":
raise RuntimeError(f"engine error: {text}")
+ def _drain(self):
+ for line in iter(self.p.stderr.readline, b""):
+ self._err.append(line.decode(errors="replace"))
+
def close(self):
try:
self.p.stdin.close()
self.p.wait(timeout=60)
except Exception:
self.p.kill()
- return self.p.stderr.read().decode(errors="replace")
+ self._pump.join(timeout=10)
+ return "".join(self._err)
@unittest.skipUnless(ENGINE.exists(), "inkling is not built")
@@ -145,7 +158,8 @@ class InklingPrefixServeTest(unittest.TestCase):
self.assertIn("[PREFIX] reusing", log,
"the second turn did not reuse the first turn's state; "
- "this test proves nothing unless it does")
+ "this test proves nothing unless it does.\n"
+ "engine said:\n" + (log or "(nothing on stderr)"))
self.assertEqual(reused, fresh,
"reusing the prefix changed the output — the engine "
"answered from a state that is not this conversation")