Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
test(inkling): prove prefix reuse is token-identical, end to end, in CI
JustVuggc3e251eb88e4
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 578ac41a..c22d8c13 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -121,6 +121,15 @@ jobs:# gate fails the moment a shared-code change regresses the engine.python3 tools/make_tiny_inkling.py tiny_inklingSNAP=tiny_inkling ./inkling 8 0 tiny_inkling/ref_inkling.json+ - name: KV prefix reuse is token-identical (serve, two turns)+ run: |+ cd c+ # Reusing a previous turn's attention state must change the timing and+ # nothing else. A wrong reuse length does not crash -- it answers from+ # another conversation's state and still reads plausibly -- so this is+ # the only gate that would catch it. Needs the fixture above, which is+ # why it runs here and not in `make check`.+ INKLING_TINY=tiny_inkling python3 -m unittest -v tests.test_inkling_prefix_serveengine-hip-syntax:name: HIP syntax checkdiff --git a/c/tests/test_inkling_prefix_serve.py b/c/tests/test_inkling_prefix_serve.pynew file mode 100644index 00000000..d95b3d1f--- /dev/null+++ b/c/tests/test_inkling_prefix_serve.py@@ -0,0 +1,120 @@+"""End-to-end proof that KV prefix reuse changes nothing but the time.++Two turns on one engine (the second reusing the first's state) must produce the+SAME tokens as the second turn alone on a cold engine. That is the whole licence+to ship the optimisation: if reuse alters even one token it is answering from a+state that belongs to a different conversation, and the reply would still look+plausible β nothing else in the tree would catch it.++Runs against the tiny random-init fixture the oracle job already builds+(tools/make_tiny_inkling.py), so it needs no checkpoint and no fast disk. The+real 469 GB model cannot serve this test on a developer machine: measured here,+a single token pulled 79 GB off disk in 9 minutes and had not finished. That is+why this gate lives in CI on a fixture rather than in a benchmark on hardware.+"""+import os+import subprocess+import sys+import time+import unittest+from pathlib import Path++HERE = Path(__file__).resolve().parent.parent+ENGINE = HERE / ("inkling.exe" if sys.platform == "win32" else "inkling")+FIXTURE = Path(os.environ.get("INKLING_TINY", HERE / "tiny_inkling"))+MAXTOK = 4+READY = b"\x01\x01READY\x01\x01\n"+++class Engine:+ """A serve-mode inkling, speaking the protocol in docs/serve_protocol.md."""++ def __init__(self, log_prefix=True):+ env = dict(os.environ, SNAP=str(FIXTURE), SERVE="1", NGEN=str(MAXTOK))+ if log_prefix:+ env["INK_PREFIX_LOG"] = "1"+ self.p = subprocess.Popen([str(ENGINE), "8"], env=env,+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,+ stderr=subprocess.PIPE, bufsize=0)+ deadline = time.time() + 300+ while time.time() < deadline:+ line = self.p.stdout.readline()+ if not line:+ err = self.p.stderr.read().decode(errors="replace")+ raise RuntimeError(f"engine exited before READY:\n{err[-2000:]}")+ if READY.strip() in line:+ return+ raise RuntimeError("engine never reported READY")++ def ask(self, rid, prompt):+ payload = prompt.encode()+ header = f"SUBMIT {rid} 0 {len(payload)} {MAXTOK} 0 1\n".encode()+ self.p.stdin.write(header + payload + b"\n")+ self.p.stdin.flush()+ chunks = []+ while True:+ line = self.p.stdout.readline()+ if not line:+ raise RuntimeError("engine closed mid-request")+ text = line.decode(errors="replace").rstrip("\n")+ kind = text.split(" ", 1)[0]+ if kind == "DATA":+ chunks.append(self.p.stdout.read(int(text.split()[2]))+ .decode(errors="replace"))+ self.p.stdout.readline() # the newline after the payload+ elif kind in ("DONE", "END"):+ return "".join(chunks)+ elif kind == "ERROR":+ raise RuntimeError(f"engine error: {text}")++ def close(self):+ try:+ self.p.stdin.close()+ self.p.wait(timeout=60)+ except Exception:+ self.p.kill()+ return self.p.stderr.read().decode(errors="replace")++[email protected](ENGINE.exists(), "inkling is not built")[email protected]((FIXTURE / "config.json").exists(),+ "tiny inkling fixture is absent (tools/make_tiny_inkling.py)")+class InklingPrefixServeTest(unittest.TestCase):+ def test_reused_prefix_yields_identical_tokens(self):+ warm = Engine()+ first = warm.ask("1", "The capital of France is")+ # Turn 2 EXTENDS what the state already holds β the shape a chat client+ # produces when it resends the transcript with a new question appended.+ second_prompt = "The capital of France is" + first + " and the capital of Spain is"+ reused = warm.ask("2", second_prompt)+ log = warm.close()++ cold = Engine(log_prefix=False)+ fresh = cold.ask("1", second_prompt)+ cold.close()++ self.assertIn("[PREFIX] reusing", log,+ "the second turn did not reuse the first turn's state; "+ "this test proves nothing unless it does")+ self.assertEqual(reused, fresh,+ "reusing the prefix changed the output β the engine "+ "answered from a state that is not this conversation")++ def test_a_diverging_prompt_is_not_reused(self):+ """The rejection path matters as much as the reuse path: a prompt that+ shares no prefix must start over, not splice itself onto stale state."""+ eng = Engine()+ eng.ask("1", "The capital of France is")+ diverged = eng.ask("2", "Completely different opening text here")+ log = eng.close()++ cold = Engine(log_prefix=False)+ fresh = cold.ask("1", "Completely different opening text here")+ cold.close()++ self.assertEqual(diverged, fresh,+ "a diverging prompt was contaminated by the previous turn")+++if __name__ == "__main__":+ unittest.main()