Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(tune): minimize replay compatibility change

dyKiUc2158805d559
diff --git a/c/autotune.py b/c/autotune.py
index 67bb3d43..b63413e3 100644
--- a/c/autotune.py
+++ b/c/autotune.py
@@ -22,9 +22,7 @@ from pathlib import Path
SCHEMA_VERSION = 1
PROMPT_RE = re.compile(r"\[PROMPT_TOKENS\]\s+\d+:\s*([0-9 ]+)")
TOKENS_RE = re.compile(r"\[TOKENS\]\s+\d+\s+generated:\s*([0-9 ]+)")
-REPLAY_TIMING_RE = re.compile(
- r"REPLAY decode:\s+(\d+)\s+tokens in\s+([0-9.]+)s.*?\|\s*[0-9.]+\s+tok/s"
-)
+TIMING_RE = re.compile(r"REPLAY decode:\s+(\d+)\s+tokens in\s+([0-9.]+)s")
SPEED_RE = re.compile(r"REPLAY decode:\s+\d+\s+tokens.*?\|\s*([0-9.]+)\s+tok/s")
HIT_RE = re.compile(r"expert hit\s+([0-9.]+)%")
LATENCY_RE = re.compile(r"latency p50\s+([0-9.]+)\s*ms.*?p99\s+([0-9.]+)\s*ms")
@@ -166,18 +164,15 @@ def candidate_steps(plan: dict, base_env: dict) -> list[tuple[str, dict]]:
def parse_replay(output: str) -> dict:
- timing = REPLAY_TIMING_RE.search(output)
speed = SPEED_RE.search(output)
- if timing:
+ if not speed:
+ raise ValueError("engine did not emit REPLAY throughput")
+ tok_s = float(speed.group(1))
+ timing = TIMING_RE.search(output)
+ if timing and float(timing.group(2)) > 0:
tokens = int(timing.group(1))
elapsed_s = float(timing.group(2))
- if elapsed_s <= 0:
- raise ValueError("engine emitted an invalid REPLAY duration")
tok_s = tokens / elapsed_s
- elif speed:
- tok_s = float(speed.group(1))
- else:
- raise ValueError("engine did not emit REPLAY throughput")
hit = HIT_RE.search(output)
latency = LATENCY_RE.search(output)
return {
diff --git a/c/tests/test_autotune.py b/c/tests/test_autotune.py
index b5f04584..bd91e676 100644
--- a/c/tests/test_autotune.py
+++ b/c/tests/test_autotune.py
@@ -73,6 +73,12 @@ class AutotuneUnitTest(unittest.TestCase):
)
self.assertEqual(result["tok_s"], 0.04)
+ def test_parse_replay_falls_back_when_elapsed_time_rounds_to_zero(self):
+ result = parse_replay(
+ "REPLAY decode: 1 tokens in 0.000s | 1234.56 tok/s"
+ )
+ self.assertEqual(result["tok_s"], 1234.56)
+
def test_profile_round_trip_and_explicit_environment_wins(self):
with tempfile.TemporaryDirectory() as directory:
engine = Path(directory) / "engine"
@@ -124,8 +130,7 @@ class AutotuneIntegrationTest(unittest.TestCase):
" pipe=os.environ.get('COLI_CUDA_PIPE','0')\n"
" speed={'0':10.0,'1':12.0,'2':11.0}[pipe]\n"
" if os.environ.get('COLI_CUDA_ASYNC') == '0': speed=9.0\n"
- " elapsed=4/speed\n"
- " print(f'REPLAY decode: 4 tokens in {elapsed:.6f}s | {speed:.2f} tok/s | expert hit 95.0%')\n"
+ " print(f'REPLAY decode: 4 tokens | {speed:.2f} tok/s | expert hit 95.0%')\n"
" print('[PROF] decode forwards: 4 | latency p50 80.0 ms | p90 90.0 ms | p99 100.0 ms | max 100.0 ms')\n",
encoding="utf-8",
)
@@ -138,10 +143,8 @@ class AutotuneIntegrationTest(unittest.TestCase):
)
self.assertTrue(profile["accepted"])
self.assertEqual(profile["winner"]["env"], {"COLI_CUDA_PIPE": "1"})
- self.assertAlmostEqual(profile["gain"], 0.20, places=5)
- self.assertAlmostEqual(
- json.loads(path.read_text())["winner"]["tok_s"], 12.0, places=4
- )
+ self.assertAlmostEqual(profile["gain"], 0.20)
+ self.assertEqual(json.loads(path.read_text())["winner"]["tok_s"], 12.0)
def test_reverse_confirmation_rejects_warmup_drift(self):
with tempfile.TemporaryDirectory() as directory:
@@ -159,8 +162,7 @@ class AutotuneIntegrationTest(unittest.TestCase):
" except FileNotFoundError: n=0\n"
" open(counter,'w').write(str(n+1))\n"
" speed=10.0+n\n"
- " elapsed=4/speed\n"
- " print(f'REPLAY decode: 4 tokens in {elapsed:.6f}s | {speed:.2f} tok/s | expert hit 95.0%')\n"
+ " print(f'REPLAY decode: 4 tokens | {speed:.2f} tok/s | expert hit 95.0%')\n"
" print('[PROF] decode forwards: 4 | latency p50 80.0 ms | p90 90.0 ms | p99 100.0 ms | max 100.0 ms')\n",
encoding="utf-8",
)