Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(tune): retain legacy throughput parsing

dyKiU8430c3801ab8
diff --git a/c/autotune.py b/c/autotune.py
index 20824080..67bb3d43 100644
--- a/c/autotune.py
+++ b/c/autotune.py
@@ -25,6 +25,7 @@ TOKENS_RE = re.compile(r"\[TOKENS\]\s+\d+\s+generated:\s*([0-9 ]+)")
REPLAY_TIMING_RE = re.compile(
r"REPLAY decode:\s+(\d+)\s+tokens in\s+([0-9.]+)s.*?\|\s*[0-9.]+\s+tok/s"
)
+SPEED_RE = re.compile(r"REPLAY decode:\s+\d+\s+tokens.*?\|\s*([0-9.]+)\s+tok/s")
HIT_RE = re.compile(r"expert hit\s+([0-9.]+)%")
LATENCY_RE = re.compile(r"latency p50\s+([0-9.]+)\s*ms.*?p99\s+([0-9.]+)\s*ms")
@@ -166,16 +167,21 @@ def candidate_steps(plan: dict, base_env: dict) -> list[tuple[str, dict]]:
def parse_replay(output: str) -> dict:
timing = REPLAY_TIMING_RE.search(output)
- if not timing:
- raise ValueError("engine did not emit REPLAY timing")
- tokens = int(timing.group(1))
- elapsed_s = float(timing.group(2))
- if elapsed_s <= 0:
- raise ValueError("engine emitted an invalid REPLAY duration")
+ speed = SPEED_RE.search(output)
+ if timing:
+ tokens = int(timing.group(1))
+ elapsed_s = float(timing.group(2))
+ if elapsed_s <= 0:
+ raise ValueError("engine emitted an invalid REPLAY duration")
+ tok_s = tokens / elapsed_s
+ elif speed:
+ tok_s = float(speed.group(1))
+ else:
+ raise ValueError("engine did not emit REPLAY throughput")
hit = HIT_RE.search(output)
latency = LATENCY_RE.search(output)
return {
- "tok_s": tokens / elapsed_s,
+ "tok_s": tok_s,
"hit_pct": float(hit.group(1)) if hit else None,
"p50_ms": float(latency.group(1)) if latency else None,
"p99_ms": float(latency.group(2)) if latency else None,
diff --git a/c/tests/test_autotune.py b/c/tests/test_autotune.py
index d0ad21b1..b5f04584 100644
--- a/c/tests/test_autotune.py
+++ b/c/tests/test_autotune.py
@@ -65,6 +65,14 @@ class AutotuneUnitTest(unittest.TestCase):
self.assertAlmostEqual(result["tok_s"], 16 / 355.556)
self.assertNotEqual(result["tok_s"], 0.04)
+ def test_parse_replay_accepts_legacy_speed_without_elapsed_time(self):
+ result = parse_replay(
+ "REPLAY decode: 16 tokens | 0.04 tok/s | expert hit 34.4%\n"
+ "[PROF] decode forwards: 16 | latency p50 4000.0 ms | p90 5000.0 ms | "
+ "p99 60000.0 ms | max 70000.0 ms"
+ )
+ self.assertEqual(result["tok_s"], 0.04)
+
def test_profile_round_trip_and_explicit_environment_wins(self):
with tempfile.TemporaryDirectory() as directory:
engine = Path(directory) / "engine"