Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(bench): separate warmup from measured serving requests
ZacharyZcR282b743b50d9
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.pyindex 5a67b455..98b5f826 100644--- a/c/tests/test_benchmark_http_serving.py+++ b/c/tests/test_benchmark_http_serving.py@@ -192,6 +192,47 @@ class BenchmarkTest(unittest.TestCase):row["first_output_seconds"] + row["dispatch_delay_seconds"])self.assertEqual(summary["arrival_timing"]["dispatch_delay_seconds"]["count"], 3)+ def test_cli_warmup_is_separate_and_failure_skips_measurement(self):+ with tempfile.TemporaryDirectory() as directory:+ workload = Path(directory) / "prompts.jsonl"+ output = Path(directory) / "report.json"+ prompts = self.workload + [{"messages": [{"role": "user", "content": "second"}]}]+ workload.write_text("".join(json.dumps(p) + "\n" for p in prompts))+ command = [sys.executable, bench.__file__, "--base-url", self.url.rsplit("/", 1)[0],+ "--model", "fixture", "--workload", str(workload), "--output", str(output),+ "--warmup-requests", "3", "--concurrency", "2", "--repeats", "2",+ "--request-rate", "100", "--slo-duration", "5"]+ for status, expected_exit in ((200, 0), (503, 1)):+ with self.subTest(status=status):+ self.server.status = status+ self.server.payloads.clear()+ completed = subprocess.run(command, capture_output=True, text=True, timeout=10)+ self.assertEqual(completed.returncode, expected_exit, completed.stderr)+ report = json.loads(output.read_text())+ warmup = report["warmup"]+ self.assertEqual(warmup["summary"]["requests"], 3)+ self.assertIsNone(warmup["summary"]["arrival_timing"])+ self.assertIsNone(warmup["summary"]["latency_slo"])+ self.assertLessEqual(self.server.peak, 2)+ self.assertCountEqual([p["messages"] for p in self.server.payloads[:3]],+ [prompts[i % 2]["messages"] for i in range(3)])+ if expected_exit:+ self.assertEqual(report["status"], "warmup_failed")+ self.assertIsNone(report["summary"])+ self.assertEqual(report["requests"], [])+ self.assertEqual(len(self.server.payloads), 3)+ else:+ self.assertEqual(report["status"], "measured")+ self.assertEqual(len(self.server.payloads), 7)+ self.assertEqual(report["summary"]["requests"], 4)+ self.assertEqual(report["summary"]["reported_successful_completion_tokens"], 12)+ self.assertEqual(report["summary"]["latency_slo"]["requests_met"], 4)+ self.assertEqual([r["scheduled_seconds"] for r in report["requests"]],+ [0, .01, .02, .03])+ completed = subprocess.run(command + ["--warmup-requests", "-1"],+ capture_output=True, text=True, timeout=10)+ self.assertEqual(completed.returncode, 2)+def test_cli_report_and_failure_exit(self):with tempfile.TemporaryDirectory() as directory:workload = Path(directory) / "prompts.jsonl"@@ -320,6 +361,35 @@ class ParsingTest(unittest.TestCase):for row, expected in zip(rows, [0, .11, .21]):self.assertAlmostEqual(row["start_seconds"], expected)+ def test_warmup_time_is_excluded_from_measured_throughput(self):+ now = [0.0]+ calls = []+ def request(_url, _payload, _key, _timeout, index, origin):+ start = now[0]+ duration = 100 if not calls else 1+ calls.append(origin)+ now[0] += duration+ return {"index": index, "start_seconds": start - origin, "success": True,+ "first_output_seconds": duration, "duration_seconds": duration,+ "completion_tokens": 3}+ with tempfile.TemporaryDirectory() as directory:+ workload = Path(directory) / "workload.jsonl"+ output = Path(directory) / "report.json"+ workload.write_text('{"messages":[{"role":"user","content":"hello"}]}\n')+ argv = [bench.__file__, "--base-url", "http://unused/v1", "--model", "fixture",+ "--workload", str(workload), "--output", str(output),+ "--warmup-requests", "1", "--repeats", "2", "--slo-duration", "2"]+ with patch.object(sys, "argv", argv), patch.object(sys, "stdout", io.StringIO()), \+ patch.object(bench.time, "perf_counter", side_effect=lambda: now[0]), \+ patch.object(bench, "request_one", side_effect=request):+ self.assertEqual(bench.main(), 0)+ report = json.loads(output.read_text())+ self.assertEqual(calls, [0, 100, 100])+ self.assertEqual(report["warmup"]["summary"]["wall_seconds"], 100)+ self.assertEqual(report["summary"]["wall_seconds"], 2)+ self.assertEqual(report["summary"]["successful_completion_tokens_per_second"], 3)+ self.assertEqual(report["summary"]["latency_slo"]["goodput_requests_per_second"], 1)+def test_request_rate_must_be_positive_and_finite(self):for value in ("0", "-1", "nan", "inf"):with self.subTest(value=value), self.assertRaises(bench.argparse.ArgumentTypeError):diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.pyindex 5f758c8a..b1154130 100644--- a/c/tools/benchmark_http_serving.py+++ b/c/tools/benchmark_http_serving.py@@ -253,6 +253,13 @@ def positive_int(value):return number+def nonnegative_int(value):+ number = int(value)+ if number < 0:+ raise argparse.ArgumentTypeError("must be nonnegative")+ return number++def positive_float(value):number = float(value)if not math.isfinite(number) or number <= 0:@@ -268,6 +275,8 @@ def main():parser.add_argument("--output", required=True, help="JSON report path")parser.add_argument("--concurrency", type=positive_int, default=1)parser.add_argument("--repeats", type=positive_int, default=1)+ parser.add_argument("--warmup-requests", type=nonnegative_int, default=0,+ help="unmeasured requests before the timed phase (default: 0)")parser.add_argument("--request-rate", type=positive_float, help="fixed scheduled arrivals per second")parser.add_argument("--max-tokens", type=positive_int, default=128)parser.add_argument("--temperature", type=float, default=0)@@ -285,23 +294,32 @@ def main():raise ValueError("output must differ from workload")except (ValueError, OSError) as exc:parser.error(str(exc))- results, summary = run(url, workload, args.model, args.concurrency, args.repeats,- args.max_tokens, args.temperature,- os.environ.get(args.api_key_env, ""), args.timeout,- args.slo_first_output, args.slo_duration, args.request_rate)+ key = os.environ.get(args.api_key_env, "")+ warmup = None+ if args.warmup_requests:+ prompts = [workload[i % len(workload)] for i in range(args.warmup_requests)]+ rows, stats = run(url, prompts, args.model, args.concurrency, 1,+ args.max_tokens, args.temperature, key, args.timeout)+ warmup = {"summary": stats, "requests": rows}+ results, summary = [], None+ if warmup is None or warmup["summary"]["failed"] == 0:+ results, summary = run(url, workload, args.model, args.concurrency, args.repeats,+ args.max_tokens, args.temperature, key, args.timeout,+ args.slo_first_output, args.slo_duration, args.request_rate)report = {"schema_version": 1, "config": {"endpoint": url, "model": args.model, "workload_sha256": digest,"workload_rows": len(workload), "concurrency": args.concurrency,"load_model": "fixed_rate" if args.request_rate is not None else "closed_loop",- "request_rate": args.request_rate,+ "request_rate": args.request_rate, "warmup_requests": args.warmup_requests,"repeats": args.repeats, "max_tokens": args.max_tokens,"temperature": args.temperature, "socket_timeout_seconds": args.timeout,"slo_first_output_seconds": args.slo_first_output, "slo_duration_seconds": args.slo_duration,"stream": True, "include_usage": True, "n": 1},- "summary": summary, "requests": results}+ "status": "warmup_failed" if summary is None else "measured",+ "warmup": warmup, "summary": summary, "requests": results}Path(args.output).write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8")- print(json.dumps(summary, indent=2))- return 1 if summary["failed"] else 0+ print(json.dumps(summary if summary is not None else warmup, indent=2))+ return 1 if summary is None or summary["failed"] else 0if __name__ == "__main__":diff --git a/docs/benchmarking.md b/docs/benchmarking.mdindex e317204b..d0d28f25 100644--- a/docs/benchmarking.md+++ b/docs/benchmarking.md@@ -118,11 +118,23 @@ Latency summaries use successful requests only, with nearest-rank p50/p95/p99and sample counts. Failed requests retain their individual durations and anyobserved output/usage.+Optional `--warmup-requests N` sends N requests before measurement, cycling+through workload rows with the same generation settings and concurrency limit.+Warmup is closed-loop even when the measured phase uses `--request-rate`.+All warmup requests finish before a fresh measurement clock and arrival schedule+start. Their rows and summary are recorded separately under `warmup`; they do+not contribute to measured latency, tokens, throughput, or SLO goodput. The+default is zero (no warmup). If any warmup request fails, the report has+`status: "warmup_failed"`, `summary: null`, and an empty measured `requests` list;+the command exits 1 without starting measurement. Successful warmup does not+prove stable performance. It can populate prefix caches, so record the same+warmup and cache policy when comparing engines or runs.+Measurement boundaries:- Without `--request-rate`, concurrency is closed-loop: at most that manyrequests are in flight, and each worker starts its next request after its- previous stream ends. There is no warmup, automatic retry, or cache flush. Repeats reuse the+ previous stream ends. There is no automatic retry or cache flush. Repeats reuse theconversations in file order; prefix caching and scheduling can affect results.- Request timing begins inside the worker, before HTTP connection setup, andexcludes waiting for a local worker. Each request uses a new connection.