Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(bench): separate warmup from measured serving requests

ZacharyZcR282b743b50d9
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.py
index 5a67b455..98b5f826 100644
--- a/c/tests/test_benchmark_http_serving.py
+++ b/c/tests/test_benchmark_http_serving.py
@@ -192,6 +192,47 @@ class BenchmarkTest(unittest.TestCase):
row["first_output_seconds"] + row["dispatch_delay_seconds"])
self.assertEqual(summary["arrival_timing"]["dispatch_delay_seconds"]["count"], 3)
+ def test_cli_warmup_is_separate_and_failure_skips_measurement(self):
+ with tempfile.TemporaryDirectory() as directory:
+ workload = Path(directory) / "prompts.jsonl"
+ output = Path(directory) / "report.json"
+ prompts = self.workload + [{"messages": [{"role": "user", "content": "second"}]}]
+ workload.write_text("".join(json.dumps(p) + "\n" for p in prompts))
+ command = [sys.executable, bench.__file__, "--base-url", self.url.rsplit("/", 1)[0],
+ "--model", "fixture", "--workload", str(workload), "--output", str(output),
+ "--warmup-requests", "3", "--concurrency", "2", "--repeats", "2",
+ "--request-rate", "100", "--slo-duration", "5"]
+ for status, expected_exit in ((200, 0), (503, 1)):
+ with self.subTest(status=status):
+ self.server.status = status
+ self.server.payloads.clear()
+ completed = subprocess.run(command, capture_output=True, text=True, timeout=10)
+ self.assertEqual(completed.returncode, expected_exit, completed.stderr)
+ report = json.loads(output.read_text())
+ warmup = report["warmup"]
+ self.assertEqual(warmup["summary"]["requests"], 3)
+ self.assertIsNone(warmup["summary"]["arrival_timing"])
+ self.assertIsNone(warmup["summary"]["latency_slo"])
+ self.assertLessEqual(self.server.peak, 2)
+ self.assertCountEqual([p["messages"] for p in self.server.payloads[:3]],
+ [prompts[i % 2]["messages"] for i in range(3)])
+ if expected_exit:
+ self.assertEqual(report["status"], "warmup_failed")
+ self.assertIsNone(report["summary"])
+ self.assertEqual(report["requests"], [])
+ self.assertEqual(len(self.server.payloads), 3)
+ else:
+ self.assertEqual(report["status"], "measured")
+ self.assertEqual(len(self.server.payloads), 7)
+ self.assertEqual(report["summary"]["requests"], 4)
+ self.assertEqual(report["summary"]["reported_successful_completion_tokens"], 12)
+ self.assertEqual(report["summary"]["latency_slo"]["requests_met"], 4)
+ self.assertEqual([r["scheduled_seconds"] for r in report["requests"]],
+ [0, .01, .02, .03])
+ completed = subprocess.run(command + ["--warmup-requests", "-1"],
+ capture_output=True, text=True, timeout=10)
+ self.assertEqual(completed.returncode, 2)
+
def test_cli_report_and_failure_exit(self):
with tempfile.TemporaryDirectory() as directory:
workload = Path(directory) / "prompts.jsonl"
@@ -320,6 +361,35 @@ class ParsingTest(unittest.TestCase):
for row, expected in zip(rows, [0, .11, .21]):
self.assertAlmostEqual(row["start_seconds"], expected)
+ def test_warmup_time_is_excluded_from_measured_throughput(self):
+ now = [0.0]
+ calls = []
+ def request(_url, _payload, _key, _timeout, index, origin):
+ start = now[0]
+ duration = 100 if not calls else 1
+ calls.append(origin)
+ now[0] += duration
+ return {"index": index, "start_seconds": start - origin, "success": True,
+ "first_output_seconds": duration, "duration_seconds": duration,
+ "completion_tokens": 3}
+ with tempfile.TemporaryDirectory() as directory:
+ workload = Path(directory) / "workload.jsonl"
+ output = Path(directory) / "report.json"
+ workload.write_text('{"messages":[{"role":"user","content":"hello"}]}\n')
+ argv = [bench.__file__, "--base-url", "http://unused/v1", "--model", "fixture",
+ "--workload", str(workload), "--output", str(output),
+ "--warmup-requests", "1", "--repeats", "2", "--slo-duration", "2"]
+ with patch.object(sys, "argv", argv), patch.object(sys, "stdout", io.StringIO()), \
+ patch.object(bench.time, "perf_counter", side_effect=lambda: now[0]), \
+ patch.object(bench, "request_one", side_effect=request):
+ self.assertEqual(bench.main(), 0)
+ report = json.loads(output.read_text())
+ self.assertEqual(calls, [0, 100, 100])
+ self.assertEqual(report["warmup"]["summary"]["wall_seconds"], 100)
+ self.assertEqual(report["summary"]["wall_seconds"], 2)
+ self.assertEqual(report["summary"]["successful_completion_tokens_per_second"], 3)
+ self.assertEqual(report["summary"]["latency_slo"]["goodput_requests_per_second"], 1)
+
def test_request_rate_must_be_positive_and_finite(self):
for value in ("0", "-1", "nan", "inf"):
with self.subTest(value=value), self.assertRaises(bench.argparse.ArgumentTypeError):
diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.py
index 5f758c8a..b1154130 100644
--- a/c/tools/benchmark_http_serving.py
+++ b/c/tools/benchmark_http_serving.py
@@ -253,6 +253,13 @@ def positive_int(value):
return number
+def nonnegative_int(value):
+ number = int(value)
+ if number < 0:
+ raise argparse.ArgumentTypeError("must be nonnegative")
+ return number
+
+
def positive_float(value):
number = float(value)
if not math.isfinite(number) or number <= 0:
@@ -268,6 +275,8 @@ def main():
parser.add_argument("--output", required=True, help="JSON report path")
parser.add_argument("--concurrency", type=positive_int, default=1)
parser.add_argument("--repeats", type=positive_int, default=1)
+ parser.add_argument("--warmup-requests", type=nonnegative_int, default=0,
+ help="unmeasured requests before the timed phase (default: 0)")
parser.add_argument("--request-rate", type=positive_float, help="fixed scheduled arrivals per second")
parser.add_argument("--max-tokens", type=positive_int, default=128)
parser.add_argument("--temperature", type=float, default=0)
@@ -285,23 +294,32 @@ def main():
raise ValueError("output must differ from workload")
except (ValueError, OSError) as exc:
parser.error(str(exc))
- results, summary = run(url, workload, args.model, args.concurrency, args.repeats,
- args.max_tokens, args.temperature,
- os.environ.get(args.api_key_env, ""), args.timeout,
- args.slo_first_output, args.slo_duration, args.request_rate)
+ key = os.environ.get(args.api_key_env, "")
+ warmup = None
+ if args.warmup_requests:
+ prompts = [workload[i % len(workload)] for i in range(args.warmup_requests)]
+ rows, stats = run(url, prompts, args.model, args.concurrency, 1,
+ args.max_tokens, args.temperature, key, args.timeout)
+ warmup = {"summary": stats, "requests": rows}
+ results, summary = [], None
+ if warmup is None or warmup["summary"]["failed"] == 0:
+ results, summary = run(url, workload, args.model, args.concurrency, args.repeats,
+ args.max_tokens, args.temperature, key, args.timeout,
+ args.slo_first_output, args.slo_duration, args.request_rate)
report = {"schema_version": 1, "config": {
"endpoint": url, "model": args.model, "workload_sha256": digest,
"workload_rows": len(workload), "concurrency": args.concurrency,
"load_model": "fixed_rate" if args.request_rate is not None else "closed_loop",
- "request_rate": args.request_rate,
+ "request_rate": args.request_rate, "warmup_requests": args.warmup_requests,
"repeats": args.repeats, "max_tokens": args.max_tokens,
"temperature": args.temperature, "socket_timeout_seconds": args.timeout,
"slo_first_output_seconds": args.slo_first_output, "slo_duration_seconds": args.slo_duration,
"stream": True, "include_usage": True, "n": 1},
- "summary": summary, "requests": results}
+ "status": "warmup_failed" if summary is None else "measured",
+ "warmup": warmup, "summary": summary, "requests": results}
Path(args.output).write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8")
- print(json.dumps(summary, indent=2))
- return 1 if summary["failed"] else 0
+ print(json.dumps(summary if summary is not None else warmup, indent=2))
+ return 1 if summary is None or summary["failed"] else 0
if __name__ == "__main__":
diff --git a/docs/benchmarking.md b/docs/benchmarking.md
index e317204b..d0d28f25 100644
--- a/docs/benchmarking.md
+++ b/docs/benchmarking.md
@@ -118,11 +118,23 @@ Latency summaries use successful requests only, with nearest-rank p50/p95/p99
and sample counts. Failed requests retain their individual durations and any
observed output/usage.
+Optional `--warmup-requests N` sends N requests before measurement, cycling
+through workload rows with the same generation settings and concurrency limit.
+Warmup is closed-loop even when the measured phase uses `--request-rate`.
+All warmup requests finish before a fresh measurement clock and arrival schedule
+start. Their rows and summary are recorded separately under `warmup`; they do
+not contribute to measured latency, tokens, throughput, or SLO goodput. The
+default is zero (no warmup). If any warmup request fails, the report has
+`status: "warmup_failed"`, `summary: null`, and an empty measured `requests` list;
+the command exits 1 without starting measurement. Successful warmup does not
+prove stable performance. It can populate prefix caches, so record the same
+warmup and cache policy when comparing engines or runs.
+
Measurement boundaries:
- Without `--request-rate`, concurrency is closed-loop: at most that many
requests are in flight, and each worker starts its next request after its
- previous stream ends. There is no warmup, automatic retry, or cache flush. Repeats reuse the
+ previous stream ends. There is no automatic retry or cache flush. Repeats reuse the
conversations in file order; prefix caching and scheduling can affect results.
- Request timing begins inside the worker, before HTTP connection setup, and
excludes waiting for a local worker. Each request uses a new connection.