Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(bench): report latency-constrained request goodput
ZacharyZcR0e939fc98663
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.pyindex 9f0daad9..117b10d4 100644--- a/c/tests/test_benchmark_http_serving.py+++ b/c/tests/test_benchmark_http_serving.py@@ -151,7 +151,8 @@ class BenchmarkTest(unittest.TestCase):output = Path(directory) / "report.json"workload.write_text(json.dumps(self.workload[0]) + "\n", encoding="utf-8")command = [sys.executable, bench.__file__, "--base-url", self.url.rsplit("/", 1)[0],- "--model", "fixture", "--workload", str(workload), "--output", str(output)]+ "--model", "fixture", "--workload", str(workload), "--output", str(output),+ "--slo-first-output", "5", "--slo-duration", "5"]for status, exit_code in ((200, 0), (503, 1)):self.server.status = statuscompleted = subprocess.run(command, capture_output=True, text=True, timeout=10)@@ -160,6 +161,8 @@ class BenchmarkTest(unittest.TestCase):self.assertEqual(report["summary"]["succeeded"], 1 - exit_code)self.assertEqual(len(report["config"]["workload_sha256"]), 64)self.assertNotIn("messages", report["config"])+ self.assertEqual(report["config"]["slo_duration_seconds"], 5)+ self.assertEqual(report["summary"]["latency_slo"]["requests_met"], 1 - exit_code)class ColibriIntegrationTest(unittest.TestCase):@@ -209,6 +212,28 @@ class ParsingTest(unittest.TestCase):with self.assertRaises(ValueError):bench.load_workload(path)+ def test_latency_slo_counts_all_attempts_in_denominator(self):+ def row(first, duration, success=True):+ return {"success": success, "first_output_seconds": first,+ "duration_seconds": duration, "completion_tokens": None}+ rows = [row(.5, 2), row(.6, 1), row(.2, 3), row(None, 1), row(.1, 1, False)]+ summary = bench.summarize(rows, 10, slo_first_output=.5, slo_duration=2)+ slo = summary["latency_slo"]+ self.assertEqual(slo["requests_met"], 1)+ self.assertEqual(slo["fraction_of_attempts"], .2)+ self.assertEqual(slo["goodput_requests_per_second"], .1)+ self.assertIsNone(summary["successful_completion_tokens_per_second"])+ self.assertEqual(bench.summarize(rows, 10, slo_duration=2)["latency_slo"]["requests_met"], 3)+ self.assertEqual(bench.summarize(rows, 10, slo_first_output=.5)["latency_slo"]["requests_met"], 2)+ self.assertIsNone(bench.summarize(rows, 10)["latency_slo"])++ def test_latency_slo_no_successes(self):+ row = {"success": False, "first_output_seconds": .1,+ "duration_seconds": .2, "completion_tokens": 10}+ slo = bench.summarize([row], 1, slo_duration=1)["latency_slo"]+ self.assertEqual(slo["requests_met"], 0)+ self.assertEqual(slo["goodput_requests_per_second"], 0)+def test_nearest_rank_distribution(self):self.assertEqual(bench.distribution(list(range(1, 101)))["p95"], 95)self.assertIsNone(bench.distribution([])["p95"])diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.pyindex 991669ea..8ea3a8fd 100644--- a/c/tools/benchmark_http_serving.py+++ b/c/tools/benchmark_http_serving.py@@ -143,12 +143,12 @@ def distribution(values):return result-def summarize(results, elapsed):+def summarize(results, elapsed, slo_first_output=None, slo_duration=None):successful = [row for row in results if row["success"]]counted = [row for row in successful if row["completion_tokens"] is not None]tokens = sum(row["completion_tokens"] for row in counted)complete = bool(successful) and len(counted) == len(successful)- return {"requests": len(results), "succeeded": len(successful),+ summary = {"requests": len(results), "succeeded": len(successful),"failed": len(results) - len(successful),"failure_rate": (len(results) - len(successful)) / len(results),"wall_seconds": elapsed,@@ -160,8 +160,23 @@ def summarize(results, elapsed):"successful_first_output_seconds": distribution([r["first_output_seconds"] for r in successful if r["first_output_seconds"] is not None])}--def run(url, workload, model, concurrency, repeats, max_tokens, temperature, key, timeout):+ summary["latency_slo"] = None+ if slo_first_output is not None or slo_duration is not None:+ met = sum(+ (slo_duration is None or row["duration_seconds"] <= slo_duration)+ and (slo_first_output is None or (+ row["first_output_seconds"] is not None+ and row["first_output_seconds"] <= slo_first_output))+ for row in successful)+ summary["latency_slo"] = {+ "first_output_seconds": slo_first_output, "duration_seconds": slo_duration,+ "requests_met": met, "fraction_of_attempts": met / len(results),+ "goodput_requests_per_second": met / elapsed}+ return summary+++def run(url, workload, model, concurrency, repeats, max_tokens, temperature, key, timeout,+ slo_first_output=None, slo_duration=None):origin = time.perf_counter()# Workers take the next request as soon as the previous stream ends. Timers# begin inside workers, excluding time waiting in the local executor queue.@@ -173,7 +188,7 @@ def run(url, workload, model, concurrency, repeats, max_tokens, temperature, keytemperature=temperature, n=1)futures.append(pool.submit(request_one, url, payload, key, timeout, index, origin))results = [future.result() for future in futures]- return results, summarize(results, time.perf_counter() - origin)+ return results, summarize(results, time.perf_counter() - origin, slo_first_output, slo_duration)def positive_int(value):@@ -202,6 +217,8 @@ def main():parser.add_argument("--temperature", type=float, default=0)parser.add_argument("--timeout", type=positive_float, default=60, help="socket operation timeout in seconds")parser.add_argument("--api-key-env", default="OPENAI_API_KEY")+ parser.add_argument("--slo-first-output", type=positive_float, help="first output latency target, seconds")+ parser.add_argument("--slo-duration", type=positive_float, help="completed request latency target, seconds")args = parser.parse_args()try:url = endpoint(args.base_url)@@ -214,12 +231,14 @@ def main():parser.error(str(exc))results, summary = run(url, workload, args.model, args.concurrency, args.repeats,args.max_tokens, args.temperature,- os.environ.get(args.api_key_env, ""), args.timeout)+ os.environ.get(args.api_key_env, ""), args.timeout,+ args.slo_first_output, args.slo_duration)report = {"schema_version": 1, "config": {"endpoint": url, "model": args.model, "workload_sha256": digest,"workload_rows": len(workload), "concurrency": args.concurrency,"repeats": args.repeats, "max_tokens": args.max_tokens,"temperature": args.temperature, "socket_timeout_seconds": args.timeout,+ "slo_first_output_seconds": args.slo_first_output, "slo_duration_seconds": args.slo_duration,"stream": True, "include_usage": True, "n": 1},"summary": summary, "requests": results}Path(args.output).write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8")diff --git a/docs/benchmarking.md b/docs/benchmarking.mdindex 21179143..3009a267 100644--- a/docs/benchmarking.md+++ b/docs/benchmarking.md@@ -150,3 +150,32 @@ and run an independent quality check: this tool deliberately does not saveresponse text or assess correctness. Retain the workload file with its hash,interleave server runs, and report repeated-run spread. Start at concurrency 1,then increase it to expose queueing and prefill interference.++### Measure throughput within a latency target++Add `--slo-first-output 1 --slo-duration 15` to require first output within one+second and protocol completion within fifteen seconds. Either flag can be used+alone; values are finite positive seconds and the boundary is inclusive.+`summary.latency_slo` reports the thresholds, `requests_met`, the fraction of+**all attempts** meeting them, and `goodput_requests_per_second` (qualifying+successful requests divided by the entire batch wall time). Without thresholds,+this field is null. Failures never qualify, even if they emitted output before+failing. When a first-output target is set, empty-output successes also do not+qualify. Missing token usage does not prevent evaluating these latency targets.++This follows the latency-constrained goodput approach used by+[vLLM's serving benchmark](https://docs.vllm.ai/en/latest/api/vllm/benchmarks/serve/),+with the client first-output boundary defined above. It does not assert TPOT or+per-token SLOs, output quality, a minimum response length, or an equivalent vLLM+TTFT definition. SLO misses alone do not change the CLI exit status: exit 1 still+means at least one protocol/transport failure. Record output-length and quality+controls alongside goodput so short or empty answers cannot masquerade as an+improvement.++Keep the load model fixed when comparing reports.+[SGLang's serving benchmark](https://github.com/sgl-project/sglang/blob/main/python/sglang/benchmark/serving.py)+also supports request-rate-driven arrivals and trace timestamps. This tool still+uses closed-loop concurrency: slow responses reduce the rate at which new+requests start. Its goodput cannot establish an open-loop arrival-rate capacity+or a production SLO guarantee. Rate-controlled arrivals and warmup/cache policy+remain separate measurement needs.