Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(bench): report latency-constrained request goodput

ZacharyZcR0e939fc98663
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.py
index 9f0daad9..117b10d4 100644
--- a/c/tests/test_benchmark_http_serving.py
+++ b/c/tests/test_benchmark_http_serving.py
@@ -151,7 +151,8 @@ class BenchmarkTest(unittest.TestCase):
output = Path(directory) / "report.json"
workload.write_text(json.dumps(self.workload[0]) + "\n", encoding="utf-8")
command = [sys.executable, bench.__file__, "--base-url", self.url.rsplit("/", 1)[0],
- "--model", "fixture", "--workload", str(workload), "--output", str(output)]
+ "--model", "fixture", "--workload", str(workload), "--output", str(output),
+ "--slo-first-output", "5", "--slo-duration", "5"]
for status, exit_code in ((200, 0), (503, 1)):
self.server.status = status
completed = subprocess.run(command, capture_output=True, text=True, timeout=10)
@@ -160,6 +161,8 @@ class BenchmarkTest(unittest.TestCase):
self.assertEqual(report["summary"]["succeeded"], 1 - exit_code)
self.assertEqual(len(report["config"]["workload_sha256"]), 64)
self.assertNotIn("messages", report["config"])
+ self.assertEqual(report["config"]["slo_duration_seconds"], 5)
+ self.assertEqual(report["summary"]["latency_slo"]["requests_met"], 1 - exit_code)
class ColibriIntegrationTest(unittest.TestCase):
@@ -209,6 +212,28 @@ class ParsingTest(unittest.TestCase):
with self.assertRaises(ValueError):
bench.load_workload(path)
+ def test_latency_slo_counts_all_attempts_in_denominator(self):
+ def row(first, duration, success=True):
+ return {"success": success, "first_output_seconds": first,
+ "duration_seconds": duration, "completion_tokens": None}
+ rows = [row(.5, 2), row(.6, 1), row(.2, 3), row(None, 1), row(.1, 1, False)]
+ summary = bench.summarize(rows, 10, slo_first_output=.5, slo_duration=2)
+ slo = summary["latency_slo"]
+ self.assertEqual(slo["requests_met"], 1)
+ self.assertEqual(slo["fraction_of_attempts"], .2)
+ self.assertEqual(slo["goodput_requests_per_second"], .1)
+ self.assertIsNone(summary["successful_completion_tokens_per_second"])
+ self.assertEqual(bench.summarize(rows, 10, slo_duration=2)["latency_slo"]["requests_met"], 3)
+ self.assertEqual(bench.summarize(rows, 10, slo_first_output=.5)["latency_slo"]["requests_met"], 2)
+ self.assertIsNone(bench.summarize(rows, 10)["latency_slo"])
+
+ def test_latency_slo_no_successes(self):
+ row = {"success": False, "first_output_seconds": .1,
+ "duration_seconds": .2, "completion_tokens": 10}
+ slo = bench.summarize([row], 1, slo_duration=1)["latency_slo"]
+ self.assertEqual(slo["requests_met"], 0)
+ self.assertEqual(slo["goodput_requests_per_second"], 0)
+
def test_nearest_rank_distribution(self):
self.assertEqual(bench.distribution(list(range(1, 101)))["p95"], 95)
self.assertIsNone(bench.distribution([])["p95"])
diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.py
index 991669ea..8ea3a8fd 100644
--- a/c/tools/benchmark_http_serving.py
+++ b/c/tools/benchmark_http_serving.py
@@ -143,12 +143,12 @@ def distribution(values):
return result
-def summarize(results, elapsed):
+def summarize(results, elapsed, slo_first_output=None, slo_duration=None):
successful = [row for row in results if row["success"]]
counted = [row for row in successful if row["completion_tokens"] is not None]
tokens = sum(row["completion_tokens"] for row in counted)
complete = bool(successful) and len(counted) == len(successful)
- return {"requests": len(results), "succeeded": len(successful),
+ summary = {"requests": len(results), "succeeded": len(successful),
"failed": len(results) - len(successful),
"failure_rate": (len(results) - len(successful)) / len(results),
"wall_seconds": elapsed,
@@ -160,8 +160,23 @@ def summarize(results, elapsed):
"successful_first_output_seconds": distribution([
r["first_output_seconds"] for r in successful if r["first_output_seconds"] is not None])}
-
-def run(url, workload, model, concurrency, repeats, max_tokens, temperature, key, timeout):
+ summary["latency_slo"] = None
+ if slo_first_output is not None or slo_duration is not None:
+ met = sum(
+ (slo_duration is None or row["duration_seconds"] <= slo_duration)
+ and (slo_first_output is None or (
+ row["first_output_seconds"] is not None
+ and row["first_output_seconds"] <= slo_first_output))
+ for row in successful)
+ summary["latency_slo"] = {
+ "first_output_seconds": slo_first_output, "duration_seconds": slo_duration,
+ "requests_met": met, "fraction_of_attempts": met / len(results),
+ "goodput_requests_per_second": met / elapsed}
+ return summary
+
+
+def run(url, workload, model, concurrency, repeats, max_tokens, temperature, key, timeout,
+ slo_first_output=None, slo_duration=None):
origin = time.perf_counter()
# Workers take the next request as soon as the previous stream ends. Timers
# begin inside workers, excluding time waiting in the local executor queue.
@@ -173,7 +188,7 @@ def run(url, workload, model, concurrency, repeats, max_tokens, temperature, key
temperature=temperature, n=1)
futures.append(pool.submit(request_one, url, payload, key, timeout, index, origin))
results = [future.result() for future in futures]
- return results, summarize(results, time.perf_counter() - origin)
+ return results, summarize(results, time.perf_counter() - origin, slo_first_output, slo_duration)
def positive_int(value):
@@ -202,6 +217,8 @@ def main():
parser.add_argument("--temperature", type=float, default=0)
parser.add_argument("--timeout", type=positive_float, default=60, help="socket operation timeout in seconds")
parser.add_argument("--api-key-env", default="OPENAI_API_KEY")
+ parser.add_argument("--slo-first-output", type=positive_float, help="first output latency target, seconds")
+ parser.add_argument("--slo-duration", type=positive_float, help="completed request latency target, seconds")
args = parser.parse_args()
try:
url = endpoint(args.base_url)
@@ -214,12 +231,14 @@ def main():
parser.error(str(exc))
results, summary = run(url, workload, args.model, args.concurrency, args.repeats,
args.max_tokens, args.temperature,
- os.environ.get(args.api_key_env, ""), args.timeout)
+ os.environ.get(args.api_key_env, ""), args.timeout,
+ args.slo_first_output, args.slo_duration)
report = {"schema_version": 1, "config": {
"endpoint": url, "model": args.model, "workload_sha256": digest,
"workload_rows": len(workload), "concurrency": args.concurrency,
"repeats": args.repeats, "max_tokens": args.max_tokens,
"temperature": args.temperature, "socket_timeout_seconds": args.timeout,
+ "slo_first_output_seconds": args.slo_first_output, "slo_duration_seconds": args.slo_duration,
"stream": True, "include_usage": True, "n": 1},
"summary": summary, "requests": results}
Path(args.output).write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8")
diff --git a/docs/benchmarking.md b/docs/benchmarking.md
index 21179143..3009a267 100644
--- a/docs/benchmarking.md
+++ b/docs/benchmarking.md
@@ -150,3 +150,32 @@ and run an independent quality check: this tool deliberately does not save
response text or assess correctness. Retain the workload file with its hash,
interleave server runs, and report repeated-run spread. Start at concurrency 1,
then increase it to expose queueing and prefill interference.
+
+### Measure throughput within a latency target
+
+Add `--slo-first-output 1 --slo-duration 15` to require first output within one
+second and protocol completion within fifteen seconds. Either flag can be used
+alone; values are finite positive seconds and the boundary is inclusive.
+`summary.latency_slo` reports the thresholds, `requests_met`, the fraction of
+**all attempts** meeting them, and `goodput_requests_per_second` (qualifying
+successful requests divided by the entire batch wall time). Without thresholds,
+this field is null. Failures never qualify, even if they emitted output before
+failing. When a first-output target is set, empty-output successes also do not
+qualify. Missing token usage does not prevent evaluating these latency targets.
+
+This follows the latency-constrained goodput approach used by
+[vLLM's serving benchmark](https://docs.vllm.ai/en/latest/api/vllm/benchmarks/serve/),
+with the client first-output boundary defined above. It does not assert TPOT or
+per-token SLOs, output quality, a minimum response length, or an equivalent vLLM
+TTFT definition. SLO misses alone do not change the CLI exit status: exit 1 still
+means at least one protocol/transport failure. Record output-length and quality
+controls alongside goodput so short or empty answers cannot masquerade as an
+improvement.
+
+Keep the load model fixed when comparing reports.
+[SGLang's serving benchmark](https://github.com/sgl-project/sglang/blob/main/python/sglang/benchmark/serving.py)
+also supports request-rate-driven arrivals and trace timestamps. This tool still
+uses closed-loop concurrency: slow responses reduce the rate at which new
+requests start. Its goodput cannot establish an open-loop arrival-rate capacity
+or a production SLO guarantee. Rate-controlled arrivals and warmup/cache policy
+remain separate measurement needs.