Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(tools): add reproducible HTTP serving benchmark

ZacharyZcR9885e893a748
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.py
new file mode 100644
index 00000000..9f0daad9
--- /dev/null
+++ b/c/tests/test_benchmark_http_serving.py
@@ -0,0 +1,218 @@
+"""HTTP benchmark contract tests; no model, GPU or third-party packages."""
+import io
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import unittest
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from tools import benchmark_http_serving as bench
+from openai_server import APIServer
+
+
+def event(value):
+ return "data: " + (value if isinstance(value, str) else json.dumps(value)) + "\n\n"
+
+
+def stream(delta=None, usage=3, finish=True, done=True):
+ text = event({"choices": [{"index": 0, "delta": {"role": "assistant"}}]})
+ text += event({"choices": [{"index": 0, "delta": delta or {}}]})
+ if finish:
+ text += event({"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]})
+ if usage is not None:
+ text += event({"choices": [], "usage": {"completion_tokens": usage}})
+ if done:
+ text += event("[DONE]")
+ return text.encode()
+
+
+class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *_args):
+ pass
+
+ def do_POST(self):
+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+ with self.server.lock:
+ self.server.payloads.append(body)
+ self.server.active += 1
+ self.server.peak = max(self.server.peak, self.server.active)
+ try:
+ if self.server.barrier:
+ self.server.barrier.wait(timeout=5)
+ self.send_response(self.server.status)
+ self.send_header("Content-Type", self.server.content_type)
+ if self.server.status == 302:
+ self.send_header("Location", "/must-not-follow")
+ self.end_headers()
+ self.wfile.write(self.server.body)
+ finally:
+ with self.server.lock:
+ self.server.active -= 1
+
+
+class BenchmarkTest(unittest.TestCase):
+ def setUp(self):
+ self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.server.lock = threading.Lock()
+ self.server.payloads = []
+ self.server.active = self.server.peak = 0
+ self.server.status = 200
+ self.server.content_type = "text/event-stream"
+ self.server.body = stream({"content": "hello"})
+ self.server.barrier = None
+ self.thread = threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": .01})
+ self.thread.start()
+ self.url = f"http://127.0.0.1:{self.server.server_port}/v1/chat/completions"
+ self.workload = [{"messages": [{"role": "user", "content": "hello"}]}]
+
+ def tearDown(self):
+ self.server.shutdown()
+ self.server.server_close()
+ self.thread.join()
+
+ def request(self):
+ return bench.request_one(self.url, self.workload[0], "", 2, 0, time.perf_counter())
+
+ def test_success_and_first_output(self):
+ for delta in ({"content": "a"}, {"reasoning_content": "b"},
+ {"reasoning": "c"}, {"tool_calls": [{"function": {"arguments": "{}"}}]}):
+ with self.subTest(delta=delta):
+ self.server.body = stream(delta)
+ row = self.request()
+ self.assertTrue(row["success"])
+ self.assertEqual(row["completion_tokens"], 3)
+ self.assertLessEqual(row["first_output_seconds"], row["duration_seconds"])
+
+ def test_empty_output_is_not_first_output(self):
+ self.server.body = stream({"content": "", "tool_calls": [{"id": "id", "function": {}}]}, usage=0)
+ row = self.request()
+ self.assertTrue(row["success"])
+ self.assertIsNone(row["first_output_seconds"])
+ self.assertEqual(row["completion_tokens"], 0)
+
+ def test_failures_are_not_successes(self):
+ bodies = [stream(done=False), stream(finish=False), b"data: {bad}\n\n",
+ event({"error": {"message": "private diagnostic"}}).encode(),
+ stream(usage=-1), stream(usage=True), b"data: [DONE]"]
+ for body in bodies:
+ with self.subTest(body=body):
+ self.server.body = body
+ row = self.request()
+ self.assertFalse(row["success"])
+ self.assertIsNotNone(row["error"])
+ self.assertNotIn("private diagnostic", json.dumps(row))
+
+ def test_http_errors_and_redirects(self):
+ for status in (429, 500, 302):
+ self.server.status = status
+ row = self.request()
+ self.assertFalse(row["success"])
+ self.assertEqual(row["http_status"], status)
+ self.assertEqual(len(self.server.payloads), 3)
+
+ def test_content_type(self):
+ self.server.content_type = "application/json"
+ self.assertFalse(self.request()["success"])
+
+ def test_missing_usage_disables_token_rate(self):
+ self.server.body = stream({"content": "a"}, usage=None)
+ row = self.request()
+ summary = bench.summarize([row], 1)
+ self.assertEqual(summary["succeeded"], 1)
+ self.assertEqual(summary["successful_requests_with_usage"], 0)
+ self.assertIsNone(summary["successful_completion_tokens_per_second"])
+
+ def test_partial_failure_accounting(self):
+ success = self.request()
+ failure = dict(success, success=False, completion_tokens=999)
+ summary = bench.summarize([success, failure], 2)
+ self.assertEqual(summary["failure_rate"], .5)
+ self.assertEqual(summary["successful_completion_tokens_per_second"], 1.5)
+ self.assertEqual(summary["successful_duration_seconds"]["count"], 1)
+ self.assertIsNone(bench.summarize([failure], 1)["successful_completion_tokens_per_second"])
+
+ def test_concurrency_and_payload(self):
+ self.server.barrier = threading.Barrier(2)
+ rows, summary = bench.run(self.url, self.workload, "test-model", 2, 4, 8, 0, "", 2)
+ self.assertEqual(summary["succeeded"], 4)
+ self.assertEqual(self.server.peak, 2)
+ self.assertEqual([r["index"] for r in rows], list(range(4)))
+ for body in self.server.payloads:
+ self.assertEqual(body, dict(self.workload[0], model="test-model", stream=True,
+ stream_options={"include_usage": True}, max_tokens=8,
+ temperature=0, n=1))
+
+ def test_cli_report_and_failure_exit(self):
+ with tempfile.TemporaryDirectory() as directory:
+ workload = Path(directory) / "prompts.jsonl"
+ output = Path(directory) / "report.json"
+ workload.write_text(json.dumps(self.workload[0]) + "\n", encoding="utf-8")
+ command = [sys.executable, bench.__file__, "--base-url", self.url.rsplit("/", 1)[0],
+ "--model", "fixture", "--workload", str(workload), "--output", str(output)]
+ for status, exit_code in ((200, 0), (503, 1)):
+ self.server.status = status
+ completed = subprocess.run(command, capture_output=True, text=True, timeout=10)
+ self.assertEqual(completed.returncode, exit_code, completed.stderr)
+ report = json.loads(output.read_text())
+ self.assertEqual(report["summary"]["succeeded"], 1 - exit_code)
+ self.assertEqual(len(report["config"]["workload_sha256"]), 64)
+ self.assertNotIn("messages", report["config"])
+
+
+class ColibriIntegrationTest(unittest.TestCase):
+ def test_real_gateway_with_fake_engine(self):
+ class Engine:
+ def generate(self, prompt, maximum, temperature, top_p, on_text,
+ cache_slot=0, cancelled=None, **kwargs):
+ kwargs["on_accept"]({"prompt_tokens": 7})
+ on_text("Hello")
+ return {"prompt_tokens": 7, "completion_tokens": 1, "length_limited": False}
+
+ server = APIServer(("127.0.0.1", 0), Engine(), "fixture", api_key="test-secret")
+ thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": .01})
+ thread.start()
+ try:
+ rows, summary = bench.run(
+ f"http://127.0.0.1:{server.server_port}/v1/chat/completions",
+ [{"messages": [{"role": "user", "content": "Hi"}]}],
+ "fixture", 1, 2, 8, 0, "test-secret", 2)
+ self.assertEqual(summary["succeeded"], 2, rows)
+ self.assertEqual(summary["reported_successful_completion_tokens"], 2)
+ self.assertEqual(summary["successful_first_output_seconds"]["count"], 2)
+ self.assertNotIn("test-secret", json.dumps(rows))
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join()
+
+
+class ParsingTest(unittest.TestCase):
+ def test_sse_multiline_comments_crlf_and_partial_eof(self):
+ raw = b': ping\r\nevent: message\r\ndata: {"choices":\r\ndata: []}\r\n\r\ndata: truncated'
+ self.assertEqual(list(bench.sse_events(io.BytesIO(raw))), ['{"choices":\n[]}'])
+
+ def test_endpoint(self):
+ self.assertEqual(bench.endpoint("http://localhost:8000/v1/"), "http://localhost:8000/v1/chat/completions")
+ for url in ("file:///v1", "http://key@host/v1", "http://host/v1?key=secret", "http://host/v1#x"):
+ with self.assertRaises(ValueError):
+ bench.endpoint(url)
+
+ def test_workload_validation(self):
+ with tempfile.TemporaryDirectory() as directory:
+ path = Path(directory) / "workload.jsonl"
+ for text in ('', '{}', '{"messages":[]}', '{"messages":[{"role":"user","content":3}]}',
+ '{"messages":[{"role":"user","content":"ok"}],"temperature":1}'):
+ path.write_text(text)
+ with self.assertRaises(ValueError):
+ bench.load_workload(path)
+
+ def test_nearest_rank_distribution(self):
+ self.assertEqual(bench.distribution(list(range(1, 101)))["p95"], 95)
+ self.assertIsNone(bench.distribution([])["p95"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.py
new file mode 100644
index 00000000..991669ea
--- /dev/null
+++ b/c/tools/benchmark_http_serving.py
@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""Bounded, closed-loop OpenAI chat streaming benchmark (stdlib only)."""
+import argparse
+import concurrent.futures
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+
+class StreamError(ValueError):
+ pass
+
+
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ return None
+
+
+def endpoint(base_url):
+ parsed = urllib.parse.urlsplit(base_url)
+ if (parsed.scheme not in ("http", "https") or not parsed.hostname
+ or parsed.username is not None or parsed.password is not None
+ or parsed.query or parsed.fragment):
+ raise ValueError("base URL must be HTTP(S), without credentials, query or fragment")
+ return base_url.rstrip("/") + "/chat/completions"
+
+
+def load_workload(path):
+ raw = Path(path).read_bytes()
+ workload = []
+ for line in raw.decode("utf-8").splitlines():
+ if not line.strip():
+ continue
+ item = json.loads(line)
+ if not isinstance(item, dict) or set(item) != {"messages"}:
+ raise ValueError("each JSONL row must contain only 'messages'")
+ messages = item["messages"]
+ if not isinstance(messages, list) or not messages:
+ raise ValueError("messages must be a nonempty list")
+ for message in messages:
+ if (not isinstance(message, dict) or set(message) != {"role", "content"}
+ or message["role"] not in ("system", "user", "assistant")
+ or not isinstance(message["content"], str)):
+ raise ValueError("messages require role and string content")
+ workload.append(item)
+ if not workload:
+ raise ValueError("workload is empty")
+ return workload, hashlib.sha256(raw).hexdigest()
+
+
+def sse_events(response):
+ """Dispatch only complete SSE events; EOF is not a successful terminator."""
+ data = []
+ for raw in response:
+ line = raw.decode("utf-8").rstrip("\r\n")
+ if not line:
+ if data:
+ yield "\n".join(data)
+ data = []
+ continue
+ field, _, value = line.partition(":")
+ if field == "data":
+ data.append(value[1:] if value.startswith(" ") else value)
+
+
+def has_output(delta):
+ if delta.get("content") or delta.get("reasoning_content") or delta.get("reasoning"):
+ return True
+ for tool in delta.get("tool_calls") or []:
+ function = tool.get("function") or {}
+ if function.get("name") or function.get("arguments"):
+ return True
+ return False
+
+
+def request_one(url, payload, key, timeout, index, origin):
+ start = time.perf_counter()
+ result = {"index": index, "start_seconds": start - origin,
+ "success": False, "http_status": None, "error": None,
+ "first_output_seconds": None, "completion_tokens": None,
+ "finish_reason": None}
+ headers = {"Content-Type": "application/json", "Accept": "text/event-stream"}
+ if key:
+ headers["Authorization"] = "Bearer " + key
+ request = urllib.request.Request(url, data=json.dumps(payload).encode(), headers=headers)
+ try:
+ opener = urllib.request.build_opener(NoRedirect)
+ with opener.open(request, timeout=timeout) as response:
+ result["http_status"] = response.status
+ if response.headers.get_content_type() != "text/event-stream":
+ raise StreamError("unexpected_content_type")
+ done = False
+ for event in sse_events(response):
+ if event == "[DONE]":
+ done = True
+ break
+ chunk = json.loads(event)
+ if "error" in chunk:
+ raise StreamError("stream_error")
+ for choice in chunk.get("choices", []):
+ if choice.get("index", 0) != 0:
+ raise StreamError("unexpected_choice")
+ if has_output(choice.get("delta") or {}):
+ if result["first_output_seconds"] is None:
+ result["first_output_seconds"] = time.perf_counter() - start
+ if choice.get("finish_reason") is not None:
+ result["finish_reason"] = choice["finish_reason"]
+ usage = chunk.get("usage")
+ if usage is not None:
+ tokens = usage.get("completion_tokens")
+ if type(tokens) is not int or tokens < 0:
+ raise StreamError("invalid_completion_tokens")
+ result["completion_tokens"] = tokens
+ if not done or result["finish_reason"] is None:
+ raise StreamError("incomplete_stream")
+ result["success"] = True
+ except StreamError as exc:
+ result["error"] = str(exc)
+ except urllib.error.HTTPError as exc:
+ result["http_status"] = exc.code
+ result["error"] = "HTTPError"
+ exc.close()
+ except Exception as exc:
+ # Do not copy server bodies, request contents or credentials into reports.
+ result["error"] = type(exc).__name__
+ result["duration_seconds"] = time.perf_counter() - start
+ return result
+
+
+def distribution(values):
+ if not values:
+ return {"count": 0, "mean": None, "p50": None, "p95": None, "p99": None}
+ ordered = sorted(values)
+ result = {"count": len(values), "mean": sum(values) / len(values)}
+ for percentile in (50, 95, 99):
+ result[f"p{percentile}"] = ordered[math.ceil(len(ordered) * percentile / 100) - 1]
+ return result
+
+
+def summarize(results, elapsed):
+ successful = [row for row in results if row["success"]]
+ counted = [row for row in successful if row["completion_tokens"] is not None]
+ tokens = sum(row["completion_tokens"] for row in counted)
+ complete = bool(successful) and len(counted) == len(successful)
+ return {"requests": len(results), "succeeded": len(successful),
+ "failed": len(results) - len(successful),
+ "failure_rate": (len(results) - len(successful)) / len(results),
+ "wall_seconds": elapsed,
+ "successful_requests_per_second": len(successful) / elapsed,
+ "successful_requests_with_usage": len(counted),
+ "reported_successful_completion_tokens": tokens,
+ "successful_completion_tokens_per_second": tokens / elapsed if complete else None,
+ "successful_duration_seconds": distribution([r["duration_seconds"] for r in successful]),
+ "successful_first_output_seconds": distribution([
+ r["first_output_seconds"] for r in successful if r["first_output_seconds"] is not None])}
+
+
+def run(url, workload, model, concurrency, repeats, max_tokens, temperature, key, timeout):
+ origin = time.perf_counter()
+ # Workers take the next request as soon as the previous stream ends. Timers
+ # begin inside workers, excluding time waiting in the local executor queue.
+ with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
+ futures = []
+ for index in range(len(workload) * repeats):
+ payload = dict(workload[index % len(workload)], model=model, stream=True,
+ stream_options={"include_usage": True}, max_tokens=max_tokens,
+ temperature=temperature, n=1)
+ futures.append(pool.submit(request_one, url, payload, key, timeout, index, origin))
+ results = [future.result() for future in futures]
+ return results, summarize(results, time.perf_counter() - origin)
+
+
+def positive_int(value):
+ number = int(value)
+ if number <= 0:
+ raise argparse.ArgumentTypeError("must be positive")
+ return number
+
+
+def positive_float(value):
+ number = float(value)
+ if not math.isfinite(number) or number <= 0:
+ raise argparse.ArgumentTypeError("must be finite and positive")
+ return number
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--base-url", required=True, help="API root including /v1")
+ parser.add_argument("--model", required=True)
+ parser.add_argument("--workload", required=True, help="JSONL rows containing messages")
+ parser.add_argument("--output", required=True, help="JSON report path")
+ parser.add_argument("--concurrency", type=positive_int, default=1)
+ parser.add_argument("--repeats", type=positive_int, default=1)
+ parser.add_argument("--max-tokens", type=positive_int, default=128)
+ parser.add_argument("--temperature", type=float, default=0)
+ parser.add_argument("--timeout", type=positive_float, default=60, help="socket operation timeout in seconds")
+ parser.add_argument("--api-key-env", default="OPENAI_API_KEY")
+ args = parser.parse_args()
+ try:
+ url = endpoint(args.base_url)
+ workload, digest = load_workload(args.workload)
+ if not math.isfinite(args.temperature) or args.temperature < 0:
+ raise ValueError("temperature must be finite and nonnegative")
+ if Path(args.output).resolve() == Path(args.workload).resolve():
+ raise ValueError("output must differ from workload")
+ except (ValueError, OSError) as exc:
+ parser.error(str(exc))
+ results, summary = run(url, workload, args.model, args.concurrency, args.repeats,
+ args.max_tokens, args.temperature,
+ os.environ.get(args.api_key_env, ""), args.timeout)
+ report = {"schema_version": 1, "config": {
+ "endpoint": url, "model": args.model, "workload_sha256": digest,
+ "workload_rows": len(workload), "concurrency": args.concurrency,
+ "repeats": args.repeats, "max_tokens": args.max_tokens,
+ "temperature": args.temperature, "socket_timeout_seconds": args.timeout,
+ "stream": True, "include_usage": True, "n": 1},
+ "summary": summary, "requests": results}
+ Path(args.output).write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8")
+ print(json.dumps(summary, indent=2))
+ return 1 if summary["failed"] else 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/docs/benchmarking.md b/docs/benchmarking.md
index 9407467c..21179143 100644
--- a/docs/benchmarking.md
+++ b/docs/benchmarking.md
@@ -75,3 +75,78 @@ platform-specific I/O constraints.
This protocol originated with the measurements and draft contributed by
[@outtodata in #867](https://github.com/JustVugg/colibri/issues/867), including
their later correction of the #863 explanation.
+
+## Compare HTTP serving with a fixed workload
+
+`c/tools/benchmark_http_serving.py` uses the OpenAI-compatible streaming chat
+endpoint and Python's standard library. Run it against Colibri, SGLang, or vLLM
+with the same workload and generation settings. This measures the whole HTTP
+request path, including server queueing and prefill, unlike a steady-decode-only
+engine benchmark. No performance comparison is implied by providing the tool.
+
+Create a UTF-8 JSONL file with one conversation per line. Rows contain only
+`messages`; messages contain a `system`, `user`, or `assistant` role and string
+`content`:
+
+```jsonl
+{"messages":[{"role":"user","content":"Explain how a CPU cache works."}]}
+{"messages":[{"role":"system","content":"Answer briefly."},{"role":"user","content":"What is a mutex?"}]}
+```
+
+From the repository root, with the server already running:
+
+```sh
+python3 c/tools/benchmark_http_serving.py \
+ --base-url http://127.0.0.1:8000/v1 --model your-served-model \
+ --workload prompts.jsonl --concurrency 4 --repeats 3 \
+ --max-tokens 128 --temperature 0 --timeout 60 --output colibri-c4.json
+```
+
+Repeat with the other server's API root, model alias, and a distinct output file.
+Authentication uses `OPENAI_API_KEY`, or the environment variable named by
+`--api-key-env`; keys and message contents are not copied into reports. The
+endpoint and model alias are recorded. URLs with credentials, queries or
+fragments are rejected, and redirects are not followed. The requested endpoint
+must support `stream_options.include_usage`; errors are recorded rather than
+silently changing the workload or retrying.
+
+The JSON report includes the workload's SHA-256, generation settings, one record
+per attempt in input order, HTTP status, failure category, relative start time,
+duration, first output time, finish reason, and reported completion tokens.
+Failed requests remain in the report, and any failure makes the command exit 1.
+Latency summaries use successful requests only, with nearest-rank p50/p95/p99
+and sample counts. Failed requests retain their individual durations and any
+observed output/usage.
+
+Measurement boundaries:
+
+- Concurrency is closed-loop: at most that many requests are in flight, and each
+ worker starts its next request after its previous stream ends. There is no
+ arrival-rate model, warmup, automatic retry, or cache flush. Repeats reuse the
+ conversations in file order; prefix caching and scheduling can affect results.
+- Request timing begins inside the worker, before HTTP connection setup, and
+ excludes waiting for a local worker. Each request uses a new connection.
+ `--timeout` limits individual socket operations, **not total request time**;
+ a stream that keeps sending data can last longer.
+- `first_output_seconds` measures receipt of the first nonempty content,
+ reasoning, or tool-function name/arguments delta. Role-only and empty deltas
+ do not count. This is client-visible first output latency, not necessarily
+ time to a visible answer or to exactly one token. Empty successful output
+ has no first-output sample. SSE chunk gaps are not reported as token latency.
+- Success requires both a finish reason and `[DONE]`. HTTP errors, stream errors,
+ malformed responses, and incomplete streams are failures. This establishes
+ protocol completion, not output correctness.
+- Token counts come only from `usage.completion_tokens`. Successful completion
+ token throughput divides those counts by the **entire batch wall time**,
+ including failed attempts. It is null if any successful request lacks usage
+ (or no request succeeds). Counts from failed streams are excluded. Inspect
+ failure rate and usage coverage alongside throughput; backend tokenizers,
+ reasoning-token accounting, and stopping policies may differ.
+
+For a meaningful comparison, record the model weights, quantization, tokenizer,
+chat template, reasoning mode, server commands/versions, cache state and hardware
+beside the report. Keep requested settings equal, check actual output lengths,
+and run an independent quality check: this tool deliberately does not save
+response text or assess correctness. Retain the workload file with its hash,
+interleave server runs, and report repeated-run spread. Start at concurrency 1,
+then increase it to expose queueing and prefill interference.