Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(tools): add reproducible HTTP serving benchmark
ZacharyZcR9885e893a748
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.pynew file mode 100644index 00000000..9f0daad9--- /dev/null+++ b/c/tests/test_benchmark_http_serving.py@@ -0,0 +1,218 @@+"""HTTP benchmark contract tests; no model, GPU or third-party packages."""+import io+import json+from pathlib import Path+import subprocess+import sys+import tempfile+import threading+import time+import unittest+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from tools import benchmark_http_serving as bench+from openai_server import APIServer+++def event(value):+ return "data: " + (value if isinstance(value, str) else json.dumps(value)) + "\n\n"+++def stream(delta=None, usage=3, finish=True, done=True):+ text = event({"choices": [{"index": 0, "delta": {"role": "assistant"}}]})+ text += event({"choices": [{"index": 0, "delta": delta or {}}]})+ if finish:+ text += event({"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]})+ if usage is not None:+ text += event({"choices": [], "usage": {"completion_tokens": usage}})+ if done:+ text += event("[DONE]")+ return text.encode()+++class Handler(BaseHTTPRequestHandler):+ def log_message(self, *_args):+ pass++ def do_POST(self):+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))+ with self.server.lock:+ self.server.payloads.append(body)+ self.server.active += 1+ self.server.peak = max(self.server.peak, self.server.active)+ try:+ if self.server.barrier:+ self.server.barrier.wait(timeout=5)+ self.send_response(self.server.status)+ self.send_header("Content-Type", self.server.content_type)+ if self.server.status == 302:+ self.send_header("Location", "/must-not-follow")+ self.end_headers()+ self.wfile.write(self.server.body)+ finally:+ with self.server.lock:+ self.server.active -= 1+++class BenchmarkTest(unittest.TestCase):+ def setUp(self):+ self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ self.server.lock = threading.Lock()+ self.server.payloads = []+ self.server.active = self.server.peak = 0+ self.server.status = 200+ self.server.content_type = "text/event-stream"+ self.server.body = stream({"content": "hello"})+ self.server.barrier = None+ self.thread = threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": .01})+ self.thread.start()+ self.url = f"http://127.0.0.1:{self.server.server_port}/v1/chat/completions"+ self.workload = [{"messages": [{"role": "user", "content": "hello"}]}]++ def tearDown(self):+ self.server.shutdown()+ self.server.server_close()+ self.thread.join()++ def request(self):+ return bench.request_one(self.url, self.workload[0], "", 2, 0, time.perf_counter())++ def test_success_and_first_output(self):+ for delta in ({"content": "a"}, {"reasoning_content": "b"},+ {"reasoning": "c"}, {"tool_calls": [{"function": {"arguments": "{}"}}]}):+ with self.subTest(delta=delta):+ self.server.body = stream(delta)+ row = self.request()+ self.assertTrue(row["success"])+ self.assertEqual(row["completion_tokens"], 3)+ self.assertLessEqual(row["first_output_seconds"], row["duration_seconds"])++ def test_empty_output_is_not_first_output(self):+ self.server.body = stream({"content": "", "tool_calls": [{"id": "id", "function": {}}]}, usage=0)+ row = self.request()+ self.assertTrue(row["success"])+ self.assertIsNone(row["first_output_seconds"])+ self.assertEqual(row["completion_tokens"], 0)++ def test_failures_are_not_successes(self):+ bodies = [stream(done=False), stream(finish=False), b"data: {bad}\n\n",+ event({"error": {"message": "private diagnostic"}}).encode(),+ stream(usage=-1), stream(usage=True), b"data: [DONE]"]+ for body in bodies:+ with self.subTest(body=body):+ self.server.body = body+ row = self.request()+ self.assertFalse(row["success"])+ self.assertIsNotNone(row["error"])+ self.assertNotIn("private diagnostic", json.dumps(row))++ def test_http_errors_and_redirects(self):+ for status in (429, 500, 302):+ self.server.status = status+ row = self.request()+ self.assertFalse(row["success"])+ self.assertEqual(row["http_status"], status)+ self.assertEqual(len(self.server.payloads), 3)++ def test_content_type(self):+ self.server.content_type = "application/json"+ self.assertFalse(self.request()["success"])++ def test_missing_usage_disables_token_rate(self):+ self.server.body = stream({"content": "a"}, usage=None)+ row = self.request()+ summary = bench.summarize([row], 1)+ self.assertEqual(summary["succeeded"], 1)+ self.assertEqual(summary["successful_requests_with_usage"], 0)+ self.assertIsNone(summary["successful_completion_tokens_per_second"])++ def test_partial_failure_accounting(self):+ success = self.request()+ failure = dict(success, success=False, completion_tokens=999)+ summary = bench.summarize([success, failure], 2)+ self.assertEqual(summary["failure_rate"], .5)+ self.assertEqual(summary["successful_completion_tokens_per_second"], 1.5)+ self.assertEqual(summary["successful_duration_seconds"]["count"], 1)+ self.assertIsNone(bench.summarize([failure], 1)["successful_completion_tokens_per_second"])++ def test_concurrency_and_payload(self):+ self.server.barrier = threading.Barrier(2)+ rows, summary = bench.run(self.url, self.workload, "test-model", 2, 4, 8, 0, "", 2)+ self.assertEqual(summary["succeeded"], 4)+ self.assertEqual(self.server.peak, 2)+ self.assertEqual([r["index"] for r in rows], list(range(4)))+ for body in self.server.payloads:+ self.assertEqual(body, dict(self.workload[0], model="test-model", stream=True,+ stream_options={"include_usage": True}, max_tokens=8,+ temperature=0, n=1))++ def test_cli_report_and_failure_exit(self):+ with tempfile.TemporaryDirectory() as directory:+ workload = Path(directory) / "prompts.jsonl"+ output = Path(directory) / "report.json"+ workload.write_text(json.dumps(self.workload[0]) + "\n", encoding="utf-8")+ command = [sys.executable, bench.__file__, "--base-url", self.url.rsplit("/", 1)[0],+ "--model", "fixture", "--workload", str(workload), "--output", str(output)]+ for status, exit_code in ((200, 0), (503, 1)):+ self.server.status = status+ completed = subprocess.run(command, capture_output=True, text=True, timeout=10)+ self.assertEqual(completed.returncode, exit_code, completed.stderr)+ report = json.loads(output.read_text())+ self.assertEqual(report["summary"]["succeeded"], 1 - exit_code)+ self.assertEqual(len(report["config"]["workload_sha256"]), 64)+ self.assertNotIn("messages", report["config"])+++class ColibriIntegrationTest(unittest.TestCase):+ def test_real_gateway_with_fake_engine(self):+ class Engine:+ def generate(self, prompt, maximum, temperature, top_p, on_text,+ cache_slot=0, cancelled=None, **kwargs):+ kwargs["on_accept"]({"prompt_tokens": 7})+ on_text("Hello")+ return {"prompt_tokens": 7, "completion_tokens": 1, "length_limited": False}++ server = APIServer(("127.0.0.1", 0), Engine(), "fixture", api_key="test-secret")+ thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": .01})+ thread.start()+ try:+ rows, summary = bench.run(+ f"http://127.0.0.1:{server.server_port}/v1/chat/completions",+ [{"messages": [{"role": "user", "content": "Hi"}]}],+ "fixture", 1, 2, 8, 0, "test-secret", 2)+ self.assertEqual(summary["succeeded"], 2, rows)+ self.assertEqual(summary["reported_successful_completion_tokens"], 2)+ self.assertEqual(summary["successful_first_output_seconds"]["count"], 2)+ self.assertNotIn("test-secret", json.dumps(rows))+ finally:+ server.shutdown()+ server.server_close()+ thread.join()+++class ParsingTest(unittest.TestCase):+ def test_sse_multiline_comments_crlf_and_partial_eof(self):+ raw = b': ping\r\nevent: message\r\ndata: {"choices":\r\ndata: []}\r\n\r\ndata: truncated'+ self.assertEqual(list(bench.sse_events(io.BytesIO(raw))), ['{"choices":\n[]}'])++ def test_endpoint(self):+ self.assertEqual(bench.endpoint("http://localhost:8000/v1/"), "http://localhost:8000/v1/chat/completions")+ for url in ("file:///v1", "http://key@host/v1", "http://host/v1?key=secret", "http://host/v1#x"):+ with self.assertRaises(ValueError):+ bench.endpoint(url)++ def test_workload_validation(self):+ with tempfile.TemporaryDirectory() as directory:+ path = Path(directory) / "workload.jsonl"+ for text in ('', '{}', '{"messages":[]}', '{"messages":[{"role":"user","content":3}]}',+ '{"messages":[{"role":"user","content":"ok"}],"temperature":1}'):+ path.write_text(text)+ with self.assertRaises(ValueError):+ bench.load_workload(path)++ def test_nearest_rank_distribution(self):+ self.assertEqual(bench.distribution(list(range(1, 101)))["p95"], 95)+ self.assertIsNone(bench.distribution([])["p95"])+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.pynew file mode 100644index 00000000..991669ea--- /dev/null+++ b/c/tools/benchmark_http_serving.py@@ -0,0 +1,231 @@+#!/usr/bin/env python3+"""Bounded, closed-loop OpenAI chat streaming benchmark (stdlib only)."""+import argparse+import concurrent.futures+import hashlib+import json+import math+import os+from pathlib import Path+import time+import urllib.error+import urllib.parse+import urllib.request+++class StreamError(ValueError):+ pass+++class NoRedirect(urllib.request.HTTPRedirectHandler):+ def redirect_request(self, req, fp, code, msg, headers, newurl):+ return None+++def endpoint(base_url):+ parsed = urllib.parse.urlsplit(base_url)+ if (parsed.scheme not in ("http", "https") or not parsed.hostname+ or parsed.username is not None or parsed.password is not None+ or parsed.query or parsed.fragment):+ raise ValueError("base URL must be HTTP(S), without credentials, query or fragment")+ return base_url.rstrip("/") + "/chat/completions"+++def load_workload(path):+ raw = Path(path).read_bytes()+ workload = []+ for line in raw.decode("utf-8").splitlines():+ if not line.strip():+ continue+ item = json.loads(line)+ if not isinstance(item, dict) or set(item) != {"messages"}:+ raise ValueError("each JSONL row must contain only 'messages'")+ messages = item["messages"]+ if not isinstance(messages, list) or not messages:+ raise ValueError("messages must be a nonempty list")+ for message in messages:+ if (not isinstance(message, dict) or set(message) != {"role", "content"}+ or message["role"] not in ("system", "user", "assistant")+ or not isinstance(message["content"], str)):+ raise ValueError("messages require role and string content")+ workload.append(item)+ if not workload:+ raise ValueError("workload is empty")+ return workload, hashlib.sha256(raw).hexdigest()+++def sse_events(response):+ """Dispatch only complete SSE events; EOF is not a successful terminator."""+ data = []+ for raw in response:+ line = raw.decode("utf-8").rstrip("\r\n")+ if not line:+ if data:+ yield "\n".join(data)+ data = []+ continue+ field, _, value = line.partition(":")+ if field == "data":+ data.append(value[1:] if value.startswith(" ") else value)+++def has_output(delta):+ if delta.get("content") or delta.get("reasoning_content") or delta.get("reasoning"):+ return True+ for tool in delta.get("tool_calls") or []:+ function = tool.get("function") or {}+ if function.get("name") or function.get("arguments"):+ return True+ return False+++def request_one(url, payload, key, timeout, index, origin):+ start = time.perf_counter()+ result = {"index": index, "start_seconds": start - origin,+ "success": False, "http_status": None, "error": None,+ "first_output_seconds": None, "completion_tokens": None,+ "finish_reason": None}+ headers = {"Content-Type": "application/json", "Accept": "text/event-stream"}+ if key:+ headers["Authorization"] = "Bearer " + key+ request = urllib.request.Request(url, data=json.dumps(payload).encode(), headers=headers)+ try:+ opener = urllib.request.build_opener(NoRedirect)+ with opener.open(request, timeout=timeout) as response:+ result["http_status"] = response.status+ if response.headers.get_content_type() != "text/event-stream":+ raise StreamError("unexpected_content_type")+ done = False+ for event in sse_events(response):+ if event == "[DONE]":+ done = True+ break+ chunk = json.loads(event)+ if "error" in chunk:+ raise StreamError("stream_error")+ for choice in chunk.get("choices", []):+ if choice.get("index", 0) != 0:+ raise StreamError("unexpected_choice")+ if has_output(choice.get("delta") or {}):+ if result["first_output_seconds"] is None:+ result["first_output_seconds"] = time.perf_counter() - start+ if choice.get("finish_reason") is not None:+ result["finish_reason"] = choice["finish_reason"]+ usage = chunk.get("usage")+ if usage is not None:+ tokens = usage.get("completion_tokens")+ if type(tokens) is not int or tokens < 0:+ raise StreamError("invalid_completion_tokens")+ result["completion_tokens"] = tokens+ if not done or result["finish_reason"] is None:+ raise StreamError("incomplete_stream")+ result["success"] = True+ except StreamError as exc:+ result["error"] = str(exc)+ except urllib.error.HTTPError as exc:+ result["http_status"] = exc.code+ result["error"] = "HTTPError"+ exc.close()+ except Exception as exc:+ # Do not copy server bodies, request contents or credentials into reports.+ result["error"] = type(exc).__name__+ result["duration_seconds"] = time.perf_counter() - start+ return result+++def distribution(values):+ if not values:+ return {"count": 0, "mean": None, "p50": None, "p95": None, "p99": None}+ ordered = sorted(values)+ result = {"count": len(values), "mean": sum(values) / len(values)}+ for percentile in (50, 95, 99):+ result[f"p{percentile}"] = ordered[math.ceil(len(ordered) * percentile / 100) - 1]+ return result+++def summarize(results, elapsed):+ successful = [row for row in results if row["success"]]+ counted = [row for row in successful if row["completion_tokens"] is not None]+ tokens = sum(row["completion_tokens"] for row in counted)+ complete = bool(successful) and len(counted) == len(successful)+ return {"requests": len(results), "succeeded": len(successful),+ "failed": len(results) - len(successful),+ "failure_rate": (len(results) - len(successful)) / len(results),+ "wall_seconds": elapsed,+ "successful_requests_per_second": len(successful) / elapsed,+ "successful_requests_with_usage": len(counted),+ "reported_successful_completion_tokens": tokens,+ "successful_completion_tokens_per_second": tokens / elapsed if complete else None,+ "successful_duration_seconds": distribution([r["duration_seconds"] for r in successful]),+ "successful_first_output_seconds": distribution([+ r["first_output_seconds"] for r in successful if r["first_output_seconds"] is not None])}+++def run(url, workload, model, concurrency, repeats, max_tokens, temperature, key, timeout):+ origin = time.perf_counter()+ # Workers take the next request as soon as the previous stream ends. Timers+ # begin inside workers, excluding time waiting in the local executor queue.+ with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:+ futures = []+ for index in range(len(workload) * repeats):+ payload = dict(workload[index % len(workload)], model=model, stream=True,+ stream_options={"include_usage": True}, max_tokens=max_tokens,+ temperature=temperature, n=1)+ futures.append(pool.submit(request_one, url, payload, key, timeout, index, origin))+ results = [future.result() for future in futures]+ return results, summarize(results, time.perf_counter() - origin)+++def positive_int(value):+ number = int(value)+ if number <= 0:+ raise argparse.ArgumentTypeError("must be positive")+ return number+++def positive_float(value):+ number = float(value)+ if not math.isfinite(number) or number <= 0:+ raise argparse.ArgumentTypeError("must be finite and positive")+ return number+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--base-url", required=True, help="API root including /v1")+ parser.add_argument("--model", required=True)+ parser.add_argument("--workload", required=True, help="JSONL rows containing messages")+ parser.add_argument("--output", required=True, help="JSON report path")+ parser.add_argument("--concurrency", type=positive_int, default=1)+ parser.add_argument("--repeats", type=positive_int, default=1)+ parser.add_argument("--max-tokens", type=positive_int, default=128)+ parser.add_argument("--temperature", type=float, default=0)+ parser.add_argument("--timeout", type=positive_float, default=60, help="socket operation timeout in seconds")+ parser.add_argument("--api-key-env", default="OPENAI_API_KEY")+ args = parser.parse_args()+ try:+ url = endpoint(args.base_url)+ workload, digest = load_workload(args.workload)+ if not math.isfinite(args.temperature) or args.temperature < 0:+ raise ValueError("temperature must be finite and nonnegative")+ if Path(args.output).resolve() == Path(args.workload).resolve():+ raise ValueError("output must differ from workload")+ except (ValueError, OSError) as exc:+ parser.error(str(exc))+ results, summary = run(url, workload, args.model, args.concurrency, args.repeats,+ args.max_tokens, args.temperature,+ os.environ.get(args.api_key_env, ""), args.timeout)+ report = {"schema_version": 1, "config": {+ "endpoint": url, "model": args.model, "workload_sha256": digest,+ "workload_rows": len(workload), "concurrency": args.concurrency,+ "repeats": args.repeats, "max_tokens": args.max_tokens,+ "temperature": args.temperature, "socket_timeout_seconds": args.timeout,+ "stream": True, "include_usage": True, "n": 1},+ "summary": summary, "requests": results}+ Path(args.output).write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8")+ print(json.dumps(summary, indent=2))+ return 1 if summary["failed"] else 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/docs/benchmarking.md b/docs/benchmarking.mdindex 9407467c..21179143 100644--- a/docs/benchmarking.md+++ b/docs/benchmarking.md@@ -75,3 +75,78 @@ platform-specific I/O constraints.This protocol originated with the measurements and draft contributed by[@outtodata in #867](https://github.com/JustVugg/colibri/issues/867), includingtheir later correction of the #863 explanation.++## Compare HTTP serving with a fixed workload++`c/tools/benchmark_http_serving.py` uses the OpenAI-compatible streaming chat+endpoint and Python's standard library. Run it against Colibri, SGLang, or vLLM+with the same workload and generation settings. This measures the whole HTTP+request path, including server queueing and prefill, unlike a steady-decode-only+engine benchmark. No performance comparison is implied by providing the tool.++Create a UTF-8 JSONL file with one conversation per line. Rows contain only+`messages`; messages contain a `system`, `user`, or `assistant` role and string+`content`:++```jsonl+{"messages":[{"role":"user","content":"Explain how a CPU cache works."}]}+{"messages":[{"role":"system","content":"Answer briefly."},{"role":"user","content":"What is a mutex?"}]}+```++From the repository root, with the server already running:++```sh+python3 c/tools/benchmark_http_serving.py \+ --base-url http://127.0.0.1:8000/v1 --model your-served-model \+ --workload prompts.jsonl --concurrency 4 --repeats 3 \+ --max-tokens 128 --temperature 0 --timeout 60 --output colibri-c4.json+```++Repeat with the other server's API root, model alias, and a distinct output file.+Authentication uses `OPENAI_API_KEY`, or the environment variable named by+`--api-key-env`; keys and message contents are not copied into reports. The+endpoint and model alias are recorded. URLs with credentials, queries or+fragments are rejected, and redirects are not followed. The requested endpoint+must support `stream_options.include_usage`; errors are recorded rather than+silently changing the workload or retrying.++The JSON report includes the workload's SHA-256, generation settings, one record+per attempt in input order, HTTP status, failure category, relative start time,+duration, first output time, finish reason, and reported completion tokens.+Failed requests remain in the report, and any failure makes the command exit 1.+Latency summaries use successful requests only, with nearest-rank p50/p95/p99+and sample counts. Failed requests retain their individual durations and any+observed output/usage.++Measurement boundaries:++- Concurrency is closed-loop: at most that many requests are in flight, and each+ worker starts its next request after its previous stream ends. There is no+ arrival-rate model, warmup, automatic retry, or cache flush. Repeats reuse the+ conversations in file order; prefix caching and scheduling can affect results.+- Request timing begins inside the worker, before HTTP connection setup, and+ excludes waiting for a local worker. Each request uses a new connection.+ `--timeout` limits individual socket operations, **not total request time**;+ a stream that keeps sending data can last longer.+- `first_output_seconds` measures receipt of the first nonempty content,+ reasoning, or tool-function name/arguments delta. Role-only and empty deltas+ do not count. This is client-visible first output latency, not necessarily+ time to a visible answer or to exactly one token. Empty successful output+ has no first-output sample. SSE chunk gaps are not reported as token latency.+- Success requires both a finish reason and `[DONE]`. HTTP errors, stream errors,+ malformed responses, and incomplete streams are failures. This establishes+ protocol completion, not output correctness.+- Token counts come only from `usage.completion_tokens`. Successful completion+ token throughput divides those counts by the **entire batch wall time**,+ including failed attempts. It is null if any successful request lacks usage+ (or no request succeeds). Counts from failed streams are excluded. Inspect+ failure rate and usage coverage alongside throughput; backend tokenizers,+ reasoning-token accounting, and stopping policies may differ.++For a meaningful comparison, record the model weights, quantization, tokenizer,+chat template, reasoning mode, server commands/versions, cache state and hardware+beside the report. Keep requested settings equal, check actual output lengths,+and run an independent quality check: this tool deliberately does not save+response text or assess correctness. Retain the workload file with its hash,+interleave server runs, and report repeated-run spread. Start at concurrency 1,+then increase it to expose queueing and prefill interference.