Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(bench): reject malformed choices and unsuccessful stream endings
ZacharyZcR2c442da4d20d
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.pyindex 14626279..5a67b455 100644--- a/c/tests/test_benchmark_http_serving.py+++ b/c/tests/test_benchmark_http_serving.py@@ -109,6 +109,34 @@ class BenchmarkTest(unittest.TestCase):self.assertIsNotNone(row["error"])self.assertNotIn("private diagnostic", json.dumps(row))+ def test_invalid_choices_do_not_count_as_success(self):+ choices = [+ [{"index": 0, "delta": {"content": 123}, "finish_reason": "stop"}],+ [{"index": 0, "delta": {"content": "ok", "reasoning": []}, "finish_reason": "stop"}],+ [{"index": 0, "delta": {}, "finish_reason": False}],+ [{"index": 0, "delta": {}, "finish_reason": ""}],+ [{"index": 0, "delta": {}, "finish_reason": "error"}],+ [{"index": False, "delta": {}, "finish_reason": "stop"}],+ [{"delta": {}, "finish_reason": "stop"}],+ [{"index": 0, "delta": {}, "finish_reason": "stop"}] * 2,+ [{"index": 0, "delta": {"tool_calls": [{"function": {"arguments": 7}}]}, "finish_reason": "tool_calls"}],+ ]+ for malformed in choices:+ with self.subTest(choices=malformed):+ self.server.body = (event({"choices": malformed}) + event("[DONE]")).encode()+ row = self.request()+ self.assertFalse(row["success"], row)+ self.assertEqual(bench.summarize([row], 1, slo_duration=1)["latency_slo"]["requests_met"], 0)++ def test_supported_finish_reasons_and_null_delta(self):+ for finish in ("stop", "length", "tool_calls", "function_call", "content_filter"):+ with self.subTest(finish=finish):+ self.server.body = (event({"choices": [{"index": 0, "delta": None,+ "finish_reason": finish}]}) + event("[DONE]")).encode()+ row = self.request()+ self.assertTrue(row["success"], row)+ self.assertIsNone(row["first_output_seconds"])+def test_http_errors_and_redirects(self):for status in (429, 500, 302):self.server.status = statusdiff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.pyindex 2d612fc4..5f758c8a 100644--- a/c/tools/benchmark_http_serving.py+++ b/c/tools/benchmark_http_serving.py@@ -69,14 +69,34 @@ def sse_events(response):data.append(value[1:] if value.startswith(" ") else value)+def nonempty_text(value):+ if value is not None and not isinstance(value, str):+ raise StreamError("invalid_output_text")+ return bool(value)++def has_output(delta):- if delta.get("content") or delta.get("reasoning_content") or delta.get("reasoning"):- return True- for tool in delta.get("tool_calls") or []:- function = tool.get("function") or {}- if function.get("name") or function.get("arguments"):- return True- return False+ if not isinstance(delta, dict):+ raise StreamError("invalid_delta")+ output = False+ for field in ("content", "reasoning_content", "reasoning"):+ output |= nonempty_text(delta.get(field))+ calls = delta.get("tool_calls")+ if calls is None:+ return output+ if not isinstance(calls, list):+ raise StreamError("invalid_tool_calls")+ for tool in calls:+ if not isinstance(tool, dict):+ raise StreamError("invalid_tool_call")+ function = tool.get("function")+ if function is None:+ continue+ if not isinstance(function, dict):+ raise StreamError("invalid_tool_function")+ for field in ("name", "arguments"):+ output |= nonempty_text(function.get(field))+ return outputdef request_one(url, payload, key, timeout, index, origin):@@ -101,18 +121,30 @@ def request_one(url, payload, key, timeout, index, origin):done = Truebreakchunk = json.loads(event)+ if not isinstance(chunk, dict):+ raise StreamError("invalid_chunk")if "error" in chunk:raise StreamError("stream_error")- for choice in chunk.get("choices", []):- if choice.get("index", 0) != 0:+ choices = chunk.get("choices", [])+ if not isinstance(choices, list) or len(choices) > 1:+ raise StreamError("invalid_choices")+ for choice in choices:+ if (not isinstance(choice, dict) or type(choice.get("index")) is not int+ or choice["index"] != 0):raise StreamError("unexpected_choice")- if has_output(choice.get("delta") or {}):+ delta = choice.get("delta")+ if has_output({} if delta is None else delta):if result["first_output_seconds"] is None:result["first_output_seconds"] = time.perf_counter() - start- if choice.get("finish_reason") is not None:- result["finish_reason"] = choice["finish_reason"]+ finish = choice.get("finish_reason")+ if finish is not None:+ if finish not in ("stop", "length", "tool_calls", "function_call", "content_filter"):+ raise StreamError("unsuccessful_finish_reason")+ result["finish_reason"] = finishusage = chunk.get("usage")if usage is not None:+ if not isinstance(usage, dict):+ raise StreamError("invalid_usage")tokens = usage.get("completion_tokens")if type(tokens) is not int or tokens < 0:raise StreamError("invalid_completion_tokens")diff --git a/docs/benchmarking.md b/docs/benchmarking.mdindex 65d5139e..e317204b 100644--- a/docs/benchmarking.md+++ b/docs/benchmarking.md@@ -133,9 +133,13 @@ Measurement boundaries:do not count. This is client-visible first output latency, not necessarilytime to a visible answer or to exactly one token. Empty successful outputhas no first-output sample. SSE chunk gaps are not reported as token latency.-- Success requires both a finish reason and `[DONE]`. HTTP errors, stream errors,- malformed responses, and incomplete streams are failures. This establishes- protocol completion, not output correctness.+- Success requires a supported finish reason (`stop`, `length`, `tool_calls`,+ `function_call`, or `content_filter`) and `[DONE]`. Error/unknown finish reasons,+ HTTP errors, stream errors, malformed responses, and incomplete streams fail.+ With `n=1`, each nonempty choices array must contain exactly one choice with+ integer index 0. Output text and tool-function fields must be strings or null.+ This establishes protocol completion, not output correctness; filtered or+ length-limited output can still count as protocol success.- Token counts come only from `usage.completion_tokens`. Successful completiontoken throughput divides those counts by the **entire batch wall time**,including failed attempts. It is null if any successful request lacks usage