Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(bench): reject malformed choices and unsuccessful stream endings

ZacharyZcR2c442da4d20d
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.py
index 14626279..5a67b455 100644
--- a/c/tests/test_benchmark_http_serving.py
+++ b/c/tests/test_benchmark_http_serving.py
@@ -109,6 +109,34 @@ class BenchmarkTest(unittest.TestCase):
self.assertIsNotNone(row["error"])
self.assertNotIn("private diagnostic", json.dumps(row))
+ def test_invalid_choices_do_not_count_as_success(self):
+ choices = [
+ [{"index": 0, "delta": {"content": 123}, "finish_reason": "stop"}],
+ [{"index": 0, "delta": {"content": "ok", "reasoning": []}, "finish_reason": "stop"}],
+ [{"index": 0, "delta": {}, "finish_reason": False}],
+ [{"index": 0, "delta": {}, "finish_reason": ""}],
+ [{"index": 0, "delta": {}, "finish_reason": "error"}],
+ [{"index": False, "delta": {}, "finish_reason": "stop"}],
+ [{"delta": {}, "finish_reason": "stop"}],
+ [{"index": 0, "delta": {}, "finish_reason": "stop"}] * 2,
+ [{"index": 0, "delta": {"tool_calls": [{"function": {"arguments": 7}}]}, "finish_reason": "tool_calls"}],
+ ]
+ for malformed in choices:
+ with self.subTest(choices=malformed):
+ self.server.body = (event({"choices": malformed}) + event("[DONE]")).encode()
+ row = self.request()
+ self.assertFalse(row["success"], row)
+ self.assertEqual(bench.summarize([row], 1, slo_duration=1)["latency_slo"]["requests_met"], 0)
+
+ def test_supported_finish_reasons_and_null_delta(self):
+ for finish in ("stop", "length", "tool_calls", "function_call", "content_filter"):
+ with self.subTest(finish=finish):
+ self.server.body = (event({"choices": [{"index": 0, "delta": None,
+ "finish_reason": finish}]}) + event("[DONE]")).encode()
+ row = self.request()
+ self.assertTrue(row["success"], row)
+ self.assertIsNone(row["first_output_seconds"])
+
def test_http_errors_and_redirects(self):
for status in (429, 500, 302):
self.server.status = status
diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.py
index 2d612fc4..5f758c8a 100644
--- a/c/tools/benchmark_http_serving.py
+++ b/c/tools/benchmark_http_serving.py
@@ -69,14 +69,34 @@ def sse_events(response):
data.append(value[1:] if value.startswith(" ") else value)
+def nonempty_text(value):
+ if value is not None and not isinstance(value, str):
+ raise StreamError("invalid_output_text")
+ return bool(value)
+
+
def has_output(delta):
- if delta.get("content") or delta.get("reasoning_content") or delta.get("reasoning"):
- return True
- for tool in delta.get("tool_calls") or []:
- function = tool.get("function") or {}
- if function.get("name") or function.get("arguments"):
- return True
- return False
+ if not isinstance(delta, dict):
+ raise StreamError("invalid_delta")
+ output = False
+ for field in ("content", "reasoning_content", "reasoning"):
+ output |= nonempty_text(delta.get(field))
+ calls = delta.get("tool_calls")
+ if calls is None:
+ return output
+ if not isinstance(calls, list):
+ raise StreamError("invalid_tool_calls")
+ for tool in calls:
+ if not isinstance(tool, dict):
+ raise StreamError("invalid_tool_call")
+ function = tool.get("function")
+ if function is None:
+ continue
+ if not isinstance(function, dict):
+ raise StreamError("invalid_tool_function")
+ for field in ("name", "arguments"):
+ output |= nonempty_text(function.get(field))
+ return output
def request_one(url, payload, key, timeout, index, origin):
@@ -101,18 +121,30 @@ def request_one(url, payload, key, timeout, index, origin):
done = True
break
chunk = json.loads(event)
+ if not isinstance(chunk, dict):
+ raise StreamError("invalid_chunk")
if "error" in chunk:
raise StreamError("stream_error")
- for choice in chunk.get("choices", []):
- if choice.get("index", 0) != 0:
+ choices = chunk.get("choices", [])
+ if not isinstance(choices, list) or len(choices) > 1:
+ raise StreamError("invalid_choices")
+ for choice in choices:
+ if (not isinstance(choice, dict) or type(choice.get("index")) is not int
+ or choice["index"] != 0):
raise StreamError("unexpected_choice")
- if has_output(choice.get("delta") or {}):
+ delta = choice.get("delta")
+ if has_output({} if delta is None else delta):
if result["first_output_seconds"] is None:
result["first_output_seconds"] = time.perf_counter() - start
- if choice.get("finish_reason") is not None:
- result["finish_reason"] = choice["finish_reason"]
+ finish = choice.get("finish_reason")
+ if finish is not None:
+ if finish not in ("stop", "length", "tool_calls", "function_call", "content_filter"):
+ raise StreamError("unsuccessful_finish_reason")
+ result["finish_reason"] = finish
usage = chunk.get("usage")
if usage is not None:
+ if not isinstance(usage, dict):
+ raise StreamError("invalid_usage")
tokens = usage.get("completion_tokens")
if type(tokens) is not int or tokens < 0:
raise StreamError("invalid_completion_tokens")
diff --git a/docs/benchmarking.md b/docs/benchmarking.md
index 65d5139e..e317204b 100644
--- a/docs/benchmarking.md
+++ b/docs/benchmarking.md
@@ -133,9 +133,13 @@ Measurement boundaries:
do not count. This is client-visible first output latency, not necessarily
time to a visible answer or to exactly one token. Empty successful output
has no first-output sample. SSE chunk gaps are not reported as token latency.
-- Success requires both a finish reason and `[DONE]`. HTTP errors, stream errors,
- malformed responses, and incomplete streams are failures. This establishes
- protocol completion, not output correctness.
+- Success requires a supported finish reason (`stop`, `length`, `tool_calls`,
+ `function_call`, or `content_filter`) and `[DONE]`. Error/unknown finish reasons,
+ HTTP errors, stream errors, malformed responses, and incomplete streams fail.
+ With `n=1`, each nonempty choices array must contain exactly one choice with
+ integer index 0. Output text and tool-function fields must be strings or null.
+ This establishes protocol completion, not output correctness; filtered or
+ length-limited output can still count as protocol success.
- Token counts come only from `usage.completion_tokens`. Successful completion
token throughput divides those counts by the **entire batch wall time**,
including failed attempts. It is null if any successful request lacks usage