Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(bench): reject choice chunks after stream completion

ZacharyZcR03f6aa8afbdf
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.py
index 98b5f826..f728f765 100644
--- a/c/tests/test_benchmark_http_serving.py
+++ b/c/tests/test_benchmark_http_serving.py
@@ -128,6 +128,22 @@ class BenchmarkTest(unittest.TestCase):
self.assertFalse(row["success"], row)
self.assertEqual(bench.summarize([row], 1, slo_duration=1)["latency_slo"]["requests_met"], 0)
+ def test_choice_after_finish_is_not_successful(self):
+ terminal = {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
+ for delta, finish in (({"content": "late output"}, None), ({}, "length")):
+ with self.subTest(delta=delta, finish=finish):
+ extra = {"choices": [{"index": 0, "delta": delta, "finish_reason": finish}]}
+ self.server.body = (event(terminal) + event(extra)
+ + event({"choices": [], "usage": {"completion_tokens": 999}})
+ + event("[DONE]")).encode()
+ row = self.request()
+ self.assertFalse(row["success"])
+ self.assertEqual(row["error"], "choice_after_finish")
+ self.assertEqual(row["finish_reason"], "stop")
+ summary = bench.summarize([row], 1, slo_duration=10)
+ self.assertEqual(summary["reported_successful_completion_tokens"], 0)
+ self.assertEqual(summary["latency_slo"]["requests_met"], 0)
+
def test_supported_finish_reasons_and_null_delta(self):
for finish in ("stop", "length", "tool_calls", "function_call", "content_filter"):
with self.subTest(finish=finish):
diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.py
index b1154130..74e32a84 100644
--- a/c/tools/benchmark_http_serving.py
+++ b/c/tools/benchmark_http_serving.py
@@ -128,6 +128,8 @@ def request_one(url, payload, key, timeout, index, origin):
choices = chunk.get("choices", [])
if not isinstance(choices, list) or len(choices) > 1:
raise StreamError("invalid_choices")
+ if choices and result["finish_reason"] is not None:
+ raise StreamError("choice_after_finish")
for choice in choices:
if (not isinstance(choice, dict) or type(choice.get("index")) is not int
or choice["index"] != 0):
diff --git a/docs/benchmarking.md b/docs/benchmarking.md
index d0d28f25..f5e2ec7f 100644
--- a/docs/benchmarking.md
+++ b/docs/benchmarking.md
@@ -150,6 +150,8 @@ Measurement boundaries:
HTTP errors, stream errors, malformed responses, and incomplete streams fail.
With `n=1`, each nonempty choices array must contain exactly one choice with
integer index 0. Output text and tool-function fields must be strings or null.
+ Once a choice has finished, further choice chunks are rejected; a trailing
+ usage chunk with empty choices is accepted.
This establishes protocol completion, not output correctness; filtered or
length-limited output can still count as protocol success.
- Token counts come only from `usage.completion_tokens`. Successful completion