Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(bench): reject choice chunks after stream completion
ZacharyZcR03f6aa8afbdf
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.pyindex 98b5f826..f728f765 100644--- a/c/tests/test_benchmark_http_serving.py+++ b/c/tests/test_benchmark_http_serving.py@@ -128,6 +128,22 @@ class BenchmarkTest(unittest.TestCase):self.assertFalse(row["success"], row)self.assertEqual(bench.summarize([row], 1, slo_duration=1)["latency_slo"]["requests_met"], 0)+ def test_choice_after_finish_is_not_successful(self):+ terminal = {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}+ for delta, finish in (({"content": "late output"}, None), ({}, "length")):+ with self.subTest(delta=delta, finish=finish):+ extra = {"choices": [{"index": 0, "delta": delta, "finish_reason": finish}]}+ self.server.body = (event(terminal) + event(extra)+ + event({"choices": [], "usage": {"completion_tokens": 999}})+ + event("[DONE]")).encode()+ row = self.request()+ self.assertFalse(row["success"])+ self.assertEqual(row["error"], "choice_after_finish")+ self.assertEqual(row["finish_reason"], "stop")+ summary = bench.summarize([row], 1, slo_duration=10)+ self.assertEqual(summary["reported_successful_completion_tokens"], 0)+ self.assertEqual(summary["latency_slo"]["requests_met"], 0)+def test_supported_finish_reasons_and_null_delta(self):for finish in ("stop", "length", "tool_calls", "function_call", "content_filter"):with self.subTest(finish=finish):diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.pyindex b1154130..74e32a84 100644--- a/c/tools/benchmark_http_serving.py+++ b/c/tools/benchmark_http_serving.py@@ -128,6 +128,8 @@ def request_one(url, payload, key, timeout, index, origin):choices = chunk.get("choices", [])if not isinstance(choices, list) or len(choices) > 1:raise StreamError("invalid_choices")+ if choices and result["finish_reason"] is not None:+ raise StreamError("choice_after_finish")for choice in choices:if (not isinstance(choice, dict) or type(choice.get("index")) is not intor choice["index"] != 0):diff --git a/docs/benchmarking.md b/docs/benchmarking.mdindex d0d28f25..f5e2ec7f 100644--- a/docs/benchmarking.md+++ b/docs/benchmarking.md@@ -150,6 +150,8 @@ Measurement boundaries:HTTP errors, stream errors, malformed responses, and incomplete streams fail.With `n=1`, each nonempty choices array must contain exactly one choice withinteger index 0. Output text and tool-function fields must be strings or null.+ Once a choice has finished, further choice chunks are rejected; a trailing+ usage chunk with empty choices is accepted.This establishes protocol completion, not output correctness; filtered orlength-limited output can still count as protocol success.- Token counts come only from `usage.completion_tokens`. Successful completion