Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(bench): account for legacy function call output
ZacharyZcRb1c9d523aae3
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.pyindex 75221094..90a8cf90 100644--- a/c/tests/test_benchmark_http_serving.py+++ b/c/tests/test_benchmark_http_serving.py@@ -90,6 +90,23 @@ class BenchmarkTest(unittest.TestCase):self.assertEqual(row["completion_tokens"], 3)self.assertLessEqual(row["first_output_seconds"], row["duration_seconds"])+ def test_legacy_function_call_output_and_validation(self):+ for function, valid, output in (({"name": "lookup"}, True, True),+ ({"arguments": "{}"}, True, True),+ ({"arguments": ""}, True, False),+ ("lookup", False, False),+ ({"arguments": 42}, False, False)):+ with self.subTest(function=function):+ chunk = {"choices": [{"index": 0, "delta": {"function_call": function},+ "finish_reason": "function_call"}]}+ self.server.body = (event(chunk) + event("[DONE]")).encode()+ row = self.request()+ self.assertEqual(row["success"], valid)+ if valid:+ self.assertEqual(row["first_output_seconds"] is not None, output)+ summary = bench.summarize([row], 1, slo_first_output=5)+ self.assertEqual(summary["latency_slo"]["requests_met"], int(output))+def test_empty_output_is_not_first_output(self):self.server.body = stream({"content": "", "tool_calls": [{"id": "id", "function": {}}]}, usage=0)row = self.request()diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.pyindex 74e32a84..438e6fe2 100644--- a/c/tools/benchmark_http_serving.py+++ b/c/tools/benchmark_http_serving.py@@ -83,9 +83,12 @@ def has_output(delta):output |= nonempty_text(delta.get(field))calls = delta.get("tool_calls")if calls is None:- return output+ calls = []if not isinstance(calls, list):raise StreamError("invalid_tool_calls")+ legacy = delta.get("function_call")+ if legacy is not None:+ calls = calls + [{"function": legacy}]for tool in calls:if not isinstance(tool, dict):raise StreamError("invalid_tool_call")diff --git a/docs/benchmarking.md b/docs/benchmarking.mdindex f5e2ec7f..23875486 100644--- a/docs/benchmarking.md+++ b/docs/benchmarking.md@@ -141,7 +141,8 @@ Measurement boundaries:`--timeout` limits individual socket operations, **not total request time**;a stream that keeps sending data can last longer.- `first_output_seconds` measures receipt of the first nonempty content,- reasoning, or tool-function name/arguments delta. Role-only and empty deltas+ reasoning, or tool-function name/arguments delta (`tool_calls` or legacy+ `function_call`). Role-only and empty deltasdo not count. This is client-visible first output latency, not necessarilytime to a visible answer or to exactly one token. Empty successful outputhas no first-output sample. SSE chunk gaps are not reported as token latency.