Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(bench): account for legacy function call output

ZacharyZcRb1c9d523aae3
diff --git a/c/tests/test_benchmark_http_serving.py b/c/tests/test_benchmark_http_serving.py
index 75221094..90a8cf90 100644
--- a/c/tests/test_benchmark_http_serving.py
+++ b/c/tests/test_benchmark_http_serving.py
@@ -90,6 +90,23 @@ class BenchmarkTest(unittest.TestCase):
self.assertEqual(row["completion_tokens"], 3)
self.assertLessEqual(row["first_output_seconds"], row["duration_seconds"])
+ def test_legacy_function_call_output_and_validation(self):
+ for function, valid, output in (({"name": "lookup"}, True, True),
+ ({"arguments": "{}"}, True, True),
+ ({"arguments": ""}, True, False),
+ ("lookup", False, False),
+ ({"arguments": 42}, False, False)):
+ with self.subTest(function=function):
+ chunk = {"choices": [{"index": 0, "delta": {"function_call": function},
+ "finish_reason": "function_call"}]}
+ self.server.body = (event(chunk) + event("[DONE]")).encode()
+ row = self.request()
+ self.assertEqual(row["success"], valid)
+ if valid:
+ self.assertEqual(row["first_output_seconds"] is not None, output)
+ summary = bench.summarize([row], 1, slo_first_output=5)
+ self.assertEqual(summary["latency_slo"]["requests_met"], int(output))
+
def test_empty_output_is_not_first_output(self):
self.server.body = stream({"content": "", "tool_calls": [{"id": "id", "function": {}}]}, usage=0)
row = self.request()
diff --git a/c/tools/benchmark_http_serving.py b/c/tools/benchmark_http_serving.py
index 74e32a84..438e6fe2 100644
--- a/c/tools/benchmark_http_serving.py
+++ b/c/tools/benchmark_http_serving.py
@@ -83,9 +83,12 @@ def has_output(delta):
output |= nonempty_text(delta.get(field))
calls = delta.get("tool_calls")
if calls is None:
- return output
+ calls = []
if not isinstance(calls, list):
raise StreamError("invalid_tool_calls")
+ legacy = delta.get("function_call")
+ if legacy is not None:
+ calls = calls + [{"function": legacy}]
for tool in calls:
if not isinstance(tool, dict):
raise StreamError("invalid_tool_call")
diff --git a/docs/benchmarking.md b/docs/benchmarking.md
index f5e2ec7f..23875486 100644
--- a/docs/benchmarking.md
+++ b/docs/benchmarking.md
@@ -141,7 +141,8 @@ Measurement boundaries:
`--timeout` limits individual socket operations, **not total request time**;
a stream that keeps sending data can last longer.
- `first_output_seconds` measures receipt of the first nonempty content,
- reasoning, or tool-function name/arguments delta. Role-only and empty deltas
+ reasoning, or tool-function name/arguments delta (`tool_calls` or legacy
+ `function_call`). Role-only and empty deltas
do not count. This is client-visible first output latency, not necessarily
time to a visible answer or to exactly one token. Empty successful output
has no first-output sample. SSE chunk gaps are not reported as token latency.