Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: timings on /v1/responses (#3291)

Patrick Schauba0fc88abd646
diff --git a/docs/DETAILS.md b/docs/DETAILS.md
index f60429bf2..61dfb584a 100644
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@ -707,6 +707,9 @@ endpoints are refused with an error that says so.
| `text.format` | `json_schema` and `json_object` use the JSON formats above (checked, not constrained) |
| `client_metadata["x-codex-turn-metadata"]` thread-title request | Opt-in `"codex_thread_titles": true`: answered without the engine as `{"title": "<the user's line, cut to 36 characters>"}` so the conversation keeps the prompt cache |
+The final event (`response.completed` or `response.incomplete`) and a non-streaming reply carry the same top-level `timings` object as
+`/v1/chat/completions`.
+
The thinking comes back as a `reasoning` output item with `reasoning_text` content. The model writes no summaries, so `summary` is
empty. With `"include": ["reasoning.encrypted_content"]` the item also carries an opaque string (base64, not encrypted).
Send reasoning items back with the conversation, as Codex does, so the prompt matches and the cache is reused. Streaming
diff --git a/serve/responses.py b/serve/responses.py
index 8b7241b98..53c4fb54f 100644
--- a/serve/responses.py
+++ b/serve/responses.py
@@ -662,10 +662,17 @@ class Assembler:
if length:
self.response["status"] = "incomplete"
self.response["incomplete_details"] = {"reason": "max_output_tokens"}
- return out + [self.event("response.incomplete", response=self.snapshot())]
+ return out + [self._with_timings(self.event("response.incomplete", response=self.snapshot()), done)]
self.response["status"] = "completed"
self.response["completed_at"] = int(time.time())
- return out + [self.event("response.completed", response=self.snapshot())]
+ return out + [self._with_timings(self.event("response.completed", response=self.snapshot()), done)]
+
+ @staticmethod
+ def _with_timings(event: dict, done: dict) -> dict:
+ """Top-level `timings` on the final event, as /v1/chat/completions has them (#107)."""
+ if done.get("timings"):
+ event["timings"] = done["timings"]
+ return event
def failed(self, message: str, code: str = "server_error") -> dict:
"""response.failed: an error after the stream started (the HTTP status is already sent)."""
@@ -683,4 +690,6 @@ def collect(events) -> dict:
for e in events:
if e is not None:
last = e
+ if last.get("timings"): # as on a whole /v1/chat/completions reply (#107)
+ return {**last["response"], "timings": last["timings"]}
return last["response"]
diff --git a/serve/test_server.py b/serve/test_server.py
index 3e60dee4e..935414f97 100644
--- a/serve/test_server.py
+++ b/serve/test_server.py
@@ -3808,6 +3808,19 @@ class UsageAndStatus(unittest.TestCase):
self.assertEqual(last["usage"]["prompt_tokens_details"]["cached_tokens"], ClockedEngine.REUSED)
self.assertEqual(last["timings"]["cache_n"], ClockedEngine.REUSED)
+ def test_responses(self):
+ # /v1/responses carries the same top-level timings as /v1/chat/completions
+ body = {"model": "x", "max_output_tokens": 20, "input": "hi"}
+ status, raw = self.request("/v1/responses", body)
+ self.assertEqual(status, 200)
+ r = json.loads(raw)
+ self.assertEqual(r["timings"]["cache_n"], ClockedEngine.REUSED)
+ self.assertEqual(r["timings"]["predicted_n"], r["usage"]["output_tokens"])
+ status, raw = self.request("/v1/responses", {**body, "stream": True})
+ last = [json.loads(line[6:]) for line in raw.decode().splitlines() if line.startswith("data: {")][-1]
+ self.assertIn(last["type"], ("response.completed", "response.incomplete"))
+ self.assertAlmostEqual(last["timings"]["predicted_per_second"], 50.0, delta=0.1) # 20 ms a token
+
def test_anthropic(self):
u = self.chat("/v1/messages")["usage"]
self.assertEqual(u["cache_read_input_tokens"], ClockedEngine.REUSED)