Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve: timings on /v1/responses (#3291)
Patrick Schauba0fc88abd646
diff --git a/docs/DETAILS.md b/docs/DETAILS.mdindex f60429bf2..61dfb584a 100644--- a/docs/DETAILS.md+++ b/docs/DETAILS.md@@ -707,6 +707,9 @@ endpoints are refused with an error that says so.| `text.format` | `json_schema` and `json_object` use the JSON formats above (checked, not constrained) || `client_metadata["x-codex-turn-metadata"]` thread-title request | Opt-in `"codex_thread_titles": true`: answered without the engine as `{"title": "<the user's line, cut to 36 characters>"}` so the conversation keeps the prompt cache |+The final event (`response.completed` or `response.incomplete`) and a non-streaming reply carry the same top-level `timings` object as+`/v1/chat/completions`.+The thinking comes back as a `reasoning` output item with `reasoning_text` content. The model writes no summaries, so `summary` isempty. With `"include": ["reasoning.encrypted_content"]` the item also carries an opaque string (base64, not encrypted).Send reasoning items back with the conversation, as Codex does, so the prompt matches and the cache is reused. Streamingdiff --git a/serve/responses.py b/serve/responses.pyindex 8b7241b98..53c4fb54f 100644--- a/serve/responses.py+++ b/serve/responses.py@@ -662,10 +662,17 @@ class Assembler:if length:self.response["status"] = "incomplete"self.response["incomplete_details"] = {"reason": "max_output_tokens"}- return out + [self.event("response.incomplete", response=self.snapshot())]+ return out + [self._with_timings(self.event("response.incomplete", response=self.snapshot()), done)]self.response["status"] = "completed"self.response["completed_at"] = int(time.time())- return out + [self.event("response.completed", response=self.snapshot())]+ return out + [self._with_timings(self.event("response.completed", response=self.snapshot()), done)]++ @staticmethod+ def _with_timings(event: dict, done: dict) -> dict:+ """Top-level `timings` on the final event, as /v1/chat/completions has them (#107)."""+ if done.get("timings"):+ event["timings"] = done["timings"]+ return eventdef failed(self, message: str, code: str = "server_error") -> dict:"""response.failed: an error after the stream started (the HTTP status is already sent)."""@@ -683,4 +690,6 @@ def collect(events) -> dict:for e in events:if e is not None:last = e+ if last.get("timings"): # as on a whole /v1/chat/completions reply (#107)+ return {**last["response"], "timings": last["timings"]}return last["response"]diff --git a/serve/test_server.py b/serve/test_server.pyindex 3e60dee4e..935414f97 100644--- a/serve/test_server.py+++ b/serve/test_server.py@@ -3808,6 +3808,19 @@ class UsageAndStatus(unittest.TestCase):self.assertEqual(last["usage"]["prompt_tokens_details"]["cached_tokens"], ClockedEngine.REUSED)self.assertEqual(last["timings"]["cache_n"], ClockedEngine.REUSED)+ def test_responses(self):+ # /v1/responses carries the same top-level timings as /v1/chat/completions+ body = {"model": "x", "max_output_tokens": 20, "input": "hi"}+ status, raw = self.request("/v1/responses", body)+ self.assertEqual(status, 200)+ r = json.loads(raw)+ self.assertEqual(r["timings"]["cache_n"], ClockedEngine.REUSED)+ self.assertEqual(r["timings"]["predicted_n"], r["usage"]["output_tokens"])+ status, raw = self.request("/v1/responses", {**body, "stream": True})+ last = [json.loads(line[6:]) for line in raw.decode().splitlines() if line.startswith("data: {")][-1]+ self.assertIn(last["type"], ("response.completed", "response.incomplete"))+ self.assertAlmostEqual(last["timings"]["predicted_per_second"], 50.0, delta=0.1) # 20 ms a token+def test_anthropic(self):u = self.chat("/v1/messages")["usage"]self.assertEqual(u["cache_read_input_tokens"], ClockedEngine.REUSED)