LLM inference in C/C++
C++ ggml
server : add timings and progress to /responses API stream (#25348)
Thomas LECONTE5eca4e3cabad
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cppindex 54fa949dfa..8d611e520d 100644--- a/tools/server/server-task.cpp+++ b/tools/server/server-task.cpp@@ -730,6 +730,10 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {}}});+ if (timings.prompt_n >= 0) {+ server_sent_events.back().at("data").push_back({"timings", timings.to_json()});+ }+return server_sent_events;}@@ -1016,6 +1020,7 @@ void server_task_result_cmpl_partial::update(task_result_state & state) {thinking_block_started = state.thinking_block_started;text_block_started = state.text_block_started;+ oai_resp_created = state.oai_resp_created;oai_resp_id = state.oai_resp_id;oai_resp_reasoning_id = state.oai_resp_reasoning_id;oai_resp_message_id = state.oai_resp_message_id;@@ -1024,6 +1029,10 @@ void server_task_result_cmpl_partial::update(task_result_state & state) {// track if the accumulated message has any reasoning contentanthropic_has_reasoning = !state.chat_msg.reasoning_content.empty();+ if (res_type == TASK_RESPONSE_TYPE_OAI_RESP && !state.oai_resp_created && (is_progress || n_decoded == 1)) {+ state.oai_resp_created = true;+ }+// Pre-compute state updates based on diffs (for next chunk)for (const common_chat_msg_diff & diff : oaicompat_msg_diffs) {if (!diff.reasoning_content_delta.empty() && !state.thinking_block_started) {@@ -1181,7 +1190,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() {json server_task_result_cmpl_partial::to_json_oaicompat_resp() {std::vector<json> events;- if (n_decoded == 1) {+ if (!oai_resp_created) {events.push_back(json {{"event", "response.created"},{"data", json {@@ -1204,6 +1213,18 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {}},}},});+ } else if (is_progress) {+ events.push_back(json {+ {"event", "response.in_progress"},+ {"data", json {+ {"type", "response.in_progress"},+ {"response", json {+ {"id", oai_resp_id},+ {"object", "response"},+ {"status", "in_progress"},+ }},+ }},+ });}for (const common_chat_msg_diff & diff : oaicompat_msg_diffs) {@@ -1302,6 +1323,17 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {});}}++ if (!events.empty()) {+ json & data = events.back().at("data");+ if (timings.prompt_n >= 0) {+ data.push_back({"timings", timings.to_json()});+ }+ if (is_progress) {+ data.push_back({"prompt_progress", progress.to_json()});+ }+ }+return events;}diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex 49f62d3866..dc6b2dac1e 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -117,6 +117,7 @@ struct task_result_state {bool text_block_started = false;// for OpenAI Responses streaming API+ bool oai_resp_created = false;const std::string oai_resp_id;const std::string oai_resp_reasoning_id;const std::string oai_resp_message_id;@@ -440,6 +441,7 @@ struct server_task_result_cmpl_partial : server_task_result {bool text_block_started = false;// for OpenAI Responses API+ bool oai_resp_created = false;std::string oai_resp_id;std::string oai_resp_reasoning_id;std::string oai_resp_message_id;diff --git a/tools/server/tests/unit/test_compat_oai_responses.py b/tools/server/tests/unit/test_compat_oai_responses.pyindex 7aab4a8ba6..14528b4874 100644--- a/tools/server/tests/unit/test_compat_oai_responses.py+++ b/tools/server/tests/unit/test_compat_oai_responses.py@@ -71,3 +71,44 @@ def test_responses_stream_with_openai_library():assert r.response.output[0].id.startswith("msg_")assert gathered_text == r.response.output_textassert match_regex("(Suddenly)+", r.response.output_text)+++def test_responses_stream_with_llama_telemetry():+ global server+ server.n_ctx = 256+ server.n_batch = 32+ server.n_slots = 1+ server.start()++ saw_progress = False+ saw_delta_timings = False+ completed = None++ res = server.make_stream_request("POST", "/responses", data={+ "input": "This is a test" * 10,+ "max_output_tokens": 8,+ "temperature": 0.8,+ "stream": True,+ "timings_per_token": True,+ "return_progress": True,+ })++ for data in res:+ if "prompt_progress" in data:+ assert data["type"] == "response.in_progress"+ assert data["prompt_progress"]["total"] > 0+ assert data["prompt_progress"]["processed"] >= data["prompt_progress"]["cache"]+ saw_progress = True+ if "timings" in data:+ assert "prompt_per_second" in data["timings"]+ assert "predicted_per_second" in data["timings"]+ if data["type"] == "response.output_text.delta":+ saw_delta_timings = True+ if data["type"] == "response.completed":+ completed = data++ assert saw_progress+ assert saw_delta_timings+ assert completed is not None+ assert "usage" in completed["response"]+ assert "timings" in completed