LLM inference in C/C++
C++ ggml
server + ui: ping silent SSE streams every 1s and kick only after 3s so slow prefill never drops healthy connections (#25241)
Pascalb5315e16e0c3
diff --git a/tools/server/README.md b/tools/server/README.mdindex e88bc5f28a..501b66123d 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -521,6 +521,8 @@ These words will not be included in the completion, so make sure to add them to`return_progress`: Include prompt processing progress in `stream` mode. The progress will be contained inside `prompt_progress` with 4 values: `total`, `cache`, `processed`, and `time_ms`. The overall progress is `processed/total`, while the actual timed progress is `(processed-cache)/(total-cache)`. The `time_ms` field contains the elapsed time in milliseconds since prompt processing started. Default: `false`+`sse_ping_interval`: Interval in seconds between SSE comment pings emitted while the stream stays silent, keeping the connection observable during long prompt processing. Overrides the server `--sse-ping-interval` setting for this request, `-1` disables pings. Default: server setting+`post_sampling_probs`: Returns the probabilities of top `n_probs` tokens after applying sampling chain.`response_fields`: A list of response fields, for example: `"response_fields": ["content", "generation_settings/n_predict"]`. If the specified field is missing, it will simply be omitted from the response without triggering an error. Note that fields with a slash will be unnested; for example, `generation_settings/n_predict` will move the field `n_predict` from the `generation_settings` object to the root of the response and give it a new name.diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 20e93258fc..bb3b91ab5e 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -4089,6 +4089,8 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(auto & rd = res->rd;auto & params = this->params;+ int32_t sse_ping_interval = params.sse_ping_interval;+try {std::vector<server_task> tasks;@@ -4139,6 +4141,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(task.params.message_spans = task.tokens.find_message_spans(delimiters);task.id_slot = json_value(data, "id_slot", -1);+ sse_ping_interval = task.params.sse_ping_interval;// OAI-compattask.params.res_type = res_type;@@ -4228,7 +4231,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(}res->status = 200;res->content_type = "text/event-stream";- res->next = [res_this = res.get(), res_type, &req, ¶ms](std::string & output) -> bool {+ res->next = [res_this = res.get(), res_type, sse_ping_interval, &req](std::string & output) -> bool {static auto format_error = [](task_response_type res_type, const json & res_json) {if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) {return format_anthropic_sse({@@ -4277,10 +4280,10 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(// receive subsequent resultsbool timeout = false;int64_t start_time = ggml_time_ms();- auto result = rd.next([&timeout, &start_time, ¶ms, &effective_should_stop]() {+ auto result = rd.next([&timeout, &start_time, sse_ping_interval, &effective_should_stop]() {if (effective_should_stop()) {return true; // should_stop condition met- } else if (params.sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)params.sse_ping_interval * 1000) {+ } else if (sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)sse_ping_interval * 1000) {timeout = true;return true; // timeout}diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cppindex 07a842bd6a..5713cc8318 100644--- a/tools/server/server-schema.cpp+++ b/tools/server/server-schema.cpp@@ -37,6 +37,10 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &add((new field_bool("return_progress", params.return_progress))->set_desc("Include prompt processing progress events in stream mode"));+ add((new field_num("sse_ping_interval", params.sse_ping_interval))+ ->set_hard_limits(-1, INT32_MAX)+ ->set_desc("Interval in seconds between SSE comment pings emitted while the stream stays silent, -1 disables pings"));+add((new field_num("n_predict", params.n_predict))->set_hard_limits(-1, INT32_MAX)->add_alias("max_completion_tokens")@@ -504,6 +508,7 @@ task_params eval_llama_cmpl_schema(params.n_cache_reuse = params_base.n_cache_reuse;params.cache_prompt = params_base.cache_prompt;params.antiprompt = params_base.antiprompt;+ params.sse_ping_interval = params_base.sse_ping_interval;// enabling this will output extra debug information in the HTTP responses from the serverparams.verbose = params_base.verbosity > 9;diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex 293bdf053a..49f62d3866 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -54,6 +54,8 @@ struct task_params {bool return_tokens = false;bool return_progress = false;+ int32_t sse_ping_interval = 30; // seconds between SSE comment pings while the stream stays silent, -1 disables+int32_t n_keep = 0; // number of tokens to keep from initial promptint32_t n_discard = 0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to halfint32_t n_predict = -1; // new tokens to predictdiff --git a/tools/ui/src/lib/constants/stream.ts b/tools/ui/src/lib/constants/stream.tsindex 3d042451fc..67951ee953 100644--- a/tools/ui/src/lib/constants/stream.ts+++ b/tools/ui/src/lib/constants/stream.ts@@ -1,3 +1,3 @@// grace window after a visibilitychange before we kick a reader whose socket likely died// while the tab was hidden. covers brief background pauses without thrashing live streams-export const STREAM_VISIBILITY_KICK_MS = 1000;+export const STREAM_VISIBILITY_KICK_MS = 3000;diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.tsindex 7dfee37731..dbd0a94f10 100644--- a/tools/ui/src/lib/services/chat.service.ts+++ b/tools/ui/src/lib/services/chat.service.ts@@ -255,6 +255,7 @@ export class ChatService {}),stream,return_progress: stream ? true : undefined,+ sse_ping_interval: stream ? 1 : undefined,tools: tools && tools.length > 0 ? tools : undefined};diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.tsindex ec695ac61c..5421f8b7f3 100644--- a/tools/ui/src/lib/types/api.d.ts+++ b/tools/ui/src/lib/types/api.d.ts@@ -265,6 +265,7 @@ export interface ApiChatCompletionRequest {stream?: boolean;model?: string;return_progress?: boolean;+ sse_ping_interval?: number;tools?: ApiChatCompletionTool[];// Reasoning parametersreasoning_format?: string;