Skip to content

LLM inference in C/C++

C++ ggml

server + ui: ping silent SSE streams every 1s and kick only after 3s so slow prefill never drops healthy connections (#25241)

Pascalb5315e16e0c3
diff --git a/tools/server/README.md b/tools/server/README.md
index e88bc5f28a..501b66123d 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -521,6 +521,8 @@ These words will not be included in the completion, so make sure to add them to
`return_progress`: Include prompt processing progress in `stream` mode. The progress will be contained inside `prompt_progress` with 4 values: `total`, `cache`, `processed`, and `time_ms`. The overall progress is `processed/total`, while the actual timed progress is `(processed-cache)/(total-cache)`. The `time_ms` field contains the elapsed time in milliseconds since prompt processing started. Default: `false`
+`sse_ping_interval`: Interval in seconds between SSE comment pings emitted while the stream stays silent, keeping the connection observable during long prompt processing. Overrides the server `--sse-ping-interval` setting for this request, `-1` disables pings. Default: server setting
+
`post_sampling_probs`: Returns the probabilities of top `n_probs` tokens after applying sampling chain.
`response_fields`: A list of response fields, for example: `"response_fields": ["content", "generation_settings/n_predict"]`. If the specified field is missing, it will simply be omitted from the response without triggering an error. Note that fields with a slash will be unnested; for example, `generation_settings/n_predict` will move the field `n_predict` from the `generation_settings` object to the root of the response and give it a new name.
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 20e93258fc..bb3b91ab5e 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -4089,6 +4089,8 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
auto & rd = res->rd;
auto & params = this->params;
+ int32_t sse_ping_interval = params.sse_ping_interval;
+
try {
std::vector<server_task> tasks;
@@ -4139,6 +4141,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
task.params.message_spans = task.tokens.find_message_spans(delimiters);
task.id_slot = json_value(data, "id_slot", -1);
+ sse_ping_interval = task.params.sse_ping_interval;
// OAI-compat
task.params.res_type = res_type;
@@ -4228,7 +4231,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
}
res->status = 200;
res->content_type = "text/event-stream";
- res->next = [res_this = res.get(), res_type, &req, &params](std::string & output) -> bool {
+ res->next = [res_this = res.get(), res_type, sse_ping_interval, &req](std::string & output) -> bool {
static auto format_error = [](task_response_type res_type, const json & res_json) {
if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) {
return format_anthropic_sse({
@@ -4277,10 +4280,10 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
// receive subsequent results
bool timeout = false;
int64_t start_time = ggml_time_ms();
- auto result = rd.next([&timeout, &start_time, &params, &effective_should_stop]() {
+ auto result = rd.next([&timeout, &start_time, sse_ping_interval, &effective_should_stop]() {
if (effective_should_stop()) {
return true; // should_stop condition met
- } else if (params.sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)params.sse_ping_interval * 1000) {
+ } else if (sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)sse_ping_interval * 1000) {
timeout = true;
return true; // timeout
}
diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp
index 07a842bd6a..5713cc8318 100644
--- a/tools/server/server-schema.cpp
+++ b/tools/server/server-schema.cpp
@@ -37,6 +37,10 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
add((new field_bool("return_progress", params.return_progress))
->set_desc("Include prompt processing progress events in stream mode"));
+ add((new field_num("sse_ping_interval", params.sse_ping_interval))
+ ->set_hard_limits(-1, INT32_MAX)
+ ->set_desc("Interval in seconds between SSE comment pings emitted while the stream stays silent, -1 disables pings"));
+
add((new field_num("n_predict", params.n_predict))
->set_hard_limits(-1, INT32_MAX)
->add_alias("max_completion_tokens")
@@ -504,6 +508,7 @@ task_params eval_llama_cmpl_schema(
params.n_cache_reuse = params_base.n_cache_reuse;
params.cache_prompt = params_base.cache_prompt;
params.antiprompt = params_base.antiprompt;
+ params.sse_ping_interval = params_base.sse_ping_interval;
// enabling this will output extra debug information in the HTTP responses from the server
params.verbose = params_base.verbosity > 9;
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 293bdf053a..49f62d3866 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -54,6 +54,8 @@ struct task_params {
bool return_tokens = false;
bool return_progress = false;
+ int32_t sse_ping_interval = 30; // seconds between SSE comment pings while the stream stays silent, -1 disables
+
int32_t n_keep = 0; // number of tokens to keep from initial prompt
int32_t n_discard = 0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to half
int32_t n_predict = -1; // new tokens to predict
diff --git a/tools/ui/src/lib/constants/stream.ts b/tools/ui/src/lib/constants/stream.ts
index 3d042451fc..67951ee953 100644
--- a/tools/ui/src/lib/constants/stream.ts
+++ b/tools/ui/src/lib/constants/stream.ts
@@ -1,3 +1,3 @@
// grace window after a visibilitychange before we kick a reader whose socket likely died
// while the tab was hidden. covers brief background pauses without thrashing live streams
-export const STREAM_VISIBILITY_KICK_MS = 1000;
+export const STREAM_VISIBILITY_KICK_MS = 3000;
diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts
index 7dfee37731..dbd0a94f10 100644
--- a/tools/ui/src/lib/services/chat.service.ts
+++ b/tools/ui/src/lib/services/chat.service.ts
@@ -255,6 +255,7 @@ export class ChatService {
}),
stream,
return_progress: stream ? true : undefined,
+ sse_ping_interval: stream ? 1 : undefined,
tools: tools && tools.length > 0 ? tools : undefined
};
diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.ts
index ec695ac61c..5421f8b7f3 100644
--- a/tools/ui/src/lib/types/api.d.ts
+++ b/tools/ui/src/lib/types/api.d.ts
@@ -265,6 +265,7 @@ export interface ApiChatCompletionRequest {
stream?: boolean;
model?: string;
return_progress?: boolean;
+ sse_ping_interval?: number;
tools?: ApiChatCompletionTool[];
// Reasoning parameters
reasoning_format?: string;