Skip to content

LLM inference in C/C++

C++ ggml

server: return 400 instead of 500 on validation error with X-Conversation-Id (#25760)

Pascalfd41bf65a2e0
diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp
index 19db04988d..f0a35b18e5 100644
--- a/tools/server/server-stream.cpp
+++ b/tools/server/server-stream.cpp
@@ -632,6 +632,13 @@ void server_res_spipe::on_complete() {
if (!spipe || next_finished) {
return;
}
+ // an empty next_orig means set_next() never ran: the request failed before streaming
+ // started, typically a params validation throw. evict the session installed by set_req()
+ // so the failed request leaves nothing behind for discovery or replay
+ if (!next_orig) {
+ g_stream_sessions.evict(server_stream_conv_id_from_headers(req->headers));
+ return;
+ }
std::string chunk;
while (!spipe->is_cancelled()) {
chunk.clear();
diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts
index 658eb5978c..ab418ed9cd 100644
--- a/tools/ui/src/lib/stores/chat.svelte.ts
+++ b/tools/ui/src/lib/stores/chat.svelte.ts
@@ -2428,7 +2428,8 @@ class ChatStore {
if (currentConfig.samplers) apiOptions.samplers = currentConfig.samplers;
- apiOptions.backend_sampling = currentConfig.backend_sampling;
+ if (hasValue(currentConfig.backend_sampling))
+ apiOptions.backend_sampling = currentConfig.backend_sampling;
if (currentConfig.customJson) apiOptions.custom = currentConfig.customJson;