Skip to content

LLM inference in C/C++

C++ ggml

server: add "X-Accel-Buffering": "no" header to streaming endpoints (#24774)

Reguna40f3aafc4599
diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp
index 5defee1f5e..4f2abab00c 100644
--- a/tools/server/server-http.cpp
+++ b/tools/server/server-http.cpp
@@ -492,6 +492,8 @@ using server_http_req_ptr = std::unique_ptr<server_http_req>;
static void process_handler_response(server_http_req_ptr && request, server_http_res_ptr & response, httplib::Response & res) {
if (response->is_stream()) {
res.status = response->status;
+ // Tell Nginx to not buffer any streamed response
+ response->headers["X-Accel-Buffering"] = "no";
set_headers(res, response->headers);
const std::string content_type = response->content_type;
// convert to shared_ptr as both chunked_content_provider() and on_complete() need to use it