LLM inference in C/C++
C++ ggml
server: add "X-Accel-Buffering": "no" header to streaming endpoints (#24774)
Reguna40f3aafc4599
diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cppindex 5defee1f5e..4f2abab00c 100644--- a/tools/server/server-http.cpp+++ b/tools/server/server-http.cpp@@ -492,6 +492,8 @@ using server_http_req_ptr = std::unique_ptr<server_http_req>;static void process_handler_response(server_http_req_ptr && request, server_http_res_ptr & response, httplib::Response & res) {if (response->is_stream()) {res.status = response->status;+ // Tell Nginx to not buffer any streamed response+ response->headers["X-Accel-Buffering"] = "no";set_headers(res, response->headers);const std::string content_type = response->content_type;// convert to shared_ptr as both chunked_content_provider() and on_complete() need to use it