LLM inference in C/C++
C++ ggml
server: add --cors-* options (#25655)
Xuan-Son Nguyen6e52db5b72b6
diff --git a/common/arg.cpp b/common/arg.cppindex 9676adafe6..b6fddae00c 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -697,7 +697,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context}};- // parse the first time to get -hf option (used for remote preset)+ // parse all CLI args now, so that -hf is available below for remote preset resolutionparse_cli_args();postprocess_cpu_params(params.cpuparams, nullptr);@@ -748,6 +748,11 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_contextparams.kv_overrides.back().key[0] = 0;}+ if (!params.server_tools.empty() && !params.cors_origins_explicit) {+ LOG_WRN("server tools are enabled, using localhost as default CORS origin (change via --cors-origins)\n");+ params.cors_origins = "localhost";+ }+// pad tensor_buft_overrides for llama_params_fit:const size_t ntbo = llama_max_tensor_buft_overrides();while (params.tensor_buft_overrides.size() < ntbo) {@@ -3047,6 +3052,42 @@ common_params_context common_params_parser_init(common_params & params, llama_exparams.public_path = value;}).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_STATIC_PATH"));+ add_opt(common_arg(+ {"--cors-origins"}, "ORIGINS",+ string_format(+ "comma-separated list of allowed origins for CORS (default: %s)\n"+ "if set to special value 'localhost', reflect the Origin header only if it is localhost",+ params.cors_origins.c_str()),+ [](common_params & params, const std::string & value) {+ params.cors_origins = value;+ params.cors_origins_explicit = true;+ }+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_ORIGINS"));+ add_opt(common_arg(+ {"--cors-methods"}, "METHODS",+ string_format("comma-separated list of allowed methods for CORS (default: %s)", params.cors_methods.c_str()),+ [](common_params & params, const std::string & value) {+ params.cors_methods = value;+ }+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_METHODS"));+ add_opt(common_arg(+ {"--cors-headers"}, "HEADERS",+ string_format("comma-separated list of allowed headers for CORS (default: %s)", params.cors_headers.c_str()),+ [](common_params & params, const std::string & value) {+ params.cors_headers = value;+ }+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_HEADERS"));+ add_opt(common_arg(+ {"--cors-credentials"},+ {"--no-cors-credentials"},+ string_format(+ "whether to allow credentials for CORS (default: %s)\n"+ "note: if this is enabled and --cors-origins is set to * (default), the Origin header will be echoed back, and credentials will always be allowed",+ params.cors_credentials ? "enabled" : "disabled"),+ [](common_params & params, bool value) {+ params.cors_credentials = value;+ }+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_CREDENTIALS"));add_opt(common_arg({"--api-prefix"}, "PREFIX",string_format("prefix path the server serves from, without the trailing slash (default: %s)", params.api_prefix.c_str()),@@ -3080,7 +3121,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex{"--tools"}, "TOOL1,TOOL2,...","experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n""specify \"all\" to enable all tools\n"- "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime",+ "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n"+ "note: for security reasons, this will limit --cors-origins to localhost by default",[](common_params & params, const std::string & value) {params.server_tools = parse_csv_row(value);}@@ -3088,7 +3130,8 @@ common_params_context common_params_parser_init(common_params & params, llama_exadd_opt(common_arg({"-ag", "--agent"},{"-no-ag", "--no-agent"},- "whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)",+ "whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)\n"+ "note: for security reasons, this will limit --cors-origins to localhost by default",[](common_params & params, bool value) {if (value) {params.server_tools = {"all"};@@ -3097,6 +3140,7 @@ common_params_context common_params_parser_init(common_params & params, llama_exparams.server_tools.clear();params.ui_mcp_proxy = false;}+ // note: do not modify cors_origins here, as the options are not evaluated in order (user may explicitly set --cors-origins before --agent)}).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_AGENT"));add_opt(common_arg(diff --git a/common/common.h b/common/common.hindex 66760005cd..bffc1767a7 100644--- a/common/common.h+++ b/common/common.h@@ -631,6 +631,14 @@ struct common_params {std::string api_prefix = ""; // NOLINTstd::string chat_template = ""; // NOLINTbool use_jinja = true; // NOLINT++ // server CORS params+ std::string cors_origins = "*";+ std::string cors_methods = "GET, POST, DELETE, OPTIONS";+ std::string cors_headers = "*";+ bool cors_credentials = true;+ bool cors_origins_explicit = false; // for --agent option+bool enable_chat_template = true;bool force_pure_content_parser = false;common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cppindex 48f903dfcb..24a38452aa 100644--- a/tools/server/server-http.cpp+++ b/tools/server/server-http.cpp@@ -47,6 +47,16 @@ static void log_server_request(const httplib::Request & req, const httplib::RespSRV_DBG("response: %s\n", res.body.c_str());}+// returns true if the Origin header value's host is localhost / 127.0.0.1 / ::1 (any port)+static bool origin_is_localhost(const std::string & origin) {+ try {+ const std::string host = common_http_parse_url(origin).host;+ return host == "localhost" || host == "127.0.0.1" || host == "::1";+ } catch (const std::exception &) {+ return false;+ }+}+// For Google Cloud Platform deployment compatibilitystruct gcp_params {bool enabled;@@ -266,13 +276,26 @@ bool server_http_context::init(const common_params & params) {};// register server middlewares- srv->set_pre_routing_handler([middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ srv->set_pre_routing_handler([¶ms, middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) {+ if (params.cors_credentials && params.cors_origins == "*") {+ // special case: echo back the Origin header to allow any origin to access the server with credentials+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ } else if (params.cors_origins == "localhost") {+ // special case: only reflect the Origin header if it is a localhost origin+ std::string origin = req.get_header_value("Origin");+ if (origin_is_localhost(origin)) {+ res.set_header("Access-Control-Allow-Origin", origin);+ } else {+ SRV_WRN("(CORS) skip non-localhost origin: %s\n", origin.c_str());+ }+ } else {+ res.set_header("Access-Control-Allow-Origin", params.cors_origins);+ }// If this is OPTIONS request, skip validation because browsers don't include Authorization headerif (req.method == "OPTIONS") {- res.set_header("Access-Control-Allow-Credentials", "true");- res.set_header("Access-Control-Allow-Methods", "GET, POST");- res.set_header("Access-Control-Allow-Headers", "*");+ res.set_header("Access-Control-Allow-Credentials", params.cors_credentials ? "true" : "false");+ res.set_header("Access-Control-Allow-Methods", params.cors_methods);+ res.set_header("Access-Control-Allow-Headers", params.cors_headers);res.set_content("", "text/html"); // blank response, no datareturn httplib::Server::HandlerResponse::Handled; // skip further processing}diff --git a/tools/server/server.cpp b/tools/server/server.cppindex c2b21120af..20effbb148 100644--- a/tools/server/server.cpp+++ b/tools/server/server.cpp@@ -303,14 +303,24 @@ int llama_server(common_params & params, int argc, char ** argv) {return res;};- // CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)- if (params.ui_mcp_proxy) {+ if (params.cors_origins == "*" && params.api_keys.empty()) {SRV_WRN("%s", "-----------------\n");- SRV_WRN("%s", "CORS proxy is enabled, do not expose server to untrusted environments\n");- SRV_WRN("%s", "This feature is EXPERIMENTAL and may be removed or changed in future versions\n");+ SRV_WRN("%s", "CORS is set to allow all origins ('*') and no API key is set\n");+ SRV_WRN("%s", "this can be a security risk (cross-origin attacks)\n");+ SRV_WRN("%s", "more info: https://github.com/ggml-org/llama.cpp/pull/25655\n");SRV_WRN("%s", "-----------------\n");+ }++ // CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)+ std::vector<std::string> warn_names;+ if (is_router_server) {+ warn_names.push_back("router mode");+ }++ if (params.ui_mcp_proxy) {ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));+ warn_names.push_back("MCP proxy (experimental)");} else {ctx_http.get ("/cors-proxy", ex_wrapper(res_403));ctx_http.post("/cors-proxy", ex_wrapper(res_403));@@ -324,17 +334,24 @@ int llama_server(common_params & params, int argc, char ** argv) {SRV_ERR("tools setup failed: %s\n", e.what());return 1;}- SRV_WRN("%s", "-----------------\n");- SRV_WRN("%s", "Built-in tools are enabled, do not expose server to untrusted environments\n");- SRV_WRN("%s", "This feature is EXPERIMENTAL and may be changed in the future\n");- SRV_WRN("%s", "-----------------\n");ctx_http.get ("/tools", ex_wrapper(tools.handle_get));ctx_http.post("/tools", ex_wrapper(tools.handle_post));+ warn_names.push_back("built-in tools (experimental)");} else {ctx_http.get ("/tools", ex_wrapper(res_403));ctx_http.post("/tools", ex_wrapper(res_403));}+ if (warn_names.size() > 0) {+ SRV_WRN("%s", "-----------------\n");+ SRV_WRN("%s", "the following feature(s) are enabled:\n");+ for (const auto & name : warn_names) {+ SRV_WRN(" %s\n", name.c_str());+ }+ SRV_WRN("%s", "do not expose the server to untrusted environments\n");+ SRV_WRN("%s", "-----------------\n");+ }+//// Handle downloading model//@@ -452,9 +469,6 @@ int llama_server(common_params & params, int argc, char ** argv) {SRV_INF("listening on %s\n", ctx_http.listening_address.c_str());if (is_router_server) {- SRV_WRN("%s", "NOTE: router mode is experimental\n");- SRV_WRN("%s", " it is not recommended to use this mode in untrusted environments\n");-if (!params.models_preset_hf.empty()) {SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str());SRV_WRN("%s", " please only use presets that you can trust! Unknown presets may be unsafe\n");diff --git a/tools/server/tests/unit/test_security.py b/tools/server/tests/unit/test_security.pyindex a0c3e214ae..ac0544575b 100644--- a/tools/server/tests/unit/test_security.py+++ b/tools/server/tests/unit/test_security.py@@ -91,7 +91,7 @@ def test_openai_library_correct_api_key():("localhost", "Access-Control-Allow-Origin", "localhost"),("web.mydomain.fr", "Access-Control-Allow-Origin", "web.mydomain.fr"),("origin", "Access-Control-Allow-Credentials", "true"),- ("web.mydomain.fr", "Access-Control-Allow-Methods", "GET, POST"),+ ("web.mydomain.fr", "Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS"),("web.mydomain.fr", "Access-Control-Allow-Headers", "*"),])def test_cors_options(origin: str, cors_header: str, cors_header_value: str):@@ -107,6 +107,70 @@ def test_cors_options(origin: str, cors_header: str, cors_header_value: str):assert res.headers[cors_header] == cors_header_value[email protected]("origin", [+ "http://localhost",+ "http://localhost:8080",+ "http://127.0.0.1",+ "http://127.0.0.1:3000",+ "http://[::1]",+ "http://[::1]:3000",+])+def test_cors_origins_localhost_reflects(origin: str):+ global server+ server = ServerPreset.router()+ server.cors_origins = "localhost"+ server.start()+ res = server.make_request("OPTIONS", "/completions", headers={+ "Origin": origin,+ "Access-Control-Request-Method": "POST",+ "Access-Control-Request-Headers": "Authorization",+ })+ assert res.status_code == 200+ assert res.headers["Access-Control-Allow-Origin"] == origin++[email protected]("origin", [+ "http://web.mydomain.fr",+ "http://evil.com",+ "http://notlocalhost",+ "http://localhost.evil.com",+])+def test_cors_origins_localhost_rejects(origin: str):+ global server+ server = ServerPreset.router()+ server.cors_origins = "localhost"+ server.start()+ res = server.make_request("OPTIONS", "/completions", headers={+ "Origin": origin,+ "Access-Control-Request-Method": "POST",+ "Access-Control-Request-Headers": "Authorization",+ })+ assert res.status_code == 200+ assert "Access-Control-Allow-Origin" not in res.headers+++def test_cors_origins_defaults_to_localhost_with_tools_enabled():+ global server+ server = ServerPreset.router()+ server.server_tools = "all"+ server.start()+ res = server.make_request("OPTIONS", "/completions", headers={+ "Origin": "http://localhost:8080",+ "Access-Control-Request-Method": "POST",+ "Access-Control-Request-Headers": "Authorization",+ })+ assert res.status_code == 200+ assert res.headers["Access-Control-Allow-Origin"] == "http://localhost:8080"++ res = server.make_request("OPTIONS", "/completions", headers={+ "Origin": "http://evil.com",+ "Access-Control-Request-Method": "POST",+ "Access-Control-Request-Headers": "Authorization",+ })+ assert res.status_code == 200+ assert "Access-Control-Allow-Origin" not in res.headers++def test_cors_proxy_only_forwards_explicit_proxy_headers():class CaptureHeadersHandler(BaseHTTPRequestHandler):def do_GET(self):diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.pyindex 8c0de384f6..f4f0e61e61 100644--- a/tools/server/tests/utils.py+++ b/tools/server/tests/utils.py@@ -114,6 +114,7 @@ class ServerProcess:backend_sampling: bool = Falsegcp_compat: bool = Falseserver_tools: str | None = None+ cors_origins: str | None = None# session variablesprocess: subprocess.Popen | None = None@@ -170,6 +171,8 @@ class ServerProcess:server_args.extend(["--models-max", self.models_max])if self.models_preset:server_args.extend(["--models-preset", self.models_preset])+ if self.cors_origins:+ server_args.extend(["--cors-origins", self.cors_origins])if self.n_batch:server_args.extend(["--batch-size", self.n_batch])if self.n_ubatch:@@ -359,7 +362,7 @@ class ServerProcess:if parse_body:try:result.body = response.json()- except JSONDecodeError:+ except (JSONDecodeError, requests.exceptions.JSONDecodeError):result.body = response.textelse:result.body = None