LLM inference in C/C++
C++ ggml
server : make models endpoints private when authentication is enabled (#26347)
s0mecodeee0ea03adf9d
diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cppindex b11dc09d0a..2ec137aa07 100644--- a/tools/server/server-http.cpp+++ b/tools/server/server-http.cpp@@ -198,8 +198,6 @@ bool server_http_context::init(const common_params & params) {std::unordered_set<std::string> endpoints {"/health","/v1/health",- "/models",- "/v1/models",};endpoints.insert(frontend_paths.begin(), frontend_paths.end());return endpoints;diff --git a/tools/server/server.cpp b/tools/server/server.cppindex 77722b9a61..01cc6633a3 100644--- a/tools/server/server.cpp+++ b/tools/server/server.cpp@@ -235,8 +235,8 @@ int llama_server(common_params & params, int argc, char ** argv) {ctx_http.get ("/metrics", ex_wrapper(routes.get_metrics));ctx_http.get ("/props", ex_wrapper(routes.get_props));ctx_http.post("/props", ex_wrapper(routes.post_props));- ctx_http.get ("/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check)- ctx_http.get ("/v1/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check)+ ctx_http.get ("/models", ex_wrapper(routes.get_models));+ ctx_http.get ("/v1/models", ex_wrapper(routes.get_models));ctx_http.post("/completion", ex_wrapper(routes.post_completions)); // legacyctx_http.post("/completions", ex_wrapper(routes.post_completions));ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai));diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.pyindex 0e1467de37..96eb87978f 100644--- a/tools/server/tests/unit/test_router.py+++ b/tools/server/tests/unit/test_router.py@@ -63,14 +63,16 @@ def test_router_chat_completion_stream(model: str, success: bool):assert content == ""-def _get_model_ids(is_reload: bool) -> set[str]:- res = server.make_request("GET", "/models" + ("?reload=1" if is_reload else ""))+def _get_model_ids(is_reload: bool, headers: dict | None = None) -> set[str]:+ res = server.make_request(+ "GET", "/models" + ("?reload=1" if is_reload else ""), headers=headers+ )assert res.status_code == 200return {item["id"] for item in res.body.get("data", [])}-def _get_model_status(model_id: str) -> str:- res = server.make_request("GET", "/models")+def _get_model_status(model_id: str, headers: dict | None = None) -> str:+ res = server.make_request("GET", "/models", headers=headers)assert res.status_code == 200for item in res.body.get("data", []):if item.get("id") == model_id or item.get("model") == model_id:@@ -78,11 +80,11 @@ def _get_model_status(model_id: str) -> str:raise AssertionError(f"Model {model_id} not found in /models response")-def _wait_for_model_status(model_id: str, desired: set[str], timeout: int = 60) -> str:+def _wait_for_model_status(model_id: str, desired: set[str], timeout: int = 60, headers: dict | None = None) -> str:deadline = time.time() + timeoutlast_status = Nonewhile time.time() < deadline:- last_status = _get_model_status(model_id)+ last_status = _get_model_status(model_id, headers=headers)if last_status in desired:return last_statustime.sleep(0.01)@@ -100,7 +102,7 @@ def _load_model_and_wait(assert load_res.status_code == 200assert isinstance(load_res.body, dict)assert load_res.body.get("success") is True- _wait_for_model_status(model_id, {"loaded"}, timeout=timeout)+ _wait_for_model_status(model_id, {"loaded"}, timeout=timeout, headers=headers)def test_router_unload_model():diff --git a/tools/server/tests/unit/test_security.py b/tools/server/tests/unit/test_security.pyindex ac0544575b..36fc439f9b 100644--- a/tools/server/tests/unit/test_security.py+++ b/tools/server/tests/unit/test_security.py@@ -15,7 +15,7 @@ def create_server():server.api_key = TEST_API_KEY[email protected]("endpoint", ["/health", "/models"])[email protected]("endpoint", ["/health"])def test_access_public_endpoint(endpoint: str):global serverserver.start()