LLM inference in C/C++
C++ ggml
server : allow model downloads at model limit fix issue #26809 (#28530)
MiaoMing Chen56381e407c0c
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex 4984f1be68..3d134acf36 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -1129,7 +1129,8 @@ void server_models::load(const std::string & name, const load_options & opts) {// exceeding models_max. Without this, the window between unload_lru()// releasing its lock and this lock_guard acquiring allows multiple// threads to each observe capacity and all proceed to load.- if (base_params.models_max > 0) {+ // Download workers do not use models_max slots.+ if (opts.mode == SERVER_CHILD_MODE_NORMAL && base_params.models_max > 0) {size_t count_active = 0;for (const auto & m : mapping) {if (m.second.meta.is_running()) {diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.pyindex e4b7f9fe48..bae1565177 100644--- a/tools/server/tests/unit/test_router.py+++ b/tools/server/tests/unit/test_router.py@@ -540,13 +540,17 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: idef test_router_download_model():- """Case 1: download a model, verify SSE events and GET /models."""+ """Case 1: download a model at the model limit, verify SSE events and GET /models."""global server+ server.models_max = 1server.start()# Ensure the model is not present before we startserver.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")+ # A download worker must not consume or evict a model slot+ _load_model_and_wait(MODEL_B, timeout=120)+sse_events: list = []stop = threading.Event()sse_ready = threading.Event()@@ -580,6 +584,7 @@ def test_router_download_model():# Model should now appear in GET /modelsids = _get_model_ids(is_reload=False)assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download"+ assert _get_model_status(MODEL_B) == "loaded"def test_router_delete_model():