LLM inference in C/C++
C++ ggml
server: prefer empty slots and keep states overwritten by child tasks (#30567)
Pascalb0d239164b26
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex d5b4a7029d..c1683d9ef6 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -1763,25 +1763,14 @@ private:}}- // find the slot that has been least recently used+ // find an empty slot, or the one that has been least recently usedif (ret == nullptr) {- int64_t t_last = -1;+ const auto free_slots = get_free_slots(1, -1);- for (server_slot & slot : slots) {- // skip the slot if it is not available- if (slot.is_processing()) {- continue;- }+ if (!free_slots.empty()) {+ ret = free_slots[0];- // select the current slot if the criteria match- if (!ret || slot.t_last_used <= t_last) {- t_last = slot.t_last_used;- ret = &slot;- }- }-- if (ret != nullptr) {- SLT_INF(*ret, "selected slot by LRU, t_last = %" PRId64 "\n", t_last);+ SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt.n_tokens(), ret->t_last_used);update_cache = true;}@@ -2480,16 +2469,24 @@ private:return true;}+ // idle slots that lose the least by being reused: empty slots first, then the least recently used onesstd::vector<server_slot *> get_free_slots(size_t n_slots_needed, int exclude_id_slot) {std::vector<server_slot *> free_slots;for (auto & slot : slots) {if (!slot.is_processing() && slot.id != exclude_id_slot) {free_slots.push_back(&slot);}- if (free_slots.size() >= n_slots_needed) {- break;- }}++ std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot * a, const server_slot * b) {+ if ((a->prompt.n_tokens() == 0) != (b->prompt.n_tokens() == 0)) {+ return a->prompt.n_tokens() == 0;+ }+ return a->t_last_used < b->t_last_used;+ });++ free_slots.resize(std::min(free_slots.size(), n_slots_needed));+return free_slots;}@@ -2519,6 +2516,11 @@ private:size_t idx = 0;GGML_ASSERT(child_slots.size() == parent_task.child_tasks.size());for (auto * slot : child_slots) {+ // keep the state that the child task is about to overwrite+ if (prompt_cache && slot->prompt_save(*prompt_cache)) {+ prompt_cache->update();+ }+int id_child = parent_task.child_tasks[idx].id;if (!launch_slot_with_task(*slot, std::move(parent_task.child_tasks[idx]))) {SRV_ERR("failed to launch slot with child task, id_task = %d\n", id_child);diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.pyindex 61c2ab0989..67e128e11e 100644--- a/tools/server/tests/unit/test_systemone.py+++ b/tools/server/tests/unit/test_systemone.py@@ -184,6 +184,44 @@ def test_systemone_prompt_cache(cache_ram, tmp_path):assert ("found better prompt" in log) == (cache_ram is not None)+def test_systemone_prompt_cache_child_slots(tmp_path):+ global server+ server = ServerPreset.tinyopenjev()+ server.n_slots = len(TEST_QUESTIONS)+ server.cache_ram = 100+ server.debug = True+ server.log_path = str(tmp_path / "server.log")+ server.start()++ states = [+ "My parcel shows delivered but it never arrived at my door.",+ "The app crashes every time I open the settings page.",+ "Please close my account and delete all my personal data.",+ ]+ angry = {"angry": TEST_QUESTIONS["angry"]}++ def decide(state, questions):+ res = server.make_request("POST", "/v1/systemone", data={"state": state, "questions": questions})+ assert res.status_code == 200++ # one state per slot, then a request whose child tasks take every slot+ for state in states:+ decide(state, angry)+ decide(TEST_STATE, TEST_QUESTIONS)++ with open(server.log_path) as f:+ n_restored = f.read().count("found better prompt with f_keep = 1.000")++ # every overwritten state comes back whole from the prompt cache+ for state in states:+ decide(state, angry)++ server.stop()++ with open(server.log_path) as f:+ assert f.read().count("found better prompt with f_keep = 1.000") == n_restored + len(states)++def test_systemone_images():global serverserver = ServerPreset.tinyopenjev()