Skip to content

LLM inference in C/C++

C++ ggml

server: prefer empty slots and keep states overwritten by child tasks (#30567)

Pascalb0d239164b26
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index d5b4a7029d..c1683d9ef6 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1763,25 +1763,14 @@ private:
}
}
- // find the slot that has been least recently used
+ // find an empty slot, or the one that has been least recently used
if (ret == nullptr) {
- int64_t t_last = -1;
+ const auto free_slots = get_free_slots(1, -1);
- for (server_slot & slot : slots) {
- // skip the slot if it is not available
- if (slot.is_processing()) {
- continue;
- }
+ if (!free_slots.empty()) {
+ ret = free_slots[0];
- // select the current slot if the criteria match
- if (!ret || slot.t_last_used <= t_last) {
- t_last = slot.t_last_used;
- ret = &slot;
- }
- }
-
- if (ret != nullptr) {
- SLT_INF(*ret, "selected slot by LRU, t_last = %" PRId64 "\n", t_last);
+ SLT_INF(*ret, "selected slot by LRU, n_tokens = %d, t_last = %" PRId64 "\n", ret->prompt.n_tokens(), ret->t_last_used);
update_cache = true;
}
@@ -2480,16 +2469,24 @@ private:
return true;
}
+ // idle slots that lose the least by being reused: empty slots first, then the least recently used ones
std::vector<server_slot *> get_free_slots(size_t n_slots_needed, int exclude_id_slot) {
std::vector<server_slot *> free_slots;
for (auto & slot : slots) {
if (!slot.is_processing() && slot.id != exclude_id_slot) {
free_slots.push_back(&slot);
}
- if (free_slots.size() >= n_slots_needed) {
- break;
- }
}
+
+ std::stable_sort(free_slots.begin(), free_slots.end(), [](const server_slot * a, const server_slot * b) {
+ if ((a->prompt.n_tokens() == 0) != (b->prompt.n_tokens() == 0)) {
+ return a->prompt.n_tokens() == 0;
+ }
+ return a->t_last_used < b->t_last_used;
+ });
+
+ free_slots.resize(std::min(free_slots.size(), n_slots_needed));
+
return free_slots;
}
@@ -2519,6 +2516,11 @@ private:
size_t idx = 0;
GGML_ASSERT(child_slots.size() == parent_task.child_tasks.size());
for (auto * slot : child_slots) {
+ // keep the state that the child task is about to overwrite
+ if (prompt_cache && slot->prompt_save(*prompt_cache)) {
+ prompt_cache->update();
+ }
+
int id_child = parent_task.child_tasks[idx].id;
if (!launch_slot_with_task(*slot, std::move(parent_task.child_tasks[idx]))) {
SRV_ERR("failed to launch slot with child task, id_task = %d\n", id_child);
diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py
index 61c2ab0989..67e128e11e 100644
--- a/tools/server/tests/unit/test_systemone.py
+++ b/tools/server/tests/unit/test_systemone.py
@@ -184,6 +184,44 @@ def test_systemone_prompt_cache(cache_ram, tmp_path):
assert ("found better prompt" in log) == (cache_ram is not None)
+def test_systemone_prompt_cache_child_slots(tmp_path):
+ global server
+ server = ServerPreset.tinyopenjev()
+ server.n_slots = len(TEST_QUESTIONS)
+ server.cache_ram = 100
+ server.debug = True
+ server.log_path = str(tmp_path / "server.log")
+ server.start()
+
+ states = [
+ "My parcel shows delivered but it never arrived at my door.",
+ "The app crashes every time I open the settings page.",
+ "Please close my account and delete all my personal data.",
+ ]
+ angry = {"angry": TEST_QUESTIONS["angry"]}
+
+ def decide(state, questions):
+ res = server.make_request("POST", "/v1/systemone", data={"state": state, "questions": questions})
+ assert res.status_code == 200
+
+ # one state per slot, then a request whose child tasks take every slot
+ for state in states:
+ decide(state, angry)
+ decide(TEST_STATE, TEST_QUESTIONS)
+
+ with open(server.log_path) as f:
+ n_restored = f.read().count("found better prompt with f_keep = 1.000")
+
+ # every overwritten state comes back whole from the prompt cache
+ for state in states:
+ decide(state, angry)
+
+ server.stop()
+
+ with open(server.log_path) as f:
+ assert f.read().count("found better prompt with f_keep = 1.000") == n_restored + len(states)
+
+
def test_systemone_images():
global server
server = ServerPreset.tinyopenjev()