Skip to content

LLM inference in C/C++

C++ ggml

server: save the draft state in the slot file (#30574)

Pascalea736c1d5c1d
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index c1683d9ef6..c20167b90b 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -2736,6 +2736,87 @@ private:
return n_read;
}
+ // the live draft state follows the checkpoints: the full draft context and the speculative state
+ // the llama state payload holds the target context only
+ static constexpr uint32_t SLOT_DRAFT_MAGIC = 0x54464453; // "SDFT"
+ static constexpr uint32_t SLOT_DRAFT_VERSION = 1;
+
+ // returns false if the draft state could not be written completely
+ bool save_slot_draft(const std::string & filepath, const server_slot & slot, size_t & n_written) const {
+ n_written = 0;
+ std::vector<uint8_t> data_dft;
+ std::vector<uint8_t> data_spec;
+ if (ctx_dft != nullptr) {
+ data_dft.resize(llama_state_seq_get_size_ext(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE));
+ llama_state_seq_get_data_ext(ctx_dft, data_dft.data(), data_dft.size(), slot.id, LLAMA_STATE_SEQ_FLAGS_NONE);
+ }
+ common_speculative_get_state(spec.get(), slot.id, data_spec);
+ if (data_dft.empty() && data_spec.empty()) {
+ return true;
+ }
+ std::ofstream ofs(std::filesystem::u8path(filepath), std::ios::binary | std::ios::app);
+ if (!ofs) {
+ SLT_WRN(slot, "failed to append the draft state to '%s'\n", filepath.c_str());
+ return false;
+ }
+ const uint32_t magic = SLOT_DRAFT_MAGIC;
+ const uint32_t version = SLOT_DRAFT_VERSION;
+ ckpt_write(ofs, &magic, sizeof(magic), n_written);
+ ckpt_write(ofs, &version, sizeof(version), n_written);
+ ckpt_write_buf(ofs, data_dft, n_written);
+ ckpt_write_buf(ofs, data_spec, n_written);
+ ofs.flush();
+ if (!ofs) {
+ SLT_WRN(slot, "failed to append the draft state to '%s' - the draft state is incomplete\n", filepath.c_str());
+ return false;
+ }
+ SLT_INF(slot, "appended the draft state (%.3f MiB) to '%s'\n", (float) n_written / 1024 / 1024, filepath.c_str());
+ return true;
+ }
+
+ // returns the number of bytes consumed, 0 if there is no usable draft state
+ // the draft starts empty unless the file holds a draft state that fits the draft context
+ size_t load_slot_draft(const std::string & filepath, size_t offset, server_slot & slot) const {
+ if (ctx_dft != nullptr) {
+ llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, -1, -1);
+ }
+ std::ifstream ifs(std::filesystem::u8path(filepath), std::ios::binary | std::ios::ate);
+ const size_t file_size = ifs ? (size_t) ifs.tellg() : 0;
+ if (!ifs || file_size < offset || !ifs.seekg(offset)) {
+ return 0;
+ }
+ const size_t n_avail = file_size - offset;
+ size_t n_read = 0;
+ uint32_t magic = 0;
+ uint32_t version = 0;
+ if (!ckpt_read(ifs, &magic, sizeof(magic), n_read) || magic != SLOT_DRAFT_MAGIC) {
+ return 0;
+ }
+ std::vector<uint8_t> data_dft;
+ std::vector<uint8_t> data_spec;
+ if (!ckpt_read(ifs, &version, sizeof(version), n_read) || version != SLOT_DRAFT_VERSION ||
+ !ckpt_read_buf(ifs, data_dft, n_avail, n_read) ||
+ !ckpt_read_buf(ifs, data_spec, n_avail, n_read)) {
+ SLT_WRN(slot, "invalid draft state in '%s' - ignored\n", filepath.c_str());
+ return 0;
+ }
+ if (!data_dft.empty()) {
+ if (ctx_dft == nullptr ||
+ llama_state_seq_set_data_ext(ctx_dft, data_dft.data(), data_dft.size(), slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) != data_dft.size()) {
+ if (ctx_dft != nullptr) {
+ llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, -1, -1);
+ }
+ SLT_WRN(slot, "draft state in '%s' does not match the draft context - dropped\n", filepath.c_str());
+ return n_read;
+ }
+ }
+ if (!data_spec.empty()) {
+ common_speculative_set_state(spec.get(), slot.id, data_spec);
+ }
+ SLT_INF(slot, "restored the draft state (%.3f MiB) from '%s'\n", (float) n_read / 1024 / 1024, filepath.c_str());
+ return n_read;
+ }
+
// returns false to decline the task, it is offered again after the decode is done
bool process_single_task(server_task && task, bool is_yielding) {
// while yielding, an encode / decode is running and only reading the server state is safe
@@ -2951,6 +3032,12 @@ private:
break;
}
+ size_t nwrite_dft = 0;
+ if (!save_slot_draft(filepath, *slot, nwrite_dft)) {
+ send_error(task, "Unable to save slot: incomplete draft state", ERROR_TYPE_SERVER);
+ break;
+ }
+
const int64_t t_end = ggml_time_us();
const double t_save_ms = (t_end - t_start) / 1000.0;
@@ -2960,7 +3047,7 @@ private:
res->filename = filename;
res->is_save = true;
res->n_tokens = slot->prompt.tokens.size();
- res->n_bytes = nwrite + nwrite_ckpt;
+ res->n_bytes = nwrite + nwrite_ckpt + nwrite_dft;
res->t_ms = t_save_ms;
queue_results.send(std::move(res));
} break;
@@ -3018,6 +3105,7 @@ private:
// nread is the end offset of the llama state payload within the file
const size_t nread_ckpt = load_slot_checkpoints(filepath, nread, *slot);
+ const size_t nread_dft = load_slot_draft(filepath, nread + nread_ckpt, *slot);
const int64_t t_end = ggml_time_us();
const double t_restore_ms = (t_end - t_start) / 1000.0;
@@ -3028,7 +3116,7 @@ private:
res->filename = filename;
res->is_save = false;
res->n_tokens = slot->prompt.tokens.size();
- res->n_bytes = nread + nread_ckpt;
+ res->n_bytes = nread + nread_ckpt + nread_dft;
res->t_ms = t_restore_ms;
queue_results.send(std::move(res));
} break;
diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py
index 33bcede934..793f1a0a7a 100644
--- a/tools/server/tests/unit/test_slot_save.py
+++ b/tools/server/tests/unit/test_slot_save.py
@@ -132,6 +132,55 @@ def test_slot_restore_legacy_token_list():
assert res.body["timings"]["prompt_n"] == 6 # only the different part is processed
+# the slot file carries the live draft state: a restored slot drafts like a slot that was never saved
+def test_slot_restore_draft_state():
+ global server
+ server.model_draft_hf_repo = server.model_hf_repo
+ server.spec_type = "draft-simple"
+ server.start()
+
+ prompt = "What is the capital of France?"
+ turn = {}
+ for id_slot in [0, 1]:
+ res = server.make_request("POST", "/completion", data={
+ "prompt": prompt,
+ "n_predict": 16,
+ "id_slot": id_slot,
+ "cache_prompt": True,
+ })
+ assert res.status_code == 200
+ turn[id_slot] = res.body["content"]
+ assert turn[0] == turn[1]
+
+ res = server.make_request("POST", "/slots/0?action=save", data={
+ "filename": "draft_slot0.bin",
+ })
+ assert res.status_code == 200
+
+ res = server.make_request("POST", "/slots/0?action=erase")
+ assert res.status_code == 200
+
+ res = server.make_request("POST", "/slots/0?action=restore", data={
+ "filename": "draft_slot0.bin",
+ })
+ assert res.status_code == 200
+
+ timings = {}
+ for id_slot in [0, 1]:
+ res = server.make_request("POST", "/completion", data={
+ "prompt": prompt + turn[id_slot] + " And",
+ "n_predict": 32,
+ "id_slot": id_slot,
+ "cache_prompt": True,
+ })
+ assert res.status_code == 200
+ timings[id_slot] = res.body["timings"]
+
+ assert timings[0]["prompt_n"] == timings[1]["prompt_n"]
+ assert timings[0]["draft_n_accepted"] > 0
+ assert timings[0]["draft_n"] == timings[1]["draft_n"]
+ assert timings[0]["draft_n_accepted"] == timings[1]["draft_n_accepted"]
+
def test_slot_erase():
global server