LLM inference in C/C++
C++ ggml
server : disable on-device spec checkpoints (#24108)
Georgi Gerganov7c158fbb4aec
diff --git a/examples/speculative-simple/speculative-simple.cpp b/examples/speculative-simple/speculative-simple.cppindex 5325bcc9e3..d87ba48beb 100644--- a/examples/speculative-simple/speculative-simple.cpp+++ b/examples/speculative-simple/speculative-simple.cpp@@ -175,7 +175,7 @@ int main(int argc, char ** argv) {llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), seq_id));if (use_ckpt_dft) {- ckpt.update_dft(ctx_dft.get(), seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ ckpt.update_dft(ctx_dft.get(), seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);}// generate a new draft@@ -196,12 +196,12 @@ int main(int argc, char ** argv) {// this allows us to restore the state if partial draft acceptance occursif (!draft.empty()) {if (use_ckpt_tgt) {- ckpt.update_tgt(ctx_tgt, seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ ckpt.update_tgt(ctx_tgt, seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);}}{- ckpt.load_dft(ctx_dft.get(), seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ ckpt.load_dft(ctx_dft.get(), seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), seq_id, ckpt.pos_max + 1, -1);}@@ -261,13 +261,13 @@ int main(int argc, char ** argv) {draft = std::move(ids);{- ckpt.load_tgt(ctx_tgt, seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ ckpt.load_tgt(ctx_tgt, seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);llama_memory_seq_rm(llama_get_memory(ctx_tgt), seq_id, ckpt.pos_max + 1, -1);}{- ckpt.load_dft(ctx_dft.get(), seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ ckpt.load_dft(ctx_dft.get(), seq_id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), seq_id, ckpt.pos_max + 1, -1);}diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 28f738c3fe..ab0d594476 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -2512,7 +2512,7 @@ private:llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id));if (use_ckpt_dft) {- slot.spec_ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ slot.spec_ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);}slot.spec_prompt = slot.prompt.tokens.get_text_tokens();@@ -2551,7 +2551,7 @@ private:if (ctx_dft) {if (use_ckpt_dft) {- ckpt.load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ ckpt.load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);}common_context_seq_rm(ctx_dft.get(), slot.id, ckpt.pos_max + 1, -1);@@ -2568,7 +2568,7 @@ private:if (use_ckpt_tgt) {//const int64_t t_start = ggml_time_us();- ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);//const int64_t t_total = ggml_time_us() - t_start;//printf("checkpoint total: %f ms\n", t_total / 1000.0);@@ -2580,7 +2580,7 @@ private:}if (use_ckpt_dft) {- ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);}}}@@ -3447,13 +3447,13 @@ private:SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size());{- ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);common_context_seq_rm(slot.ctx_tgt, slot.id, ckpt.pos_max + 1, -1);}if (slot.ctx_dft) {- ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);+ ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);common_context_seq_rm(slot.ctx_dft, slot.id, ckpt.pos_max + 1, -1);}