Skip to content

LLM inference in C/C++

C++ ggml

llama : re-reserve the sched when the nextn extraction flags change (#30020)

Piotr Wilkin (ilintar)1a3011cc0c18
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 70b7c21af8..ff2ea461c4 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -1234,6 +1234,11 @@ void llama_context::set_embeddings(bool value) {
void llama_context::set_embeddings_nextn(bool value, bool masked) {
LLAMA_LOG_DEBUG("%s: value = %d, masked = %d\n", __func__, value, masked);
+ if (cparams.embeddings_nextn != value || cparams.embeddings_nextn_masked != masked) {
+ // these flags change the graph shape
+ sched_need_reserve = true;
+ }
+
cparams.embeddings_nextn = value;
cparams.embeddings_nextn_masked = masked;
}