LLM inference in C/C++
C++ ggml
spec: fix segfault error on long prompts for eagle3 (#24707)
Ruixiang Wang1a2dea29b9cf
diff --git a/src/llama-context.cpp b/src/llama-context.cppindex 168dbabd76..529bc4a5e9 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -1382,7 +1382,7 @@ int llama_context::encode(const llama_batch & batch_inp) {const auto & hparams = model.hparams;// eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim- const int64_t n_embd = hparams.n_embd_inp();+ const int64_t n_embd = hparams.n_embd_inp_enc();const int64_t n_vocab = model.vocab.n_tokens();// note: during encode, we always pass the full sequence starting from pos = 0diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cppindex 2bf5768738..9d0683d2fe 100644--- a/src/llama-hparams.cpp+++ b/src/llama-hparams.cpp@@ -104,6 +104,10 @@ uint32_t llama_hparams::n_embd_inp() const {return n_embd_inp;}+uint32_t llama_hparams::n_embd_inp_enc() const {+ return n_embd_inp_enc_impl > 0 ? n_embd_inp_enc_impl : n_embd_inp();+}+uint32_t llama_hparams::n_embd_out() const {return n_embd_out_impl > 0 ? n_embd_out_impl : n_embd;}diff --git a/src/llama-hparams.h b/src/llama-hparams.hindex d045059a63..2eadeb2148 100644--- a/src/llama-hparams.h+++ b/src/llama-hparams.h@@ -189,6 +189,10 @@ struct llama_hparams {// input embedding dimension (0 = use n_embd)uint32_t n_embd_inp_impl = 0;+ // encoder input embedding dimension (0 = use n_embd_inp())+ // e.g. the eagle3 encoder fuses target_layers * target_hidden features+ uint32_t n_embd_inp_enc_impl = 0;+// output embedding dimension (0 = use n_embd)uint32_t n_embd_out_impl = 0;@@ -305,6 +309,9 @@ struct llama_hparams {// dimension of main + auxiliary input embeddingsuint32_t n_embd_inp() const;+ // dimension of the encoder input embeddings+ uint32_t n_embd_inp_enc() const;+// dimension of output embeddingsuint32_t n_embd_out() const;diff --git a/src/models/eagle3.cpp b/src/models/eagle3.cppindex 3321b39051..9d96fae594 100644--- a/src/models/eagle3.cpp+++ b/src/models/eagle3.cpp@@ -19,7 +19,7 @@ void llama_model_eagle3::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_TARGET_HIDDEN_SIZE, n_embd_tgt);LLAMA_LOG_INFO("%s: EAGLE3 n_embd_tgt = %u (draft n_embd = %u)\n", __func__, n_embd_tgt, hparams.n_embd);- hparams.n_embd_inp_impl = (uint32_t) target_layer_ids.size() * n_embd_tgt;+ hparams.n_embd_inp_enc_impl = (uint32_t) target_layer_ids.size() * n_embd_tgt;// eagle3 norm_before_residual (optional, default false)// compatible with Readhat eagle3 speculator model@@ -34,7 +34,7 @@ void llama_model_eagle3::load_arch_hparams(llama_model_loader & ml) {void llama_model_eagle3::load_arch_tensors(llama_model_loader &) {LLAMA_LOAD_LOCALS;- const int64_t n_embd_inp = hparams.n_embd_inp();+ const int64_t n_embd_inp = hparams.n_embd_inp_enc();const int64_t n_embd_attn_input = 2 * n_embd;// Get vocab size from the d2t tensor in the GGUF file (optional - only needed if eagle3 has different vocab_size than target)@@ -109,8 +109,8 @@ ggml_tensor * llama_model_eagle3::graph<true>::build_inp_embd_enc() const {// Input: Target model features (3 layers concatenated: low, mid, high)// Data will be provided via ubatch->embd in encode_eagle3_features()- auto inp_target = std::make_unique<llm_graph_input_embd>(hparams.n_embd_inp());- inp_target->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32,hparams.n_embd_inp(), n_tokens);+ auto inp_target = std::make_unique<llm_graph_input_embd>(hparams.n_embd_inp_enc());+ inp_target->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp_enc(), n_tokens);ggml_set_input(inp_target->embd);cur = inp_target->embd;