LLM inference in C/C++
C++ ggml
models : consolidate nextn row cropping into shared helpers (#30017)
Georgi Gerganovf0c41e0168df
diff --git a/src/llama-graph.h b/src/llama-graph.hindex 5cf74c997e..838544576f 100644--- a/src/llama-graph.h+++ b/src/llama-graph.h@@ -1053,6 +1053,16 @@ struct llm_graph_context {void cb(ggml_tensor * cur, const char * name, int il) const;+ // true when the last layer must be narrowed to the output rows before the nextn hidden state is captured+ bool crop_before_nextn(const ggml_tensor * inp_out_ids) const {+ return inp_out_ids != nullptr && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked);+ }++ // true when the nextn hidden state must be narrowed to the output rows after it is captured+ bool crop_after_nextn(const ggml_tensor * inp_out_ids) const {+ return inp_out_ids != nullptr && cparams.embeddings_nextn && !cparams.embeddings_nextn_masked;+ }+//// common//diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cppindex 907b25c67c..b04b58449f 100644--- a/src/models/bailingmoe3.cpp+++ b/src/models/bailingmoe3.cpp@@ -363,7 +363,7 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graphcb(cur, "mla_out", il);}- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -408,7 +408,7 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graphcb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cppindex cf2af012d8..3baf3950c8 100644--- a/src/models/cohere2moe.cpp+++ b/src/models/cohere2moe.cpp@@ -205,7 +205,7 @@ llama_model_cohere2moe::graph::graph(const llama_model & model, const llm_graph_1.0f / sqrtf(float(n_embd_head)), il);}- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids);@@ -269,7 +269,7 @@ llama_model_cohere2moe::graph::graph(const llama_model & model, const llm_graph_cb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cppindex 6d217ea0c9..47fc2f1ac9 100644--- a/src/models/deepseek2.cpp+++ b/src/models/deepseek2.cpp@@ -634,7 +634,7 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_pQcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);}}- if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -697,7 +697,7 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_pcb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cppindex 849c7a9a00..ade9b9ed9f 100644--- a/src/models/deepseek32.cpp+++ b/src/models/deepseek32.cpp@@ -402,7 +402,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_}// when unmasked nextn embeddings are requested, t_h_nextn must keep all rows,// so the early output masking has to be skipped (it is applied after the final norm instead)- if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -469,7 +469,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_cb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cppindex 223336d49e..4edf0c2b46 100644--- a/src/models/deepseek4.cpp+++ b/src/models/deepseek4.cpp@@ -1335,11 +1335,9 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_pggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);ggml_tensor * flat_out = inp_out_ids ? ggml_get_rows(ctx0, flat, inp_out_ids) : flat;- if (cparams.embeddings_nextn) {- ggml_tensor * h_nextn = cparams.embeddings_nextn_masked ? flat_out : inpL;- cb(h_nextn, "h_nextn", -1);- res->t_h_nextn = h_nextn;- }+ ggml_tensor * h_nextn = cparams.embeddings_nextn_masked ? flat_out : inpL;+ cb(h_nextn, "h_nextn", -1);+ res->t_h_nextn = h_nextn;if (inp_out_ids) {inpL = ggml_reshape_3d(ctx0, flat_out, n_embd, hc, n_outputs);diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cppindex 38239eba02..fbc4d2a6e2 100644--- a/src/models/gemma4.cpp+++ b/src/models/gemma4.cpp@@ -274,7 +274,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para// TODO @ngxson : strip unused token right after the last KV layer to speed up prompt processing// keep all rows when extracting unmasked nextn embeddings (MTP target needs the hidden state for every token)- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);}@@ -374,7 +374,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_paraggml_tensor * inp_this_layer = gemma4_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens]// TODO @ngxson : improve this- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_out_ids);}@@ -412,7 +412,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_paracb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cppindex 6a5132cf63..32581f8403 100644--- a/src/models/glm-dsa.cpp+++ b/src/models/glm-dsa.cpp@@ -446,7 +446,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par}// when unmasked nextn embeddings are requested, t_h_nextn must keep all rows,// so the early output masking has to be skipped (it is applied after the final norm instead)- if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -513,7 +513,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_parcb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cppindex 8cdbe10ad3..d65cace1fe 100644--- a/src/models/glm4-moe.cpp+++ b/src/models/glm4-moe.cpp@@ -365,7 +365,7 @@ llama_model_glm4_moe::graph::graph(const llama_model & model, const llm_graph_pamodel.layers[il].wo, NULL, model.layers[il].wo_s,Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);}- if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -428,7 +428,7 @@ llama_model_glm4_moe::graph::graph(const llama_model & model, const llm_graph_pacb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/glm5-next.cpp b/src/models/glm5-next.cppindex 48f4c1af88..a062deb7ad 100644--- a/src/models/glm5-next.cpp+++ b/src/models/glm5-next.cpp@@ -650,8 +650,7 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p// narrow to the output tokens, then collapse the streams// Unmasked nextn embeddings need all rows.- const bool narrow_early = inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked);- if (narrow_early) {+ if (crop_before_nextn(inp_out_ids)) {ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);flat = ggml_get_rows(ctx0, flat, inp_out_ids);inpL = ggml_reshape_3d(ctx0, flat, n_embd, hc, n_outputs);@@ -666,7 +665,7 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_pcb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (inp_out_ids && !narrow_early) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}cb(cur, "result_norm", -1);diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cppindex f6b72d843c..bb0e0a2b8a 100644--- a/src/models/hy-v3.cpp+++ b/src/models/hy-v3.cpp@@ -148,7 +148,7 @@ llama_model_hy_v3::graph::graph(const llama_model & model, const llm_graph_paramcb(cur, "attn_out", il);}- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -216,7 +216,7 @@ llama_model_hy_v3::graph::graph(const llama_model & model, const llm_graph_paramcb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cppindex ce315f956a..866959c839 100644--- a/src/models/mimo2.cpp+++ b/src/models/mimo2.cpp@@ -101,9 +101,7 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_paramggml_tensor * inp_out_ids = build_inp_out_ids();const float v_scale = hparams.f_attn_value_scale;- const bool emit_h_nextn = cparams.embeddings_nextn;const bool extract_final_inp = (size_t) n_layer < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[n_layer];- const bool crop_last_layer = inp_out_ids && (!emit_h_nextn || cparams.embeddings_nextn_masked) && !extract_final_inp;for (int il = 0; il < n_layer; ++il) {res->t_layer_inp[il] = inpL;@@ -185,7 +183,7 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param}}- if (il == n_layer - 1 && crop_last_layer) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids) && !extract_final_inp) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -236,18 +234,17 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_paramcur = inpL;if (extract_final_inp) {res->t_layer_inp[n_layer] = cur;- if (inp_out_ids && (!emit_h_nextn || cparams.embeddings_nextn_masked)) {+ if (crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}}- if (emit_h_nextn) {- cb(cur, "h_nextn", -1);- res->t_h_nextn = cur;+ // pre-norm state for MTP; set even when extraction is off+ cb(cur, "h_nextn", -1);+ res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {- cur = ggml_get_rows(ctx0, cur, inp_out_ids);- }+ if (crop_after_nextn(inp_out_ids)) {+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);}cur = build_norm(cur,diff --git a/src/models/nemotron-h-moe.cpp b/src/models/nemotron-h-moe.cppindex b9b4fdcf18..f1e3ce3b4d 100644--- a/src/models/nemotron-h-moe.cpp+++ b/src/models/nemotron-h-moe.cpp@@ -52,11 +52,6 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llama_model & model, consggml_tensor * inp_out_ids = build_inp_out_ids();- // attention fills KV over all tokens, but the MoE is position-wise: gather output rows before- // it to save FFN compute (unless unmasked embeddings_nextn needs the full-length hidden state)- const bool emit_h_nextn = cparams.embeddings_nextn;- const bool crop_before_ffn = inp_out_ids && (!emit_h_nextn || cparams.embeddings_nextn_masked);-auto * inp_attn = build_attn_inp_kv();ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);@@ -88,7 +83,7 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llama_model & model, conscb(cur, "mtp_attn_residual", il);// gather the output rows here so the MoE FFN below only runs on the positions we keep- if (crop_before_ffn) {+ if (crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}@@ -149,7 +144,7 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llama_model & model, conscb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (!crop_before_ffn && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cppindex be27650b06..cd90e24d2c 100644--- a/src/models/nemotron-h.cpp+++ b/src/models/nemotron-h.cpp@@ -220,7 +220,7 @@ llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_cur = build_ffn_layer(cur, model, il);}- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked && !extract_final_inp) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids) && !extract_final_inp) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -237,7 +237,7 @@ llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_if (extract_final_inp) {res->t_layer_inp[n_layer] = cur;- if (inp_out_ids && cparams.embeddings_nextn_masked) {+ if (crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}}@@ -248,7 +248,7 @@ llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_cb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cppindex 39660a325d..a6b63947b9 100644--- a/src/models/openai-moe.cpp+++ b/src/models/openai-moe.cpp@@ -114,7 +114,7 @@ llama_model_openai_moe::graph::graph(const llama_model & model, const llm_graph_cb(cur, "attn_out", il);}- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {// skip computing output for unused tokenscur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);@@ -154,7 +154,7 @@ llama_model_openai_moe::graph::graph(const llama_model & model, const llm_graph_res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cppindex ab98744a53..e484850dad 100644--- a/src/models/qwen35.cpp+++ b/src/models/qwen35.cpp@@ -175,7 +175,7 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_paracur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il);}- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -212,7 +212,7 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_paracb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cppindex f0f917af72..6e28028a89 100644--- a/src/models/qwen35moe.cpp+++ b/src/models/qwen35moe.cpp@@ -194,7 +194,7 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_pcur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il);}- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -232,7 +232,7 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_pcb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cppindex 340ef28f70..2d56c38ca1 100644--- a/src/models/qwen3next.cpp+++ b/src/models/qwen3next.cpp@@ -168,7 +168,7 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_pcur = build_layer_attn(inp->get_attn(), cur, inp_pos, il);}- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -206,7 +206,7 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_pcb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cppindex 71ce2d5038..aca8f6065f 100644--- a/src/models/qwen4exp.cpp+++ b/src/models/qwen4exp.cpp@@ -470,7 +470,7 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pacur = build_layer_attn(inp->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il);}- if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {// everything below is per token, so drop the rows that produce no outputcur = ggml_get_rows(ctx0, cur, inp_out_ids);inject = ggml_get_rows(ctx0, inject, inp_out_ids);@@ -499,13 +499,11 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa}// the MTP head reads the hc-wide residual, before the final mixer- if (cparams.embeddings_nextn) {- res->t_h_nextn = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]);- cb(res->t_h_nextn, "h_nextn", -1);- ggml_build_forward_expand(gf, res->t_h_nextn);- }+ res->t_h_nextn = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]);+ cb(res->t_h_nextn, "h_nextn", -1);+ ggml_build_forward_expand(gf, res->t_h_nextn);- if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {res_hc = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]);res_hc = ggml_get_rows(ctx0, res_hc, inp_out_ids);res_hc = ggml_reshape_3d(ctx0, res_hc, n_embd, hc, res_hc->ne[1]);diff --git a/src/models/step35.cpp b/src/models/step35.cppindex ca68855d8d..f4660bf196 100644--- a/src/models/step35.cpp+++ b/src/models/step35.cpp@@ -287,7 +287,7 @@ llama_model_step35::graph::graph(const llama_model & model, const llm_graph_paracb(cur, "attn_proj", il);}- if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {+ if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}@@ -349,7 +349,7 @@ llama_model_step35::graph::graph(const llama_model & model, const llm_graph_paracb(cur, "h_nextn", -1);res->t_h_nextn = cur;- if (!cparams.embeddings_nextn_masked && inp_out_ids) {+ if (crop_after_nextn(inp_out_ids)) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);}