LLM inference in C/C++
C++ ggml
model : fix qwen4exp layer inputs for DFlash drafters (#30927)
Georgi Gerganovd9ee007a06b6
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cppindex ec0b438fa3..fc9acd43e3 100644--- a/src/models/qwen4exp.cpp+++ b/src/models/qwen4exp.cpp@@ -308,6 +308,20 @@ std::unique_ptr<llm_graph_context> llama_model_qwen4exp::build_arch_graph(constreturn std::make_unique<graph>(*this, params);}+// DFlash drafters read the layer inputs as [n_embd, T] rows, so collapse the streams+static ggml_tensor * hc_stream_mean(ggml_context * ctx, ggml_tensor * x) {+ const int64_t n_embd = x->ne[0];+ const int64_t hc = x->ne[1];+ const int64_t nt = x->ne[2];++ ggml_tensor * acc = ggml_view_2d(ctx, x, n_embd, nt, x->nb[2], 0);+ for (int64_t s = 1; s < hc; ++s) {+ acc = ggml_add(ctx, acc, ggml_view_2d(ctx, x, n_embd, nt, x->nb[2], s*x->nb[1]));+ }++ return ggml_scale(ctx, acc, 1.0f / (float) hc);+}+// Hyper-connections keep hc parallel residual streams [n_embd, hc, T] in place of layer norms.// Returns the mixed [n_embd, T] stream; `inject` gets the [hc, T] scatter weights.ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(@@ -447,7 +461,11 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_paggml_build_forward_expand(gf, res_hc);for (int il = 0; il < n_layer; ++il) {- res->t_layer_inp[il] = res_hc;+ if ((size_t) il < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[il]) {+ res->t_layer_inp[il] = hc_stream_mean(ctx0, res_hc);+ cb(res->t_layer_inp[il], "layer_inp", il);+ ggml_build_forward_expand(gf, res->t_layer_inp[il]);+ }if (hparams.is_ple(il)) {res_hc = build_ple(inp->get_recr(), ple_emb, res_hc, il);