LLM inference in C/C++
C++ ggml
context : do not re-reserve the scheduler when toggling causal_attn (#28751)
Sihan Yued7ac35e1ee4
diff --git a/src/llama-context.cpp b/src/llama-context.cppindex 27b9a38d72..88851c6cba 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -1256,7 +1256,8 @@ void llama_context::set_causal_attn(bool value) {cparams.causal_attn = value;- sched_need_reserve = true;+ // no scheduler reserve needed because graph shapes must not depend on causal_attn, a flip only rebuilds the graph+ //sched_need_reserve = true;}bool llama_context::get_causal_attn() const {diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cppindex 3972ce9ce2..2adfa7a694 100644--- a/src/llama-memory-hybrid-idx.cpp+++ b/src/llama-memory-hybrid-idx.cpp@@ -277,7 +277,8 @@ void llama_memory_hybrid_idx::set_input_qsa(ggml_tensor * bias,const llama_ubatch * ubatch,uint32_t ratio,- bool blk_bias) const {+ bool blk_bias,+ bool causal_attn) const {GGML_ASSERT(ratio > 0);GGML_ASSERT(get_mem_idx() != nullptr);@@ -545,7 +546,7 @@ void llama_memory_hybrid_idx::set_input_qsa(if (blk_bias) {// a block sits wholly inside or outside the tail, so one value covers it- // the caller adds the attention mask, which drops empty, foreign and future cells+ // the caller adds the attention mask, which drops empty, foreign and, when causal, future cellsfloat * cur_blk_bias = dst_bias + i*n_blocks;for (int64_t b = 0; b < n_blocks; ++b) {@@ -555,7 +556,7 @@ void llama_memory_hybrid_idx::set_input_qsa(}// finite, so it can never meet a -inf and produce a nan- cur_blk_bias[b] = bid_idx[b] >= tail_start ? 1e9f : 0.0f;+ cur_blk_bias[b] = (causal_attn && bid_idx[b] >= tail_start) ? 1e9f : 0.0f;}// the spare block holds the unpooled cells, which are the incomplete tail, so@@ -576,7 +577,10 @@ void llama_memory_hybrid_idx::set_input_qsa(if (!cells.is_empty(j) && cells.seq_has(j, seq_id)) {const int64_t idx = ranked ? rank[j] : cells.pos_get(j);- if (idx <= q) {+ if (!causal_attn) {+ // every visible block competes on score and the unpooled cells are always selected+ v = blk_of[j] < 0 ? 1e9f : 0.0f;+ } else if (idx <= q) {// finite, so it can never meet a -inf and produce a nanv = idx >= tail_start ? 1e9f : (blk_of[j] < 0 ? -INFINITY : 0.0f);}@@ -676,8 +680,9 @@ void llama_memory_hybrid_idx_context::set_input_qsa(ggml_tensor * bias,const llama_ubatch * ubatch,uint32_t ratio,- bool blk_bias) const {+ bool blk_bias,+ bool causal_attn) const {GGML_ASSERT(mem != nullptr);- mem->set_input_qsa(cell_blk, blk_cells, blk_pos, bias, ubatch, ratio, blk_bias);+ mem->set_input_qsa(cell_blk, blk_cells, blk_pos, bias, ubatch, ratio, blk_bias, causal_attn);}diff --git a/src/llama-memory-hybrid-idx.h b/src/llama-memory-hybrid-idx.hindex 705189e7eb..3c3a788acf 100644--- a/src/llama-memory-hybrid-idx.h+++ b/src/llama-memory-hybrid-idx.h@@ -12,6 +12,8 @@// llama_memory_hybrid plus a third cache with one indexer key per token, for block-sparse attention (qwen4exp QSA)// the indexer is a side buffer over the attention cells: same size, padding, streams and slots, so cell j is one token in both+// TODO: this memory module is pending complete reimplementation - do not use for model other than Qwen4+class llama_memory_hybrid_idx : public llama_memory_hybrid {public:llama_memory_hybrid_idx(@@ -83,9 +85,10 @@ public:// bias F32 [n_kv, n_tokens/ns, ns] -inf where invisible, large where always visible// blk_bias asks for the bias per block instead: [n_blocks, n_tokens/ns, ns]// the caller then adds the attention mask, the only part of the bias that varies within a block+ // causal_attn selects the rule: causal forces the query's own block on, non-causal lets every visible block compete on scorevoid set_input_qsa(ggml_tensor * cell_blk, ggml_tensor * blk_cells, ggml_tensor * blk_pos,ggml_tensor * bias, const llama_ubatch * ubatch, uint32_t ratio,- bool blk_bias) const;+ bool blk_bias, bool causal_attn) const;private:// forget seq_id (all of it if seq_id < 0) in every cache at once, so a failed restore cannot leave the caches out of step@@ -143,7 +146,7 @@ public:void set_input_qsa(ggml_tensor * cell_blk, ggml_tensor * blk_cells, ggml_tensor * blk_pos,ggml_tensor * bias, const llama_ubatch * ubatch, uint32_t ratio,- bool blk_bias) const;+ bool blk_bias, bool causal_attn) const;private:const llama_memory_hybrid_idx * mem = nullptr;diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cppindex f33989de02..319b7b9c14 100644--- a/src/models/qwen4exp.cpp+++ b/src/models/qwen4exp.cpp@@ -6,6 +6,9 @@#include <algorithm>#include <cinttypes>+// [TAG_QWEN4_REIMPLEMENT]+// TODO: this graph implementation is pending complete reimplementation - do not use it as a reference+// bad metadata must be catchable: GGML_ASSERT aborts the whole processstatic void qwen4exp_require_nonzero(const llama_model_loader & ml, llm_kv kid, uint32_t value) {if (value == 0) {@@ -489,13 +492,13 @@ ggml_tensor * llama_model_qwen4exp::graph::build_norm_gated(// one mean-pooled indexer key scores each block; set_input resolves the cache layoutclass llama_model_qwen4exp::llm_graph_input_qsa : public llm_graph_input_i {public:- llm_graph_input_qsa(const llama_memory_hybrid_idx_context * mctx, uint32_t ratio, bool blk_bias) :- mctx(mctx), ratio(ratio), blk_bias(blk_bias) {}+ llm_graph_input_qsa(const llama_memory_hybrid_idx_context * mctx, uint32_t ratio, bool blk_bias, bool causal_attn) :+ mctx(mctx), ratio(ratio), blk_bias(blk_bias), causal_attn(causal_attn) {}virtual ~llm_graph_input_qsa() = default;void set_input(const llama_ubatch * ubatch) override {mctx->get_idx()->set_input_k_idxs(k_idxs, ubatch);- mctx->set_input_qsa(cell_blk, blk_cells, blk_pos, bias, ubatch, ratio, blk_bias);+ mctx->set_input_qsa(cell_blk, blk_cells, blk_pos, bias, ubatch, ratio, blk_bias, causal_attn);}bool can_reuse(const llm_graph_params & params) override {@@ -537,6 +540,9 @@ public:// the per-cell half of the bias is the attention mask, so only the per-block half is uploadedconst bool blk_bias;++ // this is fixed for the graph's lifetime, as causal_attn is part of the reuse key (llm_graph_params::allow_reuse)+ const bool causal_attn;};ggml_tensor * llama_model_qwen4exp::graph::build_qsa_top_k(@@ -564,11 +570,11 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_top_k(// only the "which block is visible" half of the bias varies per block// the rest is the visible/not test the attention mask already carries, so upload the per-block half only: 1/ratio of the cells- // alibi writes distances instead of a mask and non-causal keeps future cells, so both opt out+ // alibi writes distances instead of a mask, so it opts out// the mask also holds an mrope rule for the query's own position, but only 2d image positions can differ thereconst bool blk_bias = kq_mask != nullptr &&kq_mask->ne[0] == n_kv && kq_mask->ne[1] == n_tps && kq_mask->ne[3] == n_stream &&- cparams.causal_attn && !hparams.use_alibi;+ !hparams.use_alibi;// nothing above depends on the layer, so the layers sharing a ratio share one input setllm_graph_input_qsa * inp = nullptr;@@ -577,7 +583,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_top_k(if (it != qsa_inps.end()) {inp = it->second;} else {- auto qsa = std::make_unique<llm_graph_input_qsa>(mctx_hyb, (uint32_t) r, blk_bias);+ auto qsa = std::make_unique<llm_graph_input_qsa>(mctx_hyb, (uint32_t) r, blk_bias, cparams.causal_attn);qsa->k_idxs = mctx_idx->build_input_k_idxs(ctx0, ubatch);qsa->cell_blk = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_kv, n_stream);