LLM inference in C/C++
C++ ggml
mtmd: add build_inp_attn_mask (#30259)
Xuan-Son Nguyen0fd868cbcadc
diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.hindex bbee35bead..06eee46979 100644--- a/tools/mtmd/clip-graph.h+++ b/tools/mtmd/clip-graph.h@@ -101,6 +101,10 @@ struct clip_graph {ggml_tensor * build_inp_raw(int channels = 3);+ // f16 if flash attn is enabled, set it with set_input_attn_mask()+ // idx is only needed when the graph has more than one mask+ ggml_tensor * build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx = 0);+ggml_tensor * build_norm(ggml_tensor * cur,ggml_tensor * mw,diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex ca38d76df6..db6abf6337 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -588,6 +588,18 @@ ggml_tensor * clip_graph::build_inp_raw(int channels) {return inp_raw;}+static std::string get_attn_mask_name(int idx) {+ return idx == 0 ? "attn_mask" : "attn_mask_" + std::to_string(idx);+}++ggml_tensor * clip_graph::build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx) {+ const ggml_type type = flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED ? GGML_TYPE_F16 : GGML_TYPE_F32;+ ggml_tensor * mask = ggml_new_tensor_2d(ctx0, type, n_kv, n_q);+ ggml_set_name(mask, get_attn_mask_name(idx).c_str());+ ggml_set_input(mask);+ return mask;+}+ggml_tensor * clip_graph::build_norm(ggml_tensor * cur,ggml_tensor * mw,@@ -777,9 +789,8 @@ ggml_tensor * clip_graph::build_attn(k = ggml_cast(ctx0, k, GGML_TYPE_F16);v = ggml_cast(ctx0, v, GGML_TYPE_F16);- if (kq_mask) {- kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);- }+ // mask must be f16 here, use build_inp_attn_mask()+ GGML_ASSERT(!kq_mask || kq_mask->type == GGML_TYPE_F16);cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f);ggml_prec_set_acc(cur, GGML_PREC_F32);@@ -4591,6 +4602,20 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));};+ // mask from build_inp_attn_mask(), f16 if flash attn is enabled+ auto set_input_attn_mask = [&get_inp_tensor](const std::vector<float> & values, int idx = 0) {+ ggml_tensor * cur = get_inp_tensor(get_attn_mask_name(idx).c_str());+ GGML_ASSERT(ggml_nelements(cur) == (int64_t)values.size());+ if (cur->type == GGML_TYPE_F16) {+ std::vector<ggml_fp16_t> values_f16(values.size());+ ggml_fp32_to_fp16_row(values.data(), values_f16.data(), values.size());+ ggml_backend_tensor_set(cur, values_f16.data(), 0, ggml_nbytes(cur));+ } else {+ GGML_ASSERT(cur->type == GGML_TYPE_F32);+ ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));+ }+ };+auto set_input_i32 = [&get_inp_tensor](const char * name, std::vector<int32_t> & values) {ggml_tensor * cur = get_inp_tensor(name);GGML_ASSERT(cur->type == GGML_TYPE_I32);@@ -4639,7 +4664,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {}}}- set_input_f32("kq_mask", mask);+ set_input_attn_mask(mask);};// set input pixel values@@ -4753,7 +4778,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {off += s;}}- set_input_f32("muse_glimmer_sp_mask", sp_mask);+ set_input_attn_mask(sp_mask);// pixel-shuffle gather (original order): f*f spatial neighbours groupedstd::vector<int32_t> dsp; dsp.reserve(n_tok);@@ -4876,7 +4901,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {}}}- set_input_f32("vit_merger_window_mask", window_mask_data);+ set_input_attn_mask(window_mask_data);// ViT merger 2x2 downsample indicesauto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w);@@ -5061,7 +5086,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {set_input_i32("window_idx", idx);set_input_i32("inv_window_idx", inv_idx);- set_input_f32("window_mask", mask);+ set_input_attn_mask(mask);} else {for (int i = 0; i < ph * pw; i++) {idx[i] = i;@@ -5172,7 +5197,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {set_input_i32("mimovl_positions_row", positions_row);set_input_i32("mimovl_positions_col", positions_col);set_input_f32("mimovl_idx_col", idx_col);- set_input_f32("mimovl_window_mask", mask);+ set_input_attn_mask(mask);} break;case PROJECTOR_TYPE_PIXTRAL:case PROJECTOR_TYPE_KIMIVL:@@ -5366,7 +5391,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {qwen2_mask[static_cast<size_t>(i) * seq_len + j] = zero ? 0.0f : -1e9f;}}- set_input_f32("qwen2_attn_mask", qwen2_mask);+ set_input_attn_mask(qwen2_mask);}} break;case PROJECTOR_TYPE_GEMMA3:@@ -5612,8 +5637,8 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf;}}- set_input_f32("mimo_audio_full_mask", full_mask);- set_input_f32("mimo_audio_window_mask", window_mask);+ set_input_attn_mask(full_mask, 0);+ set_input_attn_mask(window_mask, 1);// input_local_transformer: block-diagonal mask + in-group positions{@@ -5636,7 +5661,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf;}}- set_input_f32("mimo_audio_local_mask", local_mask);+ set_input_attn_mask(local_mask, 2);}} break;case PROJECTOR_TYPE_LFM2A:diff --git a/tools/mtmd/models/deepseekocr2.cpp b/tools/mtmd/models/deepseekocr2.cppindex 3e8b409416..7088bb97b5 100644--- a/tools/mtmd/models/deepseekocr2.cpp+++ b/tools/mtmd/models/deepseekocr2.cpp@@ -41,9 +41,7 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {auto seq_len = inp->ne[1];// qwen2 encoder attention mask- ggml_tensor * attn_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, seq_len, seq_len);- ggml_set_name(attn_mask, "qwen2_attn_mask");- ggml_set_input(attn_mask);+ ggml_tensor * attn_mask = build_inp_attn_mask(seq_len, seq_len);ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_arange(ctx0, 0, seq_len, 1), GGML_TYPE_I32);diff --git a/tools/mtmd/models/exaone4_5.cpp b/tools/mtmd/models/exaone4_5.cppindex bd9e8c7488..d3372a1cd7 100644--- a/tools/mtmd/models/exaone4_5.cpp+++ b/tools/mtmd/models/exaone4_5.cpp@@ -58,13 +58,7 @@ ggml_cgraph * clip_graph_exaone4_5::build() {ggml_set_name(inv_window_idx, "inv_window_idx");ggml_set_input(inv_window_idx);- window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);- ggml_set_name(window_mask, "window_mask");- ggml_set_input(window_mask);-- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {- window_mask = ggml_cast(ctx0, window_mask, GGML_TYPE_F16);- }+ window_mask = build_inp_attn_mask(n_pos, n_pos);}ggml_tensor * inpL = inp;diff --git a/tools/mtmd/models/mimo-audio.cpp b/tools/mtmd/models/mimo-audio.cppindex 481b36cc8d..0415bb60d7 100644--- a/tools/mtmd/models/mimo-audio.cpp+++ b/tools/mtmd/models/mimo-audio.cpp@@ -21,13 +21,8 @@ ggml_cgraph * clip_graph_mimo_audio::build() {ggml_set_name(inp_pos, "mimo_audio_positions");ggml_set_input(inp_pos);- ggml_tensor * full_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);- ggml_set_name(full_mask, "mimo_audio_full_mask");- ggml_set_input(full_mask);-- ggml_tensor * window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);- ggml_set_name(window_mask, "mimo_audio_window_mask");- ggml_set_input(window_mask);+ ggml_tensor * full_mask = build_inp_attn_mask(n_pos, n_pos, 0);+ ggml_tensor * window_mask = build_inp_attn_mask(n_pos, n_pos, 1);build_vit_opts opts;opts.attn_mask_layers.resize(n_layer);@@ -150,9 +145,7 @@ ggml_cgraph * clip_graph_mimo_audio::build() {ggml_set_name(local_pos, "mimo_audio_local_positions");ggml_set_input(local_pos);- ggml_tensor * local_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_padded, n_padded);- ggml_set_name(local_mask, "mimo_audio_local_mask");- ggml_set_input(local_mask);+ ggml_tensor * local_mask = build_inp_attn_mask(n_padded, n_padded, 2);const float local_rope_theta = 640000.0f; // audio_config.rope_theta (differs from the encoder's)auto apply_local_rope = [&](ggml_tensor * x) {diff --git a/tools/mtmd/models/mimovl.cpp b/tools/mtmd/models/mimovl.cppindex e1fbe2671d..0cde450bf9 100644--- a/tools/mtmd/models/mimovl.cpp+++ b/tools/mtmd/models/mimovl.cpp@@ -84,13 +84,7 @@ ggml_cgraph * clip_graph_mimovl::build() {ggml_tensor * idx_col = ggml_cast(ctx0, idx_col_f, GGML_TYPE_I32);ggml_tensor * idx_col_inv = ggml_argsort(ctx0, idx_col_f, GGML_SORT_ORDER_ASC);- ggml_tensor * window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);- ggml_set_name(window_mask, "mimovl_window_mask");- ggml_set_input(window_mask);-- ggml_tensor * window_mask_attn = (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED)- ? ggml_cast(ctx0, window_mask, GGML_TYPE_F16)- : window_mask;+ ggml_tensor * window_mask = build_inp_attn_mask(n_pos, n_pos);// Reorder helper: permute patches at merge-unit granularity. The patch// sequence is laid out as n_units groups of merge_unit (=4) consecutive@@ -151,7 +145,7 @@ ggml_cgraph * clip_graph_mimovl::build() {cb(Kcur, "Kcur_rope", il);// Full layers: plain attention. Windowed layers: banded mask and per-head sinks.- ggml_tensor * mask = is_full ? nullptr : window_mask_attn;+ ggml_tensor * mask = is_full ? nullptr : window_mask;ggml_tensor * sinks = is_full ? nullptr : layer.attn_sinks;if (!is_full) {GGML_ASSERT(layer.attn_sinks != nullptr);diff --git a/tools/mtmd/models/minicpmv.cpp b/tools/mtmd/models/minicpmv.cppindex 3e9c4c2a11..16514a3379 100644--- a/tools/mtmd/models/minicpmv.cpp+++ b/tools/mtmd/models/minicpmv.cpp@@ -146,12 +146,7 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {// so each window-major group of 4 tokens only attends to itself)vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos);vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos);- vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);- ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask");- ggml_set_input(vit_merger_window_mask);- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {- vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16);- }+ vit_merger_window_mask = build_inp_attn_mask(n_pos, n_pos);// ViT merger 2x2 downsample gather indicesvit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds);diff --git a/tools/mtmd/models/muse-glimmer.cpp b/tools/mtmd/models/muse-glimmer.cppindex b201536f58..a397c1b223 100644--- a/tools/mtmd/models/muse-glimmer.cpp+++ b/tools/mtmd/models/muse-glimmer.cpp@@ -10,7 +10,7 @@// muse_glimmer_sp_perm [n_tok] i32 : window grouping permutation (applied after ln_pre)// muse_glimmer_inv_perm [n_tok] i32 : inverse of sp_perm (applied after blocks)// muse_glimmer_ds_perm [n_tok] i32 : pixel-shuffle gather (original order)-// muse_glimmer_sp_mask [n_tok, n_tok] f32 : block-diagonal window mask (sparse layers)+// attn_mask [n_tok, n_tok] f32 (f16 with flash attn) : block-diagonal window mask (sparse layers)ggml_cgraph * clip_graph_muse_glimmer::build() {const int ds = hparams.n_merge; // downsample factor (2)const int sf = hparams.muse_glimmer_sparse_factor; // 4@@ -31,9 +31,7 @@ ggml_cgraph * clip_graph_muse_glimmer::build() {ggml_tensor * inv_perm = inp_i32("muse_glimmer_inv_perm", n_tok);ggml_tensor * ds_perm = inp_i32("muse_glimmer_ds_perm", n_tok);- ggml_tensor * sp_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tok, n_tok);- ggml_set_name(sp_mask, "muse_glimmer_sp_mask");- ggml_set_input(sp_mask);+ ggml_tensor * sp_mask = build_inp_attn_mask(n_tok, n_tok);// patchify via build_inp (conv2d over raw pixels) + bilinear-resized learned pos-embggml_tensor * x = build_inp(); // [n_embd, n_tok, 1]diff --git a/tools/mtmd/models/pockettts-gen.cpp b/tools/mtmd/models/pockettts-gen.cppindex 3fd613e5f7..662feff7fb 100644--- a/tools/mtmd/models/pockettts-gen.cpp+++ b/tools/mtmd/models/pockettts-gen.cpp@@ -225,6 +225,9 @@ ggml_cgraph * clip_graph_pockettts_gen::build() {keep = ggml_mul(ctx0, keep,ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix)));ggml_tensor * kq_mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), n_kv, n_pos, 1, 1);+ if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {+ kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);+ }for (int il = 0; il < n_layer; il++) {const auto & layer = model.gen_tfm_layers[il];diff --git a/tools/mtmd/models/pockettts-spkenc.cpp b/tools/mtmd/models/pockettts-spkenc.cppindex f802d90687..280f2a97ca 100644--- a/tools/mtmd/models/pockettts-spkenc.cpp+++ b/tools/mtmd/models/pockettts-spkenc.cpp@@ -53,9 +53,7 @@ ggml_cgraph * clip_graph_pockettts_spkenc::build() {ggml_set_input(inp_pos);// the mimi transformer is causal with a sliding window, see _build_attention_mask()- ggml_tensor * kq_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, cur->ne[1], cur->ne[1]);- ggml_set_name(kq_mask, "kq_mask");- ggml_set_input(kq_mask);+ ggml_tensor * kq_mask = build_inp_attn_mask(cur->ne[1], cur->ne[1]);for (int il = 0; il < n_layer; il++) {cur = tfm_layer_forward(cur, model.layers[il], inp_pos, kq_mask, il);diff --git a/tools/mtmd/models/qwen2vl.cpp b/tools/mtmd/models/qwen2vl.cppindex 2220c2692a..4998d95bc6 100644--- a/tools/mtmd/models/qwen2vl.cpp+++ b/tools/mtmd/models/qwen2vl.cpp@@ -82,14 +82,7 @@ ggml_cgraph * clip_graph_qwen2vl::build() {ggml_set_name(inv_window_idx, "inv_window_idx");ggml_set_input(inv_window_idx);// mask for window attention- window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);- ggml_set_name(window_mask, "window_mask");- ggml_set_input(window_mask);-- // if flash attn is used, we need to pad the mask and cast to f16- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {- window_mask = ggml_cast(ctx0, window_mask, GGML_TYPE_F16);- }+ window_mask = build_inp_attn_mask(n_pos, n_pos);// inpL shape: [n_embd, n_patches_x * n_patches_y, batch_size]GGML_ASSERT(batch_size == 1);diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cppindex 84c77f4fad..9826190e68 100644--- a/tools/mtmd/models/qwen3tts-gen.cpp+++ b/tools/mtmd/models/qwen3tts-gen.cpp@@ -109,7 +109,11 @@ ggml_tensor * clip_graph_qwen3tts_gen::code_gen::causal_mask_row(int64_t n_kv_paggml_tensor * keep = ggml_tri(ctx0, ones, GGML_TRI_TYPE_LOWER_DIAG);ggml_tensor * row = ggml_view_1d(ctx0, keep, n_kv_pad, (size_t) pos * keep->nb[1]);ggml_tensor * mask = ggml_log(ctx0, row); // 0 = keep, -inf = masked- return ggml_reshape_4d(ctx0, mask, n_kv_pad, 1, 1, 1);+ mask = ggml_reshape_4d(ctx0, mask, n_kv_pad, 1, 1, 1);+ if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {+ mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);+ }+ return mask;}// talker hidden size -> predictor hidden size (small_to_mtp_projection)@@ -481,6 +485,9 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor *keep = ggml_mul(ctx0, keep, warm);ggml_tensor * mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), total_kv, N, 1, 1); // 0 = keep, -inf = masked+ if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {+ mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);+ }ggml_tensor * q_cur = ggml_reshape_4d(ctx0, q, d_head, n_head, N, 1);ggml_tensor * k_cur = ggml_reshape_4d(ctx0, k_full, d_head, n_head_kv, total_kv, 1);diff --git a/tools/mtmd/models/youtuvl.cpp b/tools/mtmd/models/youtuvl.cppindex cd8f6d446f..cda456c66b 100644--- a/tools/mtmd/models/youtuvl.cpp+++ b/tools/mtmd/models/youtuvl.cpp@@ -69,14 +69,7 @@ ggml_cgraph * clip_graph_youtuvl::build() {ggml_set_name(inv_window_idx, "inv_window_idx");ggml_set_input(inv_window_idx);// mask for window attention- window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);- ggml_set_name(window_mask, "window_mask");- ggml_set_input(window_mask);-- // if flash attn is used, we need to pad the mask and cast to f16- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {- window_mask = ggml_cast(ctx0, window_mask, GGML_TYPE_F16);- }+ window_mask = build_inp_attn_mask(n_pos, n_pos);// inpL shape: [n_embd, n_patches_x * n_patches_y, batch_size]GGML_ASSERT(batch_size == 1);