LLM inference in C/C++
C++ ggml
mtmd: add n_embd_head (#26342)
Xuan-Son Nguyendb7d8b24b519
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 650f1c8a56..d86e614d8f 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -353,6 +353,7 @@ class Keys:class Attention:HEAD_COUNT = "clip.vision.attention.head_count"HEAD_COUNT_KV = "clip.vision.attention.head_count_kv" # used by mimovl (GQA)+ HEAD_DIM = "clip.vision.attention.head_dim" # set when qkv width != n_embdLAYERNORM_EPS = "clip.vision.attention.layer_norm_epsilon"class Projector:diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex 3aa4f049f2..c5905164c3 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -1226,6 +1226,9 @@ class GGUFWriter:def add_vision_head_count_kv(self, value: int) -> None:self.add_uint32(Keys.ClipVision.Attention.HEAD_COUNT_KV, value)+ def add_vision_head_dim(self, value: int) -> None:+ self.add_uint32(Keys.ClipVision.Attention.HEAD_DIM, value)+def add_vision_attention_layernorm_eps(self, value: float) -> None:self.add_float32(Keys.ClipVision.Attention.LAYERNORM_EPS, value)diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.hindex 589fc724ed..d42b38222c 100644--- a/tools/mtmd/clip-impl.h+++ b/tools/mtmd/clip-impl.h@@ -41,6 +41,7 @@#define KEY_PROJ_DIM "clip.%s.projection_dim"#define KEY_N_HEAD "clip.%s.attention.head_count"#define KEY_N_HEAD_KV "clip.%s.attention.head_count_kv"+#define KEY_N_EMBD_HEAD "clip.%s.attention.head_dim"#define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon"#define KEY_FEATURE_LAYERS "clip.%s.feature_layer"diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.hindex fec2b01802..8b9db5101d 100644--- a/tools/mtmd/clip-model.h+++ b/tools/mtmd/clip-model.h@@ -54,6 +54,8 @@ struct clip_hparams {int32_t projection_dim = 0;int32_t n_head = 0;int32_t n_head_kv = 0;+ // 0 = derive from n_embd; set when qkv width != n_embd+ int32_t n_embd_head = 0;int32_t n_layer = 0;int32_t n_merge = 1; // number of patch merges **per-side**diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 11f9820ede..5f0d00b660 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -253,7 +253,7 @@ clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) :n_embd(hparams.n_embd),n_head(hparams.n_head),n_head_kv(hparams.n_head_kv),- d_head(n_head > 0 ? n_embd / n_head : 0),+ d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)),n_layer(hparams.n_layer),n_mmproj_embd(clip_n_mmproj_embd(ctx)),eps(hparams.eps),@@ -372,13 +372,13 @@ ggml_tensor * clip_graph::build_vit(/* nb1 */ ggml_row_size(cur->type, d_head),/* nb2 */ cur->nb[1],/* nb3 */ cur->nb[1] * n_pos,- /* offset */ ggml_row_size(cur->type, n_embd));+ /* offset */ ggml_row_size(cur->type, n_head * d_head));Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,/* nb1 */ ggml_row_size(cur->type, d_head),/* nb2 */ cur->nb[1],/* nb3 */ cur->nb[1] * n_pos,- /* offset */ ggml_row_size(cur->type, 2 * n_embd));+ /* offset */ ggml_row_size(cur->type, 2 * n_head * d_head));if (layer.q_norm) {GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]);@@ -1190,6 +1190,7 @@ struct clip_model_loader {const char * prefix = is_vision ? "vision" : "audio";get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd);get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head);+ get_u32(string_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false);get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff);get_u32(string_format(KEY_N_BLOCK, prefix), hparams.n_layer);get_u32(string_format(KEY_PROJ_DIM, prefix), hparams.projection_dim);