LLM inference in C/C++
C++ ggml
mtmd: add batching support for internvl (#24775)
Xuan-Son Nguyendb52540f730d
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex dc62232957..17079815d4 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -534,7 +534,7 @@ ggml_tensor * clip_graph::build_vit(ggml_tensor * clip_graph::build_inp() {ggml_tensor * inp_raw = build_inp_raw();ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);- inp = ggml_reshape_2d(ctx0, inp, n_patches, n_embd);+ inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch);inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));if (model.patch_bias) {inp = ggml_add(ctx0, inp, model.patch_bias);diff --git a/tools/mtmd/models/internvl.cpp b/tools/mtmd/models/internvl.cppindex 9aded3b97c..65d7d5a6b7 100644--- a/tools/mtmd/models/internvl.cpp+++ b/tools/mtmd/models/internvl.cpp@@ -8,7 +8,9 @@ ggml_cgraph * clip_graph_internvl::build() {ggml_tensor * inp = build_inp();// add CLS token- inp = ggml_concat(ctx0, inp, model.class_embedding, 1);+ ggml_tensor * cls_repeated = ggml_repeat_4d(ctx0, model.class_embedding,+ model.class_embedding->ne[0], 1, n_batch, 1);+ inp = ggml_concat(ctx0, inp, cls_repeated, 1);// The larger models use a different ViT, which uses RMS norm instead of layer norm// ref: https://github.com/ggml-org/llama.cpp/pull/13443#issuecomment-2869786188@@ -24,14 +26,15 @@ ggml_cgraph * clip_graph_internvl::build() {nullptr);// remove CLS token- cur = ggml_view_2d(ctx0, cur,- n_embd, n_patches,- ggml_row_size(cur->type, n_embd), 0);+ cur = ggml_view_3d(ctx0, cur,+ n_embd, n_patches, n_batch,+ cur->nb[1], cur->nb[2], 0);+ cur = ggml_cont(ctx0, cur);// pixel shuffle{const int scale_factor = model.hparams.n_merge;- const int bsz = 1; // batch size, always 1 for now since we don't support batching+ const int bsz = n_batch;const int height = n_patches_y;const int width = n_patches_x;GGML_ASSERT(scale_factor > 0);@@ -44,9 +47,10 @@ ggml_cgraph * clip_graph_internvl::build() {bsz);cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);// flatten to 2D- cur = ggml_cont_2d(ctx0, cur,+ cur = ggml_cont_3d(ctx0, cur,n_embd * scale_factor * scale_factor,- cur->ne[1] * cur->ne[2]);+ cur->ne[1] * cur->ne[2],+ cur->ne[3]);}// projector (always using GELU activation)diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.hindex 3a15f76829..12d5e69493 100644--- a/tools/mtmd/models/models.h+++ b/tools/mtmd/models/models.h@@ -80,6 +80,7 @@ struct clip_graph_minicpmv4_6 : clip_graph {struct clip_graph_internvl : clip_graph {clip_graph_internvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}ggml_cgraph * build() override;+ bool support_batch() const override { return true; }};struct clip_graph_nemotron_v2_vl : clip_graph {