Skip to content

LLM inference in C/C++

C++ ggml

mtmd : skip thumbnail for non-tiled LFM2 images (#27246)

Tarek Dakhranb75ecd1971bf
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index d4b5ec0d98..769b6efe61 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -999,6 +999,20 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_mi
// mtmd_image_preprocessor_lfm2
//
+mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image_u8 & img) {
+ auto const inst = get_slice_instructions(img.get_size());
+ if (!inst.slices.empty()) {
+ return mtmd_image_preprocessor_llava_uhd::preprocess(img);
+ }
+
+ // single tile: no thumbnail
+ // note: not using output.overview here because it will emit <|img_thumbnail|> token, which we don't want in this case
+ auto sliced = slice_image(img, inst);
+ mtmd_image_preproc_out output;
+ output.append(hparams, sliced.overview, true);
+ return output;
+}
+
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) {
mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
const int align_size = hparams.patch_size * hparams.n_merge;
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 2383329344..40dfea7eb4 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -145,6 +145,7 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
static constexpr int tile_size = 512;
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
+ mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
slice_instructions get_slice_instructions(const clip_image_size & original_size) override;
private: