LLM inference in C/C++
C++ ggml
mtmd : read LFM2 tiling params from GGUF metadata (#30840)
Tarek Dakhran36140dbee267
diff --git a/conversion/lfm2.py b/conversion/lfm2.pyindex 113c911168..b1f0d1d325 100644--- a/conversion/lfm2.py+++ b/conversion/lfm2.py@@ -374,6 +374,16 @@ class LFM2VLModel(MmprojModel):if resize_algo is None:raise ValueError(f"unsupported resample: {resample}")self.gguf_writer.add_vision_image_resize_algo(resize_algo)+ # tiling params from processor_config.json (do_image_splitting=False -> single tile, i.e. no tiling)+ preproc = self.preprocessor_config+ if preproc.get("do_image_splitting", True):+ self.gguf_writer.add_vision_preproc_min_tiles(preproc.get("min_tiles", 2))+ self.gguf_writer.add_vision_preproc_max_tiles(preproc.get("max_tiles", 10))+ else:+ self.gguf_writer.add_vision_preproc_min_tiles(1)+ self.gguf_writer.add_vision_preproc_max_tiles(1)+ self.gguf_writer.add_vision_preproc_tile_size(preproc.get("tile_size", 512))+ self.gguf_writer.add_vision_preproc_max_pixels_tolerance(preproc.get("max_pixels_tolerance", 2.0))@classmethoddef filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 8c19e22dab..b2d8d59e04 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -419,6 +419,8 @@ class Keys:PREPROC_MIN_TILES = "clip.vision.preproc_min_tiles"PREPROC_MAX_TILES = "clip.vision.preproc_max_tiles"PREPROC_IMAGE_SIZE = "clip.vision.preproc_image_size"+ PREPROC_TILE_SIZE = "clip.vision.preproc_tile_size"+ PREPROC_MAX_PIXELS_TOLERANCE = "clip.vision.preproc_max_pixels_tolerance"PATCH_SIZE = "clip.vision.patch_size"EMBEDDING_LENGTH = "clip.vision.embedding_length"FEED_FORWARD_LENGTH = "clip.vision.feed_forward_length"diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex 7becd74af9..2943e01869 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -1470,6 +1470,12 @@ class GGUFWriter:def add_vision_preproc_image_size(self, value: int) -> None:self.add_uint32(Keys.ClipVision.PREPROC_IMAGE_SIZE, value)+ def add_vision_preproc_tile_size(self, value: int) -> None:+ self.add_uint32(Keys.ClipVision.PREPROC_TILE_SIZE, value)++ def add_vision_preproc_max_pixels_tolerance(self, value: float) -> None:+ self.add_float32(Keys.ClipVision.PREPROC_MAX_PIXELS_TOLERANCE, value)+def add_vision_projector_query_side(self, value: int) -> None:self.add_uint32(Keys.ClipVision.Projector.QUERY_SIDE, value)diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cppindex 2d0f248741..1b7796d956 100644--- a/tests/test-mtmd-impl.cpp+++ b/tests/test-mtmd-impl.cpp@@ -96,6 +96,10 @@ MAKE_TEST(test_image_preprocessor_lfm2) {hparams.patch_size = 16;hparams.n_merge = 2;hparams.set_limit_image_tokens(64, 256);+ hparams.preproc_min_tiles = 2;+ hparams.preproc_max_tiles = 10;+ hparams.preproc_tile_size = 512;+ hparams.preproc_max_pixels_tolerance = 2.0f;// { image size, expected tiling }const std::vector<std::pair<clip_image_size, bool>> cases = {@@ -116,6 +120,12 @@ MAKE_TEST(test_image_preprocessor_lfm2) {std::string(expected ? "tiled" : "single"),std::string(actual ? "tiled" : "single"));}++ // do_image_splitting=false+ hparams.preproc_min_tiles = 1;+ hparams.preproc_max_tiles = 1;+ t.assert_equal("tiling for 1024x977 with max_tiles=1", std::string("single"),+ std::string(mtmd_image_preprocessor_lfm2::should_tile(hparams, { 1024, 977 }) ? "tiled" : "single"));}//diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.hindex 10eea6f746..8c4ffc9bb2 100644--- a/tools/mtmd/clip-impl.h+++ b/tools/mtmd/clip-impl.h@@ -56,6 +56,8 @@#define KEY_PREPROC_MIN_TILES "clip.vision.preproc_min_tiles"#define KEY_PREPROC_MAX_TILES "clip.vision.preproc_max_tiles"#define KEY_PREPROC_IMAGE_SIZE "clip.vision.preproc_image_size"+#define KEY_PREPROC_TILE_SIZE "clip.vision.preproc_tile_size"+#define KEY_PREPROC_MAX_PIXELS_TOLERANCE "clip.vision.preproc_max_pixels_tolerance"#define KEY_PATCH_SIZE "clip.vision.patch_size"#define KEY_IMAGE_MEAN "clip.vision.image_mean"#define KEY_IMAGE_STD "clip.vision.image_std"diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.hindex fefbd4f460..64e006e8d6 100644--- a/tools/mtmd/clip-model.h+++ b/tools/mtmd/clip-model.h@@ -73,7 +73,8 @@ struct clip_hparams {int32_t preproc_min_tiles = 0;int32_t preproc_max_tiles = 0;int32_t max_slice_nums = 9; // llava-uhd slice cap; per-model, carried in the GGUF- int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr)+ int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr, lfm2)+ float preproc_max_pixels_tolerance = 1.0f;resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC;resize_algo image_resize_algo_ov = RESIZE_ALGO_BICUBIC;pad_style image_pad_rf = PAD_CEIL; // padding style for the refined image (e.g. llava-1.6)diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex faeb377027..3a4dd682ab 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -1598,7 +1598,17 @@ struct clip_model_loader {hparams.image_pad_rf = PAD_NONE;}get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);+ // defaults for GGUFs converted before tiling params were written// ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json+ hparams.preproc_min_tiles = 2;+ hparams.preproc_max_tiles = 10;+ hparams.preproc_tile_size = 512;+ hparams.preproc_max_pixels_tolerance = 2.0f;+ get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false);+ get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);+ get_u32(KEY_PREPROC_TILE_SIZE, hparams.preproc_tile_size, false);+ get_f32(KEY_PREPROC_MAX_PIXELS_TOLERANCE, hparams.preproc_max_pixels_tolerance, false);+ GGML_ASSERT(hparams.preproc_min_tiles >= 1 && hparams.preproc_min_tiles <= hparams.preproc_max_tiles);hparams.set_limit_image_tokens(64, 256);} break;case PROJECTOR_TYPE_PHI4:diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cppindex e56f33e77c..86feea6bd4 100644--- a/tools/mtmd/mtmd-image.cpp+++ b/tools/mtmd/mtmd-image.cpp@@ -947,8 +947,9 @@ bool mtmd_image_preprocessor_lfm2::should_tile(const int h_bar = std::max(hparams.patch_size, round_by_factor(original_size.height));const int w_bar = std::max(hparams.patch_size, round_by_factor(original_size.width));- return static_cast<double>(h_bar) * static_cast<double>(w_bar) >- static_cast<double>(hparams.image_max_pixels) * max_pixels_tolerance;+ return hparams.preproc_max_tiles > 1 &&+ static_cast<double>(h_bar) * static_cast<double>(w_bar) >+ static_cast<double>(hparams.image_max_pixels) * hparams.preproc_max_pixels_tolerance;}mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const {@@ -959,6 +960,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf{ align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 });const bool needs_tiling = should_tile(hparams, original_size);+ const int tile_size = hparams.preproc_tile_size;if (!needs_tiling) {inst.refined_size = clip_image_size{0, 0};@@ -1009,6 +1011,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(best_ratio_diff = ratio_diff;best_ratio = ratio;} else if (ratio_diff == best_ratio_diff) {+ const int tile_size = hparams.preproc_tile_size;const float target_area = static_cast<float>(tile_size * tile_size * ratio.width * ratio.height);if (area > 0.5f * target_area) {best_ratio = ratio;@@ -1019,6 +1022,8 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(}std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() const {+ const int min_tiles = hparams.preproc_min_tiles;+ const int max_tiles = hparams.preproc_max_tiles;std::vector<clip_image_size> ratios;for (int n = min_tiles; n <= max_tiles; n++) {for (int w = 1; w <= n; w++) {diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.hindex 26193e4e4d..a9d16f1c9c 100644--- a/tools/mtmd/mtmd-image.h+++ b/tools/mtmd/mtmd-image.h@@ -191,14 +191,9 @@ protected:};// custom llava-uhd slicing logic for LFM2+// tiling params (min/max tiles, tile size, pixels tolerance) come from hparams// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.pystruct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {- // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json- static constexpr int min_tiles = 2;- static constexpr int max_tiles = 10;- static constexpr float max_pixels_tolerance = 2.0f;- static constexpr int tile_size = 512;-using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;