LLM inference in C/C++
C++ ggml
mtmd : lfm2 tiles and layout match transformers (#30261)
Yuri Khrustalev4a45de4f2c68
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cppindex 1b7796d956..6ad60213e6 100644--- a/tests/test-mtmd-impl.cpp+++ b/tests/test-mtmd-impl.cpp@@ -91,6 +91,10 @@ MAKE_TEST(test_image_preprocessor_dyn_size) {}}+static std::string image_size_to_str(const clip_image_size & size) {+ return std::to_string(size.width) + "x" + std::to_string(size.height);+}+MAKE_TEST(test_image_preprocessor_lfm2) {clip_hparams hparams;hparams.patch_size = 16;@@ -121,6 +125,32 @@ MAKE_TEST(test_image_preprocessor_lfm2) {std::string(actual ? "tiled" : "single"));}+ // { image size, expected size from HF smart_resize }+ const std::vector<std::pair<clip_image_size, clip_image_size>> resize_cases = {+ { { 400, 600 }, { 384, 608 } }, // 400 / 32 = 12.5 rounds half to even+ { { 480, 336 }, { 480, 320 } },+ { { 64, 169 }, { 160, 416 } }, // 169 * beta / 32 is exactly 13 in double+ { { 1024, 768 }, { 576, 416 } },+ };++ for (const auto & [size, expected] : resize_cases) {+ t.assert_equal("smart_resize for " + image_size_to_str(size), image_size_to_str(expected),+ image_size_to_str(mtmd_image_preprocessor_lfm2::smart_resize(hparams, size)));+ }++ // { image size, expected grid from HF }+ const std::vector<std::pair<clip_image_size, clip_image_size>> grid_cases = {+ { { 720, 1600 }, { 2, 4 } }, // 0.45 is an exact tie between 2/4 and 2/5 in double, not in float+ { { 420, 1440 }, { 1, 3 } },+ { { 255, 2160 }, { 1, 8 } },+ { { 1024, 768 }, { 3, 2 } },+ };++ for (const auto & [size, expected] : grid_cases) {+ t.assert_equal("grid for " + image_size_to_str(size), image_size_to_str(expected),+ image_size_to_str(mtmd_image_preprocessor_lfm2::get_grid_layout(hparams, size.height, size.width)));+ }+// do_image_splitting=falsehparams.preproc_min_tiles = 1;hparams.preproc_max_tiles = 1;diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 41aba280ba..26941c9a85 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -1593,10 +1593,8 @@ struct clip_model_loader {}hparams.image_resize_algo_rf = hparams.image_resize_algo;hparams.image_resize_algo_ov = hparams.image_resize_algo;- // the tiles stretch the image to the grid (d1-omni vision.py)- if (model.proj_type == PROJECTOR_TYPE_D1OMNI_V) {- hparams.image_pad_rf = PAD_NONE;- }+ // the tiles stretch the image to the grid (Lfm2VlImageProcessor, d1-omni vision.py)+ hparams.image_pad_rf = PAD_NONE;get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);// defaults for GGUFs converted before tiling params were written// ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.jsondiff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cppindex 86feea6bd4..e0cd0bf1d0 100644--- a/tools/mtmd/mtmd-image.cpp+++ b/tools/mtmd/mtmd-image.cpp@@ -952,12 +952,33 @@ bool mtmd_image_preprocessor_lfm2::should_tile(static_cast<double>(hparams.image_max_pixels) * hparams.preproc_max_pixels_tolerance;}+// same as Lfm2VlImageProcessor.smart_resize: Python's round half to even, double precision+clip_image_size mtmd_image_preprocessor_lfm2::smart_resize(+ const clip_hparams & hparams,+ const clip_image_size & original_size) {+ const int align_size = hparams.patch_size * hparams.n_merge;+ const double height = original_size.height;+ const double width = original_size.width;++ int h_bar = std::max(align_size, static_cast<int>(std::nearbyint(height / align_size)) * align_size);+ int w_bar = std::max(align_size, static_cast<int>(std::nearbyint(width / align_size)) * align_size);++ if (static_cast<double>(h_bar) * w_bar > hparams.image_max_pixels) {+ const double beta = std::sqrt(height * width / hparams.image_max_pixels);+ h_bar = std::max(align_size, static_cast<int>(std::floor(height / beta / align_size)) * align_size);+ w_bar = std::max(align_size, static_cast<int>(std::floor(width / beta / align_size)) * align_size);+ } else if (static_cast<double>(h_bar) * w_bar < hparams.image_min_pixels) {+ const double beta = std::sqrt(hparams.image_min_pixels / (height * width));+ h_bar = static_cast<int>(std::ceil(height * beta / align_size)) * align_size;+ w_bar = static_cast<int>(std::ceil(width * beta / align_size)) * align_size;+ }++ return {w_bar, h_bar};+}+mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const {mtmd_image_preprocessor_llava_uhd::slice_instructions inst;- const int align_size = hparams.patch_size * hparams.n_merge;- inst.overview_size = img_tool::calc_size_preserved_ratio(- original_size,- { align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 });+ inst.overview_size = smart_resize(hparams, original_size);const bool needs_tiling = should_tile(hparams, original_size);const int tile_size = hparams.preproc_tile_size;@@ -968,7 +989,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfreturn inst;}- const clip_image_size grid = get_grid_layout(original_size.height, original_size.width);+ const clip_image_size grid = get_grid_layout(hparams, original_size.height, original_size.width);inst.grid_size = grid;inst.refined_size = clip_image_size{tile_size * grid.width, tile_size * grid.height};@@ -996,24 +1017,26 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfreturn inst;}+// double precision as in Python, float breaks exact ties such as 720/1600 between 2/4 and 2/5clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(- float aspect_ratio,+ const clip_hparams & hparams,+ double aspect_ratio,const std::vector<clip_image_size> & target_ratios,- int width, int height) const {- float best_ratio_diff = std::numeric_limits<float>::max();+ int width, int height) {+ double best_ratio_diff = std::numeric_limits<double>::max();clip_image_size best_ratio = {1, 1};- const float area = static_cast<float>(width * height);+ const double area = static_cast<double>(width) * height;for (const auto & ratio : target_ratios) {- const float target_aspect_ratio = static_cast<float>(ratio.width) / ratio.height;- const float ratio_diff = std::abs(aspect_ratio - target_aspect_ratio);+ const double target_aspect_ratio = static_cast<double>(ratio.width) / ratio.height;+ const double ratio_diff = std::abs(aspect_ratio - target_aspect_ratio);if (ratio_diff < best_ratio_diff) {best_ratio_diff = ratio_diff;best_ratio = ratio;} else if (ratio_diff == best_ratio_diff) {- const int tile_size = hparams.preproc_tile_size;- const float target_area = static_cast<float>(tile_size * tile_size * ratio.width * ratio.height);- if (area > 0.5f * target_area) {+ const int tile_size = hparams.preproc_tile_size;+ const double target_area = static_cast<double>(tile_size * tile_size * ratio.width * ratio.height);+ if (area > 0.5 * target_area) {best_ratio = ratio;}}@@ -1021,7 +1044,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(return best_ratio;}-std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() const {+std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios(const clip_hparams & hparams) {const int min_tiles = hparams.preproc_min_tiles;const int max_tiles = hparams.preproc_max_tiles;std::vector<clip_image_size> ratios;@@ -1049,10 +1072,10 @@ std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() creturn ratios;}-clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int width) const {- const float aspect_ratio = static_cast<float>(width) / height;- const auto ratios = get_target_ratios();- return find_closest_aspect_ratio(aspect_ratio, ratios, width, height);+clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const clip_hparams & hparams, int height, int width) {+ const double aspect_ratio = static_cast<double>(width) / height;+ const auto ratios = get_target_ratios(hparams);+ return find_closest_aspect_ratio(hparams, aspect_ratio, ratios, width, height);}//diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.hindex a9d16f1c9c..3aea23ced6 100644--- a/tools/mtmd/mtmd-image.h+++ b/tools/mtmd/mtmd-image.h@@ -199,14 +199,16 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size);+ static clip_image_size smart_resize(const clip_hparams & hparams, const clip_image_size & original_size);+ static clip_image_size get_grid_layout(const clip_hparams & hparams, int height, int width);private:- clip_image_size find_closest_aspect_ratio(- float aspect_ratio,+ static clip_image_size find_closest_aspect_ratio(+ const clip_hparams & hparams,+ double aspect_ratio,const std::vector<clip_image_size> & target_ratios,- int width, int height) const;- std::vector<clip_image_size> get_target_ratios() const;- clip_image_size get_grid_layout(int height, int width) const;+ int width, int height);+ static std::vector<clip_image_size> get_target_ratios(const clip_hparams & hparams);};struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd {