LLM inference in C/C++
C++ ggml
mtmd : match the transformers Qwen-VL resize (#30736)
boshf711c8f30380
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cppindex dfa3e5e2c5..2d0f248741 100644--- a/tests/test-mtmd-impl.cpp+++ b/tests/test-mtmd-impl.cpp@@ -66,6 +66,31 @@ MAKE_TEST(test_image_preprocessor_longest_edge) {}}+MAKE_TEST(test_image_preprocessor_dyn_size) {+ // { patch_size, image size, expected size }+ // a side that is an even number plus .5 merged patches rounds down, like Python round() in smart_resize+ const std::vector<std::tuple<int, clip_image_size, clip_image_size>> cases = {+ { 16, { 768, 400 }, { 768, 384 } },+ { 16, { 800, 336 }, { 800, 320 } },+ { 16, { 400, 400 }, { 384, 384 } },+ { 16, { 640, 240 }, { 640, 256 } },+ { 14, { 966, 560 }, { 952, 560 } },+ };++ for (const auto & [patch_size, input, expected] : cases) {+ clip_hparams hparams;+ hparams.patch_size = patch_size;+ hparams.n_merge = 2;+ hparams.image_round_half_even = true;+ hparams.set_limit_image_tokens(8, 4096);++ const clip_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input);+ const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height);+ t.assert_equal(name + " width", expected.width, actual.width);+ t.assert_equal(name + " height", expected.height, actual.height);+ }+}+MAKE_TEST(test_image_preprocessor_lfm2) {clip_hparams hparams;hparams.patch_size = 16;diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.hindex 92ffe5e54f..fefbd4f460 100644--- a/tools/mtmd/clip-model.h+++ b/tools/mtmd/clip-model.h@@ -65,6 +65,7 @@ struct clip_hparams {int32_t image_max_pixels = -1;resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC;pad_style image_resize_pad = PAD_CEIL; // padding style when resizing+ bool image_round_half_even = false; // smart_resize rounds .5 to even, like Python round() in the HF processorstd::array<uint8_t, 3> image_pad_color = {0, 0, 0};// (preprocessor) for llava-uhd style modelsdiff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 793b8cd1a6..faeb377027 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -1634,6 +1634,8 @@ struct clip_model_loader {case PROJECTOR_TYPE_DOTS_OCR:{hparams.rope_theta = 10000.0f;+ hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding+ hparams.image_round_half_even = true; // smart_resize uses Python round()get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);@@ -1750,6 +1752,8 @@ struct clip_model_loader {{hparams.n_merge = 2; // default value for Qwen 2 and 2.5hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;+ hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding+ hparams.image_round_half_even = true; // smart_resize uses Python round()get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, model.proj_type == PROJECTOR_TYPE_QWEN25VL); // only 2.5 requires it// ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json@@ -1825,6 +1829,8 @@ struct clip_model_loader {{hparams.n_merge = 2; // spatial_merge_sizehparams.image_resize_algo = RESIZE_ALGO_BICUBIC;+ hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding+ hparams.image_round_half_even = true; // smart_resize uses Python round()get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);// 1D banded sliding-window radius (visual_token_window_size); required@@ -2092,6 +2098,8 @@ struct clip_model_loader {case PROJECTOR_TYPE_EXAONE4_5:{hparams.n_merge = 2;+ hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding+ hparams.image_round_half_even = true; // smart_resize uses Python round()get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false);get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cppindex 5debfe6f0d..e56f33e77c 100644--- a/tools/mtmd/mtmd-image.cpp+++ b/tools/mtmd/mtmd-image.cpp@@ -115,6 +115,7 @@ struct img_tool {int max_pixels = 0; // 0 = disabled// applied before min/max_pixels, so min_pixels can push an edge back above longest_edgeint longest_edge = 0; // 0 = disabled+ bool round_half_even = false; // Python round() instead of std::round};// calculate the size of the **resized** image, while preserving the aspect ratio and@@ -127,7 +128,10 @@ struct img_tool {return {0, 0};}- auto round_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::round(x / static_cast<float>(f))) * f; };+ auto round_by_factor = [f = opts.align_size, even = opts.round_half_even](float x) {+ const float q = x / static_cast<float>(f);+ return static_cast<int>(even ? std::nearbyint(q) : std::round(q)) * f;+ };auto ceil_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };auto floor_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::floor(x / static_cast<float>(f))) * f; };@@ -767,20 +771,24 @@ mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip// mtmd_image_preprocessor_dyn_size//-mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {+clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) {GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0);- clip_image_u8 resized_image;- const clip_image_size original_size = img.get_size();// the original pixtral model doesn't have n_mergeconst int cur_merge = hparams.n_merge;- const clip_image_size target_size = img_tool::calc_size_preserved_ratio(+ return img_tool::calc_size_preserved_ratio(original_size,{- /* align_size */ hparams.patch_size * cur_merge,- /* min_pixels */ hparams.image_min_pixels,- /* max_pixels */ hparams.image_max_pixels,- /* longest_edge */ 0,+ /* align_size */ hparams.patch_size * cur_merge,+ /* min_pixels */ hparams.image_min_pixels,+ /* max_pixels */ hparams.image_max_pixels,+ /* longest_edge */ 0,+ /* round_half_even */ hparams.image_round_half_even,});+}++mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {+ clip_image_u8 resized_image;+ const clip_image_size target_size = get_target_size(hparams, img.get_size());img_tool::resize(img, resized_image, target_size,hparams.image_resize_algo,hparams.image_resize_pad,diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.hindex 1798ce51d4..26193e4e4d 100644--- a/tools/mtmd/mtmd-image.h+++ b/tools/mtmd/mtmd-image.h@@ -131,6 +131,7 @@ struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor {// this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etcstruct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}+ static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;};