LLM inference in C/C++
C++ ggml
mtmd: lightonocr does not require longest edge to always be 1540, native resolution is the default (#30600)
Said Taghadouini2f729cfa4ca6
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cppindex 5321f2ed1e..dfa3e5e2c5 100644--- a/tests/test-mtmd-impl.cpp+++ b/tests/test-mtmd-impl.cpp@@ -45,6 +45,27 @@ struct test_registry {// mtmd_image//+MAKE_TEST(test_image_preprocessor_longest_edge) {+ clip_hparams hparams;+ hparams.image_longest_edge = 1540;+ hparams.patch_size = 14;+ hparams.n_merge = 2;++ const std::vector<std::pair<clip_image_size, clip_image_size>> cases = {+ { { 463, 463 }, { 476, 476 } },+ { { 463, 300 }, { 476, 308 } },+ { { 448, 280 }, { 448, 280 } },+ { { 2000, 1000 }, { 1540, 784 } },+ };++ for (const auto & [input, expected] : cases) {+ const clip_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input);+ const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height);+ t.assert_equal(name + " width", expected.width, actual.width);+ t.assert_equal(name + " height", expected.height, actual.height);+ }+}+MAKE_TEST(test_image_preprocessor_lfm2) {clip_hparams hparams;hparams.patch_size = 16;diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cppindex a34fd45487..5debfe6f0d 100644--- a/tools/mtmd/mtmd-image.cpp+++ b/tools/mtmd/mtmd-image.cpp@@ -863,20 +863,24 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imag// mtmd_image_preprocessor_longest_edge//-mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) const {+clip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) {GGML_ASSERT(hparams.image_longest_edge > 0);- clip_image_u8 resized_image;- const clip_image_size original_size = img.get_size();// the original pixtral model doesn't have n_mergeconst int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge;- const clip_image_size target_size = img_tool::calc_size_preserved_ratio(+ return img_tool::calc_size_preserved_ratio(original_size,{/* align_size */ hparams.patch_size * cur_merge,/* min_pixels */ std::max(0, hparams.image_min_pixels),/* max_pixels */ std::max(0, hparams.image_max_pixels),- /* longest_edge */ hparams.image_longest_edge,+ /* longest_edge */ std::min(hparams.image_longest_edge,+ std::max(original_size.width, original_size.height)),});+}++mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) const {+ clip_image_u8 resized_image;+ const clip_image_size target_size = get_target_size(hparams, img.get_size());img_tool::resize(img, resized_image, target_size,hparams.image_resize_algo,hparams.image_resize_pad,diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.hindex b318b2d420..1798ce51d4 100644--- a/tools/mtmd/mtmd-image.h+++ b/tools/mtmd/mtmd-image.h@@ -140,9 +140,10 @@ struct mtmd_image_preprocessor_glm5v : mtmd_image_preprocessor {mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;};-// similar to mtmd_image_preprocessor_dyn_size, but resize the image to have longest edge equal to hparams.image_longest_edge, while preserving aspect ratio+// similar to mtmd_image_preprocessor_dyn_size, but cap the longest edge at hparams.image_longest_edge, while preserving aspect ratiostruct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor {mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}+ static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;};