Skip to content

LLM inference in C/C++

C++ ggml

mtmd: fix LFM2 image tiling threshold (#27057)

BlackFoil7acdbb1f191d
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 08c6f5a479..3ee51b5191 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -310,6 +310,9 @@ llama_build_and_test(test-mtmd-c-api.c)
target_link_libraries(${LLAMA_TEST_NAME} PRIVATE mtmd)
unset(LLAMA_TEST_NAME)
+llama_build_and_test(test-mtmd-impl.cpp)
+target_link_libraries(test-mtmd-impl PRIVATE mtmd)
+
# GGUF model data fetcher library for tests that need real model metadata
# Only compile when cpp-httplib has SSL support (CPPHTTPLIB_OPENSSL_SUPPORT)
if (TARGET cpp-httplib)
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
new file mode 100644
index 0000000000..41cb53237e
--- /dev/null
+++ b/tests/test-mtmd-impl.cpp
@@ -0,0 +1,88 @@
+#include "testing.h"
+
+#include "mtmd-image.h"
+
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+// this test file contains:
+// 1. test cases for mtmd helpers
+// 2. test cases for internal mtmd components
+// internal headers can be included here
+
+struct test_registry {
+ using fn_t = void (*)(testing &);
+
+ struct entry {
+ std::string name;
+ fn_t fn;
+ };
+
+ static std::vector<entry> & all() {
+ static std::vector<entry> entries;
+ return entries;
+ }
+
+ test_registry(const char * name, fn_t fn) {
+ all().push_back({ name, fn });
+ }
+};
+
+#define MAKE_TEST(name) \
+ static void name(testing & t); \
+ static const test_registry test_registry_ ## name(#name, &name); \
+ static void name(testing & t)
+
+
+//
+// mtmd_image
+//
+
+MAKE_TEST(test_image_preprocessor_lfm2) {
+ clip_hparams hparams;
+ hparams.patch_size = 16;
+ hparams.n_merge = 2;
+ hparams.set_limit_image_tokens(64, 256);
+
+ // { image size, expected tiling }
+ const std::vector<std::pair<clip_image_size, bool>> cases = {
+ { { 704, 704 }, false },
+ // 720 / (patch_size * n_merge) is exactly 22.5, so this only matches HF
+ // if round_by_factor rounds half to even (22) instead of away from zero (23)
+ { { 720, 720 }, false },
+ { { 736, 736 }, true },
+ { { 1024, 977 }, true },
+ { { 1056, 384 }, false },
+ };
+
+ for (const auto & [size, expected] : cases) {
+ const bool actual = mtmd_image_preprocessor_lfm2::should_tile(hparams, size);
+
+ t.assert_equal(
+ "tiling for " + std::to_string(size.width) + "x" + std::to_string(size.height),
+ std::string(expected ? "tiled" : "single"),
+ std::string(actual ? "tiled" : "single"));
+ }
+}
+
+//
+// main
+//
+
+int main(int argc, char ** argv) {
+ testing t(std::cout);
+ t.verbose = true;
+
+ // usage: test-mtmd-impl [filter_regex]
+ for (int i = 1; i < argc; i++) {
+ t.set_filter(argv[i]);
+ }
+
+ for (const auto & e : test_registry::all()) {
+ t.test(e.name, e.fn);
+ }
+
+ return t.summary();
+}
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 95aa853eaf..aa97c768a0 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -90,6 +90,9 @@ if (BUILD_SHARED_LIBS)
set_target_properties (mtmd PROPERTIES POSITION_INDEPENDENT_CODE ON)
target_compile_definitions(mtmd PRIVATE LLAMA_BUILD)
target_compile_definitions(mtmd PUBLIC LLAMA_SHARED)
+
+ # export all symbols so that internal components can be tested by test-mtmd-impl
+ set_target_properties (mtmd PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
endif()
set(MTMD_PUBLIC_HEADERS
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 2c9ea499ce..ea85497760 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -858,6 +858,9 @@ static std::ifstream open_ifstream_binary(const std::string & fname) {
}
#endif
+// in test-mtmd-impl, we include woth common.h and this file, and these functions are duplicated
+// this is a quick fix to avoid compilation errors
+#ifndef DIRECTORY_SEPARATOR
static std::string string_format(const char * fmt, ...) {
va_list ap;
va_list ap2;
@@ -915,6 +918,7 @@ inline bool string_ends_with(std::string_view str, std::string_view suffix) {
return str.size() >= suffix.size() &&
str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
}
+#endif
//
// gguf utils
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index 769b6efe61..0d9db4f622 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -1013,14 +1013,31 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image
return output;
}
+bool mtmd_image_preprocessor_lfm2::should_tile(
+ const clip_hparams & hparams,
+ const clip_image_size & original_size) {
+ const int align_size = hparams.patch_size * hparams.n_merge;
+
+ const auto round_by_factor = [align_size](float x) {
+ // see https://github.com/ggml-org/llama.cpp/pull/27057#discussion_r3796264887
+ return static_cast<int>(std::nearbyint(static_cast<double>(x) / align_size)) * align_size;
+ };
+
+ const int h_bar = std::max(hparams.patch_size, round_by_factor(original_size.height));
+ const int w_bar = std::max(hparams.patch_size, round_by_factor(original_size.width));
+
+ return static_cast<double>(h_bar) * static_cast<double>(w_bar) >
+ static_cast<double>(hparams.image_max_pixels) * max_pixels_tolerance;
+}
+
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) {
mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
const int align_size = hparams.patch_size * hparams.n_merge;
inst.overview_size = img_tool::calc_size_preserved_ratio(
original_size,
{ align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 });
- // tile if either dimension exceeds tile_size with tolerance
- const bool needs_tiling = original_size.width > tile_size * max_pixels_tolerance || original_size.height > tile_size * max_pixels_tolerance;
+
+ const bool needs_tiling = should_tile(hparams, original_size);
if (!needs_tiling) {
inst.refined_size = clip_image_size{0, 0};
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 40dfea7eb4..732e27379d 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -148,6 +148,8 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
slice_instructions get_slice_instructions(const clip_image_size & original_size) override;
+ static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size);
+
private:
clip_image_size find_closest_aspect_ratio(
float aspect_ratio,