LLM inference in C/C++
C++ ggml
mtmd: fix longest_edge ignoring min/max pixels (#26638)
Xuan-Son Nguyen4cb22cd537a9
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.hindex 101f49cd18..7db01b576b 100644--- a/tools/mtmd/clip-model.h+++ b/tools/mtmd/clip-model.h@@ -170,6 +170,17 @@ struct clip_hparams {warmup_image_size = static_cast<int>(std::sqrt(image_max_pixels));}+ // used by longest_edge preprocessor (no model-specific value for min/max tokens)+ void set_limit_image_tokens() {+ const int patch_area = patch_size * patch_size * n_merge * n_merge;+ if (custom_image_min_tokens > 0) {+ image_min_pixels = custom_image_min_tokens * patch_area;+ }+ if (custom_image_max_tokens > 0) {+ image_max_pixels = custom_image_max_tokens * patch_area;+ }+ }+void set_warmup_n_tokens(int n_tokens) {int n_tok_per_side = static_cast<int>(std::sqrt(n_tokens));GGML_ASSERT(n_tok_per_side * n_tok_per_side == n_tokens && "n_tokens must be n*n");diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex b1360fd7d3..3b61056298 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -1434,6 +1434,7 @@ struct clip_model_loader {// use default llava-uhd preprocessing paramsget_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);+ hparams.set_limit_image_tokens();} break;case PROJECTOR_TYPE_LFM2:{@@ -1471,6 +1472,7 @@ struct clip_model_loader {get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);hparams.image_longest_edge = hparams.image_size;get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);+ hparams.set_limit_image_tokens();hparams.set_warmup_n_tokens(256); // avoid OOM on warmup} break;case PROJECTOR_TYPE_DOTS_OCR:@@ -1595,6 +1597,7 @@ struct clip_model_loader {if (hparams.image_longest_edge == 0) {hparams.image_longest_edge = 3024;}+ // note: the step3vl preprocessor slices based on a fixed window grid, so it does not support custom min/max image tokenshparams.warmup_image_size = hparams.image_size;} break;case PROJECTOR_TYPE_YOUTUVL:diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cppindex 10cfe52f56..073d83d453 100644--- a/tools/mtmd/mtmd-image.cpp+++ b/tools/mtmd/mtmd-image.cpp@@ -139,50 +139,46 @@ struct img_tool {}}- // calculate the size of the **resized** image, while preserving the aspect ratio- // the calculated size will be aligned to the nearest multiple of align_size- // if H or W size is larger than longest_edge, it will be resized to longest_edge- static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const int align_size, const int longest_edge) {- GGML_ASSERT(align_size > 0);- if (inp_size.width <= 0 || inp_size.height <= 0 || longest_edge <= 0) {- return {0, 0};- }-- float scale = std::min(static_cast<float>(longest_edge) / inp_size.width,- static_cast<float>(longest_edge) / inp_size.height);-- float target_width_f = static_cast<float>(inp_size.width) * scale;- float target_height_f = static_cast<float>(inp_size.height) * scale;-- auto ceil_by_factor = [f = align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };- int aligned_width = ceil_by_factor(target_width_f);- int aligned_height = ceil_by_factor(target_height_f);-- return {aligned_width, aligned_height};- }+ struct calc_size_opt {+ int align_size = 1;+ int min_pixels = 0; // 0 = disabled+ int max_pixels = 0; // 0 = disabled+ // applied before min/max_pixels, so min_pixels can push an edge back above longest_edge+ int longest_edge = 0; // 0 = disabled+ };- // calculate the size of the **resized** image, while preserving the aspect ratio- // the calculated size will have min_pixels <= W*H <= max_pixels- // this is referred as "smart_resize" in transformers code- static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const int align_size, const int min_pixels, const int max_pixels) {- GGML_ASSERT(align_size > 0);+ // calculate the size of the **resized** image, while preserving the aspect ratio and+ // aligning to the nearest multiple of align_size ("smart_resize" in transformers code)+ static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const calc_size_opt & opts) {+ GGML_ASSERT(opts.align_size > 0);const int width = inp_size.width;const int height = inp_size.height;+ if (width <= 0 || height <= 0) {+ return {0, 0};+ }- auto round_by_factor = [f = align_size](float x) { return static_cast<int>(std::round(x / static_cast<float>(f))) * f; };- auto ceil_by_factor = [f = align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };- auto floor_by_factor = [f = align_size](float x) { return static_cast<int>(std::floor(x / static_cast<float>(f))) * f; };+ auto round_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::round(x / static_cast<float>(f))) * f; };+ auto ceil_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };+ auto floor_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::floor(x / static_cast<float>(f))) * f; };- // always align up first- int h_bar = std::max(align_size, round_by_factor(height));- int w_bar = std::max(align_size, round_by_factor(width));+ int w_bar, h_bar;+ if (opts.longest_edge > 0) {+ const float scale = std::min(static_cast<float>(opts.longest_edge) / width,+ static_cast<float>(opts.longest_edge) / height);+ w_bar = ceil_by_factor(width * scale);+ h_bar = ceil_by_factor(height * scale);+ } else {+ // always align up first+ w_bar = std::max(opts.align_size, round_by_factor(width));+ h_bar = std::max(opts.align_size, round_by_factor(height));+ }- if (h_bar * w_bar > max_pixels) {- const auto beta = std::sqrt(static_cast<float>(height * width) / max_pixels);- h_bar = std::max(align_size, floor_by_factor(height / beta));- w_bar = std::max(align_size, floor_by_factor(width / beta));- } else if (h_bar * w_bar < min_pixels) {- const auto beta = std::sqrt(static_cast<float>(min_pixels) / (height * width));+ if (opts.max_pixels > 0 && h_bar * w_bar > opts.max_pixels) {+ const auto beta = std::sqrt(static_cast<float>(height) * width / opts.max_pixels);+ h_bar = std::max(opts.align_size, floor_by_factor(height / beta));+ w_bar = std::max(opts.align_size, floor_by_factor(width / beta));+ } else if (opts.min_pixels > 0 && h_bar * w_bar < opts.min_pixels) {+ const auto beta = std::sqrt(static_cast<float>(opts.min_pixels) / (static_cast<float>(height) * width));h_bar = ceil_by_factor(height * beta);w_bar = ceil_by_factor(width * beta);}@@ -937,9 +933,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_iconst int cur_merge = hparams.n_merge;const clip_image_size target_size = img_tool::calc_size_preserved_ratio(original_size,- hparams.patch_size * cur_merge,- hparams.image_min_pixels,- hparams.image_max_pixels);+ {+ /* align_size */ hparams.patch_size * cur_merge,+ /* min_pixels */ hparams.image_min_pixels,+ /* max_pixels */ hparams.image_max_pixels,+ /* longest_edge */ 0,+ });img_tool::resize(img, resized_image, target_size,hparams.image_resize_algo,hparams.image_resize_pad,@@ -961,8 +960,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clconst int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge;const clip_image_size target_size = img_tool::calc_size_preserved_ratio(original_size,- hparams.patch_size * cur_merge,- hparams.image_longest_edge);+ {+ /* align_size */ hparams.patch_size * cur_merge,+ /* min_pixels */ std::max(0, hparams.image_min_pixels),+ /* max_pixels */ std::max(0, hparams.image_max_pixels),+ /* longest_edge */ hparams.image_longest_edge,+ });img_tool::resize(img, resized_image, target_size,hparams.image_resize_algo,hparams.image_resize_pad,@@ -1000,8 +1003,8 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfmtmd_image_preprocessor_llava_uhd::slice_instructions inst;const int align_size = hparams.patch_size * hparams.n_merge;inst.overview_size = img_tool::calc_size_preserved_ratio(- original_size, align_size,- hparams.image_min_pixels, hparams.image_max_pixels);+ original_size,+ { align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 });// tile if either dimension exceeds tile_size with toleranceconst bool needs_tiling = original_size.width > tile_size * max_pixels_tolerance || original_size.height > tile_size * max_pixels_tolerance;@@ -1109,7 +1112,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i// CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737const clip_image_size original_size = img.get_size();const clip_image_size refined_size = img_tool::calc_size_preserved_ratio(- original_size, hparams.image_size, hparams.image_longest_edge);+ original_size,+ { hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge });// LOG_INF("%s: original size: %d x %d, refined size: %d x %d\n",// __func__, original_size.width, original_size.height,// refined_size.width, refined_size.height);