Skip to content

LLM inference in C/C++

C++ ggml

mtmd: fix idefics3 preproc (#28273)

Xuan-Son Nguyen67a17c17caa9
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index 65c24f4d47..890578978f 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -980,6 +980,56 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
//
// CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737
const clip_image_size original_size = img.get_size();
+
+ // old gguf files have no preprocessor longest size, custom token limits also need the generic size below
+ if (hparams.image_longest_edge > 0 && hparams.image_min_pixels <= 0 && hparams.image_max_pixels <= 0) {
+ const int tile_size = hparams.image_size;
+ const int longest_edge = hparams.image_longest_edge;
+ const double aspect_ratio = (double) original_size.width / original_size.height;
+
+ clip_image_size resized_size;
+ if (original_size.width >= original_size.height) {
+ resized_size.width = longest_edge;
+ resized_size.height = (int) (longest_edge / aspect_ratio);
+ resized_size.height += resized_size.height % 2;
+ } else {
+ resized_size.height = longest_edge;
+ resized_size.width = (int) (longest_edge * aspect_ratio);
+ resized_size.width += resized_size.width % 2;
+ }
+
+ const int grid_x = (resized_size.width + tile_size - 1) / tile_size;
+ const int grid_y = (resized_size.height + tile_size - 1) / tile_size;
+ const clip_image_size refined_size = clip_image_size{grid_x * tile_size, grid_y * tile_size};
+
+ clip_image_u8 resized_img;
+ img_tool::resize(img, resized_img, resized_size, hparams.image_resize_algo, PAD_NONE);
+
+ clip_image_u8 refined_img;
+ img_tool::resize(resized_img, refined_img, refined_size, hparams.image_resize_algo, PAD_NONE);
+
+ clip_image_u8 overview;
+ img_tool::resize(refined_img, overview, {tile_size, tile_size}, hparams.image_resize_algo, PAD_NONE);
+
+ std::vector<clip_image_u8> slices;
+ for (int y = 0; y < grid_y; y++) {
+ for (int x = 0; x < grid_x; x++) {
+ clip_image_u8 slice;
+ img_tool::crop(refined_img, slice, x * tile_size, y * tile_size, tile_size, tile_size);
+ slices.push_back(std::move(slice));
+ }
+ }
+
+ LOG_DBG("%s: grid size: %d x %d (%d tiles) + overview\n", __func__, grid_x, grid_y, grid_x * grid_y);
+
+ mtmd_image_preproc_out output;
+ output.append_overview(hparams, overview, true);
+ output.append(hparams, slices, true);
+ output.grid_x = grid_x;
+ output.grid_y = grid_y;
+ return output;
+ }
+
const clip_image_size refined_size = img_tool::calc_size_preserved_ratio(
original_size,
{ hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge });