LLM inference in C/C++
C++ ggml
mtmd: fix Granite4 Vision image sequence assembly (#26653)
Hemanth Battu9d57ce456c94
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.hindex 464f74ca05..2c9ea499ce 100644--- a/tools/mtmd/clip-impl.h+++ b/tools/mtmd/clip-impl.h@@ -6,6 +6,7 @@#include <array>#include <climits>+#include <cmath>#include <cstdarg>#include <cinttypes>#include <string>@@ -642,9 +643,25 @@ struct mtmd_serialization; // forward declarationstruct clip_image_f32 {// marks the global view in e.g., DeepSeek-OCR Modelsbool add_viewsep = false;- // whether a learned newline (or EOI) token should be appended after the image (eg Granite4 Vision)+ // appends a learned newline (or EOI) token after the image+ // no model uses it now (Granite4 Vision moved to anyres), kept for future modelsbool add_newline = false;+ // llava-next "anyres" tiling, used by Granite4 Vision+ // the whole grid is encoded and assembled in a single graph+ // NOTE: excluded from serialized: a deserialized image is always a placeholder, which is never encoded+ struct anyres_info {+ int grid_x = 0; // tiles per row, 0 means the image is not tiled+ int grid_y = 0; // tiles per column+ int orig_nx = 0; // size of the source image, used to drop the padding tokens+ int orig_ny = 0;++ bool is_tiled() const {+ return grid_x > 0 && grid_y > 0;+ }+ };+ anyres_info anyres;+clip_image_size get_size() const {return { nx_, ny_ };}@@ -726,6 +743,25 @@ struct clip_image_f32 {}};+// token area kept after removing the padding added by the anyres resize+// ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L109+static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h,+ int & off_x, int & off_y, int & out_w, int & out_h) {+ off_x = 0;+ off_y = 0;+ out_w = cur_w;+ out_h = cur_h;+ if ((float) orig_w / orig_h > (float) cur_w / cur_h) {+ const int new_h = (int) std::floor((double) orig_h * cur_w / orig_w + 1e-7);+ off_y = (cur_h - new_h) / 2;+ out_h = cur_h - 2 * off_y;+ } else {+ const int new_w = (int) std::floor((double) orig_w * cur_h / orig_h + 1e-7);+ off_x = (cur_w - new_w) / 2;+ out_w = cur_w - 2 * off_x;+ }+}+//// logging//diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex c08b41d590..7530ed6133 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -4229,18 +4229,20 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {case PROJECTOR_TYPE_GRANITE4_VISION:{// Per-tile output token count: each projector block outputs- // query_side^2 tokens per window × n^2 windows.- // For 384×384 input: n = 24/8 = 3, query_side = 4 → 144.+ // query_side^2 tokens per window x n^2 windows.+ // For 384x384 input: n = 24/8 = 3, query_side = 4 -> 144.const int window_side = ctx->model.hparams.downsample_window_side;const int query_side = ctx->model.hparams.downsample_query_side;const int side = img->nx() / params.patch_size;const int n = side / window_side;- n_patches = (query_side * n) * (query_side * n);- if (img->add_newline) {- // For single-tile case: append 1 newline row.- // For multi-tile rowwise: handled by caller, but here we- // report the per-tile count including one trailing newline.- n_patches += 1;+ const int out_side = query_side * n;+ n_patches = out_side * out_side;+ if (img->anyres.is_tiled()) {+ // overview tile, then the unpadded tile grid with one newline per row+ int off_x, off_y, w, h;+ clip_anyres_unpad(img->anyres.grid_x * out_side, img->anyres.grid_y * out_side,+ img->anyres.orig_nx, img->anyres.orig_ny, off_x, off_y, w, h);+ n_patches += h * (w + 1);}} break;default:@@ -5505,10 +5507,18 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {return idx;};+ // the same permutation is applied to every tile of the stacked imageauto upload = [&](const std::string & name, const std::vector<int32_t> & idx) {ggml_tensor * t = ggml_graph_get_tensor(gf, name.c_str());GGML_ASSERT(t);- ggml_backend_tensor_set(t, idx.data(), 0, idx.size() * sizeof(int32_t));+ GGML_ASSERT(ggml_nelements(t) % (int64_t) idx.size() == 0);+ const int n_rep = ggml_nelements(t) / idx.size();+ std::vector<int32_t> buf;+ buf.reserve(idx.size() * n_rep);+ for (int i = 0; i < n_rep; ++i) {+ buf.insert(buf.end(), idx.begin(), idx.end());+ }+ ggml_backend_tensor_set(t, buf.data(), 0, ggml_nbytes(t));};// Stage 1b only uses block 0's permutations; future stagesdiff --git a/tools/mtmd/models/granite4-vision.cpp b/tools/mtmd/models/granite4-vision.cppindex 1b252543c0..a75f1cee9a 100644--- a/tools/mtmd/models/granite4-vision.cpp+++ b/tools/mtmd/models/granite4-vision.cpp@@ -14,18 +14,39 @@* Stage 1a: SigLIP vision tower (N layers, post-norm)* Stage 1b: WindowQFormer blocks (deepstack + spatial)* Stage 1c: Concatenate and pack outputs- * Stage 1d: Append newline tokens if add_newline is set+ * Stage 1d: Assemble the anyres tiles into one token sequence*/// ---------------------------------------------------------------------------// Member method implementations// ---------------------------------------------------------------------------+// split the stacked tiles into the batch axis, then run the usual patch embedding+ggml_tensor * clip_graph_granite4_vision::build_tile_inp() {+ ggml_tensor * inp_raw = build_inp_raw();++ if (n_tiles > 1) {+ const int px = img.nx();+ inp_raw = ggml_reshape_4d(ctx0, inp_raw, px * px, n_tiles, 3, 1);+ inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 2, 1, 3));+ inp_raw = ggml_reshape_4d(ctx0, inp_raw, px, px, 3, n_tiles);+ }++ ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);+ inp = ggml_reshape_3d(ctx0, inp, tile_side * tile_side, n_embd, n_tiles);+ inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));+ if (model.patch_bias) {+ inp = ggml_add(ctx0, inp, model.patch_bias);+ }+ return inp;+}+ggml_tensor * clip_graph_granite4_vision::gather(ggml_tensor * src,const std::string & name,int idx_len) {- ggml_tensor * idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, idx_len);+ // one index row per tile, all rows hold the same permutation+ ggml_tensor * idx = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, idx_len, n_tiles);ggml_set_name(idx, name.c_str());ggml_set_input(idx);return ggml_get_rows(ctx0, src, idx);@@ -36,12 +57,15 @@ ggml_tensor * clip_graph_granite4_vision::interp_down(int side,int new_side) {const int n_embd = src->ne[0];- ggml_tensor * t = ggml_reshape_4d(ctx0, src, n_embd, side, side, 1);+ ggml_tensor * t = ggml_reshape_4d(ctx0, src, n_embd, side, side, n_tiles);t = ggml_cont(ctx0, ggml_permute(ctx0, t, 2, 0, 1, 3));+ // fold the tile axis into the channel axis, ggml_pool_2d only pools the first two axes+ t = ggml_reshape_3d(ctx0, t, side, side, n_embd * n_tiles);const int kernel = side / new_side;t = ggml_pool_2d(ctx0, t, GGML_OP_POOL_AVG, kernel, kernel, kernel, kernel, 0, 0);+ t = ggml_reshape_4d(ctx0, t, new_side, new_side, n_embd, n_tiles);t = ggml_cont(ctx0, ggml_permute(ctx0, t, 1, 2, 0, 3));- return ggml_reshape_2d(ctx0, t, n_embd, new_side * new_side);+ return ggml_reshape_3d(ctx0, t, n_embd, new_side * new_side, n_tiles);}// ---------------------------------------------------------------------------@@ -63,6 +87,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block(const int n = image_side / window_side;const int new_side = n * query_side;const int n_windows = n * n;+ const int n_win_all = n_windows * n_tiles; // windows of every tile, batched togetherconst int enc_len = window_side * window_side;const int query_len = query_side * query_side;@@ -82,7 +107,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block(ggml_tensor * enc_flat = gather(x,"g4v_blk" + std::to_string(bid) + "_win_idx",image_side * image_side);- enc = ggml_reshape_3d(ctx0, enc_flat, n_embd, enc_len, n_windows);+ enc = ggml_reshape_3d(ctx0, enc_flat, n_embd, enc_len, n_win_all);}cbx(enc, "enc");@@ -104,7 +129,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block(ggml_tensor * dw_flat = gather(d,"g4v_blk" + std::to_string(bid) + "_qwin_idx",new_side * new_side);- ggml_tensor * dw = ggml_reshape_3d(ctx0, dw_flat, n_embd, query_len, n_windows);+ ggml_tensor * dw = ggml_reshape_3d(ctx0, dw_flat, n_embd, query_len, n_win_all);q_in = ggml_add(ctx0, dw, blk.qf_proj_query);}cbx(q_in, "query_embeds");@@ -140,12 +165,12 @@ ggml_tensor * clip_graph_granite4_vision::build_block(ggml_tensor * K = linear(q, pl.k_w, pl.k_b);ggml_tensor * V = linear(q, pl.v_w, pl.v_b);- Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_windows);- K = ggml_reshape_4d(ctx0, K, d_h, n_head, nq, n_windows);- V = ggml_reshape_4d(ctx0, V, d_h, n_head, nq, n_windows);+ Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_win_all);+ K = ggml_reshape_4d(ctx0, K, d_h, n_head, nq, n_win_all);+ V = ggml_reshape_4d(ctx0, V, d_h, n_head, nq, n_win_all);sa_out = build_attn(pl.o_w, pl.o_b, Q, K, V, nullptr, scale, bid);- sa_out = ggml_reshape_3d(ctx0, sa_out, n_embd, nq, n_windows);+ sa_out = ggml_reshape_3d(ctx0, sa_out, n_embd, nq, n_win_all);sa_out = ggml_add(ctx0, sa_out, q);sa_out = build_norm(sa_out, pl.ln_1_w, pl.ln_1_b,@@ -166,13 +191,13 @@ ggml_tensor * clip_graph_granite4_vision::build_block(ggml_tensor * K = linear(e_in, pl.cross_attn_k_w, pl.cross_attn_k_b);ggml_tensor * V = linear(e_in, pl.cross_attn_v_w, pl.cross_attn_v_b);- Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_windows);- K = ggml_reshape_4d(ctx0, K, d_h, n_head, nkv, n_windows);- V = ggml_reshape_4d(ctx0, V, d_h, n_head, nkv, n_windows);+ Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_win_all);+ K = ggml_reshape_4d(ctx0, K, d_h, n_head, nkv, n_win_all);+ V = ggml_reshape_4d(ctx0, V, d_h, n_head, nkv, n_win_all);ca_out = build_attn(pl.cross_attn_o_w, pl.cross_attn_o_b,Q, K, V, nullptr, scale, bid);- ca_out = ggml_reshape_3d(ctx0, ca_out, n_embd, nq, n_windows);+ ca_out = ggml_reshape_3d(ctx0, ca_out, n_embd, nq, n_win_all);ca_out = ggml_add(ctx0, ca_out, sa_out);ca_out = build_norm(ca_out, pl.cross_attn_norm_w, pl.cross_attn_norm_b,@@ -183,13 +208,13 @@ ggml_tensor * clip_graph_granite4_vision::build_block(// 6c. FFNggml_tensor * ffn;{- ggml_tensor * t = ggml_reshape_2d(ctx0, ca_out, n_embd, query_len * n_windows);+ ggml_tensor * t = ggml_reshape_2d(ctx0, ca_out, n_embd, query_len * n_win_all);t = build_mm(pl.ff_up_w, t);if (pl.ff_up_b) t = ggml_add(ctx0, t, pl.ff_up_b);t = ggml_gelu_erf(ctx0, t);t = build_mm(pl.ff_down_w, t);if (pl.ff_down_b) t = ggml_add(ctx0, t, pl.ff_down_b);- t = ggml_reshape_3d(ctx0, t, n_embd, query_len, n_windows);+ t = ggml_reshape_3d(ctx0, t, n_embd, query_len, n_win_all);ffn = ggml_add(ctx0, t, ca_out);ffn = build_norm(ffn, pl.ln_2_w, pl.ln_2_b, NORM_TYPE_NORMAL, qformer_eps, bid);}@@ -198,7 +223,7 @@ ggml_tensor * clip_graph_granite4_vision::build_block(// 7. _unwin back to rasterggml_tensor * unwinned;{- ggml_tensor * flat = ggml_reshape_2d(ctx0, ffn, n_embd, query_len * n_windows);+ ggml_tensor * flat = ggml_reshape_3d(ctx0, ffn, n_embd, query_len * n_windows, n_tiles);unwinned = gather(flat,"g4v_blk" + std::to_string(bid) + "_unwin_idx",new_side * new_side);@@ -244,13 +269,42 @@ ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0)return ggml_reshape_2d(ctx0, nl_row_2d, n_mmproj_embd, 1);}-// Append a single newline row at the end of the tile output.-ggml_tensor * clip_graph_granite4_vision::append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output) {- // For the single-tile case, append one newline row at the end.- // For the multi-tile rowwise case, this will be called per-tile- // (though currently only the single-tile path uses it).- ggml_tensor * nl_row = build_newline_row(ctx0);- return ggml_concat(ctx0, tile_output, nl_row, 1);+// Assemble [overview, tile(0,0), tile(0,1), ...] into one token sequence:+// the overview tokens first, then the tile grid read in raster order with one newline per row.+// ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L266+ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cur, int out_side) {+ const int n_dim = cur->ne[0];+ const int grid_x = anyres.grid_x;+ const int grid_y = anyres.grid_y;+ const int cur_w = grid_x * out_side;+ const int cur_h = grid_y * out_side;+ GGML_ASSERT(cur->ne[1] == out_side * out_side);+ GGML_ASSERT(cur->ne[2] == 1 + grid_x * grid_y);++ ggml_tensor * base = ggml_view_2d(ctx0, cur, n_dim, out_side * out_side, cur->nb[1], 0);++ ggml_tensor * tiles = ggml_view_3d(ctx0, cur, n_dim, out_side * out_side, grid_x * grid_y,+ cur->nb[1], cur->nb[2], cur->nb[2]);++ // (n_dim*out_side, out_side, grid_x, grid_y) -> interleave the tiles of a grid row+ tiles = ggml_reshape_4d(ctx0, tiles, n_dim * out_side, out_side, grid_x, grid_y);+ tiles = ggml_cont(ctx0, ggml_permute(ctx0, tiles, 0, 2, 1, 3));+ tiles = ggml_reshape_3d(ctx0, tiles, n_dim, cur_w, cur_h);++ // drop the tokens that only cover the padding added when resizing to the grid+ int off_x, off_y, w, h;+ clip_anyres_unpad(cur_w, cur_h, anyres.orig_nx, anyres.orig_ny, off_x, off_y, w, h);+ if (w != cur_w || h != cur_h) {+ tiles = ggml_cont(ctx0, ggml_view_3d(ctx0, tiles, n_dim, w, h,+ tiles->nb[1], tiles->nb[2],+ off_x * tiles->nb[1] + off_y * tiles->nb[2]));+ }++ ggml_tensor * nl = ggml_repeat_4d(ctx0, build_newline_row(ctx0), n_dim, 1, h, 1);+ tiles = ggml_concat(ctx0, tiles, nl, 1);+ tiles = ggml_reshape_2d(ctx0, tiles, n_dim, (w + 1) * h);++ return ggml_concat(ctx0, base, tiles, 1);}ggml_cgraph * clip_graph_granite4_vision::build() {@@ -260,10 +314,12 @@ ggml_cgraph * clip_graph_granite4_vision::build() {GGML_ASSERT(!model.qf_proj_blocks.empty());// --- Stage 1a: SigLIP encoder producing intermediate hidden states ---- ggml_tensor * inp = build_inp();+ ggml_tensor * inp = build_tile_inp();inp = ggml_add(ctx0, inp, model.position_embeddings);cb(inp, "pos_embed", -1);+ const int tile_n_patches = tile_side * tile_side;+ggml_tensor * inpL = inp;std::vector<ggml_tensor *> layer_outs(n_layer, nullptr);@@ -281,12 +337,13 @@ ggml_cgraph * clip_graph_granite4_vision::build() {ggml_tensor * Vcur = build_mm(layer.v_w, cur);if (layer.v_b) Vcur = ggml_add(ctx0, Vcur, layer.v_b);- Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_patches);- Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_patches);- Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_patches);+ Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, tile_n_patches, n_tiles);+ Kcur = ggml_reshape_4d(ctx0, Kcur, d_head, n_head, tile_n_patches, n_tiles);+ Vcur = ggml_reshape_4d(ctx0, Vcur, d_head, n_head, tile_n_patches, n_tiles);cur = build_attn(layer.o_w, layer.o_b,Qcur, Kcur, Vcur, nullptr, kq_scale, il);+ cur = ggml_reshape_3d(ctx0, cur, n_embd, tile_n_patches, n_tiles);cur = ggml_add(ctx0, cur, inpL);inpL = cur;@@ -318,7 +375,7 @@ ggml_cgraph * clip_graph_granite4_vision::build() {ggml_tensor * stream = build_block(blk, h, bid,hparams.proj_spatial_offsets[bid],- n_patches_x,+ tile_side,hparams.downsample_window_side,hparams.downsample_query_side,qformer_eps);@@ -326,10 +383,11 @@ ggml_cgraph * clip_graph_granite4_vision::build() {mmproj = mmproj ? ggml_concat(ctx0, mmproj, stream, 0) : stream;}- // --- Stage 1d: Append newline tokens if add_newline is set ---- if (add_newline) {- mmproj = append_rowwise_newlines(ctx0, mmproj);- ggml_set_name(mmproj, "g4v_mmproj_out_nl");+ // --- Stage 1d: assemble the tiles and weave in the newline tokens ---+ if (anyres.is_tiled()) {+ const int out_side = tile_side / hparams.downsample_window_side * hparams.downsample_query_side;+ mmproj = build_anyres_assembly(mmproj, out_side);+ ggml_set_name(mmproj, "g4v_mmproj_out_anyres");} else {ggml_set_name(mmproj, "g4v_mmproj_out");}diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.hindex ed8c1ea518..3631d849b6 100644--- a/tools/mtmd/models/models.h+++ b/tools/mtmd/models/models.h@@ -402,16 +402,19 @@ struct clip_graph_exaone4_5 : clip_graph {struct clip_graph_granite4_vision : clip_graph {clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img): clip_graph(ctx, img),- add_newline(img.add_newline) {}+ anyres(img.anyres),+ n_tiles(img.ny() / img.nx()),+ tile_side(img.nx() / patch_size) {}ggml_cgraph * build() override;private:- // The graph is per-tile since only batch-size 1 is supported in clip. As- // such, this value is set at construct time based on the tile that will be- // encoded, then used during build to determine how to handle newlines.- const bool add_newline;+ // the input image is a stack of tiles on the Y axis: [overview, tile(0,0), tile(0,1), ...]+ const clip_image_f32::anyres_info anyres;+ const int n_tiles;+ const int tile_side; // patches per tile side+ ggml_tensor * build_tile_inp();ggml_tensor * gather(ggml_tensor * src, const std::string & name, int idx_len);ggml_tensor * interp_down(ggml_tensor * src, int side, int new_side);ggml_tensor * build_block(const qf_block & blk, ggml_tensor * h, int bid,@@ -419,7 +422,7 @@ private:int query_side, float qformer_eps);ggml_tensor * build_newline_row(ggml_context * ctx0);- ggml_tensor * append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output);+ ggml_tensor * build_anyres_assembly(ggml_tensor * cur, int out_side);};struct clip_graph_muse_glimmer : clip_graph {diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cppindex 02a0a29cd8..e795b47e06 100644--- a/tools/mtmd/mtmd-image.cpp+++ b/tools/mtmd/mtmd-image.cpp@@ -1602,17 +1602,50 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im}mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) {- auto output = mtmd_image_preprocessor_llava_uhd::preprocess(img);- if (output.entries.size() == 0) {- // Single-tile (overview only): append one newline row.- output.overview.add_newline = true;- } else {- // Multi-tile: overview gets no newline, grid tiles get one.- output.overview.add_newline = false;- for (size_t i = 0; i < output.entries.size(); ++i) {- output.entries[i].add_newline = true;+ GGML_ASSERT(!hparams.image_res_candidates.empty());++ const clip_image_size orig_size = img.get_size();+ const int tile_size = hparams.image_size;++ // llava-next always encodes an overview plus a grid of tiles, even for small images+ const clip_image_size refined_size = select_best_resolution(orig_size, hparams.image_res_candidates);+ const int grid_x = refined_size.width / tile_size;+ const int grid_y = refined_size.height / tile_size;++ clip_image_u8 overview;+ img_tool::resize(img, overview, {tile_size, tile_size}, hparams.image_resize_algo_ov,+ hparams.image_pad_ov, hparams.image_pad_color_ov);++ clip_image_u8 refined;+ img_tool::resize(img, refined, refined_size, hparams.image_resize_algo_rf,+ hparams.image_pad_rf, hparams.image_pad_color_rf);++ // stack the overview and the tiles on the Y axis, so the whole grid goes through one graph+ clip_image_u8 stacked;+ stacked.set_size({tile_size, tile_size * (1 + grid_x * grid_y)}, false);+ auto copy_tile = [&](const clip_image_u8 & src, int src_x, int src_y, int dst_idx) {+ for (int py = 0; py < tile_size; py++) {+ for (int px = 0; px < tile_size; px++) {+ stacked.set_pixel(px, dst_idx * tile_size + py, src.get_pixel(src_x + px, src_y + py));+ }+ }+ };+ copy_tile(overview, 0, 0, 0);+ for (int ty = 0; ty < grid_y; ty++) {+ for (int tx = 0; tx < grid_x; tx++) {+ copy_tile(refined, tx * tile_size, ty * tile_size, 1 + ty * grid_x + tx);}}++ LOG_DBG("%s: grid size: %d x %d (%d tiles) + overview\n", __func__, grid_x, grid_y, grid_x * grid_y);++ mtmd_image_preproc_out output;+ output.append(hparams, stacked, true);+ auto & entry = output.entries.back();+ entry.anyres.grid_x = grid_x;+ entry.anyres.grid_y = grid_y;+ entry.anyres.orig_nx = orig_size.width;+ entry.anyres.orig_ny = orig_size.height;return output;}diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.hindex 0669aa1129..2383329344 100644--- a/tools/mtmd/mtmd-image.h+++ b/tools/mtmd/mtmd-image.h@@ -85,9 +85,6 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {protected:clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false);-private:- clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max);-/*** Selects the best resolution from a list of possible resolutions based on the original size.*@@ -104,6 +101,9 @@ private:* @return The best fit resolution*/clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions);++private:+ clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max);int ensure_divide(int length, int patch_size);clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false);clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio);@@ -225,7 +225,7 @@ struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor {mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;};-// similar to llava_uhd, but has add_newline+// llava-next "anyres": stacks the overview and all tiles into one image, assembled by clip in a single graphstruct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd {mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex 4b9c45d626..6596b2785f 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -891,10 +891,10 @@ struct mtmd_context {} break;case PROJECTOR_TYPE_GRANITE4_VISION:{- img_beg = "<image>";- img_end = "";+ // ... (image embeddings) \n ...+ img_beg = "";+ img_end = "\n";image_preproc = std::make_unique<mtmd_image_preprocessor_granite>(ctx_v);- ov_img_first = true;} break;default:throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj));