LLM inference in C/C++
C++ ggml
mtmd: use ggml_rope_set_offset (#27521)
Xuan-Son Nguyenb21e4de74567
diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.hindex 2cf1b683a7..bbee35bead 100644--- a/tools/mtmd/clip-graph.h+++ b/tools/mtmd/clip-graph.h@@ -137,9 +137,15 @@ struct clip_graph {int il,ggml_tensor * sinks = nullptr) const;- // implementation of the 2D RoPE without adding a new op in ggml- // this is not efficient (use double the memory), but works on all backends- // TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065+ // implementation of the 2D RoPE using two ggml_rope_ext calls+ //+ // unlike GGML_ROPE_TYPE_VISION which forces NEOX ordering, this rotates adjacent pairs (normal ordering)+ //+ // example:+ // given a single head with size = 8 --> [00000000]+ // dims [0, 4) rotate with pos_a, dims [4, 8) rotate with pos_b --> [aaaabbbb]+ // interleave_freq = false --> both halves use the same inv_freq set (like GGML_ROPE_TYPE_VISION)+ // interleave_freq = true --> first half uses even inv_freq, second half uses odd inv_freq (used by pixtral)ggml_tensor * build_rope_2d(ggml_context * ctx0,ggml_tensor * cur,diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 9977ed4909..89ca65a7bb 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -819,8 +819,6 @@ ggml_tensor * clip_graph::build_attn(}// implementation of the 2D RoPE without adding a new op in ggml-// this is not efficient (use double the memory), but works on all backends-// TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065ggml_tensor * clip_graph::build_rope_2d(ggml_context * ctx0,ggml_tensor * cur,@@ -829,9 +827,7 @@ ggml_tensor * clip_graph::build_rope_2d(const float freq_base,const bool interleave_freq) {- const int64_t n_dim = cur->ne[0];- const int64_t n_head = cur->ne[1];- const int64_t n_pos = cur->ne[2];+ const int64_t n_dim = cur->ne[0];// for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos)// we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3@@ -845,46 +841,30 @@ ggml_tensor * clip_graph::build_rope_2d(? std::pow(freq_base, (float)-2/n_dim): 1.0;- // first half- ggml_tensor * first;- {- first = ggml_view_3d(ctx0, cur,- n_dim/2, n_head, n_pos,- cur->nb[1],- cur->nb[2],- 0);- first = ggml_rope_ext(- ctx0,- first,- pos_a, // positions- nullptr, // freq factors- n_dim/2, // n_dims- 0, 0, freq_base,- 1.0f, 0.0f, 1.0f, 0.0f, 0.0f- );- }-- // second half- ggml_tensor * second;- {- second = ggml_view_3d(ctx0, cur,- n_dim/2, n_head, n_pos,- cur->nb[1],- cur->nb[2],- n_dim/2 * ggml_element_size(cur));- second = ggml_rope_ext(- ctx0,- second,- pos_b, // positions- nullptr, // freq factors- n_dim/2, // n_dims- 0, 0, freq_base,- freq_scale_odd,- 0.0f, 1.0f, 0.0f, 0.0f- );- }+ // first half, dims [0, n_dim/2)+ cur = ggml_rope_ext(+ ctx0,+ cur,+ pos_a, // positions+ nullptr, // freq factors+ n_dim/2, // n_dims+ 0, 0, freq_base,+ 1.0f, 0.0f, 1.0f, 0.0f, 0.0f+ );++ // second half, dims [n_dim/2, n_dim)+ cur = ggml_rope_ext(+ ctx0,+ cur,+ pos_b, // positions+ nullptr, // freq factors+ n_dim/2, // n_dims+ 0, 0, freq_base,+ freq_scale_odd,+ 0.0f, 1.0f, 0.0f, 0.0f+ );+ cur = ggml_rope_set_offset(cur, n_dim/2);- cur = ggml_concat(ctx0, first, second, 0);return cur;}diff --git a/tools/mtmd/models/gemma4v.cpp b/tools/mtmd/models/gemma4v.cppindex 87cbd43fc5..4484389470 100644--- a/tools/mtmd/models/gemma4v.cpp+++ b/tools/mtmd/models/gemma4v.cpp@@ -44,51 +44,31 @@ ggml_cgraph * clip_graph_gemma4v::build() {// similar to build_rope_2d, but use neox orderingauto add_pos = [&](ggml_tensor * cur, const clip_layer &) {- const int64_t n_dim = cur->ne[0];- const int64_t n_head = cur->ne[1];- const int64_t n_pos = cur->ne[2];-- // first half- ggml_tensor * first;- {- first = ggml_view_4d(ctx0, cur,- n_dim/2, n_head, n_pos, n_batch,- cur->nb[1],- cur->nb[2],- cur->nb[3],- 0);- first = ggml_rope_ext(- ctx0,- first,- pos_x, // positions- nullptr, // freq factors- n_dim/2, // n_dims- GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,- 1.0f, 0.0f, 1.0f, 0.0f, 0.0f- );- }-- // second half- ggml_tensor * second;- {- second = ggml_view_4d(ctx0, cur,- n_dim/2, n_head, n_pos, n_batch,- cur->nb[1],- cur->nb[2],- cur->nb[3],- n_dim/2 * ggml_element_size(cur));- second = ggml_rope_ext(- ctx0,- second,- pos_y, // positions- nullptr, // freq factors- n_dim/2, // n_dims- GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,- 1.0f, 0.0f, 1.0f, 0.0f, 0.0f- );- }-- cur = ggml_concat(ctx0, first, second, 0);+ const int64_t n_dim = cur->ne[0];++ // first half, dims [0, n_dim/2)+ cur = ggml_rope_ext(+ ctx0,+ cur,+ pos_x, // positions+ nullptr, // freq factors+ n_dim/2, // n_dims+ GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,+ 1.0f, 0.0f, 1.0f, 0.0f, 0.0f+ );++ // second half, dims [n_dim/2, n_dim)+ cur = ggml_rope_ext(+ ctx0,+ cur,+ pos_y, // positions+ nullptr, // freq factors+ n_dim/2, // n_dims+ GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,+ 1.0f, 0.0f, 1.0f, 0.0f, 0.0f+ );+ cur = ggml_rope_set_offset(cur, n_dim/2);+return cur;};diff --git a/tools/mtmd/models/minimax-m3.cpp b/tools/mtmd/models/minimax-m3.cppindex 447621754e..256e531057 100644--- a/tools/mtmd/models/minimax-m3.cpp+++ b/tools/mtmd/models/minimax-m3.cpp@@ -2,30 +2,22 @@ggml_tensor * clip_graph_minimax_m3::apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) {- const int64_t Hn = x->ne[1];- const int64_t P = x->ne[2];- const size_t es = ggml_element_size(x);- const int dh = (int) x->ne[0];- const int axd = 2 * ((2 * (dh / 2) / 3) / 2);+ const int dh = (int) x->ne[0];+ const int axd = 2 * ((2 * (dh / 2) / 3) / 2);- GGML_ASSERT(x->nb[0] == es);GGML_ASSERT(3 * axd <= dh);const float th = hparams.rope_theta;// layout of x is [t, h, w, pad]// t is unrotated, h and w are rotated, pad is unrotated- // note: everything from n_dims onward untouched, so w and pad are rotated in one call.- auto sl = [&](int off, int n) {- return ggml_cont(ctx0, ggml_view_3d(ctx0, x, n, Hn, P, x->nb[1], x->nb[2], (size_t) off * es));- };- ggml_tensor * t = sl(0, axd);- ggml_tensor * h = sl(axd, axd);- ggml_tensor * w = sl(2 * axd, dh - 2 * axd); // w + pad-- h = ggml_rope_ext(ctx0, h, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);- w = ggml_rope_ext(ctx0, w, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);- return ggml_concat(ctx0, ggml_concat(ctx0, t, h, 0), w, 0);+ x = ggml_rope_ext(ctx0, x, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);+ x = ggml_rope_set_offset(x, axd);++ x = ggml_rope_ext(ctx0, x, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);+ x = ggml_rope_set_offset(x, 2 * axd);++ return x;}ggml_cgraph * clip_graph_minimax_m3::build() {