LLM inference in C/C++
C++ ggml
model: Add EXAONE 4.5 implementations (#21733)
Junwon Hwang48b88c3b0057
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex cfaa24ba1a..2220057402 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -58,6 +58,7 @@ TEXT_MODEL_MAP: dict[str, str] = {"Ernie4_5_ForCausalLM": "ernie","Ernie4_5_MoeForCausalLM": "ernie","EuroBertModel": "bert",+ "Exaone4_5_ForConditionalGeneration": "exaone","Exaone4ForCausalLM": "exaone","ExaoneForCausalLM": "exaone","ExaoneMoEForCausalLM": "exaone",@@ -240,6 +241,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {"DeepseekOCR2ForCausalLM": "deepseek","DeepseekOCRForCausalLM": "deepseek","DotsOCRForCausalLM": "dotsocr",+ "Exaone4_5_ForConditionalGeneration": "exaone","Gemma3ForConditionalGeneration": "gemma","Gemma3nForConditionalGeneration": "gemma","Gemma4ForConditionalGeneration": "gemma",diff --git a/conversion/base.py b/conversion/base.pyindex 18c3ddde22..55682c827d 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -2593,7 +2593,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st# Step3-VL keeps text config under text_config but uses a custom top-level architecture.# For text conversion we route to a dedicated text-only class.# TODO: refactor this later to avoid adding exception here- if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM"):+ if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration"):return arch# if "architectures" is found in the sub-config, use that insteaddiff --git a/conversion/exaone.py b/conversion/exaone.pyindex aa1313e2f8..b21f027842 100644--- a/conversion/exaone.py+++ b/conversion/exaone.py@@ -3,14 +3,15 @@ from __future__ import annotationsimport mathfrom pathlib import Path-from typing import Iterable, TYPE_CHECKING+from typing import Callable, Iterable, TYPE_CHECKINGimport torchif TYPE_CHECKING:from torch import Tensor-from .base import ModelBase, TextModel, gguf+from .base import MmprojModel, ModelBase, TextModel, gguf+from .qwenvl import Qwen2VLVisionModel@ModelBase.register("ExaoneForCausalLM")@@ -208,3 +209,97 @@ class ExaoneMoEModel(Exaone4Model):experts = [k for d in self._experts for k in d.keys()]if len(experts) > 0:raise ValueError(f"Unprocessed experts: {experts}")++[email protected]("Exaone4_5_ForConditionalGeneration")+class Exaone4_5_TextModel(Exaone4Model):+ """Text tower of EXAONE 4.5; Tensors match EXAONE4"""++ model_arch = gguf.MODEL_ARCH.EXAONE4++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ n_nextn = int(self.hparams.get("num_nextn_predict_layers", 0) or 0)+ if n_nextn > 0:+ self.block_count = self.hparams["num_hidden_layers"] + n_nextn+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ n_nextn = int(self.hparams.get("num_nextn_predict_layers", 0) or 0)+ if n_nextn > 0:+ self.gguf_writer.add_nextn_predict_layers(n_nextn)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if name.startswith("mtp."):+ n_nextn = int(self.hparams.get("num_nextn_predict_layers", 0) or 0)+ if n_nextn <= 0:+ return+ nh = self.hparams["num_hidden_layers"]+ if ".layers." in name:+ share = self.hparams.get("mtp_share_layers", False)+ mtp_bid = bid if bid is not None else 0+ if share:+ for k in range(n_nextn):+ nn = name.replace(f"mtp.layers.{mtp_bid}", f"model.layers.{nh + k}")+ yield from super().modify_tensors(data_torch, nn, nh + k)+ return+ name = name.replace(f"mtp.layers.{mtp_bid}", f"model.layers.{mtp_bid + nh}")+ else:+ remapper = {+ "mtp.fc": gguf.MODEL_TENSOR.NEXTN_EH_PROJ,+ "mtp.pre_fc_norm_embedding": gguf.MODEL_TENSOR.NEXTN_ENORM,+ "mtp.pre_fc_norm_hidden": gguf.MODEL_TENSOR.NEXTN_HNORM,+ "mtp.norm": gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ }+ _n = Path(name)+ key = _n.stem+ if key not in remapper:+ return+ for bid_mtp in range(nh, self.block_count):+ mapped_name = self.format_tensor_name(remapper[key], bid_mtp, suffix=_n.suffix)+ yield from ModelBase.modify_tensors(self, data_torch, mapped_name, bid_mtp)+ return++ yield from super().modify_tensors(data_torch, name, bid)++[email protected]("Exaone4_5_ForConditionalGeneration")+class Exaone4_5VisionModel(Qwen2VLVisionModel):+ """Vision tower for EXAONE 4.5; Qwen2-VL-style ViT (GQA) + patch merger"""++ @classmethod+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+ name, gen = item+ name = name.replace("model.visual.", "visual.", 1)+ return super().filter_tensors((name, gen))++ def set_gguf_parameters(self):+ MmprojModel.set_gguf_parameters(self)+ assert self.hparams_vision is not None+ hparams = self.hparams_vision+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.EXAONE4_5)+ self.gguf_writer.add_vision_use_silu(True)+ self.gguf_writer.add_vision_min_pixels(self.preprocessor_config["min_pixels"])+ self.gguf_writer.add_vision_max_pixels(self.preprocessor_config["max_pixels"])+ num_kv_head = self.find_vparam(["num_key_value_heads"], optional=True)+ if num_kv_head is not None:+ self.gguf_writer.add_vision_head_count_kv(num_kv_head)+ eps = hparams.get("rms_norm_eps", self.global_config.get("rms_norm_eps", 1e-6))+ self.gguf_writer.add_vision_attention_layernorm_eps(eps)+ if (window_size := hparams.get("window_size")) is not None:+ self.gguf_writer.add_vision_window_size(window_size)+ fullatt_block_indexes = hparams.get("fullatt_block_indexes")+ if fullatt_block_indexes:+ n_wa_pattern = fullatt_block_indexes[0] + 1+ for i in range(1, len(fullatt_block_indexes)):+ if fullatt_block_indexes[i] - fullatt_block_indexes[i - 1] != n_wa_pattern:+ raise ValueError(f"Invalid EXAONE4.5 fullatt_block_indexes: {fullatt_block_indexes}")+ self.gguf_writer.add_vision_n_wa_pattern(n_wa_pattern)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if ".qkv." in name:+ yield from ModelBase.modify_tensors(self, data_torch, name, bid)+ return++ yield from Qwen2VLVisionModel.modify_tensors(self, data_torch, name, bid)diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex b4dfd58382..8aed0d7667 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -3310,6 +3310,13 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.FFN_DOWN,MODEL_TENSOR.FFN_UP,MODEL_TENSOR.FFN_POST_NORM,+ # NextN/MTP tensors - preserved but unused+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,],MODEL_ARCH.EXAONE_MOE: [MODEL_TENSOR.TOKEN_EMBD,@@ -4318,6 +4325,7 @@ class VisionProjectorType:LLAMA4 = "llama4"QWEN2VL = "qwen2vl_merger"QWEN25VL = "qwen2.5vl_merger"+ EXAONE4_5 = "exaone4_5"QWEN3VL = "qwen3vl_merger"STEP3VL = "step3vl"ULTRAVOX = "ultravox"diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cppindex 499e22dde8..b5030eb054 100644--- a/src/models/exaone4.cpp+++ b/src/models/exaone4.cpp@@ -15,6 +15,9 @@ void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+ ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);+ GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");+ hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;switch (hparams.n_layer) {case 30: type = LLM_TYPE_1_2B; break;@@ -38,21 +41,37 @@ void llama_model_exaone4::load_arch_tensors(llama_model_loader &) {}for (int i = 0; i < n_layer; ++i) {+ const bool is_nextn = hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers;+ int flags = 0;+ if (is_nextn) {+ // NextN/MTP layers are preserved in GGUF but are not executed yet.+ flags |= TENSOR_SKIP;+ }+auto & layer = layers[i];- create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);- layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, flags);+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, flags);++ if (!is_nextn) {+ layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));+ }- layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));+ layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, flags);+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, flags);+ layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, flags);- layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);- layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);- layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, flags);+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, flags);+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, flags);+ layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, flags);- layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);- layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);- layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);- layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0);+ if (is_nextn) {+ layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, flags);+ layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, flags);+ layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, flags);+ layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd}, flags | TENSOR_NOT_REQUIRED);+ }}}@@ -90,7 +109,11 @@ llama_model_exaone4::graph<iswa>::graph(const llama_model & model, const llm_gra}ggml_tensor * inp_out_ids = build_inp_out_ids();- for (int il = 0; il < n_layer; ++il) {+ // MTP / NextN tail blocks are loaded for compatibility but not executed (same as exaone-moe).+ const int n_layer_main = int(n_layer) - int(hparams.nextn_predict_layers);+ GGML_ASSERT(n_layer_main > 0);++ for (int il = 0; il < n_layer_main; ++il) {ggml_tensor * inpSA = inpL;// use RoPE for SWA layers or non-SWA models@@ -126,7 +149,7 @@ llama_model_exaone4::graph<iswa>::graph(const llama_model & model, const llm_graQcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);cb(cur, "attn_out", il);}- if (il == n_layer - 1 && inp_out_ids) {+ if (il == n_layer_main - 1 && inp_out_ids) {cur = ggml_get_rows(ctx0, cur, inp_out_ids);inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);}diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txtindex 14808d4221..61510747cc 100644--- a/tools/mtmd/CMakeLists.txt+++ b/tools/mtmd/CMakeLists.txt@@ -18,6 +18,7 @@ add_library(mtmdmodels/cogvlm.cppmodels/conformer.cppmodels/dotsocr.cpp+ models/exaone4_5.cppmodels/gemma4a.cppmodels/gemma4v.cppmodels/glm4v.cppdiff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.hindex 14398dc486..b7fdcc42f1 100644--- a/tools/mtmd/clip-impl.h+++ b/tools/mtmd/clip-impl.h@@ -347,6 +347,7 @@ enum projector_type {PROJECTOR_TYPE_KIMIK25,PROJECTOR_TYPE_NEMOTRON_V2_VL,PROJECTOR_TYPE_HUNYUANVL,+ PROJECTOR_TYPE_EXAONE4_5,PROJECTOR_TYPE_MINICPMV4_6,PROJECTOR_TYPE_GRANITE_SPEECH,PROJECTOR_TYPE_MIMOVL,@@ -396,6 +397,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {{ PROJECTOR_TYPE_YASA2, "yasa2"},{ PROJECTOR_TYPE_KIMIK25, "kimik25"},{ PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},+ { PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"},{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 7bb702b95c..3eeda41155 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -884,6 +884,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32{builder = std::make_unique<clip_graph_qwen3vl>(ctx, img);} break;+ case PROJECTOR_TYPE_EXAONE4_5:+ {+ builder = std::make_unique<clip_graph_exaone4_5>(ctx, img);+ } break;case PROJECTOR_TYPE_MIMOVL:{builder = std::make_unique<clip_graph_mimovl>(ctx, img);@@ -1556,6 +1560,19 @@ struct clip_model_loader {hparams.audio_window_len = 400;hparams.audio_hop_len = 160;} break;+ case PROJECTOR_TYPE_EXAONE4_5:+ {+ hparams.n_merge = 2;+ get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);+ get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false);+ get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);+ get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);+ hparams.set_warmup_n_tokens(46 * 46);+ if (hparams.rope_theta <= 0.0f) {+ hparams.rope_theta = 10000.0f;+ }+ get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);+ } break;case PROJECTOR_TYPE_GEMMA4A:{// Gemma4 feature_extraction_gemma4.py:@@ -1799,6 +1816,7 @@ struct clip_model_loader {|| model.proj_type == PROJECTOR_TYPE_LDPV2|| model.proj_type == PROJECTOR_TYPE_QWEN2VL|| model.proj_type == PROJECTOR_TYPE_QWEN25VL+ || model.proj_type == PROJECTOR_TYPE_EXAONE4_5|| model.proj_type == PROJECTOR_TYPE_GLM_EDGE|| model.proj_type == PROJECTOR_TYPE_GEMMA3|| model.proj_type == PROJECTOR_TYPE_IDEFICS3@@ -1948,6 +1966,7 @@ struct clip_model_loader {} break;case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:+ case PROJECTOR_TYPE_EXAONE4_5:{model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));@@ -3079,6 +3098,7 @@ int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 *case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_EXAONE4_5:case PROJECTOR_TYPE_MIMOVL:case PROJECTOR_TYPE_GLM4V:case PROJECTOR_TYPE_PADDLEOCR:@@ -3100,6 +3120,7 @@ int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 *case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_EXAONE4_5:case PROJECTOR_TYPE_MIMOVL:case PROJECTOR_TYPE_GLM4V:case PROJECTOR_TYPE_PADDLEOCR:@@ -3179,6 +3200,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * imcase PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_EXAONE4_5:case PROJECTOR_TYPE_MIMOVL:case PROJECTOR_TYPE_GLM4V:case PROJECTOR_TYPE_YOUTUVL:@@ -3675,11 +3697,15 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_imaset_input_i32("positions", positions);} break;case PROJECTOR_TYPE_QWEN25VL:+ case PROJECTOR_TYPE_EXAONE4_5:case PROJECTOR_TYPE_YOUTUVL:{// pw * ph = number of tokens output by ViT after apply patch merger// ipw * ipw = number of vision token been processed inside ViT- const bool use_window_attn = ctx->model.proj_type == PROJECTOR_TYPE_QWEN25VL ? hparams.n_wa_pattern > 0 : !hparams.wa_layer_indexes.empty();+ const bool use_window_attn =+ (ctx->model.proj_type == PROJECTOR_TYPE_QWEN25VL || ctx->model.proj_type == PROJECTOR_TYPE_EXAONE4_5)+ ? hparams.n_wa_pattern > 0+ : !hparams.wa_layer_indexes.empty();const int merge_ratio = 2;const int pw = image_size_width / patch_size / merge_ratio;const int ph = image_size_height / patch_size / merge_ratio;@@ -4262,6 +4288,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {return ctx->model.mm_model_mlp_3_w->ne[1];case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:+ case PROJECTOR_TYPE_EXAONE4_5:case PROJECTOR_TYPE_JANUS_PRO:case PROJECTOR_TYPE_YOUTUVL:return ctx->model.mm_1_b->ne[0];diff --git a/tools/mtmd/models/exaone4_5.cpp b/tools/mtmd/models/exaone4_5.cppnew file mode 100644index 0000000000..7bfbaca996--- /dev/null+++ b/tools/mtmd/models/exaone4_5.cpp@@ -0,0 +1,170 @@+// similar to qwen2vl, except for GQA attention+#include "models.h"++ggml_cgraph * clip_graph_exaone4_5::build() {+ GGML_ASSERT(model.patch_bias == nullptr);+ GGML_ASSERT(model.class_embedding == nullptr);++ const int batch_size = 1;+ const bool use_window_attn = hparams.n_wa_pattern > 0;+ const int n_wa_pattern = hparams.n_wa_pattern;+ const int n_pos = n_patches;+ const int num_position_ids = n_pos * 4;++ const norm_type norm_t = NORM_TYPE_RMS;++ const int64_t n_kv_head = hparams.n_head_kv > 0 ? hparams.n_head_kv : n_head;+ GGML_ASSERT(n_head % n_kv_head == 0);++ int rope_sections[4] = { d_head / 4, d_head / 4, d_head / 4, d_head / 4 };+ const float rope_freq_base = hparams.rope_theta > 0.0f ? hparams.rope_theta : 10000.0f;++ ggml_tensor * inp_raw = build_inp_raw();+ ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);++ GGML_ASSERT(img.nx % (patch_size * 2) == 0);+ GGML_ASSERT(img.ny % (patch_size * 2) == 0);++ {+ ggml_tensor * inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);+ inp = ggml_add(ctx0, inp, inp_1);+ inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);+ inp = ggml_cont_4d(+ ctx0, inp,+ n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);+ inp = ggml_reshape_4d(+ ctx0, inp,+ n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));+ inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);+ inp = ggml_cont_3d(+ ctx0, inp,+ n_embd, n_patches_x * n_patches_y, batch_size);+ }++ ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);+ ggml_set_name(positions, "positions");+ ggml_set_input(positions);++ ggml_tensor * window_mask = nullptr;+ ggml_tensor * window_idx = nullptr;+ ggml_tensor * inv_window_idx = nullptr;++ if (use_window_attn) {+ window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);+ ggml_set_name(window_idx, "window_idx");+ ggml_set_input(window_idx);++ inv_window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);+ ggml_set_name(inv_window_idx, "inv_window_idx");+ ggml_set_input(inv_window_idx);++ window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);+ ggml_set_name(window_mask, "window_mask");+ ggml_set_input(window_mask);++ if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {+ window_mask = ggml_cast(ctx0, window_mask, GGML_TYPE_F16);+ }+ }++ ggml_tensor * inpL = inp;++ if (use_window_attn) {+ GGML_ASSERT(batch_size == 1);+ inpL = ggml_reshape_2d(ctx0, inpL, n_embd * 4, n_patches_x * n_patches_y * batch_size / 4);+ inpL = ggml_get_rows(ctx0, inpL, inv_window_idx);+ inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_patches_x * n_patches_y, batch_size);+ }++ for (int il = 0; il < n_layer; il++) {+ const auto & layer = model.layers[il];+ const bool full_attn = use_window_attn ? (il + 1) % n_wa_pattern == 0 : true;+ ggml_tensor * cur = inpL;++ cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);+ cb(cur, "ln1", il);++ {+ GGML_ASSERT(layer.qkv_w != nullptr);+ cur = build_mm(layer.qkv_w, cur);+ if (layer.qkv_b) {+ cur = ggml_add(ctx0, cur, layer.qkv_b);+ }++ const int64_t n_embd_kv = d_head * n_kv_head;+ ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_patches,+ ggml_row_size(cur->type, d_head),+ cur->nb[1],+ 0);+ ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, d_head, n_kv_head, n_patches,+ ggml_row_size(cur->type, d_head),+ cur->nb[1],+ ggml_row_size(cur->type, n_embd));+ ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, d_head, n_kv_head, n_patches,+ ggml_row_size(cur->type, d_head),+ cur->nb[1],+ ggml_row_size(cur->type, n_embd + n_embd_kv));++ cb(Qcur, "Qcur", il);+ cb(Kcur, "Kcur", il);+ cb(Vcur, "Vcur", il);++ Qcur = ggml_rope_multi(+ ctx0, Qcur, positions, nullptr,+ d_head / 2, rope_sections, GGML_ROPE_TYPE_VISION, 32768, rope_freq_base, 1, 0, 1, 32, 1);+ Kcur = ggml_rope_multi(+ ctx0, Kcur, positions, nullptr,+ d_head / 2, rope_sections, GGML_ROPE_TYPE_VISION, 32768, rope_freq_base, 1, 0, 1, 32, 1);++ cb(Qcur, "Qcur_rope", il);+ cb(Kcur, "Kcur_rope", il);+ cb(Vcur, "Vcur", il);++ ggml_tensor * attn_mask = full_attn ? nullptr : window_mask;+ cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, attn_mask, kq_scale, il);+ cb(cur, "attn_out", il);+ }++ cur = ggml_add(ctx0, cur, inpL);+ inpL = cur;++ cb(cur, "ffn_inp", il);++ cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);+ cb(cur, "ffn_inp_normed", il);++ cur = build_ffn(cur,+ layer.ff_up_w, layer.ff_up_b,+ layer.ff_gate_w, layer.ff_gate_b,+ layer.ff_down_w, layer.ff_down_b,+ hparams.ffn_op, il);++ cb(cur, "ffn_out", il);++ cur = ggml_add(ctx0, inpL, cur);+ cb(cur, "layer_out", il);++ inpL = cur;+ }++ ggml_tensor * embeddings = inpL;+ embeddings = build_norm(embeddings, model.post_ln_w, model.post_ln_b, norm_t, eps, n_layer);+ embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);+ embeddings = build_ffn(embeddings,+ model.mm_0_w, model.mm_0_b,+ nullptr, nullptr,+ model.mm_1_w, model.mm_1_b,+ FFN_GELU,+ -1);++ if (use_window_attn) {+ GGML_ASSERT(batch_size == 1);+ embeddings = ggml_reshape_2d(ctx0, embeddings, hparams.projection_dim, n_patches_x * n_patches_y / 4);+ embeddings = ggml_get_rows(ctx0, embeddings, window_idx);+ embeddings = ggml_reshape_3d(ctx0, embeddings, hparams.projection_dim, n_patches_x * n_patches_y / 4, batch_size);+ }++ ggml_build_forward_expand(gf, embeddings);++ return gf;+}diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.hindex a856882c27..da37bc6505 100644--- a/tools/mtmd/models/models.h+++ b/tools/mtmd/models/models.h@@ -196,3 +196,8 @@ struct clip_graph_kimik25 : clip_graph {ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode);};++struct clip_graph_exaone4_5 : clip_graph {+ clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}+ ggml_cgraph * build() override;+};diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex b3401634fd..94c3be82f0 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -505,6 +505,13 @@ struct mtmd_context {img_end = "<|hy_place▁holder▁no▁101|>";image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);} break;+ case PROJECTOR_TYPE_EXAONE4_5:+ {+ // <vision> ... (image embeddings) ... </vision>+ img_beg = "<vision>";+ img_end = "</vision>";+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);+ } break;default:throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj));}