Skip to content

LLM inference in C/C++

C++ ggml

model: Add support for SenseNova U1 model (#28919)

Maphist07775ffb7e48f
diff --git a/conversion/__init__.py b/conversion/__init__.py
index bddfe3c1f5..369ce7afda 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -17,6 +17,7 @@ __all__ = [
TEXT_MODEL_MAP: dict[str, str] = {
+ "NEOChatModel": "sensenova_u1",
"AfmoeForCausalLM": "afmoe",
"LagunaForCausalLM": "laguna",
"ApertusForCausalLM": "llama",
@@ -367,6 +368,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"MuseSparkForConditionalGeneration": "muse_spark",
"PaddleOCRVisionModel": "ernie",
"Phi4ForCausalLMV": "phi",
+ "NEOVisionModel": "sensenova_u1",
"Qwen2AudioForConditionalGeneration": "ultravox",
"Qwen2VLForConditionalGeneration": "qwenvl",
"Qwen2VLModel": "qwenvl",
diff --git a/conversion/base.py b/conversion/base.py
index dec1d43222..2cbaed700f 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -3143,7 +3143,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
# Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
# Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
# latent MoE, situ, ...). Route on the top-level architecture instead.
- if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"):
+ if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"):
return arch
# if "architectures" is found in the sub-config, use that instead
diff --git a/conversion/sensenova_u1.py b/conversion/sensenova_u1.py
new file mode 100644
index 0000000000..bde5f9d9b3
--- /dev/null
+++ b/conversion/sensenova_u1.py
@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import torch
+
+from .base import MmprojModel, ModelBase, gguf
+from .qwen import Qwen2Model
+
+
[email protected]("sensenova/SenseNova-U1.5-8B-MoT")
[email protected]("NEOChatModel")
+class SenseNovaU1Model(Qwen2Model):
+ model_arch = gguf.MODEL_ARCH.SENSENOVA_U1
+
+ @classmethod
+ def filter_tensors(cls, item):
+ name, _ = item
+ if not name.startswith("language_model.") or "_mot_gen" in name:
+ return None
+ return super().filter_tensors(item)
+
+ def set_gguf_parameters(self):
+ if int(self.hparams.get("num_experts") or 0) > 1:
+ raise ValueError("SenseNova U1 conversion currently supports the dense understanding branch")
+ if self.rope_parameters.get("rope_type", "default") not in ("default", None):
+ raise ValueError("SenseNova U1 scaled RoPE is not supported")
+ if self.hparams.get("attention_bias", False):
+ raise ValueError("SenseNova U1 attention bias is not supported")
+ if self.hparams.get("use_sliding_window", False):
+ raise ValueError("SenseNova U1 sliding attention is not supported")
+ super().set_gguf_parameters()
+ self.gguf_writer.add_rope_freq_base_spatial(self.hparams["rope_theta_hw"])
+
+ def get_tensors(self):
+ for name, gen in self.model_tensors.items():
+ if name.endswith((".q_norm_hw.weight", ".k_norm_hw.weight")):
+ continue
+ data = gen()
+ if name.endswith((".q_norm.weight", ".k_norm.weight")):
+ spatial = self.model_tensors[name.replace("_norm.weight", "_norm_hw.weight")]()
+ # Keep one weight vector; the graph normalizes its two halves independently.
+ data = torch.cat((data, spatial), dim=0)
+ yield name, data
+
+
[email protected]("NEOVisionModel")
[email protected]("sensenova/SenseNova-U1.5-8B-MoT")
+class SenseNovaU1VisionModel(MmprojModel):
+ def __init__(self, dir_model: Path, *args, **kwargs):
+ hparams = kwargs.pop("hparams", None)
+ if hparams is None:
+ hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
+ else:
+ hparams = dict(hparams)
+
+ vision_config = dict(hparams["vision_config"])
+ # NEOVisionModel has no transformer blocks
+ # these defaults satisfy the generic mmproj metadata contract without replacing checkpoint values
+ vision_config.setdefault("image_size", 0)
+ vision_config.setdefault("intermediate_size", 0)
+ vision_config.setdefault("num_hidden_layers", 0)
+ vision_config.setdefault("num_attention_heads", 1)
+ hparams["vision_config"] = vision_config
+ super().__init__(dir_model, *args, hparams=hparams, **kwargs)
+
+ self.preprocessor_config.setdefault("image_mean", [0.485, 0.456, 0.406])
+ self.preprocessor_config.setdefault("image_std", [0.229, 0.224, 0.225])
+
+ @classmethod
+ def filter_tensors(cls, item):
+ name, gen = item
+ if not name.startswith("vision_model."):
+ return None
+ return super().filter_tensors((name, gen))
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_clip_projector_type("sensenova_u1")
+ vision_config = self.hparams["vision_config"]
+ # These defaults match the known U1.5 checkpoint when older configs omit fields.
+ hidden_size = int(vision_config.get("hidden_size", 1024))
+ min_pixels = int(vision_config.get("min_pixels", 65536))
+ max_pixels = int(vision_config.get("max_pixels", 16777216))
+ downsample_ratio = float(vision_config.get("downsample_ratio", 0.5))
+ if downsample_ratio <= 0:
+ raise ValueError(f"SenseNova U1 downsample_ratio must be positive, got {downsample_ratio}")
+ merge_size = round(1.0 / downsample_ratio)
+ if abs((1.0 / downsample_ratio) - merge_size) > 1e-6:
+ raise ValueError(f"SenseNova U1 downsample_ratio must have an integer reciprocal, got {downsample_ratio}")
+ norm_eps = float(vision_config.get("layer_norm_eps", vision_config.get("rms_norm_eps", 1e-6)))
+ self.gguf_writer.add_vision_head_dim(hidden_size)
+ self.gguf_writer.add_vision_attention_layernorm_eps(norm_eps)
+ self.gguf_writer.add_vision_min_pixels(min_pixels)
+ self.gguf_writer.add_vision_max_pixels(max_pixels)
+ self.gguf_writer.add_vision_spatial_merge_size(merge_size)
+
+ def modify_tensors(self, data_torch, name, bid):
+ tensor_names = {
+ "vision_model.embeddings.patch_embedding.weight": "v.patch_embd.weight",
+ "vision_model.embeddings.patch_embedding.bias": "v.patch_embd.bias",
+ "vision_model.embeddings.dense_embedding.weight": self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0),
+ "vision_model.embeddings.dense_embedding.bias": self.format_tensor_name(
+ gguf.MODEL_TENSOR.V_MMPROJ, 0, suffix=".bias"
+ ),
+ }
+ if name not in tensor_names:
+ raise ValueError(f"Unexpected SenseNova U1 vision tensor: {name!r}")
+ return [(tensor_names[name], data_torch)]
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0):
+ return gguf.GGMLQuantizationType.F16 if self.ftype == gguf.LlamaFileType.MOSTLY_F16 else gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 2840348cba..619a15fc96 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -263,6 +263,7 @@ class Keys:
DIMENSION_SECTIONS = "{arch}.rope.dimension_sections"
SECTION_ORDER = "{arch}.rope.section_order"
FREQ_BASE = "{arch}.rope.freq_base"
+ FREQ_BASE_SPATIAL = "{arch}.rope.freq_base_spatial"
FREQ_BASE_SWA = "{arch}.rope.freq_base_swa"
SCALING_TYPE = "{arch}.rope.scaling.type"
SCALING_FACTOR = "{arch}.rope.scaling.factor"
@@ -573,6 +574,7 @@ class MODEL_ARCH(IntEnum):
QWEN2 = auto()
QWEN2MOE = auto()
QWEN2VL = auto()
+ SENSENOVA_U1 = auto()
QWEN3 = auto()
QWEN3MOE = auto()
QWEN3NEXT = auto()
@@ -1390,6 +1392,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.QWEN2: "qwen2",
MODEL_ARCH.QWEN2MOE: "qwen2moe",
MODEL_ARCH.QWEN2VL: "qwen2vl",
+ MODEL_ARCH.SENSENOVA_U1: "sensenova_u1",
MODEL_ARCH.QWEN3: "qwen3",
MODEL_ARCH.QWEN3MOE: "qwen3moe",
MODEL_ARCH.QWEN3NEXT: "qwen3next",
@@ -2916,6 +2919,24 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_DOWN_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
],
+ MODEL_ARCH.SENSENOVA_U1: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
MODEL_ARCH.QWEN3: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -6485,6 +6506,7 @@ KEY_ATTENTION_LAYERNORM_RMS_EPS = Keys.Attention.LAYERNORM_RMS_EPS
# RoPE
KEY_ROPE_DIMENSION_COUNT = Keys.Rope.DIMENSION_COUNT
KEY_ROPE_FREQ_BASE = Keys.Rope.FREQ_BASE
+KEY_ROPE_FREQ_BASE_SPATIAL = Keys.Rope.FREQ_BASE_SPATIAL
KEY_ROPE_SCALING_TYPE = Keys.Rope.SCALING_TYPE
KEY_ROPE_SCALING_FACTOR = Keys.Rope.SCALING_FACTOR
KEY_ROPE_SCALING_ORIG_CTX_LEN = Keys.Rope.SCALING_ORIG_CTX_LEN
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index e8a96e5710..d003922116 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1167,6 +1167,9 @@ class GGUFWriter:
def add_rope_freq_base(self, value: float) -> None:
self.add_float32(Keys.Rope.FREQ_BASE.format(arch=self.arch), value)
+ def add_rope_freq_base_spatial(self, value: float) -> None:
+ self.add_float32(Keys.Rope.FREQ_BASE_SPATIAL.format(arch=self.arch), value)
+
def add_rope_scaling_type(self, value: RopeScalingType) -> None:
self.add_string(Keys.Rope.SCALING_TYPE.format(arch=self.arch), value.value)
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 4534be22f3..1c11536b51 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -33,6 +33,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_QWEN2, "qwen2" },
{ LLM_ARCH_QWEN2MOE, "qwen2moe" },
{ LLM_ARCH_QWEN2VL, "qwen2vl" },
+ { LLM_ARCH_SENSENOVA_U1, "sensenova_u1" },
{ LLM_ARCH_QWEN3, "qwen3" },
{ LLM_ARCH_QWEN3MOE, "qwen3moe" },
{ LLM_ARCH_QWEN3NEXT, "qwen3next" },
@@ -339,6 +340,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_ROPE_DIMENSION_SECTIONS, "%s.rope.dimension_sections" },
{ LLM_KV_ROPE_SECTION_ORDER, "%s.rope.section_order" },
{ LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" },
+ { LLM_KV_ROPE_FREQ_BASE_SPATIAL, "%s.rope.freq_base_spatial" },
{ LLM_KV_ROPE_FREQ_BASE_SWA, "%s.rope.freq_base_swa" },
{ LLM_KV_ROPE_SCALE_LINEAR, "%s.rope.scale_linear" },
{ LLM_KV_ROPE_SCALING_TYPE, "%s.rope.scaling.type" },
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 5a8ca42500..afceb8ea6f 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -38,6 +38,7 @@ enum llm_arch {
LLM_ARCH_QWEN2,
LLM_ARCH_QWEN2MOE,
LLM_ARCH_QWEN2VL,
+ LLM_ARCH_SENSENOVA_U1,
LLM_ARCH_QWEN3,
LLM_ARCH_QWEN3MOE,
LLM_ARCH_QWEN3NEXT,
@@ -344,6 +345,7 @@ enum llm_kv {
LLM_KV_ROPE_DIMENSION_SECTIONS,
LLM_KV_ROPE_SECTION_ORDER,
LLM_KV_ROPE_FREQ_BASE,
+ LLM_KV_ROPE_FREQ_BASE_SPATIAL,
LLM_KV_ROPE_FREQ_BASE_SWA,
LLM_KV_ROPE_SCALE_LINEAR,
LLM_KV_ROPE_SCALING_TYPE,
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 46b1d2ae27..d620fcd9c8 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -15,6 +15,7 @@
bool llama_model_saver_supports_arch(llm_arch arch) {
switch (arch) {
+ case LLM_ARCH_SENSENOVA_U1:
case LLM_ARCH_GEMMA3N:
case LLM_ARCH_BITNET:
case LLM_ARCH_T5:
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 6712f0f22f..3797c567dc 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -110,6 +110,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_qwen2vl(params);
case LLM_ARCH_QWEN2MOE:
return new llama_model_qwen2moe(params);
+ case LLM_ARCH_SENSENOVA_U1:
+ return new llama_model_sensenova_u1(params);
case LLM_ARCH_QWEN3:
return new llama_model_qwen3(params);
case LLM_ARCH_QWEN3MOE:
@@ -3640,6 +3642,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
// DSV4 DSpark drafters use DeepSeek-V4's normal RoPE; legacy DFlash backbones are NeoX
return model->hparams.dsv4_hc_mult > 0 ? LLAMA_ROPE_TYPE_NORM : LLAMA_ROPE_TYPE_NEOX;
+ case LLM_ARCH_SENSENOVA_U1:
case LLM_ARCH_QWEN2VL:
case LLM_ARCH_PADDLEOCR:
return LLAMA_ROPE_TYPE_MROPE;
diff --git a/src/models/models.h b/src/models/models.h
index 9aa450deab..47b0203da1 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -614,6 +614,21 @@ struct llama_model_qwen2moe : public llama_model_base {
};
+struct llama_model_sensenova_u1 : public llama_model_base {
+ llama_model_sensenova_u1(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ float rope_freq_base_spatial = 10000.0f;
+
+ struct graph : public llm_graph_context {
+ graph(const llama_model_sensenova_u1 & model, const llm_graph_params & params);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
struct llama_model_qwen3 : public llama_model_base {
llama_model_qwen3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/src/models/sensenova-u1.cpp b/src/models/sensenova-u1.cpp
new file mode 100644
index 0000000000..c42c604e8e
--- /dev/null
+++ b/src/models/sensenova-u1.cpp
@@ -0,0 +1,190 @@
+#include "models.h"
+
+namespace {
+class llm_graph_input_pos_sensenova_u1 : public llm_graph_input_pos {
+public:
+ llm_graph_input_pos_sensenova_u1() : llm_graph_input_pos(4) {}
+
+ void set_input(const llama_ubatch * ubatch) override {
+ if (ubatch->pos && pos) {
+ const int64_t n_tokens = ubatch->n_tokens;
+
+ // U1 expects text-token positions in the temporal plane only: [t, 0, 0, 0].
+ // The generic M-RoPE input uses [t, t, t, 0], so U1 needs a custom input class.
+ if (ubatch->token) {
+ std::vector<llama_pos> pos_data(n_tokens*n_pos_per_embd, 0);
+ for (int i = 0; i < n_tokens; ++i) {
+ pos_data[i] = ubatch->pos[i];
+ }
+ ggml_backend_tensor_set(pos, pos_data.data(), 0, pos_data.size()*ggml_element_size(pos));
+ } else {
+ ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_embd*ggml_element_size(pos));
+ }
+ }
+ }
+};
+}
+
+void llama_model_sensenova_u1::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ ml.get_key(LLM_KV_ROPE_FREQ_BASE_SPATIAL, rope_freq_base_spatial);
+ if (hparams.n_embd_head_k() % 4 != 0 || hparams.n_embd_head_v() != hparams.n_embd_head_k()) {
+ throw std::runtime_error("SenseNova U1 requires equal Q/K/V head sizes divisible by four");
+ }
+ switch (hparams.n_layer()) {
+ case 42: type = LLM_TYPE_8B; break;
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+}
+
+void llama_model_sensenova_u1::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+ // output
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed
+ if (output == NULL) {
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+ layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_sensenova_u1::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
+
+llama_model_sensenova_u1::graph::graph(const llama_model_sensenova_u1 & model, const llm_graph_params & params) : llm_graph_context(params) {
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ auto normalize = [&](ggml_tensor * x, ggml_tensor * weight) {
+ x = ggml_rms_norm(ctx0, x, hparams.f_norm_rms_eps);
+
+ return ggml_mul(ctx0, x, weight);
+ };
+
+ inpL = build_inp_embd(model.tok_embd);
+
+ ggml_tensor * pos[3]{};
+ auto inp = std::make_unique<llm_graph_input_pos_sensenova_u1>();
+ auto & inp_pos = inp->pos;
+ inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, (int64_t)n_tokens*hparams.n_pos_per_embd());
+ ggml_set_input(inp_pos);
+ res->add_input(std::move(inp));
+
+ for (int axis = 0; axis < 3; ++axis) {
+ pos[axis] = ggml_view_1d(ctx0, inp_pos, n_tokens, axis*n_tokens*sizeof(int32_t));
+ }
+
+ auto normalize_and_rotate = [&](ggml_tensor * x, ggml_tensor * weight) {
+ const int64_t heads = x->ne[1];
+ x = ggml_reshape_4d(ctx0, x, n_embd_head/2, 2, heads, n_tokens);
+ x = normalize(x, ggml_reshape_2d(ctx0, weight, n_embd_head/2, 2));
+ x = ggml_reshape_3d(ctx0, x, n_embd_head, heads, n_tokens);
+
+ // Normalize temporal and spatial halves separately; rotate time, height, then width.
+ for (int axis = 0; axis < 3; ++axis) {
+ const int dims = axis == 0 ? n_embd_head/2 : n_embd_head/4;
+ const int offset = axis == 0 ? 0 : n_embd_head/2 + (axis - 1)*n_embd_head/4;
+ const float base = axis == 0 ? freq_base : model.rope_freq_base_spatial;
+ x = ggml_rope_ext(ctx0, x, pos[axis], nullptr, dims, GGML_ROPE_TYPE_NEOX,
+ n_ctx_orig, base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f);
+ ggml_rope_set_offset(x, offset);
+ }
+ return x;
+ };
+
+ auto * inp_attn = build_attn_inp_kv();
+
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ for (int il = 0; il < n_layer; ++il) {
+ res->t_layer_inp[il] = inpL;
+
+ ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = normalize(inpL, model.layers[il].attn_norm);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
+
+ Qcur = normalize_and_rotate(Qcur, model.layers[il].attn_q_norm);
+ Kcur = normalize_and_rotate(Kcur, model.layers[il].attn_k_norm);
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ cur = build_attn(inp_attn,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
+ }
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = normalize(ffn_inp, model.layers[il].ffn_norm);
+ cb(cur, "ffn_norm", il);
+
+ cur = build_ffn(cur,
+ model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s,
+ model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,
+ model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+ cur = inpL;
+
+ cur = normalize(cur, model.output_norm);
+
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ // lm_head
+ cur = build_lora_mm(model.output, cur, model.output_s);
+
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+}
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 21d27e17d9..c86b1315a6 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -329,6 +329,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
// partial rotary: n_rot must not exceed the indexer key length (64)
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64));
}
+ if (arch == LLM_ARCH_SENSENOVA_U1) {
+ ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SPATIAL, 10000.0f);
+ }
ms.add_kv(LLM_KV_ATTENTION_CLAMP_KQV, 1.0f);
ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f);
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 398b73904f..bd7fb73566 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -55,6 +55,7 @@ add_library(mtmd
models/paddleocr.cpp
models/pixtral.cpp
models/qwen2vl.cpp
+ models/sensenova-u1.cpp
models/minimax-m3.cpp
models/qwen3vl.cpp
models/ling3vl.cpp
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 71d349cc59..6ca4652808 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -547,6 +547,7 @@ enum projector_type {
PROJECTOR_TYPE_POCKETTTS_SPKENC,
PROJECTOR_TYPE_POCKETTTS_GEN,
PROJECTOR_TYPE_MUSE_GLIMMER,
+ PROJECTOR_TYPE_SENSENOVA_U1,
PROJECTOR_TYPE_COHERE2V,
PROJECTOR_TYPE_MUSE_SPARK_AUDIO,
PROJECTOR_TYPE_UNKNOWN,
@@ -621,6 +622,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_POCKETTTS_SPKENC, "pockettts_spkenc"},
{ PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"},
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
+ { PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"},
{ PROJECTOR_TYPE_COHERE2V, "cohere2v"},
{ PROJECTOR_TYPE_MUSE_SPARK_AUDIO, "muse-spark-audio"},
};
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 359ec5ac36..869e3fff82 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -997,6 +997,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_qwen3vl>(ctx, img);
} break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ builder = std::make_unique<clip_graph_sensenova_u1>(ctx, img);
+ } break;
case PROJECTOR_TYPE_LING3VL:
{
builder = std::make_unique<clip_graph_ling3vl>(ctx, img);
@@ -1795,6 +1799,20 @@ struct clip_model_loader {
LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
}
} break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ hparams.n_merge = 2;
+ hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
+ get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
+ get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
+ get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
+ hparams.rope_theta = 10000.0f;
+ hparams.set_limit_image_tokens(16, 4096);
+ if (hparams.n_merge != 2) {
+ throw std::runtime_error(string_format("%s: SenseNova U1 requires spatial_merge_size=2, got %u", __func__, hparams.n_merge));
+ }
+ hparams.set_warmup_n_tokens(16*16);
+ } break;
case PROJECTOR_TYPE_MINIMAX_M3:
{
hparams.n_merge = 2; // spatial_merge_size
@@ -3837,6 +3855,11 @@ struct clip_model_loader {
}
} break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+ } break;
default:
GGML_ASSERT(false && "unknown projector type");
}
@@ -4311,6 +4334,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
@@ -4341,6 +4365,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
@@ -4426,6 +4451,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
@@ -5525,6 +5551,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_i32("pos_w", pos_data);
} break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ const int n_patches_per_row = image_size_width / patch_size;
+ std::vector<int> pos_data(n_pos);
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = i / n_patches_per_row;
+ }
+ set_input_i32("pos_y", pos_data);
+ for (int i = 0; i < n_pos; i++) {
+ pos_data[i] = i % n_patches_per_row;
+ }
+ set_input_i32("pos_x", pos_data);
+ } break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
// set the 2D positions (mrope layout, only the first 2 channels are used)
@@ -6411,6 +6450,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_JANUS_PRO:
case PROJECTOR_TYPE_YOUTUVL:
return ctx->model.mm_1_b->ne[0];
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ return ctx->model.hparams.projection_dim;
case PROJECTOR_TYPE_QWEN3VL:
// main path + deepstack paths
return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers);
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 2944486059..55e525cdc0 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -50,6 +50,11 @@ struct clip_graph_qwen3vl : clip_graph_qwen2vl {
ggml_cgraph * build() override;
};
+struct clip_graph_sensenova_u1 : clip_graph {
+ clip_graph_sensenova_u1(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+ ggml_cgraph * build() override;
+};
+
struct clip_graph_ling3vl : clip_graph_qwen3vl {
clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/models/sensenova-u1.cpp b/tools/mtmd/models/sensenova-u1.cpp
new file mode 100644
index 0000000000..57875c6edb
--- /dev/null
+++ b/tools/mtmd/models/sensenova-u1.cpp
@@ -0,0 +1,40 @@
+#include "models.h"
+
+ggml_cgraph * clip_graph_sensenova_u1::build() {
+ GGML_ASSERT(model.patch_embeddings_0 != nullptr);
+ GGML_ASSERT(model.patch_bias != nullptr);
+ GGML_ASSERT(model.mm_0_w != nullptr);
+ GGML_ASSERT(model.mm_0_b != nullptr);
+ GGML_ASSERT(hparams.n_merge == 2);
+
+ ggml_tensor * pos_x = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
+ ggml_set_name(pos_x, "pos_x");
+ ggml_set_input(pos_x);
+
+ ggml_tensor * pos_y = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
+ ggml_set_name(pos_y, "pos_y");
+ ggml_set_input(pos_y);
+
+ ggml_tensor * inp = build_inp();
+ inp = ggml_gelu(ctx0, inp);
+
+ // Apply adjacent-pair 2D RoPE to patch embeddings before downsampling.
+ inp = ggml_reshape_4d(ctx0, inp, n_embd, 1, n_patches, n_batch);
+ inp = build_rope_2d(ctx0, inp, pos_x, pos_y, hparams.rope_theta, false);
+ inp = ggml_reshape_3d(ctx0, inp, n_embd, n_patches, n_batch);
+ cb(inp, "patch_embd_rope", -1);
+
+ inp = ggml_reshape_4d(ctx0, inp, n_embd, n_patches_x, n_patches_y, n_batch);
+ inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
+ inp = ggml_conv_2d(ctx0, model.mm_0_w, inp,
+ hparams.n_merge, hparams.n_merge, 0, 0, 1, 1);
+
+ const int n_output_patches = n_patches / (hparams.n_merge * hparams.n_merge);
+ inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 2, 0, 3));
+ inp = ggml_reshape_3d(ctx0, inp, n_mmproj_embd, n_output_patches, n_batch);
+ inp = ggml_add(ctx0, inp, model.mm_0_b);
+ cb(inp, "proj_out", -1);
+
+ ggml_build_forward_expand(gf, inp);
+ return gf;
+}
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 9e2c5f3c92..9460f15fe2 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -225,6 +225,7 @@ enum mtmd_pos_type {
MTMD_POS_TYPE_NORMAL, // number of positions equals to number of tokens
MTMD_POS_TYPE_MROPE, // qwen-vl mrope style, each image takes max(t,h,w) position indexes
MTMD_POS_TYPE_HUNYUANVL, // HunyuanVL mrope + BOI/EOI/newline layout with XD-RoPE dim-3
+ MTMD_POS_TYPE_SENSENOVA_U1, // U1 uses zero-based H/W and one temporal position per image
MTMD_POS_TYPE_CANVAS, // MiniCPM-V 4.7: overview + slices in one chunk, sharing one 2D canvas (see mtmd_image_tokens::canvas_tile_grid)
MTMD_POS_TYPE_COUNT, // for validation
};
@@ -751,6 +752,12 @@ struct mtmd_context {
img_end = "]<]end of image[>[";
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
} break;
+ case PROJECTOR_TYPE_SENSENOVA_U1:
+ {
+ img_beg = "<img>";
+ img_end = "</img>";
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ } break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
img_beg = "<|image_start|>";
@@ -1615,10 +1622,15 @@ struct mtmd_tokenizer {
image_tokens->ny = 1;
}
image_tokens->pos = ctx->pos_type;
+ // U1 uses M-RoPE, but unlike Qwen its image H/W coordinates are
+ // zero-based and the whole image occupies one temporal position.
+ if (ctx->proj_type_v() == PROJECTOR_TYPE_SENSENOVA_U1) {
+ image_tokens->pos = MTMD_POS_TYPE_SENSENOVA_U1;
+ }
// HunyuanVL wraps the image grid with BOI/EOI and adds one newline per row,
// and uses XD-RoPE (dim-3 = image index). Override the position type so that
// n_tokens() and mtmd_image_tokens_get_decoder_pos pick the HunyuanVL layout.
- if (ctx->proj_type_v() == PROJECTOR_TYPE_HUNYUANVL) {
+ else if (ctx->proj_type_v() == PROJECTOR_TYPE_HUNYUANVL) {
image_tokens->pos = MTMD_POS_TYPE_HUNYUANVL;
image_tokens->image_idx = n_images_added;
GGML_ASSERT(n_tokens == (size_t)image_tokens->n_tokens());
@@ -2398,6 +2410,7 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk
}
case PROJECTOR_TYPE_GEMMA4UV:
case PROJECTOR_TYPE_GEMMA3:
+ case PROJECTOR_TYPE_SENSENOVA_U1:
case PROJECTOR_TYPE_DEEPSEEK4V:
return true;
default:
@@ -2761,6 +2774,13 @@ mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * ima
pos.y = pos_0 + i;
pos.z = pos_0 + i;
} break;
+ case MTMD_POS_TYPE_SENSENOVA_U1:
+ {
+ pos.t = pos_0;
+ pos.x = i % image_tokens->nx;
+ pos.y = i / image_tokens->nx;
+ pos.z = 0;
+ } break;
case MTMD_POS_TYPE_HUNYUANVL:
{
// HunyuanVL layout: [BOI] [row0 tokens + newline] ... [row(ny-1) tokens + newline] [EOI]
@@ -2813,6 +2833,8 @@ llama_pos mtmd_image_tokens_get_n_pos(const mtmd_image_tokens * image_tokens) {
return std::max(image_tokens->nx, image_tokens->ny);
case MTMD_POS_TYPE_NORMAL:
return image_tokens->n_tokens();
+ case MTMD_POS_TYPE_SENSENOVA_U1:
+ return 1;
case MTMD_POS_TYPE_HUNYUANVL:
// HunyuanVL: the sequential (dim-0) position advances by the full token count
// (includes BOI/EOI and row newline tokens), not by max(nx, ny)