Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

gguf better support for transformers5.0 and fix bug of Qwen3Next (#1474)

Heng Guoab14698cf83a
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 8bdae36b2..2efe78914 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -2139,7 +2139,7 @@ class BaseCompressor(object):
)
exit(-1)
elif total_cnt < nsamples:
- logger.warning(
+ logger.warning_once(
f"An insufficient number of samples likely reduces the accuracy of the quantized model. "
f"Target samples count is {nsamples}, while valid samples count is {total_cnt}"
)
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index b0a83b1d5..73ed63206 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -546,9 +546,16 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model
# from auto_round.export.export_to_gguf.convert import ModelBase, get_model_architecture
convert_hf_to_gguf = LazyImport("auto_round.export.export_to_gguf.convert_hf_to_gguf")
- model_architecture = convert_hf_to_gguf.get_model_architecture(
- hparams=model.config.to_dict(), model_type=model_type
- )
+ try:
+ model_architecture = convert_hf_to_gguf.get_model_architecture(
+ hparams=model.config.to_dict(), model_type=model_type
+ )
+ except AttributeError as e:
+ raise ImportError(
+ "Please use the latest gguf-py, you can use the following command to install it:\n"
+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"
+ " && pip install . sentencepiece"
+ )
try:
if model_type != ModelType.TEXT:
model_class_vision = convert_hf_to_gguf.ModelBase.from_model_architecture(
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 1ebbbdfd8..bd28d89ef 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -218,6 +218,13 @@ def _quant_data_with_args(
return data
+def need_modify_tensor(cls, name):
+ hf_arch = getattr(cls, "hf_arch", "")
+ if hf_arch == "Qwen3NextForCausalLM" and "in_proj_qkvz.weight" in name:
+ return True
+ return False
+
+
def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=None):
"""
@@ -254,7 +261,7 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, d
attr_tensor = getattr(module, attr)
if not isinstance(attr_tensor, torch.Tensor):
continue
- if hasattr(cls, "permute"):
+ if hasattr(cls, "permute") or need_modify_tensor(cls, name):
bs = module.weight.shape[0]
attr_tensors_dict = dict(cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid))
attr_tensor = attr_tensors_dict[new_name]
@@ -408,7 +415,6 @@ def prepare_tensors(cls):
clean_weight_list = []
modify_name = _special_name_handle(cls, name)
- orig_device = data_torch.device
data_torch = data_torch.to("cpu")
for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):
skip = False
@@ -621,7 +627,7 @@ def prepare_tensors(cls):
f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}" f" --> {data_qtype.name}, shape = {shape_str}"
)
if not (hasattr(cls, "current_packing_block") and cls.current_packing_block is not None):
- clear_memory(device_list=[orig_device])
+ clear_memory(device_list=[cls.device])
cls.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)
diff --git a/auto_round/export/export_to_gguf/convert_hf_to_gguf.py b/auto_round/export/export_to_gguf/convert_hf_to_gguf.py
index b823580ce..162e05dad 100644
--- a/auto_round/export/export_to_gguf/convert_hf_to_gguf.py
+++ b/auto_round/export/export_to_gguf/convert_hf_to_gguf.py
@@ -14,7 +14,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
from __future__ import annotations
import argparse
@@ -70,10 +69,8 @@ except ImportError:
"`pip install mistral-common[image,audio]` to install it."
)
-
logger = logging.getLogger("hf-to-gguf")
-
###### MODEL DEFINITIONS ######
@@ -170,18 +167,24 @@ class ModelBase:
self.model_name = model_name
self.dir_model_card = dir_model # overridden in convert_lora_to_gguf.py
- # Apply heuristics to figure out typical tensor encoding based on first layer tensor encoding type
+ # Apply heuristics to figure out typical tensor encoding based on first tensor's dtype
+ # NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie.
if self.ftype == gguf.LlamaFileType.GUESSED:
- # NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie.
- _, first_tensor = next(self.get_tensors())
- if first_tensor.dtype == torch.float16:
- logger.info(f"choosing --outtype f16 from first tensor type ({first_tensor.dtype})")
- self.ftype = gguf.LlamaFileType.MOSTLY_F16
- else:
- logger.info(f"choosing --outtype bf16 from first tensor type ({first_tensor.dtype})")
- self.ftype = gguf.LlamaFileType.MOSTLY_BF16
+ for _, tensor in self.get_tensors():
+ if tensor.dim() < 2:
+ continue
- self.dequant_model()
+ if tensor.dtype == torch.bfloat16:
+ self.ftype = gguf.LlamaFileType.MOSTLY_BF16
+ logger.info("heuristics detected bfloat16 tensor dtype, setting --outtype bf16")
+ break
+ elif tensor.dtype == torch.float16:
+ self.ftype = gguf.LlamaFileType.MOSTLY_F16
+ logger.info("heuristics detected float16 tensor dtype, setting --outtype f16")
+ break
+ else:
+ self.ftype = gguf.LlamaFileType.MOSTLY_F16
+ logger.info("heuristics unable to detect tensor dtype, defaulting to --outtype f16")
# Configure GGUF Writer
self.gguf_writer = gguf.GGUFWriter(
@@ -246,6 +249,8 @@ class ModelBase:
if weight_map is None or not isinstance(weight_map, dict):
raise ValueError(f"Can't load 'weight_map' from {index_name!r}")
tensor_names_from_index.update(weight_map.keys())
+ part_dict: dict[str, None] = dict.fromkeys(weight_map.values(), None)
+ part_names = sorted(part_dict.keys())
else:
weight_map = {}
else:
@@ -255,9 +260,7 @@ class ModelBase:
logger.info(f"gguf: indexing model part '{part_name}'")
ctx: ContextManager[Any]
if is_safetensors:
- from safetensors import safe_open
-
- ctx = cast(ContextManager[Any], safe_open(self.dir_model / part_name, framework="pt", device="cpu"))
+ ctx = cast(ContextManager[Any], gguf.utility.SafetensorsLocal(self.dir_model / part_name))
else:
ctx = contextlib.nullcontext(
torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True)
@@ -268,18 +271,22 @@ class ModelBase:
for name in model_part.keys():
if is_safetensors:
+ data: gguf.utility.LocalTensor = model_part[name]
if self.lazy:
- data = model_part.get_slice(name)
- data_gen = lambda data=data: LazyTorchTensor.from_safetensors_slice(data) # noqa: E731
+ data_gen = lambda data=data: LazyTorchTensor.from_local_tensor(data) # noqa: E731
else:
- data = model_part.get_tensor(name)
- data_gen = lambda data=data: data # noqa: E731
+ dtype = LazyTorchTensor._dtype_str_map[data.dtype]
+ data_gen = (
+ lambda data=data, dtype=dtype: torch.from_numpy(data.mmap_bytes())
+ .view(dtype)
+ .reshape(data.shape)
+ ) # noqa: E731
else:
- data = model_part[name]
+ data_torch: Tensor = model_part[name]
if self.lazy:
- data_gen = lambda data=data: LazyTorchTensor.from_eager(data) # noqa: E731
+ data_gen = lambda data=data_torch: LazyTorchTensor.from_eager(data) # noqa: E731
else:
- data_gen = lambda data=data: data # noqa: E731
+ data_gen = lambda data=data_torch: data # noqa: E731
tensors[name] = data_gen
# verify tensor name presence and identify potentially missing files
@@ -290,9 +297,7 @@ class ModelBase:
extra = sorted(tensor_names_from_parts.difference(tensor_names_from_index))
missing_files = sorted(set(weight_map[n] for n in missing if n in weight_map))
if len(extra) == 0 and len(missing_files) > 0:
- raise ValueError(
- f"Missing or incomplete model files: {missing_files}\n" f"Missing tensors: {missing}"
- )
+ raise ValueError(f"Missing or incomplete model files: {missing_files}\nMissing tensors: {missing}")
else:
raise ValueError(
"Mismatch between weight map and model parts for tensor names:\n"
@@ -321,15 +326,14 @@ class ModelBase:
# The scale is inverted
return data / scale.float()
- def dequant_simple(weight: Tensor, scale: Tensor) -> Tensor:
+ def dequant_simple(weight: Tensor, scale: Tensor, block_size: Sequence[int] | None = None) -> Tensor:
scale = scale.float()
- if weight_block_size := quant_config.get("weight_block_size"):
- # TODO: make sure it's a list of integers
- for i, size in enumerate(weight_block_size):
+ if block_size is not None:
+ for i, size in enumerate(block_size):
scale = scale.repeat_interleave(size, i)
- # unpad the scale (e.g. when the tensor size isn't a multiple of the block size)
- scale = scale[tuple(slice(0, size) for size in weight.shape)]
+ # unpad the scale (e.g. when the tensor size isn't a multiple of the block size)
+ scale = scale[tuple(slice(0, size) for size in weight.shape)]
return weight.float() * scale
@@ -373,6 +377,47 @@ class ModelBase:
return (scales[g_idx].float() * (weight - zeros[g_idx]).float()).T
+ def dequant_packed(
+ w: Tensor,
+ scale: Tensor,
+ shape_tensor: Tensor,
+ zero_point: Tensor | None,
+ num_bits: int,
+ group_size: int,
+ ):
+ assert w.dtype == torch.int32
+ shape = tuple(shape_tensor.tolist())
+ assert len(shape) == 2
+ mask = (1 << num_bits) - 1
+
+ shifts = torch.arange(0, 32 - (num_bits - 1), num_bits, dtype=torch.int32)
+ if self.lazy:
+ shifts = LazyTorchTensor.from_eager(shifts)
+
+ if zero_point is None:
+ offset = 1 << (num_bits - 1)
+ else:
+ assert len(zero_point.shape) == 2
+ offset = (zero_point.unsqueeze(1) >> shifts.reshape(1, -1, 1)) & mask
+ offset = offset.reshape(-1, zero_point.shape[1])
+ # trim padding, and prepare for broadcast
+ # NOTE: the zero-point is packed along dim 0
+ offset = offset[: shape[0], :].unsqueeze(-1)
+
+ # extract values
+ # NOTE: the weights are packed along dim 1
+ unpacked = (w.unsqueeze(-1) >> shifts.reshape(1, 1, -1)) & mask
+ unpacked = unpacked.reshape(shape[0], -1)
+
+ # trim padding
+ unpacked = unpacked[:, : shape[1]]
+
+ # prepare for broadcast of the scale
+ unpacked = unpacked.reshape(shape[0], (unpacked.shape[-1] + group_size - 1) // group_size, group_size)
+ unpacked = unpacked - offset
+
+ return (unpacked * scale.unsqueeze(-1).float()).reshape(shape)
+
if quant_method == "bitnet":
for name in self.model_tensors.keys():
if name.endswith(".weight_scale"):
@@ -382,12 +427,24 @@ class ModelBase:
self.model_tensors[weight_name] = lambda w=w, s=s: dequant_bitnet(w(), s())
tensors_to_remove.append(name)
elif quant_method == "fp8":
+ block_size = quant_config.get("weight_block_size")
for name in self.model_tensors.keys():
if name.endswith(".weight_scale_inv"):
weight_name = name.removesuffix("_scale_inv")
w = self.model_tensors[weight_name]
s = self.model_tensors[name]
- self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s())
+ self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
+ tensors_to_remove.append(name)
+ if name.endswith(".activation_scale"): # unused
+ tensors_to_remove.append(name)
+ # mistral format
+ if name.endswith(".qscale_weight"):
+ weight_name = name.removesuffix("qscale_weight") + "weight"
+ w = self.model_tensors[weight_name]
+ s = self.model_tensors[name]
+ self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
+ tensors_to_remove.append(name)
+ if name.endswith(".qscale_act"):
tensors_to_remove.append(name)
elif quant_method == "gptq":
for name in self.model_tensors.keys():
@@ -409,6 +466,60 @@ class ModelBase:
".scales",
)
]
+ elif quant_method == "compressed-tensors":
+ quant_format = quant_config["format"]
+ groups = quant_config["config_groups"]
+ if len(groups) > 1:
+ raise NotImplementedError("Can't handle multiple config groups for compressed-tensors yet")
+ weight_config = tuple(groups.values())[0]["weights"]
+
+ if (
+ quant_format == "float-quantized"
+ or quant_format == "int-quantized"
+ or quant_format == "naive-quantized"
+ ):
+ block_size = weight_config.get("block_structure", None)
+ strategy = weight_config.get("strategy")
+ assert strategy == "channel" or strategy == "block"
+ assert weight_config.get("group_size") is None # didn't find a model using this yet
+ for name in self.model_tensors.keys():
+ if name.endswith(".weight_scale"):
+ weight_name = name.removesuffix("_scale")
+ w = self.model_tensors[weight_name]
+ s = self.model_tensors[name]
+ self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), block_size)
+ tensors_to_remove.append(name)
+ elif quant_format == "pack-quantized":
+ assert weight_config.get("strategy") == "group"
+ assert weight_config.get("type", "int") == "int"
+ num_bits = weight_config.get("num_bits")
+ group_size = weight_config.get("group_size")
+ assert isinstance(num_bits, int)
+ assert isinstance(group_size, int)
+ for name in self.model_tensors.keys():
+ if name.endswith(".weight_packed"):
+ base_name = name.removesuffix("_packed")
+ w = self.model_tensors[name]
+ scale = self.model_tensors[base_name + "_scale"]
+ shape = self.model_tensors[base_name + "_shape"]
+ zero_point = self.model_tensors.get(base_name + "_zero_point", lambda: None)
+ new_tensors[base_name] = (
+ lambda w=w, scale=scale, shape=shape, zero_point=zero_point: dequant_packed(
+ w(),
+ scale(),
+ shape(),
+ zero_point(),
+ num_bits,
+ group_size,
+ )
+ )
+ tensors_to_remove += [base_name + n for n in ("_packed", "_shape", "_scale")]
+ if (base_name + "_zero_point") in self.model_tensors:
+ tensors_to_remove.append(base_name + "_zero_point")
+ else:
+ raise NotImplementedError(
+ f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported"
+ )
else:
raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}")
@@ -458,7 +569,6 @@ class ModelBase:
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
del bid # unused
-
return [(self.map_tensor_name(name), data_torch)]
def tensor_force_quant(
@@ -473,7 +583,13 @@ class ModelBase:
return ()
def prepare_tensors(self):
- max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")
+ self.dequant_model()
+
+ # Handle empty tensor_map for models with block_count=0 (like MobileNetV5)
+ if self.tensor_map.mapping:
+ max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")
+ else:
+ max_name_len = len("vision_encoder.weight,") # Default reasonable length
for name, data_torch in chain(self.generate_extra_tensors(), self.get_tensors()):
# we don't need these
@@ -512,6 +628,7 @@ class ModelBase:
self.match_model_tensor_name(new_name, key, bid)
for key in (
gguf.MODEL_TENSOR.FFN_GATE_INP,
+ gguf.MODEL_TENSOR.FFN_GATE_INP_SHEXP,
gguf.MODEL_TENSOR.POS_EMBD,
gguf.MODEL_TENSOR.TOKEN_TYPES,
gguf.MODEL_TENSOR.SSM_CONV1D,
@@ -528,9 +645,13 @@ class ModelBase:
gguf.MODEL_TENSOR.A_ENC_EMBD_POS,
gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,
gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,
+ # Kimi KDA conv weights should be F32
+ gguf.MODEL_TENSOR.SSM_CONV1D_Q,
+ gguf.MODEL_TENSOR.SSM_CONV1D_K,
+ gguf.MODEL_TENSOR.SSM_CONV1D_V,
)
)
- or not new_name.endswith(".weight")
+ or new_name[-7:] not in (".weight", ".lora_a", ".lora_b")
):
data_qtype = gguf.GGMLQuantizationType.F32
@@ -663,9 +784,15 @@ class ModelBase:
if "llm_config" in config:
# rename for InternVL
config["text_config"] = config["llm_config"]
+ if "lm_config" in config:
+ # rename for GlmASR
+ config["text_config"] = config["lm_config"]
if "thinker_config" in config:
# rename for Qwen2.5-Omni
config["text_config"] = config["thinker_config"]["text_config"]
+ if "lfm" in config:
+ # rename for LFM2-Audio
+ config["text_config"] = config["lfm"]
return config
@classmethod
@@ -713,6 +840,26 @@ class TextModel(ModelBase):
self.block_count = self.find_hparam(["n_layers", "num_hidden_layers", "n_layer", "num_layers"])
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+ self.rope_parameters = self.hparams.get("rope_parameters", self.hparams.get("rope_scaling")) or {}
+
+ rope_theta = self.find_hparam(
+ ["global_rope_theta", "rope_global_theta", "rope_theta_global", "rope_theta", "rotary_emb_base"],
+ optional=True,
+ )
+ local_rope_theta = self.find_hparam(
+ ["local_rope_theta", "rope_local_theta", "rope_theta_local", "swa_rope_theta", "rope_local_base_freq"],
+ optional=True,
+ )
+
+ # Ensure "rope_theta" and "rope_type" is mirrored in rope_parameters
+ if "full_attention" not in self.rope_parameters and "sliding_attention" not in self.rope_parameters:
+ if local_rope_theta is not None:
+ self.rope_parameters["sliding_attention"] = {"rope_theta": local_rope_theta}
+ if "rope_theta" not in self.rope_parameters and rope_theta is not None:
+ self.rope_parameters["rope_theta"] = rope_theta
+ if "rope_type" not in self.rope_parameters and (rope_type := self.rope_parameters.get("type")) is not None:
+ self.rope_parameters["rope_type"] = rope_type
+
@classmethod
def __init_subclass__(cls):
# can't use an abstract property, because overriding it without type errors
@@ -771,7 +918,17 @@ class TextModel(ModelBase):
self.gguf_writer.add_block_count(self.block_count)
if (
- n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx", "n_positions", "max_length"], optional=True)
+ n_ctx := self.find_hparam(
+ [
+ "max_position_embeddings",
+ "n_ctx",
+ "n_positions",
+ "max_length",
+ "max_sequence_length",
+ "model_max_length",
+ ],
+ optional=True,
+ )
) is not None:
self.gguf_writer.add_context_length(n_ctx)
logger.info(f"gguf: context length = {n_ctx}")
@@ -792,9 +949,58 @@ class TextModel(ModelBase):
self.gguf_writer.add_head_count_kv(n_head_kv)
logger.info(f"gguf: key-value head count = {n_head_kv}")
- if (rope_theta := self.hparams.get("rope_theta")) is not None:
+ # TODO: Handle "sliding_attention" similarly when models start implementing it
+ rope_params = self.rope_parameters.get("full_attention", self.rope_parameters)
+ if (rope_type := rope_params.get("rope_type")) is not None:
+ rope_factor = rope_params.get("factor")
+ rope_gguf_type = gguf.RopeScalingType.NONE
+ if rope_type == "linear" and rope_factor is not None:
+ rope_gguf_type = gguf.RopeScalingType.LINEAR
+ self.gguf_writer.add_rope_scaling_type(rope_gguf_type)
+ self.gguf_writer.add_rope_scaling_factor(rope_factor)
+ elif rope_type == "yarn" and rope_factor is not None:
+ rope_gguf_type = gguf.RopeScalingType.YARN
+ self.gguf_writer.add_rope_scaling_type(rope_gguf_type)
+ self.gguf_writer.add_rope_scaling_factor(rope_factor)
+ self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_params["original_max_position_embeddings"])
+ if (yarn_ext_factor := rope_params.get("extrapolation_factor")) is not None:
+ self.gguf_writer.add_rope_scaling_yarn_ext_factor(yarn_ext_factor)
+ if (
+ yarn_attn_factor := rope_params.get("attention_factor", rope_params.get("attn_factor"))
+ ) is not None:
+ self.gguf_writer.add_rope_scaling_yarn_attn_factor(yarn_attn_factor)
+ if (yarn_beta_fast := rope_params.get("beta_fast")) is not None:
+ self.gguf_writer.add_rope_scaling_yarn_beta_fast(yarn_beta_fast)
+ if (yarn_beta_slow := rope_params.get("beta_slow")) is not None:
+ self.gguf_writer.add_rope_scaling_yarn_beta_slow(yarn_beta_slow)
+ # self.gguf_writer.add_rope_scaling_yarn_log_mul(rope_params["mscale_all_dim"])
+ elif rope_type == "su" or rope_type == "longrope":
+ rope_gguf_type = gguf.RopeScalingType.LONGROPE
+ self.gguf_writer.add_rope_scaling_type(rope_gguf_type)
+ elif rope_type == "dynamic":
+ # HunYuan, handled in model class
+ pass
+ elif rope_type.lower() == "llama3":
+ # Handled in generate_extra_tensors
+ pass
+ else:
+ logger.warning(f"Unknown RoPE type: {rope_type}")
+ logger.info(f"gguf: rope scaling type = {rope_gguf_type.name}")
+
+ if "mrope_section" in self.rope_parameters:
+ mrope_section = self.rope_parameters["mrope_section"]
+ # Pad to 4 dimensions [time, height, width, extra]
+ while len(mrope_section) < 4:
+ mrope_section.append(0)
+ self.gguf_writer.add_rope_dimension_sections(mrope_section[:4])
+ logger.info(f"gguf: mrope sections: {mrope_section[:4]}")
+
+ if (rope_theta := rope_params.get("rope_theta")) is not None:
self.gguf_writer.add_rope_freq_base(rope_theta)
logger.info(f"gguf: rope theta = {rope_theta}")
+ if (local_rope_theta := self.rope_parameters.get("sliding_attention", {}).get("rope_theta")) is not None:
+ self.gguf_writer.add_rope_freq_base_swa(local_rope_theta)
+ logger.info(f"gguf: rope theta swa = {local_rope_theta}")
if (f_rms_eps := self.find_hparam(["rms_norm_eps", "norm_eps"], optional=True)) is not None:
self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)
logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")
@@ -803,10 +1009,12 @@ class TextModel(ModelBase):
) is not None:
self.gguf_writer.add_layer_norm_eps(f_norm_eps)
logger.info(f"gguf: layer norm epsilon = {f_norm_eps}")
- if (n_experts := self.hparams.get("num_local_experts")) is not None:
+ if (n_experts := self.find_hparam(["num_local_experts", "num_experts"], optional=True)) is not None:
self.gguf_writer.add_expert_count(n_experts)
logger.info(f"gguf: expert count = {n_experts}")
- if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:
+ if (
+ n_experts_used := self.find_hparam(["num_experts_per_tok", "num_experts_per_token"], optional=True)
+ ) is not None:
self.gguf_writer.add_expert_used_count(n_experts_used)
logger.info(f"gguf: experts used count = {n_experts_used}")
if (n_expert_groups := self.hparams.get("n_group")) is not None:
@@ -816,6 +1024,20 @@ class TextModel(ModelBase):
self.gguf_writer.add_expert_group_used_count(n_group_used)
logger.info(f"gguf: expert groups used count = {n_group_used}")
+ if (
+ score_func := self.find_hparam(
+ ["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func"],
+ optional=True,
+ )
+ ) is not None:
+ if score_func == "sigmoid":
+ self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+ elif score_func == "softmax":
+ self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
+ else:
+ raise ValueError(f"Unsupported expert score gating function value: {score_func}")
+ logger.info(f"gguf: expert score gating function = {score_func}")
+
if (head_dim := self.hparams.get("head_dim")) is not None:
self.gguf_writer.add_key_length(head_dim)
self.gguf_writer.add_value_length(head_dim)
@@ -850,7 +1072,8 @@ class TextModel(ModelBase):
)
seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))
- seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>")) # deepseek-coder
+ # deepseek-coder
+ seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))
# TODO: should these be marked as UNUSED instead? (maybe not)
seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">")) # gemma{,-2}
@@ -889,8 +1112,8 @@ class TextModel(ModelBase):
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))
if previous_token != token:
logger.info(
- f"{repr(previous_token)} is encoded and decoded back to {repr(token)} "
- f"using AutoTokenizer"
+ f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using"
+ " AutoTokenizer"
)
if added_tokens_decoder[i].special or self.does_token_look_special(token):
@@ -898,7 +1121,7 @@ class TextModel(ModelBase):
else:
# NOTE: this was added for Gemma.
# Encoding and decoding the tokens above isn't sufficient for this case.
- token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")
+ token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spaces
toktypes.append(gguf.TokenType.USER_DEFINED)
else:
toktypes.append(gguf.TokenType.NORMAL)
@@ -916,7 +1139,12 @@ class TextModel(ModelBase):
# we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can
# use in llama.cpp to implement the same pre-tokenizer
- chktxt = "\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български ''''''```````\"\"\"\"......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL" # pylint: disable=C0301
+ chktxt = (
+ "\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅"
+ " 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁"
+ " ?我想在apple工作1314151天~ ------======= нещо на Български ''''''```````\"\"\"\"......!!!!!!?????? I've"
+ " been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL"
+ )
chktok = tokenizer.encode(chktxt)
chkhsh = sha256(str(chktok).encode()).hexdigest()
@@ -941,6 +1169,9 @@ class TextModel(ModelBase):
if chkhsh == "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902":
# ref: https://huggingface.co/zai-org/GLM-4.5-Air
res = "glm4"
+ if chkhsh == "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267":
+ # ref: https://huggingface.co/zai-org/GLM-4.7-Flash
+ res = "glm4"
if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":
# ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0
res = "minerva-7b"
@@ -971,6 +1202,9 @@ class TextModel(ModelBase):
if chkhsh == "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273":
# ref: https://huggingface.co/alvarobartt/grok-2-tokenizer
res = "grok-2"
+ if chkhsh == "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df":
+ # ref: https://huggingface.co/aari1995/German_Semantic_V3
+ res = "jina-v2-de"
if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":
# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
res = "llama-bpe"
@@ -1010,6 +1244,9 @@ class TextModel(ModelBase):
if chkhsh == "9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8":
# ref: https://huggingface.co/CohereForAI/c4ai-command-r-v01
res = "command-r"
+ if chkhsh == "d772b220ace2baec124bed8cfafce0ead7d6c38a4b65ef11261cf9d5d62246d1":
+ # ref: https://huggingface.co/CohereLabs/tiny-aya-base
+ res = "tiny_aya"
if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea":
# ref: https://huggingface.co/Qwen/Qwen1.5-7B
res = "qwen2"
@@ -1046,6 +1283,9 @@ class TextModel(ModelBase):
if chkhsh == "b53802fb28e26d645c3a310b34bfe07da813026ec7c7716883404d5e0f8b1901":
# ref: https://huggingface.co/core42/jais-13b
res = "jais"
+ if chkhsh == "bc5108ee1eb6a3d600cadd065f63190fbd0554dbc9e4bbd6a0d977970afc8d2a":
+ # ref: https://huggingface.co/inceptionai/Jais-2-8B-Chat
+ res = "jais-2"
if chkhsh == "7b3e7548e4308f52a76e8229e4e6cc831195d0d1df43aed21ac6c93da05fec5f":
# ref: https://huggingface.co/WisdomShell/CodeShell-7B
res = "codeshell"
@@ -1121,12 +1361,42 @@ class TextModel(ModelBase):
if chkhsh == "a1e163ecab2e718a4c829d1148b6e86824ec36163bb71941c3dca9cd5ac25756":
# ref: https://huggingface.co/JetBrains/Mellum-4b-base
res = "mellum"
+ if chkhsh == "a0b64b4385f123663873756336c085744376d015ff328bb1d901598f63c44152":
+ # ref: https://huggingface.co/answerdotai/ModernBERT-base
+ res = "modern-bert"
+ if chkhsh == "49fc0303c9e0d2c2c565c510f64b2d9b271276acdcdadff733249eda9f7d59df":
+ # ref: https://huggingface.co/arcee-ai/Trinity-Tokenizer
+ res = "afmoe"
if chkhsh == "9b1be57e70d20d9501b2b3186e792d81181ae36ada3903c26f9fea418cf87206":
# ref: https://huggingface.co/inclusionAI/Ling-mini-base-2.0
res = "bailingmoe2"
if chkhsh == "53e325976a6e142379c19b09afcae354f2f496f147afa8f9e189a33fe4e3024e":
# ref: https://huggingface.co/ibm-granite/granite-docling-258M
res = "granite-docling"
+ if chkhsh == "f4f37b6c8eb9ea29b3eac6bb8c8487c5ab7885f8d8022e67edc1c68ce8403e95":
+ # ref: https://huggingface.co/MiniMaxAI/MiniMax-M2
+ res = "minimax-m2"
+ if chkhsh == "4a2e2abae11ca2b86d570fc5b44be4d5eb5e72cc8f22dd136a94b37da83ab665":
+ # ref: https://huggingface.co/KORMo-Team/KORMo-tokenizer
+ res = "kormo"
+ if chkhsh == "9d70134b369a70e5735009b6de918f7581b5211f7c074d1f89f753aea8248af1":
+ # ref: https://huggingface.co/tencent/Youtu-LLM-2B
+ res = "youtu"
+ if chkhsh == "16389f0a1f51ee53e562ffd51c371dc508639ab0e4261502071836e50e223e91":
+ # ref: https://huggingface.co/upstage/Solar-Open-100B
+ res = "solar-open"
+ if chkhsh == "6c81ce329e0802883b22eabab0d3fa48357337ef1ecb45443828bf1f6254833f":
+ # ref: https://huggingface.co/LGAI-EXAONE/K-EXAONE-236B-A23B
+ res = "exaone-moe"
+ if chkhsh == "d30d75d9059f1aa2c19359de71047b3ae408c70875e8a3ccf8c5fba56c9d8af4":
+ # ref: https://huggingface.co/Qwen/Qwen3.5-9B-Instruct
+ res = "qwen35"
+ if chkhsh == "b4b8ca1f9769494fbd956ebc4c249de6131fb277a4a3345a7a92c7dd7a55808d":
+ # ref: https://huggingface.co/jdopensource/JoyAI-LLM-Flash
+ res = "joyai-llm"
+ if chkhsh == "e4d54df1ebc1f2b91acd986c5b51aa50837d5faf7c7398e73c1f9e9ee5d19869":
+ # ref: https://huggingface.co/kakaocorp/kanana-2-30b-a3b-instruct-2601
+ res = "kanana2"
if res is None:
logger.warning("\n")
@@ -1313,7 +1583,7 @@ class TextModel(ModelBase):
if token_data.get("special") or self.does_token_look_special(token):
toktypes[token_id] = SentencePieceTokenTypes.CONTROL
else:
- token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")
+ token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spaces
toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED
scores[token_id] = -1000.0
@@ -1467,6 +1737,40 @@ class TextModel(ModelBase):
raise NotImplementedError("Only MEAN, CLS, and LAST pooling types supported")
self.gguf_writer.add_pooling_type(pooling_type)
+ def _set_vocab_glmedge(self):
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ tokens, toktypes, tokpre = self.get_vocab_base()
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+ special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])
+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab.add_to_gguf(self.gguf_writer)
+
+ def _set_vocab_glm(self):
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ tokens, toktypes, tokpre = self.get_vocab_base()
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+ # Special tokens
+ # Note: Using <|endoftext|> (151329) for eot causes endless generation
+ special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # 151331
+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # 151336
+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # 151329
+ special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338
+ special_vocab.add_to_gguf(self.gguf_writer)
+
def _set_vocab_interns1(self):
tokens: list[str] = []
toktypes: list[int] = []
@@ -1499,8 +1803,8 @@ class TextModel(ModelBase):
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))
if previous_token != token:
logger.info(
- f"{repr(previous_token)} is encoded and decoded back to {repr(token)} "
- "using AutoTokenizer"
+ f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using"
+ " AutoTokenizer"
)
if added_tokens_decoder[i].special or self.does_token_look_special(token):
@@ -1520,6 +1824,156 @@ class TextModel(ModelBase):
special_vocab._set_special_token("bos", 151643)
special_vocab.add_to_gguf(self.gguf_writer)
+ def _set_vocab_mistral(self):
+ if not _mistral_common_installed:
+ raise ImportError(_mistral_import_error_msg)
+
+ vocab = MistralVocab(self.dir_model)
+ logger.info(f"Converting tokenizer {vocab.tokenizer_type} of size {vocab.vocab_size}.")
+
+ self.gguf_writer.add_tokenizer_model(vocab.gguf_tokenizer_model)
+
+ tokens = []
+ scores = []
+ toktypes = []
+
+ for text, score, toktype in vocab.all_tokens():
+ tokens.append(text)
+ scores.append(score)
+ toktypes.append(toktype)
+
+ assert len(tokens) == vocab.vocab_size, f"token count ({len(tokens)}) != vocab size ({vocab.vocab_size})"
+
+ if vocab.tokenizer_type == MistralTokenizerType.tekken:
+ self.gguf_writer.add_tokenizer_pre("tekken")
+ self.gguf_writer.add_token_merges(vocab.extract_vocab_merges_from_model())
+
+ logger.info(
+ f"Setting bos, eos, unk and pad token IDs to {vocab.bos_id}, {vocab.eos_id}, {vocab.unk_id},"
+ f" {vocab.pad_id}."
+ )
+
+ self.gguf_writer.add_bos_token_id(vocab.bos_id)
+ self.gguf_writer.add_eos_token_id(vocab.eos_id)
+ self.gguf_writer.add_unk_token_id(vocab.unk_id)
+ self.gguf_writer.add_pad_token_id(vocab.pad_id)
+
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_scores(scores)
+ self.gguf_writer.add_token_types(toktypes)
+ self.gguf_writer.add_vocab_size(vocab.vocab_size)
+
+ self.gguf_writer.add_add_bos_token(True)
+ self.gguf_writer.add_add_eos_token(False)
+
+ local_template_file_path = self.dir_model / "chat_template.jinja"
+
+ if self.is_mistral_format and local_template_file_path.is_file():
+ # Ministral-3 and other new Mistral models come with chat templates.
+ # ref: https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512/tree/main
+ logger.info("Using an existing Mistral local chat template.")
+
+ with open(local_template_file_path, "r", encoding="utf-8") as f:
+ template = f.read()
+ elif not self.is_mistral_format or not self.disable_mistral_community_chat_template:
+ template_dir = Path(__file__).parent / "models/templates/"
+
+ # Log only for Mistral format that the official tokenization and detokenization is via `mistral-common`.
+ if self.is_mistral_format:
+ logger.info(
+ "Using a Mistral community chat template. These templates can be subject to errors in early days or"
+ " weeks after a release. Mistral recommends to use `mistral-common` to perform tokenization and"
+ " detokenization."
+ )
+ template = MistralModel.get_community_chat_template(vocab, template_dir, self.is_mistral_format)
+ else:
+ logger.info(
+ "Not using a Mistral local or community chat template. Ensure to perform the tokenization and"
+ " detokenization via `mistral-common`."
+ )
+ template = None
+
+ if template is not None:
+ self.gguf_writer.add_chat_template(template)
+
+ def _set_vocab_plamo(self):
+ # PLaMo models use a custom tokenizer with a .jsonl file
+ tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl"
+ tokenizer_config_path = self.dir_model / "tokenizer_config.json"
+
+ if not tokenizer_jsonl_path.is_file():
+ raise FileNotFoundError(f"PLaMo tokenizer file not found: {tokenizer_jsonl_path}")
+
+ # Load tokenizer config
+ with open(tokenizer_config_path, "r", encoding="utf-8") as f:
+ tokenizer_config = json.load(f)
+
+ # Load tokens from JSONL file (actually a list format)
+ tokens = []
+ scores = []
+ toktypes = []
+
+ with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f:
+ for line_num, line in enumerate(f):
+ if line.strip():
+ token_data = json.loads(line)
+ # Format: [token, score, type, ?, ?, ?, ?]
+ token = token_data[0].encode("utf-8")
+ score = float(token_data[1])
+ token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL"
+
+ tokens.append(token)
+ scores.append(score)
+
+ if token_type_str == "UNKNOWN":
+ toktypes.append(gguf.TokenType.UNKNOWN)
+ elif token_type_str == "CONTROL":
+ toktypes.append(gguf.TokenType.CONTROL)
+ elif token_type_str == "BYTE":
+ toktypes.append(gguf.TokenType.BYTE)
+ else:
+ token_str = token_data[0]
+ if token_str.startswith("<|plamo:") and token_str.endswith("|>"):
+ toktypes.append(gguf.TokenType.CONTROL)
+ else:
+ toktypes.append(gguf.TokenType.NORMAL)
+
+ vocab_size = self.hparams["vocab_size"]
+ if vocab_size > len(tokens):
+ pad_count = vocab_size - len(tokens)
+ logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")
+ for i in range(1, pad_count + 1):
+ tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))
+ scores.append(-1000.0)
+ toktypes.append(gguf.TokenType.UNUSED)
+
+ self.gguf_writer.add_tokenizer_model("plamo2")
+ self.gguf_writer.add_tokenizer_pre("default")
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_scores(scores)
+ self.gguf_writer.add_token_types(toktypes)
+
+ if "bos_token" in tokenizer_config and tokenizer_config["bos_token"] is not None:
+ token_id = tokens.index(tokenizer_config["bos_token"].encode("utf-8"))
+ self.gguf_writer.add_bos_token_id(token_id)
+ if "eos_token" in tokenizer_config and tokenizer_config["eos_token"] is not None:
+ token_id = tokens.index(tokenizer_config["eos_token"].encode("utf-8"))
+ self.gguf_writer.add_eos_token_id(token_id)
+ if "pad_token" in tokenizer_config and tokenizer_config["pad_token"] is not None:
+ token_id = tokens.index(tokenizer_config["pad_token"].encode("utf-8"))
+ self.gguf_writer.add_pad_token_id(token_id)
+ if "sep_token" in tokenizer_config and tokenizer_config["sep_token"] is not None:
+ token_id = tokens.index(tokenizer_config["sep_token"].encode("utf-8"))
+ self.gguf_writer.add_sep_token_id(token_id)
+ if "unk_token" in tokenizer_config and tokenizer_config["unk_token"] is not None:
+ token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8"))
+ self.gguf_writer.add_unk_token_id(token_id)
+
+ # Add <|plamo:op|> as EOT to ensure appropriate end of generation
+ self.gguf_writer.add_eot_token_id(4)
+
+ self.gguf_writer.add_add_space_prefix(False)
+
class MmprojModel(ModelBase):
model_type = ModelType.MMPROJ
@@ -1527,7 +1981,15 @@ class MmprojModel(ModelBase):
preprocessor_config: dict[str, Any]
global_config: dict[str, Any]
- n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]
+ n_block_keys = [
+ "n_layers",
+ "num_hidden_layers",
+ "n_layer",
+ "num_layers",
+ "depth",
+ "encoder_layers",
+ "vt_num_hidden_layers",
+ ]
has_vision_encoder: bool = True # by default
has_audio_encoder: bool = False
@@ -1576,16 +2038,42 @@ class MmprojModel(ModelBase):
# load preprocessor config
self.preprocessor_config = {}
- if not self.is_mistral_format:
- with open(self.dir_model / "preprocessor_config.json", "r", encoding="utf-8") as f:
- self.preprocessor_config = json.load(f)
+
+ # prefer preprocessor_config.json if possible
+ preprocessor_config_path = self.dir_model / "preprocessor_config.json"
+ if preprocessor_config_path.is_file():
+ with open(preprocessor_config_path, "r", encoding="utf-8") as f:
+ cfg = json.load(f)
+ # move media_proc_cfg to root level for compat
+ if "media_proc_cfg" in cfg:
+ cfg = {
+ **cfg,
+ **cfg["media_proc_cfg"],
+ }
+ # merge configs
+ self.preprocessor_config = {**self.preprocessor_config, **cfg}
+
+ # prefer processor_config.json if possible
+ processor_config_path = self.dir_model / "processor_config.json"
+ if processor_config_path.is_file():
+ with open(processor_config_path, "r", encoding="utf-8") as f:
+ cfg = json.load(f)
+ # move image_processor to root level for compat
+ if "image_processor" in cfg:
+ cfg = {
+ **cfg,
+ **cfg["image_processor"],
+ }
+ # merge configs
+ self.preprocessor_config = {**self.preprocessor_config, **cfg}
def get_vision_config(self) -> dict[str, Any] | None:
config_name = "vision_config" if not self.is_mistral_format else "vision_encoder"
return self.global_config.get(config_name)
def get_audio_config(self) -> dict[str, Any] | None:
- return self.global_config.get("audio_config")
+ mm_config_key = "whisper_config" if "whisper_config" in self.hparams else "audio_config"
+ return self.global_config.get(mm_config_key)
def set_type(self):
self.gguf_writer.add_type(gguf.GGUFType.MMPROJ)
@@ -1620,10 +2108,14 @@ class MmprojModel(ModelBase):
self.image_size = self.find_vparam(["image_size"])
self.gguf_writer.add_vision_image_size(self.image_size)
self.gguf_writer.add_vision_patch_size(self.find_vparam(["patch_size"]))
- self.gguf_writer.add_vision_embedding_length(self.find_vparam(["hidden_size"]))
- self.gguf_writer.add_vision_feed_forward_length(self.find_vparam(["intermediate_size"]))
+ self.gguf_writer.add_vision_embedding_length(self.find_vparam(["hidden_size", "vt_hidden_size"]))
+ self.gguf_writer.add_vision_feed_forward_length(
+ self.find_vparam(["intermediate_size", "vt_intermediate_size"])
+ )
self.gguf_writer.add_vision_block_count(self.find_vparam(self.n_block_keys))
- self.gguf_writer.add_vision_head_count(self.find_vparam(["num_attention_heads"]))
+ self.gguf_writer.add_vision_head_count(
+ self.find_vparam(["num_attention_heads", "num_heads", "vt_num_attention_heads"])
+ )
# preprocessor config
image_mean = (
@@ -1668,7 +2160,7 @@ class MmprojModel(ModelBase):
def tensor_force_quant(self, name, new_name, bid, n_dims):
del bid, name, n_dims # unused
- if ".patch_embd.weight" in new_name:
+ if ".patch_embd.weight" in new_name or ".patch_merger.weight" in new_name:
return (
gguf.GGMLQuantizationType.F16
if self.ftype == gguf.LlamaFileType.MOSTLY_F16
@@ -1682,11 +2174,9 @@ class GPTNeoXModel(TextModel):
model_arch = gguf.MODEL_ARCH.GPTNEOX
def set_gguf_parameters(self):
- block_count = self.hparams["num_hidden_layers"]
-
self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])
self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])
self.gguf_writer.add_rope_dimension_count(
int(self.hparams["rotary_pct"] * (self.hparams["hidden_size"] // self.hparams["num_attention_heads"])),
@@ -1696,13 +2186,9 @@ class GPTNeoXModel(TextModel):
self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))
n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))
- tensors: list[tuple[str, Tensor]] = []
-
if re.match(r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.weight", name):
qkv_weights = data_torch.reshape((n_head, 3, n_embed // n_head, n_embed))
data_torch = torch.cat(
@@ -1726,9 +2212,7 @@ class GPTNeoXModel(TextModel):
)
logger.info("re-format attention.linear_qkv.bias")
- tensors.append((self.map_tensor_name(name), data_torch))
-
- return tensors
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("BloomForCausalLM", "BloomModel")
@@ -1741,22 +2225,18 @@ class BloomModel(TextModel):
self.gguf_writer.add_context_length(self.hparams.get("seq_length", n_embed))
self.gguf_writer.add_embedding_length(n_embed)
self.gguf_writer.add_feed_forward_length(4 * n_embed)
- self.gguf_writer.add_block_count(self.hparams["n_layer"])
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_head_count(n_head)
self.gguf_writer.add_head_count_kv(n_head)
self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])
self.gguf_writer.add_file_type(self.ftype)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))
n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))
name = re.sub(r"transformer\.", "", name)
- tensors: list[tuple[str, Tensor]] = []
-
if re.match(r"h\.\d+\.self_attention\.query_key_value\.weight", name):
qkv_weights = data_torch.reshape((n_head, 3, n_embed // n_head, n_embed))
data_torch = torch.cat(
@@ -1780,9 +2260,7 @@ class BloomModel(TextModel):
)
logger.info("re-format attention.linear_qkv.bias")
- tensors.append((self.map_tensor_name(name), data_torch))
-
- return tensors
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("MPTForCausalLM")
@@ -1801,10 +2279,9 @@ class MPTModel(TextModel):
self.gguf_writer.add_unk_token_id(0)
def set_gguf_parameters(self):
- block_count = self.hparams["n_layers"]
self.gguf_writer.add_context_length(self.hparams["max_seq_len"])
self.gguf_writer.add_embedding_length(self.hparams["d_model"])
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_feed_forward_length(4 * self.hparams["d_model"])
self.gguf_writer.add_head_count(self.hparams["n_heads"])
if kv_n_heads := self.hparams["attn_config"].get("kv_n_heads"):
@@ -1818,15 +2295,13 @@ class MPTModel(TextModel):
self.gguf_writer.add_max_alibi_bias(0.0)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
if "scales" in name:
new_name = self.map_tensor_name(name, try_suffixes=(".weight", ".bias", ".scales"))
new_name = new_name.replace("scales", "act.scales")
else:
new_name = self.map_tensor_name(name, try_suffixes=(".weight", ".bias"))
- return [(new_name, data_torch)]
+ yield from super().modify_tensors(data_torch, new_name, bid)
@ModelBase.register("OrionForCausalLM")
@@ -1837,7 +2312,6 @@ class OrionModel(TextModel):
self._set_vocab_sentencepiece()
def set_gguf_parameters(self):
- block_count = self.hparams["num_hidden_layers"]
head_count = self.hparams["num_attention_heads"]
head_count_kv = self.hparams.get("num_key_value_heads", head_count)
@@ -1855,7 +2329,7 @@ class OrionModel(TextModel):
self.gguf_writer.add_tensor_data_layout("Meta AI original pth")
self.gguf_writer.add_context_length(ctx_length)
self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])
self.gguf_writer.add_head_count(head_count)
self.gguf_writer.add_head_count_kv(head_count_kv)
@@ -1870,45 +2344,18 @@ class BaichuanModel(TextModel):
self._set_vocab_sentencepiece()
def set_gguf_parameters(self):
- block_count = self.hparams["num_hidden_layers"]
+ super().set_gguf_parameters()
+
+ self.gguf_writer.add_tensor_data_layout("Meta AI original pth")
+ self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
head_count = self.hparams["num_attention_heads"]
head_count_kv = self.hparams.get("num_key_value_heads", head_count)
- ctx_length = 0
- if "max_sequence_length" in self.hparams:
- ctx_length = self.hparams["max_sequence_length"]
- elif "max_position_embeddings" in self.hparams:
- ctx_length = self.hparams["max_position_embeddings"]
- elif "model_max_length" in self.hparams:
- ctx_length = self.hparams["model_max_length"]
- else:
- raise ValueError("gguf: can not find ctx length parameter.")
-
- self.gguf_writer.add_tensor_data_layout("Meta AI original pth")
- self.gguf_writer.add_context_length(ctx_length)
- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])
- self.gguf_writer.add_block_count(block_count)
- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])
- self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])
- self.gguf_writer.add_head_count(head_count)
- self.gguf_writer.add_head_count_kv(head_count_kv)
- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])
- self.gguf_writer.add_file_type(self.ftype)
-
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
-
- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- head_count = self.hparams["num_attention_heads"]
- head_count_kv = self.hparams.get("num_key_value_heads", head_count)
-
- tensors: list[tuple[str, Tensor]] = []
-
if bid is not None and name == f"model.layers.{bid}.self_attn.W_pack.weight":
logger.info(f"Unpacking and permuting layer {bid}")
- tensors = [
+ yield from [
(
self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid),
self._reverse_hf_permute_part(data_torch, 0, head_count, head_count),
@@ -1920,9 +2367,7 @@ class BaichuanModel(TextModel):
(self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), self._reverse_hf_part(data_torch, 2)),
]
else:
- tensors = [(self.map_tensor_name(name), data_torch)]
-
- return tensors
+ yield from self.modify_tensors(data_torch, self.map_tensor_name(name), bid)
def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor:
if n_kv_head is not None and n_head != n_kv_head:
@@ -2002,39 +2447,12 @@ class XverseModel(TextModel):
special_vocab.add_to_gguf(self.gguf_writer)
def set_gguf_parameters(self):
- block_count = self.hparams["num_hidden_layers"]
- head_count = self.hparams["num_attention_heads"]
- head_count_kv = self.hparams.get("num_key_value_heads", head_count)
-
- ctx_length = 0
- if "max_sequence_length" in self.hparams:
- ctx_length = self.hparams["max_sequence_length"]
- elif "max_position_embeddings" in self.hparams:
- ctx_length = self.hparams["max_position_embeddings"]
- elif "model_max_length" in self.hparams:
- ctx_length = self.hparams["model_max_length"]
- else:
- raise ValueError("gguf: can not find ctx length parameter.")
+ super().set_gguf_parameters()
self.gguf_writer.add_tensor_data_layout("Meta AI original pth")
- self.gguf_writer.add_context_length(ctx_length)
- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])
- self.gguf_writer.add_block_count(block_count)
- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])
self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])
- self.gguf_writer.add_head_count(head_count)
- self.gguf_writer.add_head_count_kv(head_count_kv)
- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])
- self.gguf_writer.add_file_type(self.ftype)
-
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
head_count = self.hparams["num_attention_heads"]
head_count_kv = self.hparams.get("num_key_value_heads", head_count)
@@ -2044,7 +2462,7 @@ class XverseModel(TextModel):
if name.endswith("k_proj.weight"):
data_torch = self._reverse_hf_permute(data_torch, head_count, head_count_kv)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor:
if n_kv_head is not None and n_head != n_kv_head:
@@ -2062,10 +2480,6 @@ class FalconModel(TextModel):
model_arch = gguf.MODEL_ARCH.FALCON
def set_gguf_parameters(self):
- block_count = self.hparams.get("num_hidden_layers")
- if block_count is None:
- block_count = self.hparams["n_layer"] # old name
-
n_head = self.hparams.get("num_attention_heads")
if n_head is None:
n_head = self.hparams["n_head"] # old name
@@ -2078,15 +2492,13 @@ class FalconModel(TextModel):
self.gguf_writer.add_tensor_data_layout("jploski") # qkv tensor transform
self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])
self.gguf_writer.add_feed_forward_length(4 * self.hparams["hidden_size"])
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_head_count(n_head)
self.gguf_writer.add_head_count_kv(n_head_kv)
self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])
self.gguf_writer.add_file_type(self.ftype)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
# QKV tensor transform
# The original query_key_value tensor contains n_head_kv "kv groups",
# each consisting of n_head/n_head_kv query weights followed by one key
@@ -2108,7 +2520,7 @@ class FalconModel(TextModel):
v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
data_torch = torch.cat((q, k, v)).reshape_as(data_torch)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("GPTBigCodeForCausalLM")
@@ -2116,12 +2528,10 @@ class StarCoderModel(TextModel):
model_arch = gguf.MODEL_ARCH.STARCODER
def set_gguf_parameters(self):
- block_count = self.hparams["n_layer"]
-
self.gguf_writer.add_context_length(self.hparams["n_positions"])
self.gguf_writer.add_embedding_length(self.hparams["n_embd"])
self.gguf_writer.add_feed_forward_length(4 * self.hparams["n_embd"])
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_head_count(self.hparams["n_head"])
self.gguf_writer.add_head_count_kv(1)
self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])
@@ -2152,14 +2562,12 @@ class RefactModel(TextModel):
multiple_of = 256
ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)
- block_count = self.hparams["n_layer"]
-
# refact uses Alibi. So this is from config.json which might be used by training.
self.gguf_writer.add_context_length(self.hparams["n_positions"])
self.gguf_writer.add_embedding_length(self.hparams["n_embd"])
self.gguf_writer.add_feed_forward_length(ff_dim)
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_head_count(self.hparams["n_head"])
self.gguf_writer.add_head_count_kv(1)
self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"])
@@ -2175,26 +2583,30 @@ class RefactModel(TextModel):
n_head_kv = 1
head_dim = self.hparams["n_embd"] // n_head
- tensors: list[tuple[str, Tensor]] = []
-
if bid is not None:
if name == f"transformer.h.{bid}.attn.kv.weight":
- tensors.append(
- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), data_torch[: n_head_kv * head_dim])
+ yield from super().modify_tensors(
+ data_torch[: n_head_kv * head_dim], self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), bid
)
- tensors.append(
- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), data_torch[n_head_kv * head_dim :])
+ yield from super().modify_tensors(
+ data_torch[n_head_kv * head_dim :], self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), bid
)
- elif name == f"transformer.h.{bid}.attn.q.weight":
- tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), data_torch))
- elif name == f"transformer.h.{bid}.mlp.gate_up_proj.weight":
- tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), data_torch[:ff_dim]))
- tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), data_torch[ff_dim:]))
-
- if len(tensors) == 0:
- tensors.append((self.map_tensor_name(name), data_torch))
+ return
+ if name == f"transformer.h.{bid}.attn.q.weight":
+ yield from super().modify_tensors(
+ data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), bid
+ )
+ return
+ if name == f"transformer.h.{bid}.mlp.gate_up_proj.weight":
+ yield from super().modify_tensors(
+ data_torch[:ff_dim], self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), bid
+ )
+ yield from super().modify_tensors(
+ data_torch[ff_dim:], self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), bid
+ )
+ return
- return tensors
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM")
@@ -2210,11 +2622,10 @@ class StableLMModel(TextModel):
def set_gguf_parameters(self):
hparams = self.hparams
- block_count = hparams["num_hidden_layers"]
self.gguf_writer.add_context_length(hparams["max_position_embeddings"])
self.gguf_writer.add_embedding_length(hparams["hidden_size"])
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])
rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"])
self.gguf_writer.add_rope_dimension_count(
@@ -2246,7 +2657,7 @@ class StableLMModel(TextModel):
if len(self._q_norms[bid]) >= n_head:
return self._stack_qk_norm(bid, n_head, self._q_norms[bid], "q_layernorm")
else:
- return []
+ return
if name.find("k_layernorm.norms") != -1:
assert bid is not None
@@ -2259,9 +2670,9 @@ class StableLMModel(TextModel):
if len(self._k_norms[bid]) >= n_kv_head:
return self._stack_qk_norm(bid, n_kv_head, self._k_norms[bid], "k_layernorm")
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def _stack_qk_norm(self, bid: int, n_head: int, norms: dict[str, Tensor], layer_name: str = "q_layernorm"):
data: list[Tensor] = []
@@ -2273,9 +2684,8 @@ class StableLMModel(TextModel):
data_torch = torch.stack(data, dim=0)
merged_name = f"model.layers.{bid}.self_attn.{layer_name}.weight"
- new_name = self.map_tensor_name(merged_name)
- return [(new_name, data_torch)]
+ yield from super().modify_tensors(data_torch, merged_name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -2297,6 +2707,7 @@ class StableLMModel(TextModel):
"VLlama3ForCausalLM",
"LlavaForConditionalGeneration",
"VoxtralForConditionalGeneration",
+ "IQuestCoderForCausalLM",
"LlamaModel",
)
class LlamaModel(TextModel):
@@ -2308,68 +2719,13 @@ class LlamaModel(TextModel):
# fix for SmolVLM2, missing `num_attention_heads` in config.json
if self.hf_arch == "VLlama3ForCausalLM":
self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 32)
-
- def _set_vocab_mistral(self):
- if not _mistral_common_installed:
- raise ImportError(_mistral_import_error_msg)
-
- vocab = MistralVocab(self.dir_model)
- logger.info(f"Converting tokenizer {vocab.tokenizer_type} of size {vocab.vocab_size}.")
-
- self.gguf_writer.add_tokenizer_model(vocab.gguf_tokenizer_model)
-
- tokens = []
- scores = []
- toktypes = []
-
- for text, score, toktype in vocab.all_tokens():
- tokens.append(text)
- scores.append(score)
- toktypes.append(toktype)
-
- assert len(tokens) == vocab.vocab_size, f"token count ({len(tokens)}) != vocab size ({vocab.vocab_size})"
-
- if vocab.tokenizer_type == MistralTokenizerType.tekken:
- self.gguf_writer.add_tokenizer_pre("tekken")
- self.gguf_writer.add_token_merges(vocab.extract_vocab_merges_from_model())
-
- logger.info(
- f"Setting bos, eos, unk and pad token IDs to {vocab.bos_id}, {vocab.eos_id}, {vocab.unk_id},"
- f" {vocab.pad_id}."
- )
-
- self.gguf_writer.add_bos_token_id(vocab.bos_id)
- self.gguf_writer.add_eos_token_id(vocab.eos_id)
- self.gguf_writer.add_unk_token_id(vocab.unk_id)
- self.gguf_writer.add_pad_token_id(vocab.pad_id)
-
- self.gguf_writer.add_token_list(tokens)
- self.gguf_writer.add_token_scores(scores)
- self.gguf_writer.add_token_types(toktypes)
- self.gguf_writer.add_vocab_size(vocab.vocab_size)
-
- self.gguf_writer.add_add_bos_token(True)
- self.gguf_writer.add_add_eos_token(False)
-
- template_dir = Path(__file__).parent / "models/templates/"
-
- if not self.is_mistral_format or not self.disable_mistral_community_chat_template:
- # Log only for Mistral format that the official tokenization and detokenization is via `mistral-common`.
- if self.is_mistral_format:
- logger.info(
- "Using a Mistral community chat template. These templates can be subject to "
- "errors in early days or weeks after a release. "
- "Mistral recommends to use `mistral-common` to perform tokenization and detokenization."
- )
- template = MistralModel.get_community_chat_template(vocab, template_dir, self.is_mistral_format)
- self.gguf_writer.add_chat_template(template)
- else:
- logger.info(
- "Not using a Mistral community chat template. Ensure to perform the tokenization and"
- " detokenization via `mistral-common`."
- )
+ hparams = ModelBase.load_hparams(self.dir_model, is_mistral_format=False)
+ self.origin_hf_arch = hparams.get("architectures", [None])[0]
def set_vocab(self):
+ if self.origin_hf_arch == "GlmasrModel":
+ return self._set_vocab_glmedge()
+
if self.is_mistral_format:
return self._set_vocab_mistral()
@@ -2420,11 +2776,6 @@ class LlamaModel(TextModel):
rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
self.gguf_writer.add_rope_dimension_count(rope_dim)
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
-
@staticmethod
def permute(weights: Tensor, n_head: int, n_head_kv: int | None):
if n_head_kv is not None and n_head != n_head_kv:
@@ -2446,6 +2797,7 @@ class LlamaModel(TextModel):
"vision_language_adapter.",
"patch_merger.",
"pre_mm_projector_norm",
+ "audio_encoder.",
]
is_multimodal_tensor = (
@@ -2458,7 +2810,7 @@ class LlamaModel(TextModel):
)
if is_multimodal_tensor:
- return [] # skip vision tensors
+ return # skip vision tensors
elif self.hf_arch == "LlamaModel":
name = "model." + name
elif name.startswith("model.text_model"):
@@ -2484,8 +2836,6 @@ class LlamaModel(TextModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for wid in ["w1", "w2", "w3"]:
data: list[Tensor] = []
@@ -2499,26 +2849,24 @@ class LlamaModel(TextModel):
merged_name = f"layers.{bid}.feed_forward.experts.{wid}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
- if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):
- if rope_scaling.get("rope_type", "").lower() == "llama3":
- base = self.hparams.get("rope_theta", 10000.0)
+ if rope_params := self.rope_parameters.get("full_attention", self.rope_parameters):
+ if rope_params.get("rope_type", "").lower() == "llama3":
+ base = rope_params.get("rope_theta", 10000.0)
if (dim := self.hparams.get("head_dim")) is None:
dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
- factor = rope_scaling.get("factor", 8.0)
- low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)
- high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)
+ factor = rope_params.get("factor", 8.0)
+ low_freq_factor = rope_params.get("low_freq_factor", 1.0)
+ high_freq_factor = rope_params.get("high_freq_factor", 4.0)
old_context_len = self.hparams.get("original_max_position_embeddings", 8192)
low_freq_wavelen = old_context_len / low_freq_factor
@@ -2558,11 +2906,67 @@ class ArceeModel(LlamaModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
self._try_set_pooling_type()
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
+
+
[email protected]("AfmoeForCausalLM")
+class AfmoeModel(LlamaModel):
+ model_arch = gguf.MODEL_ARCH.AFMOE
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ # MoE parameters
+ if (n_shared_experts := self.hparams.get("num_shared_experts")) is not None:
+ self.gguf_writer.add_expert_shared_count(n_shared_experts)
+ if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
+ self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
+ if (n_dense_layers := self.hparams.get("num_dense_layers")) is not None:
+ self.gguf_writer.add_leading_dense_block_count(n_dense_layers)
+
+ # Route normalization and scaling
+ if (route_norm := self.hparams.get("route_norm")) is not None:
+ self.gguf_writer.add_expert_weights_norm(route_norm)
+ if (route_scale := self.hparams.get("route_scale")) is not None:
+ self.gguf_writer.add_expert_weights_scale(route_scale)
+
+ # Sliding window attention
+ if (sliding_window := self.hparams.get("sliding_window")) is not None:
+ self.gguf_writer.add_sliding_window(sliding_window)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Handle expert weights - they're already merged in the HF format
+ # process the experts separately
+ if name.find("mlp.experts") != -1:
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
+ assert bid is not None
+
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) >= n_experts * 3:
+ # merge the experts into a single 3d tensor
+ for w_name in ["gate_proj", "up_proj", "down_proj"]:
+ data: list[Tensor] = []
+
+ for xid in range(n_experts):
+ ename_to_retrieve = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"
+ data.append(self._experts[bid][ename_to_retrieve])
+ del self._experts[bid][ename_to_retrieve]
+
+ data_torch = torch.stack(data, dim=0)
+ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+ yield from ModelBase.modify_tensors(self, data_torch, merged_name, bid)
+
+ return
+ else:
+ return
+
+ if name.endswith(".expert_bias"):
+ name = name.replace(".expert_bias", ".expert_bias.bias")
+
+ yield from ModelBase.modify_tensors(self, data_torch, name, bid)
@ModelBase.register(
@@ -2619,7 +3023,6 @@ class LlavaVisionModel(MmprojModel):
self.gguf_writer.add_vision_spatial_merge_size(self.global_config["spatial_merge_size"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
n_head = (
self.hparams["num_attention_heads"]
if not self.is_mistral_format
@@ -2642,7 +3045,8 @@ class LlavaVisionModel(MmprojModel):
data_torch = LlamaModel.permute(data_torch, n_head, n_head)
if name.endswith(("k_proj.weight", "k_proj.bias")) and not self.is_mistral_format:
data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
embed_key = "embed_tokens.weight" if not self.is_mistral_format else "tok_embeddings.weight"
if self.img_break_tok_id > 0 and embed_key in name:
@@ -2650,13 +3054,14 @@ class LlavaVisionModel(MmprojModel):
# for pixtral model, we need to extract the [IMG_BREAK] token embedding
img_break_embd = data_torch[self.img_break_tok_id]
name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK]
- return [(self.map_tensor_name(name), img_break_embd)]
+ yield from super().modify_tensors(img_break_embd, name, bid)
- return [] # skip other tensors
+ return # skip other tensors
@ModelBase.register("Idefics3ForConditionalGeneration", "SmolVLMForConditionalGeneration")
class SmolVLMModel(MmprojModel):
+
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.hparams["model_type"] == "smolvlm_vision":
@@ -2683,13 +3088,12 @@ class SmolVLMModel(MmprojModel):
return super().tensor_force_quant(name, new_name, bid, n_dims)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
is_vision_tensor = "vision_tower" in name or "vision_model" in name or "model.connector" in name
if is_vision_tensor:
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
- return [] # skip other tensors
+ return # skip other tensors
@ModelBase.register(
@@ -2728,22 +3132,22 @@ class Llama4Model(LlamaModel):
name_gate = name.replace("gate_up_proj", "gate_proj.weight")
dim_half = data_torch.shape[-1] // 2
gate_proj_weight, up_proj_weight = data_torch.transpose(-1, -2).split(dim_half, dim=-2)
- return [
- (self.map_tensor_name(name_gate), gate_proj_weight),
- (self.map_tensor_name(name_up), up_proj_weight),
- ]
+ yield from super().modify_tensors(gate_proj_weight, name_gate, bid)
+ yield from super().modify_tensors(up_proj_weight, name_up, bid)
+ return
if name.endswith("down_proj"):
name += ".weight"
data_torch = data_torch.transpose(-1, -2)
if "multi_modal_projector" in name or "vision_model" in name:
- return []
- return super().modify_tensors(data_torch, name, bid)
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Llama4ForConditionalGeneration")
class Llama4VisionModel(MmprojModel):
+
def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.LLAMA4)
@@ -2753,27 +3157,49 @@ class Llama4VisionModel(MmprojModel):
self.gguf_writer.add_vision_use_gelu(True)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
if "multi_modal_projector" in name or "vision_model" in name:
# process vision tensors
if "positional_embedding_vlm" in name and ".weight" not in name:
name += ".weight"
if "multi_modal_projector.linear_1" in name:
# despite the name with number postfix, this is a single fully connected layer
- return [(gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_MMPROJ_FC] + ".weight", data_torch)]
- return [(self.map_tensor_name(name), data_torch)]
- return []
+ yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_MMPROJ_FC] + ".weight", data_torch)
+ else:
+ yield from super().modify_tensors(data_torch, name, bid)
[email protected]("Mistral3ForConditionalGeneration")
+ "Mistral3ForConditionalGeneration",
+ "Ministral3ForCausalLM",
+)
class Mistral3Model(LlamaModel):
- model_arch = gguf.MODEL_ARCH.LLAMA
+ model_arch = gguf.MODEL_ARCH.MISTRAL3
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # for compatibility, we use LLAMA arch for older models
+ # TODO: remove this once everyone has migrated to newer version of llama.cpp
+ if self.hparams.get("model_type") != "ministral3":
+ self.model_arch = gguf.MODEL_ARCH.LLAMA
+ self.gguf_writer.arch = gguf.MODEL_ARCH_NAMES[self.model_arch]
+ self.gguf_writer.add_architecture()
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ rope_params = self.rope_parameters
+ if self.hparams.get("model_type") == "ministral3":
+ assert rope_params, "ministral3 must have 'rope_parameters' config"
+ assert rope_params["rope_type"] == "yarn", "ministral3 rope_type must be 'yarn'"
+ self.gguf_writer.add_rope_scaling_yarn_log_mul(rope_params["mscale_all_dim"])
+ self.gguf_writer.add_attn_temperature_scale(rope_params["llama_4_scaling_beta"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
name = name.replace("language_model.", "")
if "multi_modal_projector" in name or "vision_tower" in name:
- return []
- return super().modify_tensors(data_torch, name, bid)
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("DeciLMForCausalLM")
@@ -2864,7 +3290,7 @@ class DeciModel(TextModel):
assert self.block_count == len(self._num_kv_heads)
assert self.block_count == len(self._num_heads)
assert self.block_count == len(self._ffn_dims)
- if (rope_theta := self.hparams.get("rope_theta")) is not None:
+ if (rope_theta := self.rope_parameters.get("rope_theta")) is not None:
self.gguf_writer.add_rope_freq_base(rope_theta)
self.gguf_writer.add_head_count_kv(self._num_kv_heads)
self.gguf_writer.add_head_count(self._num_heads)
@@ -2889,11 +3315,6 @@ class DeciModel(TextModel):
rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
self.gguf_writer.add_rope_dimension_count(rope_dim)
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
-
@staticmethod
def permute(weights: Tensor, n_head: int, n_head_kv: int | None):
if n_head_kv is not None and n_head != n_head_kv:
@@ -2921,19 +3342,19 @@ class DeciModel(TextModel):
data_torch = DeciModel.permute(data_torch, n_head, n_head)
if name.endswith(("k_proj.weight", "k_proj.bias")):
data_torch = DeciModel.permute(data_torch, n_head, n_kv_head)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
- if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):
- if rope_scaling.get("rope_type", "").lower() == "llama3":
- base = self.hparams.get("rope_theta", 10000.0)
+ if rope_params := self.rope_parameters.get("full_attention", self.rope_parameters):
+ if rope_params.get("rope_type", "").lower() == "llama3":
+ base = rope_params.get("rope_theta", 10000.0)
if (dim := self.hparams.get("head_dim")) is None:
dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
- factor = rope_scaling.get("factor", 8.0)
- low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)
- high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)
+ factor = rope_params.get("factor", 8.0)
+ low_freq_factor = rope_params.get("low_freq_factor", 1.0)
+ high_freq_factor = rope_params.get("high_freq_factor", 4.0)
old_context_len = self.hparams.get("original_max_position_embeddings", 8192)
low_freq_wavelen = old_context_len / low_freq_factor
@@ -2998,7 +3419,7 @@ class BitnetModel(TextModel):
# transform weight into 1/0/-1 (in fp32)
data_torch = self.weight_quant(data_torch)
- yield (new_name, data_torch)
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("GrokForCausalLM", "Grok1ForCausalLM")
@@ -3057,11 +3478,11 @@ class GrokModel(TextModel):
_cur_expert = ""
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- tensors: list[tuple[str, Tensor]] = []
+ deferred: list[tuple[Tensor, str, int | None]] = []
is_expert = ".moe." in name or ".block_sparse_moe.experts." in name
if not is_expert:
- tensors.append((self.map_tensor_name(name), data_torch))
+ deferred.append((data_torch, name, bid))
# process the experts separately
if is_expert or self._cur_expert:
@@ -3076,11 +3497,11 @@ class GrokModel(TextModel):
if name in self._experts[bid]:
self._cur_expert = name
self._experts[bid][name].append(data_torch)
- return []
+ return
elif is_expert:
self._cur_expert = name
self._experts[bid][name] = [data_torch]
- return []
+ return
else:
self._cur_expert = ""
@@ -3102,11 +3523,10 @@ class GrokModel(TextModel):
merged_name = f"transformer.decoder_layer.{bid}.moe.{wid[0]}.weight"
- new_name = self.map_tensor_name(merged_name)
+ yield from super().modify_tensors(data_torch, merged_name, bid)
- yield (new_name, data_torch)
-
- yield from tensors
+ for t in deferred:
+ yield from super().modify_tensors(*t)
@ModelBase.register("DbrxForCausalLM")
@@ -3116,7 +3536,7 @@ class DbrxModel(TextModel):
def set_gguf_parameters(self):
ffn_config = self.hparams["ffn_config"]
attn_config = self.hparams["attn_config"]
- self.gguf_writer.add_block_count(self.hparams["n_layers"])
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_context_length(self.hparams["max_seq_len"])
self.gguf_writer.add_embedding_length(self.hparams["d_model"])
@@ -3138,8 +3558,6 @@ class DbrxModel(TextModel):
logger.info(f"gguf: file type = {self.ftype}")
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
n_expert = self.hparams["ffn_config"]["moe_num_experts"]
n_ff = self.hparams["ffn_config"]["ffn_hidden_size"]
n_embd = self.hparams["d_model"]
@@ -3172,7 +3590,7 @@ class DbrxModel(TextModel):
# https://huggingface.co/databricks/dbrx-instruct/blob/main/model.safetensors.index.json#L15
new_name = self.map_tensor_name(name if not experts else name + ".weight", try_suffixes=(".weight",))
- return [(new_name, data_torch)]
+ yield from super().modify_tensors(data_torch, new_name, bid)
def tensor_force_quant(
self, name: str, new_name: str, bid: int | None, n_dims: int
@@ -3197,10 +3615,6 @@ class MiniCPMModel(TextModel):
logit_scale = self.hparams["hidden_size"] / self.hparams["dim_model_base"]
self.gguf_writer.add_logit_scale(logit_scale)
logger.info(f"gguf: (minicpm) logit_scale = {logit_scale}")
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "longrope":
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LONGROPE)
- logger.info(f"gguf: (minicpm) rope_scaling_type = {gguf.RopeScalingType.LONGROPE}")
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
rope_dims = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
@@ -3229,18 +3643,16 @@ class MiniCPMModel(TextModel):
self._set_vocab_sentencepiece()
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
n_head = self.hparams["num_attention_heads"]
n_kv_head = self.hparams.get("num_key_value_heads")
# HF models permute some of the tensors, so we need to undo that
- if name.endswith(("q_proj.weight")):
+ if name.endswith("q_proj.weight"):
data_torch = LlamaModel.permute(data_torch, n_head, n_head)
- if name.endswith(("k_proj.weight")):
+ if name.endswith("k_proj.weight"):
data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("MiniCPM3ForCausalLM")
@@ -3333,19 +3745,14 @@ class QwenModel(TextModel):
def set_vocab(self):
self._set_vocab_qwen()
- def set_gguf_parameters(self):
- self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])
- self.gguf_writer.add_block_count(self.hparams["num_hidden_layers"])
- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])
- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])
- self.gguf_writer.add_rope_freq_base(self.hparams["rotary_emb_base"])
- self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])
- self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])
- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"])
- self.gguf_writer.add_file_type(self.ftype)
-
[email protected]("Qwen2Model", "Qwen2ForCausalLM", "Qwen2AudioForConditionalGeneration")
+ "Qwen2Model",
+ "Qwen2ForCausalLM",
+ "Qwen2AudioForConditionalGeneration",
+ "KORMoForCausalLM",
+ "AudioFlamingo3ForConditionalGeneration",
+)
class Qwen2Model(TextModel):
model_arch = gguf.MODEL_ARCH.QWEN2
@@ -3358,11 +3765,6 @@ class Qwen2Model(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
self._try_set_pooling_type()
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if self.hf_arch == "Qwen2Model":
@@ -3378,7 +3780,7 @@ class Qwen2Model(TextModel):
or name.startswith("model.multi_modal_projector")
):
# skip vision and audio tensors
- return []
+ return
yield from super().modify_tensors(data_torch, name, bid)
@@ -3436,12 +3838,6 @@ class DreamModel(TextModel):
# Dream models use non-causal attention for diffusion
self.gguf_writer.add_causal_attention(False)
- # Handle RoPE scaling similar to Qwen2
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
# Add Dream-specific parameters
mask_token_id = self.hparams.get("mask_token_id")
@@ -3564,6 +3960,13 @@ class Ernie4_5Model(TextModel):
def set_vocab(self):
self._set_vocab_sentencepiece()
+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"
+ if tokenizer_config_file.is_file():
+ with open(tokenizer_config_file, "r", encoding="utf-8") as f:
+ tokenizer_config_json = json.load(f)
+ if "add_prefix_space" in tokenizer_config_json:
+ self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])
+
def set_gguf_parameters(self):
super().set_gguf_parameters()
@@ -3573,6 +3976,10 @@ class Ernie4_5Model(TextModel):
if (head_dim := self.hparams.get("head_dim")) is None:
head_dim = self.hparams["hidden_size"] // num_heads
+ if "mlp_AR" in name or "vision_model" in name:
+ # skip vision model and projector tensors
+ return
+
if "ernie." in name:
name = name.replace("ernie.", "model.")
# split the qkv weights
@@ -3587,23 +3994,20 @@ class Ernie4_5Model(TextModel):
q_proj_weight, k_proj_weight, v_proj_weight = data_torch.split(
[total_q_dim, total_k_dim, total_v_dim], dim=0
)
- return [
- (self.map_tensor_name(name_q), q_proj_weight),
- (self.map_tensor_name(name_k), k_proj_weight),
- (self.map_tensor_name(name_v), v_proj_weight),
- ]
+ yield from super().modify_tensors(q_proj_weight, name_q, bid)
+ yield from super().modify_tensors(k_proj_weight, name_k, bid)
+ yield from super().modify_tensors(v_proj_weight, name_v, bid)
# split the up_gate_proj into gate and up
# up_gate_proj shape: [2 * intermediate_size, hidden_size]
- if "up_gate_proj" in name:
+ elif "up_gate_proj" in name:
name_up = name.replace("up_gate_proj.weight", "up_proj.weight")
name_gate = name.replace("up_gate_proj.weight", "gate_proj.weight")
dim_half = data_torch.shape[0] // 2
gate_proj_weight, up_proj_weight = data_torch.split(dim_half, dim=0)
- return [
- (self.map_tensor_name(name_gate), gate_proj_weight),
- (self.map_tensor_name(name_up), up_proj_weight),
- ]
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(gate_proj_weight, name_gate, bid)
+ yield from super().modify_tensors(up_proj_weight, name_up, bid)
+ else:
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Ernie4_5_MoeForCausalLM")
@@ -3642,20 +4046,20 @@ class Ernie4_5MoeModel(Ernie4_5Model):
# skip Multi-Token Prediction (MTP) layers (again, same as DeepseekV2)
match = re.match(r"model.mtp_block.(\d+)", name)
if match:
- return []
+ return
# skip all other MTP tensors for now
match = re.match(r"model.mtp_emb_norm.(\d+)", name)
if match:
- return []
+ return
match = re.match(r"model.mtp_hidden_norm.(\d+)", name)
if match:
- return []
+ return
match = re.match(r"model.mtp_linear_proj.(\d+)", name)
if match:
- return []
+ return
# process the experts separately
if name.find("mlp.experts") != -1:
@@ -3668,8 +4072,6 @@ class Ernie4_5MoeModel(Ernie4_5Model):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["gate_proj", "up_proj", "down_proj"]:
data: list[Tensor] = []
@@ -3681,13 +4083,9 @@ class Ernie4_5MoeModel(Ernie4_5Model):
data_torch = torch.stack(data, dim=0)
merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
- tensors.append((new_name, data_torch))
-
- return tensors
- else:
- return []
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ else:
+ yield from ModelBase.modify_tensors(self, data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -3699,7 +4097,49 @@ class Ernie4_5MoeModel(Ernie4_5Model):
raise ValueError(f"Unprocessed experts: {experts}")
[email protected]("PaddleOCRVLForConditionalGeneration")
+class PaddleOCRModel(Ernie4_5Model):
+ model_arch = gguf.MODEL_ARCH.PADDLEOCR
+
+
[email protected]("PaddleOCRVisionModel")
+class PaddleOCRVisionModel(MmprojModel):
+ # PaddleOCR-VL uses a modified version of Siglip
+ min_pixels: int = 0
+ max_pixels: int = 0
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ assert self.hparams_vision is not None
+ self.min_pixels = self.preprocessor_config["min_pixels"]
+ self.max_pixels = self.preprocessor_config["max_pixels"]
+ self.hparams_vision["image_size"] = int(math.sqrt(self.max_pixels))
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ assert self.hparams_vision is not None
+ hparams = self.hparams_vision
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.PADDLEOCR)
+ self.gguf_writer.add_vision_max_pixels(self.max_pixels)
+ self.gguf_writer.add_vision_min_pixels(self.min_pixels)
+ self.gguf_writer.add_vision_use_gelu(True)
+ self.gguf_writer.add_vision_attention_layernorm_eps(hparams.get("rms_norm_eps", 1e-6))
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ name = name.replace("visual.", "model.")
+
+ if "vision_model" in name or "mlp_AR" in name:
+ if "packing_position_embedding" in name:
+ return # unused
+ elif "vision_model.head" in name:
+ # we don't yet support image embeddings for this model
+ return
+ else:
+ yield from super().modify_tensors(data_torch, name, bid)
+ return # skip other tensors
+
+
"Qwen2VLModel",
"Qwen2VLForConditionalGeneration",
"Qwen2_5_VLForConditionalGeneration",
@@ -3710,9 +4150,6 @@ class Qwen2VLModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- mrope_section = self.hparams["rope_scaling"]["mrope_section"]
- mrope_section += [0] * max(0, 4 - len(mrope_section))
- self.gguf_writer.add_rope_dimension_sections(mrope_section)
def set_vocab(self):
try:
@@ -3721,7 +4158,6 @@ class Qwen2VLModel(TextModel):
self._set_vocab_gpt2()
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
if name.startswith("thinker."):
name = name.replace("thinker.", "")
if (
@@ -3731,12 +4167,13 @@ class Qwen2VLModel(TextModel):
or name.startswith("token2wav")
):
# skip multimodal tensors
- return []
- return [(self.map_tensor_name(name), data_torch)]
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Qwen2VLModel", "Qwen2VLForConditionalGeneration", "Qwen2_5_VLForConditionalGeneration")
class Qwen2VLVisionModel(MmprojModel):
+
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert self.hparams_vision is not None
@@ -3781,7 +4218,6 @@ class Qwen2VLVisionModel(MmprojModel):
return super().tensor_force_quant(name, new_name, bid, n_dims)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
if name.startswith("visual."):
# process visual tensors
# split QKV tensors if needed
@@ -3795,23 +4231,18 @@ class Qwen2VLVisionModel(MmprojModel):
wq = data_torch[:c]
wk = data_torch[c : c * 2]
wv = data_torch[c * 2 :]
- return [
- (self.map_tensor_name(name.replace("qkv", "q")), wq),
- (self.map_tensor_name(name.replace("qkv", "k")), wk),
- (self.map_tensor_name(name.replace("qkv", "v")), wv),
- ]
+ yield from super().modify_tensors(wq, name.replace("qkv", "q"), bid)
+ yield from super().modify_tensors(wk, name.replace("qkv", "k"), bid)
+ yield from super().modify_tensors(wv, name.replace("qkv", "v"), bid)
elif "patch_embed.proj.weight" in name:
# split Conv3D into Conv2Ds
c1, c2, kt, kh, kw = data_torch.shape
del c1, c2, kh, kw # unused
assert kt == 2, "Current implementation only support temporal_patch_size of 2"
- return [
- (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight", data_torch[:, :, 0, ...]),
- (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch[:, :, 1, ...]),
- ]
+ yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight", data_torch[:, :, 0, ...])
+ yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch[:, :, 1, ...])
else:
- return [(self.map_tensor_name(name), data_torch)]
- return [] # skip other tensors
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Qwen2_5OmniModel")
@@ -3865,14 +4296,15 @@ class Qwen25OmniModel(Qwen2VLVisionModel):
# transpose conv1 and conv2 bias
data_torch = data_torch.unsqueeze(-1)
if "audio_bos_eos_token" in name:
- return []
- return [(self.map_tensor_name(name), data_torch)]
-
- return super().modify_tensors(data_torch, name, bid)
+ # this tensor is left unused in transformers code
+ # https://github.com/huggingface/transformers/blob/6e3063422c4b1c014aa60c32b9254fd2902f0f28/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py#L1809
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("InternVisionModel")
class InternVisionModel(MmprojModel):
+
def set_gguf_parameters(self):
assert self.hparams_vision is not None
if isinstance(self.hparams_vision["image_size"], list):
@@ -3915,7 +4347,6 @@ class InternVisionModel(MmprojModel):
return name
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
vision_prefix = ["vision_model", "mlp", "model.vision_tower", "model.multi_modal_projector"]
# deal with intern-s1 special case
name = self._mapping_interns1_name(name)
@@ -3937,13 +4368,97 @@ class InternVisionModel(MmprojModel):
wq = data_torch[:c]
wk = data_torch[c : c * 2]
wv = data_torch[c * 2 :]
- return [
- (self.map_tensor_name(name.replace("attn.qkv", "self_attn.q_proj")), wq),
- (self.map_tensor_name(name.replace("attn.qkv", "self_attn.k_proj")), wk),
- (self.map_tensor_name(name.replace("attn.qkv", "self_attn.v_proj")), wv),
- ]
- return [(self.map_tensor_name(name), data_torch)]
- return [] # skip other tensors
+ yield from super().modify_tensors(wq, name.replace("attn.qkv", "self_attn.q_proj"), bid)
+ yield from super().modify_tensors(wk, name.replace("attn.qkv", "self_attn.k_proj"), bid)
+ yield from super().modify_tensors(wv, name.replace("attn.qkv", "self_attn.v_proj"), bid)
+ else:
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+ "NemotronH_Nano_VL_V2",
+ "RADIOModel",
+)
+class NemotronNanoV2VLModel(MmprojModel):
+ # ViT-Huge architecture parameters for RADIO v2.5-h
+ _vit_hidden_size = 1280
+ _vit_intermediate_size = 5120
+ _vit_num_layers = 32
+ _vit_num_heads = 16
+
+ def get_vision_config(self) -> dict[str, Any] | None:
+ # RADIO config doesn't have standard ViT parameters, so they need to be constructed manually
+ vision_config = self.global_config.get("vision_config")
+ if vision_config is None:
+ return None
+ # Add ViT-H parameters
+ vision_config = {
+ **vision_config,
+ "hidden_size": self._vit_hidden_size,
+ "intermediate_size": self._vit_intermediate_size,
+ "num_hidden_layers": self._vit_num_layers,
+ "num_attention_heads": self._vit_num_heads,
+ "image_size": self.global_config.get("force_image_size", 512),
+ }
+ return vision_config
+
+ def set_gguf_parameters(self):
+ if "image_mean" not in self.preprocessor_config:
+ self.preprocessor_config["image_mean"] = [0.485, 0.456, 0.406]
+ if "image_std" not in self.preprocessor_config:
+ self.preprocessor_config["image_std"] = [0.229, 0.224, 0.225]
+
+ super().set_gguf_parameters()
+ hparams = self.global_config
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_V2_VL)
+ self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
+ self.gguf_writer.add_vision_use_gelu(True)
+ downsample_ratio = hparams.get("downsample_ratio", 0.5)
+ self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / downsample_ratio))
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if ".position_embd." in new_name or "pos_embed" in new_name:
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if "input_conditioner" in name:
+ return
+
+ # RADIO's pos_embed doesn't have .weight suffix, but clip.cpp expects it
+ if "patch_generator.pos_embed" in name:
+ if not name.endswith(".weight"):
+ name += ".weight"
+ # Downsample position embeddings for fixed 512x512 image size
+ import torch.nn.functional as F
+
+ n_embd = self.hparams["hidden_size"]
+ image_size = self.global_config.get("force_image_size", 512)
+ patch_size = self.hparams["patch_size"]
+ target_patches_per_side = image_size // patch_size # 32
+ max_patches_per_side = int((data_torch.shape[1]) ** 0.5) # 128
+ if target_patches_per_side != max_patches_per_side:
+ # Reshape to grid, interpolate, flatten back
+ data_torch = data_torch.reshape(1, max_patches_per_side, max_patches_per_side, n_embd)
+ data_torch = data_torch.permute(0, 3, 1, 2).float() # [1, n_embd, 128, 128]
+ data_torch = F.interpolate(
+ data_torch,
+ size=(target_patches_per_side, target_patches_per_side),
+ mode="bilinear",
+ align_corners=True,
+ )
+ data_torch = data_torch.permute(0, 2, 3, 1) # [1, 32, 32, n_embd]
+ data_torch = data_torch.reshape(1, target_patches_per_side * target_patches_per_side, n_embd)
+
+ # Reshape linear patch embedding to conv2d format for ggml_conv_2d
+ # From [n_embd, patch_size*patch_size*3] to [n_embd, 3, patch_size, patch_size]
+ if "patch_generator.embedder" in name:
+ patch_size = self.hparams["patch_size"]
+ n_embd = self.hparams["hidden_size"]
+ data_torch = data_torch.reshape(n_embd, 3, patch_size, patch_size)
+
+ if name.startswith("vision_model.radio_model.model.") or name.startswith("mlp1."):
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("WavTokenizerDec")
@@ -3951,19 +4466,17 @@ class WavTokenizerDecModel(TextModel):
model_arch = gguf.MODEL_ARCH.WAVTOKENIZER_DEC
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
if (
name.endswith("codebook.cluster_size")
or name.endswith("codebook.embed_avg")
or name.endswith("codebook.inited")
):
logger.debug(f"Skipping {name!r}")
- return []
+ return
logger.info(f"{self.map_tensor_name(name)} -> {data_torch.shape}")
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def set_vocab(self):
self._set_vocab_none()
@@ -3991,36 +4504,57 @@ class Qwen2MoeModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- if (n_experts := self.hparams.get("num_experts")) is not None:
- self.gguf_writer.add_expert_count(n_experts)
if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
logger.info(f"gguf: expert feed forward length = {moe_intermediate_size}")
if (shared_expert_intermediate_size := self.hparams.get("shared_expert_intermediate_size")) is not None:
self.gguf_writer.add_expert_shared_feed_forward_length(shared_expert_intermediate_size)
logger.info(f"gguf: expert shared feed forward length = {shared_expert_intermediate_size}")
- # YaRN is not enabled by default
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
_experts: list[dict[str, Tensor]] | None = None
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# process the experts separately
name = name.replace("language_model.", "") # InternVL
+
+ # handle aggregated expert tensors
+ # GGUF stores dimensions reversed from PyTorch, so:
+ # PyTorch (A,B,C) -> GGUF writes [C,B,A] -> GGML reads ne={C,B,A}
+ # Input shapes from HF: (n_expert, n_ff_exp, n_embd) or (n_expert, n_embd, n_ff_exp)
+ # Expected GGML ne: {n_embd, n_ff_exp, n_expert} for gate/up, {n_ff_exp, n_embd, n_expert} for down
+ if name.endswith("mlp.experts.down_proj") or name.endswith("mlp.experts.down_proj.weight"):
+ mapped = f"{name}.weight" if not name.endswith(".weight") else name
+ # HF: [n_expert, n_embd, n_ff] -> GGML: {n_ff, n_embd, n_expert}
+ yield from super().modify_tensors(data_torch, mapped, bid)
+ return
+
+ if name.endswith("mlp.experts.gate_up_proj") or name.endswith("mlp.experts.gate_up_proj.weight"):
+ if data_torch.ndim < 3 or data_torch.shape[-2] % 2 != 0:
+ raise ValueError(f"Unexpected gate_up_proj shape for {name}: {tuple(data_torch.shape)}")
+ # HF: [n_expert, 2*n_ff, n_embd] -> split on dim=-2
+ n_ff = data_torch.shape[-2] // 2
+ gate = data_torch[..., :n_ff, :].contiguous()
+ up = data_torch[..., n_ff:, :].contiguous()
+ # gate/up: [n_expert, n_ff, n_embd] -> GGML: {n_embd, n_ff, n_expert}
+ base_name = name.removesuffix(".weight").removesuffix(".gate_up_proj")
+ mapped_gate = f"{base_name}.gate_proj.weight"
+ mapped_up = f"{base_name}.up_proj.weight"
+ yield from super().modify_tensors(gate, mapped_gate, bid)
+ yield from super().modify_tensors(up, mapped_up, bid)
+ return
+
if (
name.startswith("mlp")
or name.startswith("vision_model")
or name.startswith("model.vision_tower")
or name.startswith("model.multi_modal_projector")
+ or name.startswith("model.visual")
):
# skip visual tensors
- return []
+ return
+
if name.find("experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -4029,8 +4563,6 @@ class Qwen2MoeModel(TextModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["down_proj", "gate_proj", "up_proj"]:
data: list[Tensor] = []
@@ -4044,14 +4576,12 @@ class Qwen2MoeModel(TextModel):
merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -4120,9 +4650,13 @@ class Qwen3Model(Qwen2Model):
[
{
"name": "rerank",
- "template": '<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>\n' # pylint: disable=C0301
- "<|im_start|>user\n<Instruct>: Given a web search query, retrieve relevant passages that answer the query\n<Query>: {query}\n<Document>: {document}<|im_end|>\n" # pylint: disable=C0301
- "<|im_start|>assistant\n<think>\n\n</think>\n\n",
+ "template": (
+ "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query"
+ ' and the Instruct provided. Note that the answer can only be "yes" or'
+ ' "no".<|im_end|>\n<|im_start|>user\n<Instruct>: Given a web search query, retrieve'
+ " relevant passages that answer the query\n<Query>: {query}\n<Document>:"
+ " {document}<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
+ ),
}
]
)
@@ -4136,7 +4670,7 @@ class Qwen3Model(Qwen2Model):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if "model.vision_" in name:
# skip multimodal tensors
- return []
+ return
if self.is_rerank:
is_tied_head = self.is_tied_embeddings and "embed_tokens" in name
@@ -4146,13 +4680,12 @@ class Qwen3Model(Qwen2Model):
gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.CLS_OUT] + ".weight",
self._get_cls_out_tensor(data_torch),
)
+ yield cls_out_head
if is_tied_head:
- embed = (self.map_tensor_name(name), data_torch)
- return [cls_out_head, embed]
- if is_real_head:
- return [cls_out_head]
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
- return super().modify_tensors(data_torch, name, bid)
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Qwen3MoeForCausalLM")
@@ -4173,12 +4706,398 @@ class Qwen3MoeModel(Qwen2MoeModel):
super().set_vocab()
[email protected]("Qwen3NextForCausalLM")
+class Qwen3NextModel(Qwen2MoeModel):
+ model_arch = gguf.MODEL_ARCH.QWEN3NEXT
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_ssm_conv_kernel(self.hparams["linear_conv_kernel_dim"])
+ self.gguf_writer.add_ssm_state_size(self.hparams["linear_key_head_dim"])
+ self.gguf_writer.add_ssm_group_count(self.hparams["linear_num_key_heads"])
+ self.gguf_writer.add_ssm_time_step_rank(self.hparams["linear_num_value_heads"])
+ self.gguf_writer.add_ssm_inner_size(
+ self.hparams["linear_value_head_dim"] * self.hparams["linear_num_value_heads"]
+ )
+ self.gguf_writer.add_full_attention_interval(self.hparams.get("full_attention_interval", 4))
+ if (rope_dim := self.hparams.get("head_dim")) is None:
+ rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
+ self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.25)))
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.startswith("mtp"):
+ return # ignore MTP layers for now
+ if name.endswith(".A_log"):
+ data_torch = -torch.exp(data_torch)
+ elif name.endswith(".dt_bias"):
+ name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"
+ elif "conv1d" in name:
+ data_torch = data_torch.squeeze()
+ elif name.endswith("norm.weight") and not name.endswith("linear_attn.norm.weight"):
+ data_torch = data_torch + 1
+
+ if "in_proj_qkvz.weight" in name:
+ # original order: [q, k, v, z] * head_count
+ # corrected order: [q * head_count, k * head_count, v * head_count, z * head_count]
+ head_k_dim = self.hparams["linear_key_head_dim"]
+ head_v_dim = self.hparams["linear_value_head_dim"]
+ num_v_heads = self.hparams["linear_num_value_heads"]
+ num_k_heads = self.hparams["linear_num_key_heads"]
+ hidden_size = self.hparams["hidden_size"]
+ split_arg_list_qkvz = [
+ head_k_dim, # q partition
+ head_k_dim, # k partition
+ (num_v_heads // num_k_heads * head_v_dim), # v partition
+ (num_v_heads // num_k_heads * head_v_dim), # z partition
+ ]
+ # view as (n_embd, head_count, [q+k+v+z])
+ data_torch = data_torch.permute(1, 0).contiguous()
+ data_torch = data_torch.view(-1, num_k_heads, sum(split_arg_list_qkvz))
+ # split into q, k, v, z
+ q, k, v, z = torch.split(data_torch, split_arg_list_qkvz, dim=-1)
+ # flatten dim + head_count
+ q = q.contiguous().view(hidden_size, -1)
+ k = k.contiguous().view(hidden_size, -1)
+ v = v.contiguous().view(hidden_size, -1)
+ z = z.contiguous().view(hidden_size, -1)
+ # stack back
+ qkv = torch.cat([q, k, v], dim=-1).permute(1, 0).contiguous()
+ z = z.permute(1, 0).contiguous()
+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid, ".weight"), qkv)
+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_GATE, bid, ".weight"), z)
+ else:
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+class RND1Model(Qwen2MoeModel):
+ model_arch = gguf.MODEL_ARCH.RND1
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ # RND1 specific parameters
+ # RND1 uses bidirectional attention
+ self.gguf_writer.add_causal_attention(False)
+
+ if (mask_token_id := self.hparams.get("mask_token_id")) is not None:
+ self.gguf_writer.add_mask_token_id(mask_token_id)
+
+
+ "Qwen3VLForConditionalGeneration",
+ "Qwen3VLMoeForConditionalGeneration",
+ "Qwen3_5ForConditionalGeneration",
+ "Qwen3_5MoeForConditionalGeneration",
+)
+class Qwen3VLVisionModel(MmprojModel):
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ assert self.hparams_vision is not None
+ # Compute image_size if not present
+ if "image_size" not in self.hparams_vision:
+ # For Qwen3VL/Qwen3VLMoe, compute from num_position_embeddings
+ num_pos = self.hparams_vision.get("num_position_embeddings", 2304)
+ patch_size = self.hparams_vision.get("patch_size", 16)
+ # num_position_embeddings = (image_size / patch_size) ** 2
+ # So image_size = sqrt(num_position_embeddings) * patch_size
+ image_size = int(num_pos**0.5 * patch_size)
+ self.hparams_vision["image_size"] = image_size
+
+ # Rename config values for compatibility
+ self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_heads")
+ self.hparams_vision["num_hidden_layers"] = self.hparams_vision.get("depth")
+
+ self.is_deepstack_layers = [False] * int(self.hparams_vision["num_hidden_layers"] or 0)
+ for idx in self.hparams_vision.get("deepstack_visual_indexes", []):
+ self.is_deepstack_layers[idx] = True
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN3VL)
+ self.gguf_writer.add_vision_use_gelu(True)
+
+ if self.hparams_vision is not None:
+ merge_size = self.hparams_vision.get("spatial_merge_size")
+ if merge_size is not None:
+ self.gguf_writer.add_vision_spatial_merge_size(int(merge_size))
+
+ # Use text config's rms_norm_eps for vision attention layernorm eps
+ rms_norm_eps = self.global_config.get("text_config", {}).get("rms_norm_eps", 1e-6)
+ self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps)
+
+ if self.is_deepstack_layers:
+ self.gguf_writer.add_vision_is_deepstack_layers(self.is_deepstack_layers)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ assert self.hparams_vision is not None
+ # Skip text model tensors - they go in the text model file
+ if name.startswith("model.language_model.") or name.startswith("lm_head."):
+ return
+
+ # Skip MTP tensors
+ if name.startswith("mtp."):
+ return
+
+ if name.startswith("model.visual."):
+ name = name.replace("model.visual.", "visual.", 1)
+
+ if name.startswith("visual.deepstack_merger_list."):
+ prefix, rest = name.split(".", maxsplit=3)[2:]
+ # prefix is the layer index, convert to absolute clip layer index!
+ idx = self.hparams_vision.get("deepstack_visual_indexes", [])[int(prefix)]
+ target = rest
+
+ tensor_type: gguf.MODEL_TENSOR
+ if target.startswith("norm."):
+ tensor_type = gguf.MODEL_TENSOR.V_DS_NORM
+ suffix = target.split(".", 1)[1]
+ elif target.startswith("linear_fc1."):
+ tensor_type = gguf.MODEL_TENSOR.V_DS_FC1
+ suffix = target.split(".", 1)[1]
+ elif target.startswith("linear_fc2."):
+ tensor_type = gguf.MODEL_TENSOR.V_DS_FC2
+ suffix = target.split(".", 1)[1]
+ else:
+ raise ValueError(f"Unexpected deepstack tensor: {name}")
+
+ new_name = self.format_tensor_name(tensor_type, idx, suffix=f".{suffix}")
+ yield from super().modify_tensors(data_torch, new_name, bid)
+ return
+
+ if name.startswith("visual.merger."):
+ suffix = name.split(".", 2)[2]
+ if suffix.startswith("linear_fc"):
+ fc_idx_str, tail = suffix.split(".", 1)
+ fc_num = int(fc_idx_str.replace("linear_fc", ""))
+ # Qwen3VL has linear_fc1 and linear_fc2
+ # Map to indices 0 and 2 (matching Qwen2VL which uses indices 0 and 2)
+ if fc_num == 1:
+ fc_idx = 0
+ elif fc_num == 2:
+ fc_idx = 2
+ else:
+ raise ValueError(f"unexpected fc index {fc_num} in {name}")
+ new_name = self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, fc_idx, suffix=f".{tail}")
+ elif suffix.startswith("norm."):
+ new_name = self.format_tensor_name(gguf.MODEL_TENSOR.V_POST_NORM, suffix=f".{suffix.split('.', 1)[1]}")
+ else:
+ raise ValueError(f"Unexpected merger tensor: {name}")
+ yield (new_name, data_torch)
+ return
+
+ if name == "visual.patch_embed.proj.weight":
+ # split Conv3D into Conv2Ds along temporal dimension
+ c1, c2, kt, _, _ = data_torch.shape
+ del c1, c2
+ if kt != 2:
+ raise ValueError("Current implementation only supports temporal_patch_size of 2")
+ yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight", data_torch[:, :, 0, ...])
+ yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch[:, :, 1, ...])
+ return
+
+ if name == "visual.patch_embed.proj.bias":
+ # Include the bias - it's used by the C++ code
+ yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".bias", data_torch)
+ return
+
+ if name.startswith("visual."):
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
+
+ # Fall back to parent class for other tensors
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+ "Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration", "GlmOcrForConditionalGeneration"
+)
+class Glm4VVisionModel(Qwen3VLVisionModel):
+
+ def set_gguf_parameters(self):
+ MmprojModel.set_gguf_parameters(self) # skip Qwen3VLVisionModel parameters
+ assert self.hparams_vision is not None
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GLM4V)
+
+ hidden_act = str(self.hparams_vision.get("hidden_act", "")).lower()
+ if hidden_act == "gelu":
+ self.gguf_writer.add_vision_use_gelu(True)
+ elif hidden_act == "silu":
+ self.gguf_writer.add_vision_use_silu(True)
+
+ rms_norm_eps = self.hparams_vision.get("rms_norm_eps", 1e-5)
+ self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.startswith("model.visual."):
+ name = name.replace("model.visual.", "visual.")
+ if name.startswith("visual.merger."):
+ yield from ModelBase.modify_tensors(self, data_torch, name, bid)
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("Qwen3VLForConditionalGeneration")
+class Qwen3VLTextModel(Qwen3Model):
+ model_arch = gguf.MODEL_ARCH.QWEN3VL
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ # Handle MRoPE (Multi-axis Rotary Position Embedding) for Qwen3-VL
+ vision_config = self.hparams.get("vision_config", {})
+ deepstack_layer_num = len(vision_config.get("deepstack_visual_indexes", []))
+ self.gguf_writer.add_num_deepstack_layers(deepstack_layer_num)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Skip vision tensors - they go in the mmproj file
+ if name.startswith("model.visual."):
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("Qwen3VLMoeForConditionalGeneration")
+class Qwen3VLMoeTextModel(Qwen3MoeModel):
+ model_arch = gguf.MODEL_ARCH.QWEN3VLMOE
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ vision_config = self.hparams.get("vision_config", {})
+ deepstack_layer_num = len(vision_config.get("deepstack_visual_indexes", []))
+ self.gguf_writer.add_num_deepstack_layers(deepstack_layer_num)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Skip vision tensors - they go in the mmproj file
+ if name.startswith("model.visual."):
+ return
+
+ # Qwen3VL has transposed packed tensors, so we treat it differently from general Qwen2MoE packed tensors
+ if name.endswith("mlp.experts.down_proj") or name.endswith("mlp.experts.down_proj.weight"):
+ name = name.replace("language_model.", "")
+ mapped = f"{name}.weight" if not name.endswith(".weight") else name
+ permuted = data_torch.permute(0, 2, 1).contiguous()
+ yield from ModelBase.modify_tensors(self, permuted, mapped, bid)
+ return
+
+ if name.endswith("mlp.experts.gate_up_proj") or name.endswith("mlp.experts.gate_up_proj.weight"):
+ name = name.replace("language_model.", "")
+ if data_torch.ndim < 3 or data_torch.shape[-1] % 2 != 0:
+ raise ValueError(f"Unexpected gate_up_proj shape for {name}: {tuple(data_torch.shape)}")
+ split_dim = data_torch.shape[-1] // 2
+ gate = data_torch[..., :split_dim].contiguous()
+ up = data_torch[..., split_dim:].contiguous()
+ # Input gate/up: (n_expert=128, n_embd=2048, n_ff_exp=768)
+ # Want GGML ne: {n_embd, n_ff_exp, n_expert} = {2048, 768, 128}
+ # Need PyTorch: (128, 768, 2048) [reversed of GGML]
+ # So: permute(0, 2, 1): (128, 2048, 768) -> (128, 768, 2048)
+ base_name = name.removesuffix(".weight")
+ base = base_name.rsplit(".", 1)[0]
+ mapped_gate = f"{base}.gate_proj.weight"
+ mapped_up = f"{base}.up_proj.weight"
+ perm_gate = gate.permute(0, 2, 1).contiguous()
+ perm_up = up.permute(0, 2, 1).contiguous()
+ yield from ModelBase.modify_tensors(self, perm_gate, mapped_gate, bid)
+ yield from ModelBase.modify_tensors(self, perm_up, mapped_up, bid)
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+class _LinearAttentionVReorderBase(Qwen3NextModel):
+ model_arch = gguf.MODEL_ARCH.QWEN3NEXT # overridden by subclasses
+ """reorders V heads from grouped to tiled order for ggml broadcast
+
+ see https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306
+
+ Linear attention may has num_k_heads < num_v_heads. The HF weights store
+ V heads grouped by K head: [G0_v0..v{r-1}, G1_v0..v{r-1}, ...].
+ ggml binary ops use tiled broadcast: [K0, K1, ..., K0, K1, ...].
+ We reorder V heads to tiled order so ggml_repeat can replace the expensive
+ interleaved repeat: [G0_v0, G1_v0, ..., G0_v1, G1_v1, ...].
+ """
+
+ @staticmethod
+ def _reorder_v_heads(tensor: Tensor, dim: int, num_k_heads: int, num_v_per_k: int, head_dim: int) -> Tensor:
+ """Reorder V heads from grouped (by K head) to tiled order along the given dimension."""
+ shape = list(tensor.shape)
+ if dim < 0:
+ dim += len(shape)
+ new_shape = shape[:dim] + [num_k_heads, num_v_per_k, head_dim] + shape[dim + 1 :]
+ tensor = tensor.reshape(*new_shape)
+ perm = list(range(len(new_shape)))
+ perm[dim], perm[dim + 1] = perm[dim + 1], perm[dim]
+ return tensor.permute(*perm).contiguous().reshape(*shape)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ num_k_heads = self.hparams.get("linear_num_key_heads", 0)
+ num_v_heads = self.hparams.get("linear_num_value_heads", 0)
+
+ if num_k_heads > 0 and num_v_heads > 0 and num_k_heads != num_v_heads and "linear_attn." in name:
+ head_k_dim = self.hparams["linear_key_head_dim"]
+ head_v_dim = self.hparams["linear_value_head_dim"]
+ num_v_per_k = num_v_heads // num_k_heads
+
+ if ".in_proj_qkv." in name:
+ # QKV weight: reorder only the V rows
+ q_dim = head_k_dim * num_k_heads
+ k_dim = head_k_dim * num_k_heads
+ q = data_torch[:q_dim]
+ k = data_torch[q_dim : q_dim + k_dim]
+ v = data_torch[q_dim + k_dim :]
+ v = self._reorder_v_heads(v, 0, num_k_heads, num_v_per_k, head_v_dim)
+ data_torch = torch.cat([q, k, v], dim=0)
+
+ elif ".in_proj_z." in name:
+ # Z gate weight: reorder rows (num_v_heads * head_v_dim)
+ data_torch = self._reorder_v_heads(data_torch, 0, num_k_heads, num_v_per_k, head_v_dim)
+
+ elif ".in_proj_b." in name or ".in_proj_a." in name:
+ # Beta/Alpha weight: reorder rows (num_v_heads, head_dim=1)
+ data_torch = self._reorder_v_heads(data_torch, 0, num_k_heads, num_v_per_k, 1)
+
+ elif ".A_log" in name or ".dt_bias" in name or ".dt_proj" in name:
+ # A_log / dt_bias: 1D parameters with num_v_heads elements
+ if data_torch.ndim == 1:
+ data_torch = self._reorder_v_heads(
+ data_torch.unsqueeze(-1), 0, num_k_heads, num_v_per_k, 1
+ ).squeeze(-1)
+ else:
+ data_torch = self._reorder_v_heads(data_torch, -1, num_k_heads, num_v_per_k, 1)
+
+ elif ".conv1d" in name:
+ # Conv1d kernel: reorder only the V channel portion
+ data = data_torch.squeeze()
+ qk_channels = head_k_dim * num_k_heads * 2
+ qk_part = data[:qk_channels]
+ v_part = data[qk_channels:]
+ v_part = self._reorder_v_heads(v_part, 0, num_k_heads, num_v_per_k, head_v_dim)
+ data_torch = torch.cat([qk_part, v_part], dim=0)
+
+ elif ".out_proj." in name:
+ # Out projection weight: reorder columns (input dimension)
+ data_torch = self._reorder_v_heads(data_torch, 1, num_k_heads, num_v_per_k, head_v_dim)
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("Qwen3_5ForConditionalGeneration")
+class Qwen3_5TextModel(_LinearAttentionVReorderBase):
+ model_arch = gguf.MODEL_ARCH.QWEN35
+
+
[email protected]("Qwen3_5MoeForConditionalGeneration")
+class Qwen3_5MoeTextModel(_LinearAttentionVReorderBase):
+ model_arch = gguf.MODEL_ARCH.QWEN35MOE
+
+
@ModelBase.register("GPT2LMHeadModel")
class GPT2Model(TextModel):
model_arch = gguf.MODEL_ARCH.GPT2
def set_gguf_parameters(self):
- self.gguf_writer.add_block_count(self.hparams["n_layer"])
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_context_length(self.hparams["n_ctx"])
self.gguf_writer.add_embedding_length(self.hparams["n_embd"])
self.gguf_writer.add_feed_forward_length(4 * self.hparams["n_embd"])
@@ -4187,22 +5106,17 @@ class GPT2Model(TextModel):
self.gguf_writer.add_file_type(self.ftype)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
- tensors: list[tuple[str, Tensor]] = []
-
# we don't need these
if name.endswith((".attn.bias", ".attn.masked_bias")):
- return tensors
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight", ".c_proj.weight")):
data_torch = data_torch.transpose(1, 0)
new_name = self.map_tensor_name(name)
- tensors.append((new_name, data_torch))
-
- return tensors
+ yield from super().modify_tensors(data_torch, new_name, bid)
@ModelBase.register("PhiForCausalLM")
@@ -4210,8 +5124,6 @@ class Phi2Model(TextModel):
model_arch = gguf.MODEL_ARCH.PHI2
def set_gguf_parameters(self):
- block_count = self.find_hparam(["num_hidden_layers", "n_layer"])
-
rot_pct = self.find_hparam(["partial_rotary_factor"])
n_embd = self.find_hparam(["hidden_size", "n_embd"])
n_head = self.find_hparam(["num_attention_heads", "n_head"])
@@ -4220,7 +5132,7 @@ class Phi2Model(TextModel):
self.gguf_writer.add_embedding_length(n_embd)
self.gguf_writer.add_feed_forward_length(4 * n_embd)
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_head_count(n_head)
self.gguf_writer.add_head_count_kv(n_head)
self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_epsilon", "layer_norm_eps"]))
@@ -4305,8 +5217,8 @@ class Phi3MiniModel(TextModel):
if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:
if tokens[token_id] != token:
logger.warning(
- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '
- f' -> {token.decode("utf-8")!r}'
+ f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} ->'
+ f' {token.decode("utf-8")!r}'
)
tokens[token_id] = token
scores[token_id] = -1000.0
@@ -4325,8 +5237,8 @@ class Phi3MiniModel(TextModel):
if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:
if tokens[token_id] != token:
logger.warning(
- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '
- f'-> {token.decode("utf-8")!r}'
+ f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} ->'
+ f' {token.decode("utf-8")!r}'
)
tokens[token_id] = token
scores[token_id] = -1000.0
@@ -4344,8 +5256,6 @@ class Phi3MiniModel(TextModel):
special_vocab.add_to_gguf(self.gguf_writer)
def set_gguf_parameters(self):
- block_count = self.find_hparam(["num_hidden_layers", "n_layer"])
-
n_embd = self.find_hparam(["hidden_size", "n_embd"])
n_head = self.find_hparam(["num_attention_heads", "n_head"])
n_head_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"])
@@ -4359,12 +5269,14 @@ class Phi3MiniModel(TextModel):
self.gguf_writer.add_rope_scaling_orig_ctx_len(orig_max_pos_embds)
self.gguf_writer.add_embedding_length(n_embd)
self.gguf_writer.add_feed_forward_length(self.find_hparam(["intermediate_size"]))
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_head_count(n_head)
self.gguf_writer.add_head_count_kv(n_head_kv)
self.gguf_writer.add_layer_norm_rms_eps(rms_eps)
self.gguf_writer.add_rope_dimension_count(rope_dims)
- self.gguf_writer.add_rope_freq_base(self.find_hparam(["rope_theta"]))
+ self.gguf_writer.add_rope_freq_base(
+ self.rope_parameters.get("full_attention", self.rope_parameters)["rope_theta"]
+ )
self.gguf_writer.add_file_type(self.ftype)
sliding_window = self.hparams.get("sliding_window")
# use zero value of sliding_window to distinguish Phi-4 from other PHI3 models
@@ -4408,8 +5320,8 @@ class Phi3MiniModel(TextModel):
if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:
raise ValueError(
- f"The length of rope long and short factors must be {rope_dims / 2}. "
- "long_factors = {len(long_factors)}, short_factors = {len(short_factors)}."
+ f"The length of rope long and short factors must be {rope_dims / 2}. long_factors ="
+ f" {len(long_factors)}, short_factors = {len(short_factors)}."
)
yield (
@@ -4430,13 +5342,13 @@ class PhiMoeModel(Phi3MiniModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])
- self.gguf_writer.add_expert_count(self.hparams["num_local_experts"])
+ self.gguf_writer.add_expert_used_count(self.find_hparam(["num_experts_per_tok", "num_experts_per_token"]))
+ self.gguf_writer.add_expert_count(self.find_hparam(["num_local_experts", "num_experts"]))
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# process the experts separately
if name.find("block_sparse_moe.experts") != -1:
- n_experts = self.hparams["num_local_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -4445,8 +5357,6 @@ class PhiMoeModel(Phi3MiniModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["w1", "w2", "w3"]:
data: list[Tensor] = []
@@ -4460,14 +5370,12 @@ class PhiMoeModel(Phi3MiniModel):
merged_name = f"model.layers.{bid}.block_sparse_moe.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -4488,12 +5396,11 @@ class PlamoModel(TextModel):
def set_gguf_parameters(self):
hparams = self.hparams
- block_count = hparams["num_hidden_layers"]
self.gguf_writer.add_context_length(4096) # not in config.json
self.gguf_writer.add_embedding_length(hparams["hidden_size"])
self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_head_count(hparams["num_attention_heads"])
self.gguf_writer.add_head_count_kv(5) # hparams["num_key_value_heads"]) is wrong
self.gguf_writer.add_layer_norm_rms_eps(hparams["rms_norm_eps"])
@@ -4514,8 +5421,6 @@ class PlamoModel(TextModel):
return data_torch
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
new_name = self.map_tensor_name(name)
# shuffle for broadcasting of gqa in ggml_mul_mat
@@ -4524,7 +5429,7 @@ class PlamoModel(TextModel):
elif new_name.endswith("attn_output.weight"):
data_torch = self.shuffle_attn_output_weight(data_torch)
- return [(new_name, data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Plamo2ForCausalLM", "PLaMo2ForCausalLM")
@@ -4532,106 +5437,25 @@ class Plamo2Model(TextModel):
model_arch = gguf.MODEL_ARCH.PLAMO2
def set_vocab(self):
- # PLaMo 2 uses a custom tokenizer with a .jsonl file
- # We need to handle this specially
- tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl"
- tokenizer_config_path = self.dir_model / "tokenizer_config.json"
-
- if not tokenizer_jsonl_path.is_file():
- raise FileNotFoundError(f"PLaMo 2 tokenizer file not found: {tokenizer_jsonl_path}")
+ self._set_vocab_plamo()
- # Load tokenizer config
- with open(tokenizer_config_path, "r", encoding="utf-8") as f:
- tokenizer_config = json.load(f)
+ def set_gguf_parameters(self):
+ hparams = self.hparams
+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
- # Load tokens from JSONL file (actually a list format)
- tokens = []
- scores = []
- toktypes = []
-
- with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f:
- for line_num, line in enumerate(f):
- if line.strip():
- token_data = json.loads(line)
- # Format: [token, score, type, ?, ?, ?, ?]
- token = token_data[0].encode("utf-8")
- score = float(token_data[1])
- token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL"
-
- tokens.append(token)
- scores.append(score)
-
- # Map token type strings to GGUF token types
- if token_type_str == "UNKNOWN":
- toktypes.append(gguf.TokenType.UNKNOWN)
- elif token_type_str == "CONTROL":
- toktypes.append(gguf.TokenType.CONTROL)
- elif token_type_str == "BYTE":
- toktypes.append(gguf.TokenType.BYTE)
- else:
- # Check for PLaMo-2 special tokens
- token_str = token_data[0]
- if token_str.startswith("<|plamo:") and token_str.endswith("|>"):
- toktypes.append(gguf.TokenType.CONTROL)
- else:
- toktypes.append(gguf.TokenType.NORMAL)
-
- vocab_size = self.hparams["vocab_size"]
- if vocab_size > len(tokens):
- pad_count = vocab_size - len(tokens)
- logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")
- for i in range(1, pad_count + 1):
- tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))
- scores.append(-1000.0)
- toktypes.append(gguf.TokenType.UNUSED)
-
- # Use "plamo2" tokenizer type for PLaMo-2's custom Aho-Corasick tokenizer
- self.gguf_writer.add_tokenizer_model("plamo2")
- self.gguf_writer.add_tokenizer_pre("default")
- self.gguf_writer.add_token_list(tokens)
- self.gguf_writer.add_token_scores(scores)
- self.gguf_writer.add_token_types(toktypes)
-
- # Add special tokens from config
- if "bos_token" in tokenizer_config and tokenizer_config["bos_token"] is not None:
- token_id = tokens.index(tokenizer_config["bos_token"].encode("utf-8"))
- self.gguf_writer.add_bos_token_id(token_id)
- if "eos_token" in tokenizer_config and tokenizer_config["eos_token"] is not None:
- token_id = tokens.index(tokenizer_config["eos_token"].encode("utf-8"))
- self.gguf_writer.add_eos_token_id(token_id)
- if "pad_token" in tokenizer_config and tokenizer_config["pad_token"] is not None:
- token_id = tokens.index(tokenizer_config["pad_token"].encode("utf-8"))
- self.gguf_writer.add_pad_token_id(token_id)
- if "sep_token" in tokenizer_config and tokenizer_config["sep_token"] is not None:
- token_id = tokens.index(tokenizer_config["sep_token"].encode("utf-8"))
- self.gguf_writer.add_sep_token_id(token_id)
- if "unk_token" in tokenizer_config and tokenizer_config["unk_token"] is not None:
- token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8"))
- self.gguf_writer.add_unk_token_id(token_id)
-
- # Add <|plamo:op|> as EOT to ensure appropriate end of generation
- self.gguf_writer.add_eot_token_id(4)
-
- self.gguf_writer.add_add_space_prefix(False)
-
- def set_gguf_parameters(self):
- hparams = self.hparams
- block_count = hparams["num_hidden_layers"]
- self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
-
- # Which layers are Mamba layers
- # PLaMo 2 uses mamba_step to indicate the pattern (e.g., 2 means every other layer)
- # This logic matches modeling_plamo.py's is_mamba function
- mamba_step = hparams.get("mamba_step", 2)
- mamba_enabled = hparams.get("mamba_enabled", True)
- num_key_value_heads = []
- num_attention_heads = []
+ # Which layers are Mamba layers
+ # PLaMo 2 uses mamba_step to indicate the pattern (e.g., 2 means every other layer)
+ # This logic matches modeling_plamo.py's is_mamba function
+ mamba_step = hparams.get("mamba_step", 2)
+ mamba_enabled = hparams.get("mamba_enabled", True)
+ num_key_value_heads = []
+ num_attention_heads = []
if mamba_enabled:
- for i in range(block_count):
- if block_count <= (mamba_step // 2):
+ for i in range(self.block_count):
+ if self.block_count <= (mamba_step // 2):
# use attention in last layer
- is_mamba = i != block_count - 1
+ is_mamba = i != self.block_count - 1
else:
is_mamba = (i % mamba_step) != (mamba_step // 2)
if is_mamba:
@@ -4649,9 +5473,9 @@ class Plamo2Model(TextModel):
self.gguf_writer.add_embedding_length(hparams.get("hidden_size", 4096))
self.gguf_writer.add_key_length(hparams.get("hidden_size_per_head", 128))
self.gguf_writer.add_value_length(hparams.get("hidden_size_per_head", 128))
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_layer_norm_rms_eps(hparams.get("rms_norm_eps", 1e-06))
- self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 10000))
+ self.gguf_writer.add_rope_freq_base(self.rope_parameters.get("rope_theta", 10000))
# Mamba parameters
self.gguf_writer.add_ssm_state_size(hparams.get("mamba_d_state", 64))
@@ -4666,8 +5490,6 @@ class Plamo2Model(TextModel):
self.gguf_writer.add_file_type(self.ftype)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
if name.endswith(".A_log"):
data_torch = -torch.exp(data_torch)
elif name.endswith(".dt_bias"):
@@ -4696,9 +5518,56 @@ class Plamo2Model(TextModel):
elif name.endswith(".norm.weight"):
data_torch += 1.0
- new_name = self.map_tensor_name(name)
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("Plamo3ForCausalLM", "PLaMo3ForCausalLM")
+class Plamo3Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.PLAMO3
+
+ def set_vocab(self):
+ self._set_vocab_plamo()
+
+ tokenizer_config_path = self.dir_model / "tokenizer_config.json"
+ tokenizer_config = {}
+
+ if tokenizer_config_path.is_file():
+ with open(tokenizer_config_path, encoding="utf-8") as f:
+ tokenizer_config = json.load(f)
+
+ chat_template = tokenizer_config.get("chat_template")
+ chat_template_jinja = self.dir_model / "chat_template.jinja"
+
+ if chat_template_jinja.is_file():
+ with open(chat_template_jinja, encoding="utf-8") as f:
+ chat_template = f.read()
+
+ if chat_template:
+ self.gguf_writer.add_chat_template(chat_template)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
+ if (sliding_window := self.find_hparam(["window_size", "sliding_window"], optional=True)) is not None:
+ self.gguf_writer.add_sliding_window(sliding_window)
+ self.gguf_writer.add_sliding_window_pattern(self.hparams["sliding_window_pattern"])
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+
+ if name.endswith(".pre_mixer_norm.weight"):
+ data_torch = data_torch + 1.0
+ elif name.endswith(".post_mixer_norm.weight"):
+ data_torch = data_torch + 1.0 / 5
+ elif name.endswith(".pre_mlp_norm.weight"):
+ data_torch = data_torch + 1.0
+ elif name.endswith(".post_mlp_norm.weight"):
+ data_torch = data_torch + 1.0 / (5**1.5)
+ elif name.endswith((".mixer.q_norm.weight", ".mixer.k_norm.weight")):
+ data_torch = data_torch + 1.0
+ elif name.endswith(".norm.weight"):
+ data_torch = data_torch + 1.0
- return [(new_name, data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("CodeShellForCausalLM")
@@ -4706,12 +5575,10 @@ class CodeShellModel(TextModel):
model_arch = gguf.MODEL_ARCH.CODESHELL
def set_gguf_parameters(self):
- block_count = self.hparams["n_layer"]
-
self.gguf_writer.add_context_length(self.hparams["n_positions"])
self.gguf_writer.add_embedding_length(self.hparams["n_embd"])
self.gguf_writer.add_feed_forward_length(4 * self.hparams["n_embd"])
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_head_count(self.hparams["n_head"])
self.gguf_writer.add_head_count_kv(self.hparams["num_query_groups"])
self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])
@@ -4721,6 +5588,231 @@ class CodeShellModel(TextModel):
self.gguf_writer.add_rope_scaling_factor(1.0)
[email protected]("KimiLinearModel", "KimiLinearForCausalLM")
+class KimiLinearModel(TextModel):
+ """Kimi-Linear model with hybrid MLA+KDA architecture"""
+
+ model_arch = gguf.MODEL_ARCH.KIMI_LINEAR
+
+ _experts: list[dict[str, Tensor]] | None = None
+
+ def set_vocab(self):
+ try:
+ self._set_vocab_gpt2()
+ return
+ except Exception:
+ pass
+
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ tokpre = self.get_vocab_base_pre(tokenizer)
+
+ if tokpre == "kimi-k2":
+ # Build merges list using the approach similar to HunYuanMoE
+ merges = []
+ vocab = {}
+ mergeable_ranks = tokenizer.model._mergeable_ranks
+ for token, rank in mergeable_ranks.items():
+ vocab[QwenModel.token_bytes_to_string(token)] = rank
+ if len(token) == 1:
+ continue
+ merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)
+ if len(merged) == 2:
+ merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))
+ # Build token list
+ vocab_size = self.hparams["vocab_size"]
+ special_tokens = tokenizer.special_tokens
+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
+ tokens: list[str] = []
+ toktypes: list[int] = []
+
+ for i in range(vocab_size):
+ if i not in reverse_vocab:
+ tokens.append(f"[PAD{i}]")
+ toktypes.append(gguf.TokenType.UNUSED)
+ else:
+ token = reverse_vocab[i]
+ tokens.append(token)
+ if i in special_tokens.values():
+ toktypes.append(gguf.TokenType.CONTROL)
+ else:
+ toktypes.append(gguf.TokenType.NORMAL)
+
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+ self.gguf_writer.add_token_merges(merges)
+
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)
+ special_vocab.add_to_gguf(self.gguf_writer)
+ # override eos id in config.json with tiktoken eos id
+ self.gguf_writer.add_eos_token_id(tokenizer.eos_id)
+ else:
+ raise NotImplementedError(f"Deepseek pre-tokenizer {tokpre!r} is not supported yet!")
+
+ def set_gguf_parameters(self):
+ # note: To enable MLA KV cache, attention needs to be converted into MQA (ie: GQA with 1 group)
+ self.hparams["num_key_value_heads"] = 1
+
+ super().set_gguf_parameters()
+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
+
+ # KDA & MLA params
+ # Get ssm_d_conv from linear_attn_config.short_conv_kernel_size or ssm_d_conv
+ linear_attn_config = self.hparams["linear_attn_config"]
+ # n_head == 0 for KDA layers, n_head > 0 for MLA layers
+ # full_attention_layers list will be used to distinguish layer type
+ _num_kv_heads = list()
+ _full_attn_layers = linear_attn_config["full_attn_layers"]
+ for il in range(self.hparams["num_hidden_layers"]):
+ if il + 1 in _full_attn_layers:
+ _num_kv_heads.append(self.hparams["num_key_value_heads"])
+ else:
+ _num_kv_heads.append(0)
+ assert len(_num_kv_heads) == self.hparams["num_hidden_layers"]
+ self.gguf_writer.add_head_count_kv(_num_kv_heads)
+
+ if (ssm_d_conv := linear_attn_config.get("short_conv_kernel_size")) is not None:
+ self.gguf_writer.add_ssm_conv_kernel(ssm_d_conv)
+ if (kda_head_dim := linear_attn_config.get("head_dim")) is not None:
+ self.gguf_writer.add_kda_head_dim(kda_head_dim)
+
+ # MLA params - use add_* methods that handle arch substitution
+ # Support both HuggingFace naming (q_lora_rank, kv_lora_rank) and internal naming (n_lora_q, n_lora_kv)
+ if (q_lora_rank := self.find_hparam(["q_lora_rank", "n_lora_q"], optional=True)) is not None:
+ self.gguf_writer.add_q_lora_rank(q_lora_rank)
+ # To enable MLA KV cache, MLA needs to be converted into MQA with larger heads, then decompresses to MHA
+ kv_lora_rank = self.find_hparam(["kv_lora_rank", "n_lora_kv"], optional=False)
+ self.gguf_writer.add_kv_lora_rank(kv_lora_rank)
+
+ # MLA head dimensions
+ # Support HuggingFace naming: qk_nope_head_dim, qk_rope_head_dim, v_head_dim
+ qk_nope_head_dim = self.hparams.get("qk_nope_head_dim")
+ # Rotation - use qk_rope_head_dim for Kimi
+ qk_rope_head_dim = self.find_hparam(["qk_rope_head_dim", "n_rot"], optional=False)
+ self.gguf_writer.add_rope_dimension_count(qk_rope_head_dim)
+ self.gguf_writer.add_key_length(kv_lora_rank + qk_rope_head_dim)
+ v_head_dim = self.hparams.get("v_head_dim")
+
+ # Calculate n_embd_head_k_mla = qk_nope_head_dim + qk_rope_head_dim
+ if (n_embd_head_k_mla := self.find_hparam(["n_embd_head_k_mla"], optional=True)) is not None:
+ self.gguf_writer.add_key_length_mla(n_embd_head_k_mla)
+ elif qk_nope_head_dim is not None:
+ n_embd_head_k_mla = qk_nope_head_dim + qk_rope_head_dim
+ self.gguf_writer.add_key_length_mla(n_embd_head_k_mla)
+
+ # n_embd_head_v_mla = v_head_dim
+ if (n_embd_head_v_mla := self.hparams.get("n_embd_head_v_mla")) is not None:
+ self.gguf_writer.add_value_length_mla(n_embd_head_v_mla)
+ elif v_head_dim is not None:
+ self.gguf_writer.add_value_length_mla(v_head_dim)
+
+ # moe_intermediate_size (1024 for Kimi)
+ self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
+ # num_shared_experts (1 for Kimi)
+ self.gguf_writer.add_expert_shared_count(self.hparams["num_shared_experts"])
+ # first_k_dense_replace (1 for Kimi - first layer uses dense MLP)
+ self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"])
+ # Routed scaling factor (expert_weights_scale = 2.446 for Kimi)
+ self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+ if self._experts is not None:
+ experts = [k for d in self._experts for k in d.keys()]
+ if len(experts) > 0:
+ raise ValueError(f"Unprocessed experts: {experts}")
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ logger.info(f"Processing {name}: shape before = {tuple(data_torch.shape)}")
+
+ # Handle KDA conv1d weights
+ # HuggingFace/vLLM stores as [d_inner, d_conv] (2D), memory layout: conv_step changes fastest
+ # llama.cpp expects ggml ne = [d_conv, 1, d_inner, 1], memory layout: ne[0]=d_conv changes fastest
+ # GGUF reverses numpy shape when writing, so numpy (1, d_inner, 1, d_conv) -> ggml ne = [d_conv, 1, d_inner, 1]
+ # Memory layouts match: both have conv_step (d_conv) changing fastest
+ if name.endswith((".q_conv1d.weight", ".k_conv1d.weight", ".v_conv1d.weight")):
+ # HF shape: [d_inner, d_conv] e.g. [4096, 4]
+ # Target numpy shape: (1, d_inner, 1, d_conv) -> ggml ne = [d_conv, 1, d_inner, 1]
+ if data_torch.ndim == 2:
+ d_inner, d_conv = data_torch.shape
+ # Reshape to (1, d_inner, 1, d_conv) - memory layout preserved (d_conv fastest)
+ data_torch = data_torch.reshape(1, d_inner, 1, d_conv)
+ logger.info(
+ f"Reshaped conv1d weight {name}: [d_inner={d_inner}, d_conv={d_conv}] -> numpy"
+ f" {tuple(data_torch.shape)} -> ggml ne=[{d_conv}, 1, {d_inner}, 1]"
+ )
+ elif data_torch.ndim == 3:
+ # Already 3D [d_inner, 1, d_conv] from unsqueeze
+ d_inner, _, d_conv = data_torch.shape
+ data_torch = data_torch.reshape(1, d_inner, 1, d_conv)
+ logger.info(
+ f"Reshaped conv1d weight {name}: [d_inner={d_inner}, 1, d_conv={d_conv}] -> numpy"
+ f" {tuple(data_torch.shape)} -> ggml ne=[{d_conv}, 1, {d_inner}, 1]"
+ )
+
+ # Kimi specific bias
+ if name.endswith("e_score_correction_bias"):
+ name = name.replace("e_score_correction_bias", "e_score_correction.bias")
+
+ # Handle A_log: iHF stores as [1, 1, num_heads, 1]
+ # llama.cpp expects ggml ne = [1, num_heads, 1, 1]
+ # GGUF reverses numpy shape: numpy (1, 1, num_heads, 1) -> ggml ne = [1, num_heads, 1, 1]
+ if name.endswith(".A_log"):
+ data_torch = -torch.exp(data_torch)
+ if name.endswith(".dt_bias"):
+ name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"
+ logger.info("Changed dt_bias to dt_proj.bias")
+
+ # process the experts separately
+ if name.find("block_sparse_moe.experts") != -1:
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
+ assert bid is not None
+
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) >= n_experts * 3:
+ # merge the experts into a single 3d tensor
+ # w1: gate, w2: down, w3: up
+ for wid, tname in [
+ ("w1", gguf.MODEL_TENSOR.FFN_GATE_EXP),
+ ("w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP),
+ ("w3", gguf.MODEL_TENSOR.FFN_UP_EXP),
+ ]:
+ data: list[Tensor] = []
+ for xid in range(n_experts):
+ ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{wid}.weight"
+ data.append(self._experts[bid][ename])
+ del self._experts[bid][ename]
+ data_torch = torch.stack(data, dim=0)
+ new_name = self.format_tensor_name(tname, bid)
+ yield from super().modify_tensors(data_torch, new_name, bid)
+ return
+
+ # note: MLA with the absorption optimization, needs these two split and k_b_proj transposed
+ if name.endswith("kv_b_proj.weight"):
+ name_kb = name.replace("kv_b_proj", "k_b_proj")
+ name_vb = name.replace("kv_b_proj", "v_b_proj")
+ n_head_kv = self.hparams["num_key_value_heads"]
+ v_head_dim = self.find_hparam(["n_embd_head_v_mla", "v_head_dim"], optional=False)
+ qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
+ logger.info("Split kv_b n_head_kv %d\n" % n_head_kv)
+ assert data_torch.shape[0] == n_head_kv * (v_head_dim + qk_nope_head_dim)
+ kv_b = data_torch.view(n_head_kv, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])
+ k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
+ k_b = k_b.transpose(1, 2)
+ yield from super().modify_tensors(k_b, name_kb, bid)
+ yield from super().modify_tensors(v_b, name_vb, bid)
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("InternLM2ForCausalLM")
class InternLM2Model(TextModel):
model_arch = gguf.MODEL_ARCH.INTERNLM2
@@ -4743,7 +5835,7 @@ class InternLM2Model(TextModel):
logger.error(f"Error: Missing {tokenizer_path}")
sys.exit(1)
- sentencepiece_model = model.ModelProto() # pylint: disable=E1101
+ sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue]
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix
@@ -4806,8 +5898,8 @@ class InternLM2Model(TextModel):
if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:
if tokens[token_id] != token:
logger.warning(
- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '
- f'-> {token.decode("utf-8")!r}'
+ f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} ->'
+ f' {token.decode("utf-8")!r}'
)
tokens[token_id] = token
scores[token_id] = -1000.0
@@ -4829,8 +5921,8 @@ class InternLM2Model(TextModel):
if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:
if tokens[token_id] != token:
logger.warning(
- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '
- f'-> {token.decode("utf-8")!r}'
+ f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} ->'
+ f' {token.decode("utf-8")!r}'
)
tokens[token_id] = token
scores[token_id] = -1000.0
@@ -4859,21 +5951,6 @@ class InternLM2Model(TextModel):
special_vocab.add_to_gguf(self.gguf_writer)
- def set_gguf_parameters(self):
- self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])
- self.gguf_writer.add_block_count(self.hparams["num_hidden_layers"])
- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])
- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])
- self.gguf_writer.add_rope_freq_base(self.hparams["rope_theta"])
- self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])
- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])
- self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"])
- self.gguf_writer.add_file_type(self.ftype)
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
-
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
num_heads = self.hparams["num_attention_heads"]
num_kv_heads = self.hparams["num_key_value_heads"]
@@ -4885,7 +5962,7 @@ class InternLM2Model(TextModel):
name = name.replace("language_model.", "") # InternVL
if name.startswith("mlp") or name.startswith("vision_model"):
# skip visual tensors
- return []
+ return
if bid is not None and f"model.layers.{bid}.attention.wqkv" in name:
qkv = data_torch
@@ -4893,18 +5970,15 @@ class InternLM2Model(TextModel):
qkv = qkv.reshape((num_groups, q_per_kv + 2, head_dim, n_embd))
q, k, v = qkv[:, :q_per_kv], qkv[:, -2], qkv[:, -1]
- # The model weights of q and k require additional reshape.
q = LlamaModel.permute(q.reshape((-1, q.shape[-1])), num_heads, num_heads)
k = LlamaModel.permute(k.reshape((-1, k.shape[-1])), num_heads, num_kv_heads)
v = v.reshape((-1, v.shape[-1]))
- return [
- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), q),
- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), k),
- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), v),
- ]
+ yield from super().modify_tensors(q, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), bid)
+ yield from super().modify_tensors(k, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), bid)
+ yield from super().modify_tensors(v, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), bid)
else:
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("InternLM3ForCausalLM")
@@ -4950,23 +6024,18 @@ class InternLM3Model(TextModel):
rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
self.gguf_writer.add_rope_dimension_count(rope_dim)
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
-
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
n_head = self.hparams["num_attention_heads"]
n_kv_head = self.hparams.get("num_key_value_heads")
name = name.replace("language_model.", "") # InternVL
if name.startswith("mlp") or name.startswith("vision_model"):
# skip visual tensors
- return []
+ return
if name.endswith(("q_proj.weight", "q_proj.bias")):
data_torch = LlamaModel.permute(data_torch, n_head, n_head)
if name.endswith(("k_proj.weight", "k_proj.bias")):
data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("BertModel", "BertForMaskedLM", "CamembertModel", "BertForSequenceClassification")
@@ -5001,14 +6070,15 @@ class BertModel(TextModel):
self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))
# convert to phantom space vocab
- def phantom(tok):
- if tok.startswith("[") and tok.endswith("]"):
+ def phantom(tok, toktype):
+ if toktype == gguf.TokenType.CONTROL:
return tok
if tok.startswith("##"):
return tok[2:]
return "\u2581" + tok
- tokens = list(map(phantom, tokens))
+ assert len(tokens) == len(toktypes)
+ tokens = list(map(phantom, tokens, toktypes))
# add vocab to gguf
self.gguf_writer.add_tokenizer_model("bert")
@@ -5021,8 +6091,6 @@ class BertModel(TextModel):
special_vocab.add_to_gguf(self.gguf_writer)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
name = name.removeprefix("bert.")
if name.endswith(".gamma"):
@@ -5033,13 +6101,13 @@ class BertModel(TextModel):
# we are only using BERT for embeddings so we don't need the pooling layer
if name in ("embeddings.position_ids", "pooler.dense.weight", "pooler.dense.bias"):
- return [] # we don't need these
+ return # we don't need these
if name.startswith("cls.predictions"):
- return []
+ return
if name.startswith("cls.seq_relationship"):
- return []
+ return
if self.cls_out_labels:
# For BertForSequenceClassification (direct projection layer)
@@ -5049,7 +6117,7 @@ class BertModel(TextModel):
if name == "classifier.bias":
name = "classifier.out_proj.bias"
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def _xlmroberta_tokenizer_init(self) -> None:
# we need the pad_token_id to know how to chop down position_embd matrix
@@ -5097,7 +6165,7 @@ class BertModel(TextModel):
vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size)
else:
- sentencepiece_model = model.ModelProto() # pylint: disable=E1101
+ sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue]
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM
@@ -5205,9 +6273,9 @@ class DistilBertModel(BertModel):
# These layers act as MLM head, so we don't need them
if name.startswith("vocab_"):
- return []
+ return
- return super().modify_tensors(data_torch, name, bid)
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("RobertaModel", "RobertaForSequenceClassification")
@@ -5249,7 +6317,7 @@ class RobertaModel(BertModel):
if self._position_offset is not None:
data_torch = data_torch[self._position_offset :, :]
- return super().modify_tensors(data_torch, name, bid)
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("NomicBertModel")
@@ -5304,25 +6372,24 @@ class NomicBertModel(BertModel):
) -> Iterable[tuple[str, torch.Tensor]]:
# If the tensor is an experts bias tensor, skip it by returning an empty list.
if "mlp.experts.bias" in name:
- return [] # Explicitly return an empty list.
+ return # Explicitly return.
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
if "mlp.experts.mlp.w1" in name:
- data_torch = data_torch.view(self.hparams["num_experts"], self.hparams["n_inner"], self.hparams["n_embd"])
+ data_torch = data_torch.view(n_experts, self.hparams["n_inner"], self.hparams["n_embd"])
name += ".weight"
if "mlp.experts.mlp.w2" in name:
- data_torch = data_torch.view(self.hparams["num_experts"], self.hparams["n_inner"], self.hparams["n_embd"])
+ data_torch = data_torch.view(n_experts, self.hparams["n_inner"], self.hparams["n_embd"])
data_torch = data_torch.transpose(1, 2)
name += ".weight"
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def set_gguf_parameters(self):
super().set_gguf_parameters()
- self.gguf_writer.add_rope_freq_base(self.hparams["rotary_emb_base"])
if self.is_moe:
self.gguf_writer.add_moe_every_n_layers(self.hparams["moe_every_n_layers"])
- self.gguf_writer.add_expert_count(self.hparams["num_experts"])
self.gguf_writer.add_expert_used_count(self.hparams["moe_top_k"])
def _is_tokenizer_xlmroberta(self) -> bool:
@@ -5352,15 +6419,15 @@ class NeoBert(BertModel):
self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)
logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")
- self.gguf_writer.add_pooling_type(gguf.PoolingType.CLS)
+ self.gguf_writer.add_pooling_type(gguf.PoolingType.CLS) # https://huggingface.co/chandar-lab/NeoBERT#how-to-use
def modify_tensors(self, data_torch, name, bid):
if name.startswith("decoder."):
- return []
+ return
name = name.removeprefix("model.")
- return super().modify_tensors(data_torch, name, bid)
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("XLMRobertaModel", "XLMRobertaForSequenceClassification")
@@ -5421,7 +6488,7 @@ class XLMRobertaModel(BertModel):
if name.endswith(".0.lora_A") or name.endswith(".0.lora_B"):
if name.startswith("pooler.dense"):
- return []
+ return
num_loras = data_torch.size(0)
assert num_loras == len(self._lora_names)
@@ -5437,16 +6504,14 @@ class XLMRobertaModel(BertModel):
new_name = new_name[:-1] + ("a" if new_name[-1:] == "b" else "b")
lora_writer.add_tensor(new_name, data.float().numpy(), raw_dtype=gguf.GGMLQuantizationType.F32)
- return []
+ return
- return super().modify_tensors(data_torch, name, bid)
+ yield from super().modify_tensors(data_torch, name, bid)
def set_gguf_parameters(self):
super().set_gguf_parameters()
# jina-embeddings-v3
- if rotary_emb_base := self.hparams.get("rotary_emb_base"):
- self.gguf_writer.add_rope_freq_base(rotary_emb_base)
lora_alpha = self.hparams.get("lora_alpha")
if lora_prompt_prefixes := self.hparams.get("task_instructions"):
assert self._lora_files and all(lora_name in lora_prompt_prefixes for lora_name in self._lora_files.keys())
@@ -5488,11 +6553,10 @@ class GemmaModel(TextModel):
def set_gguf_parameters(self):
hparams = self.hparams
- block_count = hparams["num_hidden_layers"]
self.gguf_writer.add_context_length(hparams["max_position_embeddings"])
self.gguf_writer.add_embedding_length(hparams["hidden_size"])
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])
self.gguf_writer.add_head_count(hparams["num_attention_heads"])
self.gguf_writer.add_head_count_kv(
@@ -5504,18 +6568,16 @@ class GemmaModel(TextModel):
self.gguf_writer.add_file_type(self.ftype)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
# lm_head is not used in llama.cpp, while autoawq will include this tensor in model
# To prevent errors, skip loading lm_head.weight.
if name == "lm_head.weight":
logger.debug(f"Skipping get tensor {name!r} in safetensors so that convert can end normally.")
- return []
+ return
if name.endswith("norm.weight"):
data_torch = data_torch + 1
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Gemma2ForCausalLM")
@@ -5529,11 +6591,10 @@ class Gemma2Model(TextModel):
def set_gguf_parameters(self):
hparams = self.hparams
- block_count = hparams["num_hidden_layers"]
self.gguf_writer.add_context_length(hparams["max_position_embeddings"])
self.gguf_writer.add_embedding_length(hparams["hidden_size"])
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])
self.gguf_writer.add_head_count(hparams["num_attention_heads"])
self.gguf_writer.add_head_count_kv(
@@ -5548,18 +6609,16 @@ class Gemma2Model(TextModel):
self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
# lm_head is not used in llama.cpp, while autoawq will include this tensor in model
# To prevent errors, skip loading lm_head.weight.
if name == "lm_head.weight":
logger.debug(f"Skipping get tensor {name!r} in safetensors so that convert can end normally.")
- return []
+ return
if name.endswith("norm.weight"):
data_torch = data_torch + 1
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Gemma3ForCausalLM", "Gemma3ForConditionalGeneration")
@@ -5568,38 +6627,34 @@ class Gemma3Model(TextModel):
norm_shift = 1.0 # Gemma3RMSNorm adds 1.0 to the norm value
def set_vocab(self):
- self._set_vocab_sentencepiece()
-
- self.gguf_writer.add_add_space_prefix(False)
+ if (self.dir_model / "tokenizer.model").is_file():
+ self._set_vocab_sentencepiece()
+ self.gguf_writer.add_add_space_prefix(False)
+ else:
+ self._set_vocab_gpt2()
def set_gguf_parameters(self):
+ super().set_gguf_parameters()
hparams = self.hparams
- block_count = hparams["num_hidden_layers"]
# some default values are not specified in the hparams
self.gguf_writer.add_context_length(hparams.get("max_position_embeddings", 131072))
- self.gguf_writer.add_embedding_length(hparams["hidden_size"])
- self.gguf_writer.add_block_count(block_count)
- self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])
self.gguf_writer.add_head_count(hparams.get("num_attention_heads", 8))
self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("rms_norm_eps", 1e-6))
self.gguf_writer.add_key_length(hparams.get("head_dim", 256))
self.gguf_writer.add_value_length(hparams.get("head_dim", 256))
- self.gguf_writer.add_file_type(self.ftype)
- self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 1_000_000.0)) # for global layers
+ self.gguf_writer.add_rope_freq_base(
+ self.rope_parameters.get("full_attention", self.rope_parameters).get("rope_theta", 1_000_000.0)
+ ) # for global layers
# attn_logit_softcapping is removed in Gemma3
assert hparams.get("attn_logit_softcapping") is None
- self.gguf_writer.add_sliding_window(hparams["sliding_window"])
+ if final_logit_softcap := hparams.get("final_logit_softcapping"):
+ self.gguf_writer.add_final_logit_softcapping(final_logit_softcap)
+ if hparams.get("sliding_window_pattern") != 1:
+ self.gguf_writer.add_sliding_window(hparams["sliding_window"])
self.gguf_writer.add_head_count_kv(hparams.get("num_key_value_heads", 4))
- if hparams.get("rope_scaling") is not None:
- assert hparams["rope_scaling"]["rope_type"] == "linear"
- # important: this rope_scaling is only applied for global layers, and not used by 1B model
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
- self.gguf_writer.add_rope_scaling_factor(hparams["rope_scaling"]["factor"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
if "language_model." in name:
name = name.replace("language_model.", "")
@@ -5609,12 +6664,14 @@ class Gemma3Model(TextModel):
or name.startswith("multimodal_projector.")
or name.startswith("vision_model.")
):
- return [] # skip vision tensors
+ return # skip vision tensors
# remove OOV (out-of-vocabulary) rows in token_embd
if "embed_tokens.weight" in name:
- vocab = self._create_vocab_sentencepiece()
- tokens = vocab[0]
+ if (self.dir_model / "tokenizer.model").is_file():
+ tokens = self._create_vocab_sentencepiece()[0]
+ else:
+ tokens = self.get_vocab_base()[0]
data_torch = data_torch[: len(tokens)]
# ref code in Gemma3RMSNorm
@@ -5623,7 +6680,7 @@ class Gemma3Model(TextModel):
if name.endswith("norm.weight"):
data_torch = data_torch + self.norm_shift
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Gemma3TextModel")
@@ -5652,6 +6709,8 @@ class EmbeddingGemma(Gemma3Model):
if mod_conf_file.is_file():
with open(mod_conf_file, encoding="utf-8") as mod_conf_json_file:
mod_conf = json.load(mod_conf_json_file)
+ # hparams dense_2_feat_out and dense_3_feat_in are required
+ # when loading model's dense weights
prefix = self._get_dense_prefix(mod_path)
if mod_conf["in_features"] is not None and mod_conf["out_features"] is not None:
self.dense_features_dims[prefix] = (
@@ -5707,6 +6766,7 @@ class EmbeddingGemma(Gemma3Model):
@ModelBase.register("Gemma3ForConditionalGeneration")
class Gemma3VisionModel(MmprojModel):
+
def set_gguf_parameters(self):
super().set_gguf_parameters()
hparams = self.hparams
@@ -5734,10 +6794,8 @@ class Gemma3VisionModel(MmprojModel):
return super().tensor_force_quant(name, new_name, bid, n_dims)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
if "vision_model.head." in name:
- return [] # skip redundant tensors for tinygemma3
+ return # skip redundant tensors for tinygemma3
if (
name.startswith("multi_modal_projector.")
@@ -5755,12 +6813,200 @@ class Gemma3VisionModel(MmprojModel):
logger.info(f"Correcting norm value for '{name}'")
data_torch = data_torch + 1
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
+
+ return # skip other tensors
+
+
+class ConformerAudioModel(MmprojModel):
+ _batch_norm_tensors: list[dict[str, Tensor]] | None = None
+
+ @staticmethod
+ def is_audio_tensor(name: str):
+ return any(
+ p in name for p in ["audio", "codebook", "conformer", "depth_embedding", "depthformer", "depth_linear"]
+ )
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if ConformerAudioModel.is_audio_tensor(name):
+ if ".conv" in name or "_conv" in name and ".weight" in name:
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # fold running_mean, running_var and eps into weight and bias for batch_norm
+ if "batch_norm" in name:
+ if self._batch_norm_tensors is None:
+ self._batch_norm_tensors = [{} for _ in range(self.block_count)]
+ assert bid is not None
+ self._batch_norm_tensors[bid][name] = data_torch
+
+ if len(self._batch_norm_tensors[bid]) < 5:
+ return
+
+ weight = self._batch_norm_tensors[bid][f"conformer.layers.{bid}.conv.batch_norm.weight"]
+ bias = self._batch_norm_tensors[bid][f"conformer.layers.{bid}.conv.batch_norm.bias"]
+ running_mean = self._batch_norm_tensors[bid][f"conformer.layers.{bid}.conv.batch_norm.running_mean"]
+ running_var = self._batch_norm_tensors[bid][f"conformer.layers.{bid}.conv.batch_norm.running_var"]
+ eps = 1e-5 # default value
- return [] # skip other tensors
+ a = weight / torch.sqrt(running_var + eps)
+ b = bias - running_mean * a
+ yield from super().modify_tensors(a, f"conformer.layers.{bid}.conv.batch_norm.weight", bid)
+ yield from super().modify_tensors(b, f"conformer.layers.{bid}.conv.batch_norm.bias", bid)
+ return
+
+ # reshape conv weights
+ if name.startswith("conformer.pre_encode.conv.") and name.endswith(".bias"):
+ data_torch = data_torch[:, None, None]
+ if "conv.depthwise_conv" in name and name.endswith(".weight"):
+ assert data_torch.shape[1] == 1
+ data_torch = data_torch.reshape(data_torch.shape[0], data_torch.shape[2])
+ if "conv.pointwise_conv" in name and name.endswith(".weight"):
+ assert data_torch.shape[2] == 1
+ data_torch = data_torch.reshape(data_torch.shape[0], data_torch.shape[1])
+
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Gemma3nForConditionalGeneration")
+class Gemma3nVisionAudioModel(ConformerAudioModel):
+ has_audio_encoder = True
+ has_vision_encoder = True
+
+ # Double indexed mapping for MobileNetV5 blocks (not supported by tensor_mapping.py)
+ # This is the only known model having this, so we prefer implementing it outside of tensor_mapping.py
+ block_tensor_mapping = {
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.conv_exp.weight": "v.blk.{bid}.{sid}.conv_exp.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.bn1.weight": "v.blk.{bid}.{sid}.bn1.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.conv_pwl.weight": "v.blk.{bid}.{sid}.conv_pwl.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.bn2.weight": "v.blk.{bid}.{sid}.bn2.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.dw_start.conv.weight": (
+ "v.blk.{bid}.{sid}.dw_start.conv.weight"
+ ),
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.dw_start.bn.weight": "v.blk.{bid}.{sid}.dw_start.bn.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.dw_mid.conv.weight": "v.blk.{bid}.{sid}.dw_mid.conv.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.dw_mid.bn.weight": "v.blk.{bid}.{sid}.dw_mid.bn.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.pw_exp.conv.weight": "v.blk.{bid}.{sid}.pw_exp.conv.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.pw_exp.bn.weight": "v.blk.{bid}.{sid}.pw_exp.bn.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.pw_proj.conv.weight": "v.blk.{bid}.{sid}.pw_proj.conv.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.pw_proj.bn.weight": "v.blk.{bid}.{sid}.pw_proj.bn.weight",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.layer_scale.gamma": "v.blk.{bid}.{sid}.layer_scale.gamma",
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.query.proj.weight": (
+ "v.blk.{bid}.{sid}.attn.query.proj.weight"
+ ),
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.key.proj.weight": (
+ "v.blk.{bid}.{sid}.attn.key.proj.weight"
+ ),
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.value.proj.weight": (
+ "v.blk.{bid}.{sid}.attn.value.proj.weight"
+ ),
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.output.proj.weight": (
+ "v.blk.{bid}.{sid}.attn.output.proj.weight"
+ ),
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.key.down_conv.weight": (
+ "v.blk.{bid}.{sid}.attn.key.down_conv.weight"
+ ),
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.key.norm.weight": (
+ "v.blk.{bid}.{sid}.attn.key.norm.weight"
+ ),
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.value.down_conv.weight": (
+ "v.blk.{bid}.{sid}.attn.value.down_conv.weight"
+ ),
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.attn.value.norm.weight": (
+ "v.blk.{bid}.{sid}.attn.value.norm.weight"
+ ),
+ "model.vision_tower.timm_model.blocks.{bid}.{sid}.norm.weight": "v.blk.{bid}.{sid}.norm.weight",
+ }
+
+ def __init__(self, *args, **kwargs):
+ # Parent init will call find_hparam which now returns 0 for empty keys
+ super().__init__(*args, **kwargs)
+ assert self.hparams_vision is not None
+ self.hparams_vision["n_layers"] = 128 # fake value for audio encoder, vision encoder doesn't use it
+ self.hparams_vision["intermediate_size"] = self.hparams_vision.get("intermediate_size", 2048) * 4
+ self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_attention_heads", 8)
+
+ # MobileNetV5 does not use image_mean/std
+ self.preprocessor_config["image_mean"] = [0.0, 0.0, 0.0]
+ self.preprocessor_config["image_std"] = [1.0, 1.0, 1.0]
+ self.hparams_vision["image_size"] = self.preprocessor_config.get("size", {"height": 768, "width": 768})[
+ "height"
+ ]
+
+ # Image sequence length (256 tokens = 16x16 for Gemma3n)
+ image_seq_length = self.preprocessor_config.get("image_seq_length", 256)
+ image_size = self.hparams_vision["image_size"]
+ self.hparams_vision["patch_size"] = image_size // image_seq_length
+
+ # remap audio hparams
+ assert self.hparams_audio is not None
+ self.hparams_audio["n_layers"] = self.hparams_audio["conf_num_hidden_layers"]
+ self.hparams_audio["num_attention_heads"] = self.hparams_audio["conf_num_attention_heads"]
+ self.hparams_audio["feat_in"] = self.hparams_audio["input_feat_size"]
+ self.hparams_audio["intermediate_size"] = self.hparams_audio.get("intermediate_size", 6144)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ # vision params
+ self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.GEMMA3NV)
+ self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-6))
+
+ # audio params
+ assert self.hparams_audio is not None
+ self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.GEMMA3NA)
+ self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["feat_in"])
+ self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ # Force quantization settings for specific tensor types
+ if "input_projection" in name or "input_proj" in name:
+ return gguf.GGMLQuantizationType.F16
+ if ".embeddings." in name or "stem" in name:
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def custom_map(self, name: str) -> str:
+ """Parses names like model.vision_tower.timm_model.blocks.1.2.suffix and applies template mapping."""
+ parts = name.split(".")
+ # MobileNet blocks have at least 7 parts: model, vision_tower, timm_model, blocks, bid, sid, and suffix
+ if len(parts) >= 7:
+ bid, sid = parts[4], parts[5]
+ suffix = ".".join(parts[6:])
+ template = f"model.vision_tower.timm_model.blocks.{{bid}}.{{sid}}.{suffix}"
+ if template in self.block_tensor_mapping:
+ return self.block_tensor_mapping[template].format(bid=bid, sid=sid)
+
+ raise ValueError(f"Unknown name: {name}")
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if ConformerAudioModel.is_audio_tensor(name):
+ name = name.replace("model.audio_tower.conformer.", "conformer.layers.")
+ yield from super().modify_tensors(data_torch, name, bid)
+
+ # Gemma3n uses
+ # - model.embed_vision.* for projection layers
+ # - model.vision_tower.* for vision encoder
+ # Skip non-vision tensors
+ if not (name.startswith("model.embed_vision.") or name.startswith("model.vision_tower.")):
+ return
+
+ if name.startswith("model.vision_tower.timm_model.blocks."):
+ # Double-indexed block tensors through custom logic
+ yield (self.custom_map(name), data_torch)
+ return
+ else:
+ # Route non-repeating (conv_stem, msfa, embedding, etc.) and un-caught through tensor_mapping.py
+ new_name = self.map_tensor_name(name)
+
+ if new_name.endswith("conv_stem.conv.bias") or new_name.endswith("layer_scale.gamma"):
+ data_torch = data_torch.unsqueeze(0).unsqueeze(-1).unsqueeze(-1) # [1, C, 1, 1]
+
+ yield from ModelBase.modify_tensors(self, data_torch, new_name, bid)
+
+
[email protected]("Gemma3nForCausalLM", "Gemma3nForConditionalGeneration")
class Gemma3NModel(Gemma3Model):
model_arch = gguf.MODEL_ARCH.GEMMA3N
norm_shift = 0.0 # same value with Gemma3p5RMSNorm scale_shift on python code
@@ -5783,8 +7029,25 @@ class Gemma3NModel(Gemma3Model):
]
def set_vocab(self):
+ # For Gemma3n multimodal models, we need the FULL vocab_size (262400)
+ # which includes special tokens from 262144-262399 for vision/audio.
+ # The vocab_size_per_layer_input (262144) is only the embedding size per layer.
+ # Temporarily override the hparams lookup order to prioritize vocab_size.
+
+ # Store original vocab_size_per_layer_input if it exists
+ vocab_size_per_layer_input = self.hparams.get("vocab_size_per_layer_input")
+
+ # Temporarily remove vocab_size_per_layer_input to force using vocab_size
+ if vocab_size_per_layer_input is not None:
+ del self.hparams["vocab_size_per_layer_input"]
+
+ # Call parent set_vocab which will now use vocab_size (262400)
super().set_vocab()
+ # Restore vocab_size_per_layer_input for later use
+ if vocab_size_per_layer_input is not None:
+ self.hparams["vocab_size_per_layer_input"] = vocab_size_per_layer_input
+
def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_altup_active_idx(self.hparams["altup_active_idx"])
@@ -5818,10 +7081,40 @@ class Gemma3NModel(Gemma3Model):
# TODO: implement self.prediction_coefs.weight.clamp_(...)
if "language_model." not in name:
- return [] # skip non-language model tensors
+ return # skip non-language model tensors
+
+ # Pad token embeddings for vision/audio special tokens (262144-262399)
+ if "embed_tokens.weight" in name or "embed_tokens_per_layer" in name:
+ # Move to CPU to avoid meta device issues during padding
+ data_torch = data_torch.to(device="cpu")
+
+ vocab_size = self.hparams.get("vocab_size", 262400)
+ current_size = data_torch.shape[0] # First dimension is vocab_size
+
+ if current_size < vocab_size:
+ # Pad with zeros for vision/audio tokens (they get embeddings from vision tower)
+ padding_size = vocab_size - current_size
+ tensor_type = "per-layer embeddings" if "per_layer" in name else "token embeddings"
+ logger.info(
+ f"Padding {tensor_type} shape {list(data_torch.shape)} from {current_size} to {vocab_size} (adding"
+ f" {padding_size} vision/audio token slots)"
+ )
+
+ # Create padding with zeros (vision tokens won't use these embeddings)
+ padding = torch.zeros(
+ (padding_size, data_torch.shape[1]), dtype=data_torch.dtype, device=data_torch.device
+ )
+ data_torch = torch.cat([data_torch, padding], dim=0)
+
+ # Continue with normal processing
+ name = name.replace("language_model.", "")
+ yield from ModelBase.modify_tensors(self, data_torch, name, bid)
+ return
if "altup_unembed_projections" in name:
data_torch = data_torch.to(device="cpu")
+ # altup_unembed matrices are [hidden_size, hidden_size], NOT vocab-based
+ # They should NOT be padded
if ".0." in name:
self._altup_unembd[0] = data_torch
elif ".1." in name:
@@ -5832,9 +7125,10 @@ class Gemma3NModel(Gemma3Model):
raise ValueError(f"Unknown name: {name}")
out = self._stack_matrices(self._altup_unembd)
if out is not None:
- return [(self.map_tensor_name("model.altup_unembed_projections.weight"), out)]
+ yield from ModelBase.modify_tensors(self, out, "model.altup_unembed_projections.weight", bid)
+ return
else:
- return []
+ return
if "altup_projections" in name:
data_torch = data_torch.to(device="cpu")
@@ -5848,11 +7142,12 @@ class Gemma3NModel(Gemma3Model):
raise ValueError(f"Unknown name: {name}")
out = self._stack_matrices(self._altup_proj)
if out is not None:
- return [(self.map_tensor_name("model.altup_projections.weight"), out)]
+ yield from ModelBase.modify_tensors(self, out, "model.altup_projections.weight", bid)
+ return
else:
- return []
+ return
- return super().modify_tensors(data_torch, name, bid)
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Starcoder2ForCausalLM")
@@ -5868,7 +7163,6 @@ class Rwkv6Model(TextModel):
self._set_vocab_rwkv_world()
def set_gguf_parameters(self):
- block_count = self.hparams["num_hidden_layers"]
head_size = self.hparams["head_size"]
hidden_size = self.hparams["hidden_size"]
layer_norm_eps = self.hparams["layer_norm_epsilon"]
@@ -5884,7 +7178,7 @@ class Rwkv6Model(TextModel):
# RWKV isn't context limited
self.gguf_writer.add_context_length(1048576)
self.gguf_writer.add_embedding_length(hidden_size)
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_layer_norm_eps(layer_norm_eps)
self.gguf_writer.add_rescale_every_n_layers(rescale_every_n_layers)
self.gguf_writer.add_wkv_head_size(head_size)
@@ -5961,7 +7255,6 @@ class RWKV6Qwen2Model(Rwkv6Model):
self._set_vocab_gpt2()
def set_gguf_parameters(self):
- block_count = self.hparams["num_hidden_layers"]
num_attention_heads = self.hparams["num_attention_heads"]
num_key_value_heads = self.hparams["num_key_value_heads"]
hidden_size = self.hparams["hidden_size"]
@@ -5974,7 +7267,7 @@ class RWKV6Qwen2Model(Rwkv6Model):
# RWKV isn't context limited
self.gguf_writer.add_context_length(1048576)
self.gguf_writer.add_embedding_length(hidden_size)
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_wkv_head_size(head_size)
self.gguf_writer.add_time_mix_extra_dim(time_mix_extra_dim)
self.gguf_writer.add_time_decay_extra_dim(time_decay_extra_dim)
@@ -6015,7 +7308,6 @@ class Rwkv7Model(TextModel):
return max(1, round(hidden_size**exponent * multiplier / 32)) * 32
def set_gguf_parameters(self):
- block_count = self.hparams["num_hidden_layers"]
try:
head_size = self.hparams["head_size"]
layer_norm_eps = self.hparams["layer_norm_epsilon"]
@@ -6074,7 +7366,7 @@ class Rwkv7Model(TextModel):
# RWKV isn't context limited
self.gguf_writer.add_context_length(1048576)
self.gguf_writer.add_embedding_length(hidden_size)
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_layer_norm_eps(layer_norm_eps)
self.gguf_writer.add_wkv_head_size(head_size)
self.gguf_writer.add_decay_lora_rank(lora_rank_decay)
@@ -6175,7 +7467,6 @@ class ARwkv7Model(Rwkv7Model):
self._set_vocab_gpt2()
def set_gguf_parameters(self):
- block_count = self.hparams["num_hidden_layers"]
hidden_size = self.hparams["hidden_size"]
head_size = self.hparams["head_size"]
rms_norm_eps = self.hparams["rms_norm_eps"]
@@ -6192,7 +7483,7 @@ class ARwkv7Model(Rwkv7Model):
# RWKV isn't context limited
self.gguf_writer.add_context_length(1048576)
self.gguf_writer.add_embedding_length(hidden_size)
- self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)
self.gguf_writer.add_wkv_head_size(head_size)
self.gguf_writer.add_decay_lora_rank(lora_rank_decay)
@@ -6207,6 +7498,17 @@ class ARwkv7Model(Rwkv7Model):
self.gguf_writer.add_head_count(0)
[email protected]("MaincoderForCausalLM")
+class MaincoderModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.MAINCODER
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ if (head_dim := self.hparams.get("head_dim")) is not None:
+ self.gguf_writer.add_rope_dimension_count(head_dim)
+
+
@ModelBase.register("MambaForCausalLM", "MambaLMHeadModel", "FalconMambaForCausalLM")
class MambaModel(TextModel):
model_arch = gguf.MODEL_ARCH.MAMBA
@@ -6285,11 +7587,11 @@ class MambaModel(TextModel):
if self._tok_embd is not None and new_name == output_name:
if torch.equal(self._tok_embd, data_torch):
logger.debug(f"{output_name} is equivalent to {tok_embd_name}, omitting")
- return []
+ return
elif new_name == tok_embd_name:
self._tok_embd = data_torch
- return [(new_name, data_torch)]
+ yield from super().modify_tensors(data_torch, new_name, bid)
@ModelBase.register("Mamba2ForCausalLM")
@@ -6303,6 +7605,8 @@ class Mamba2Model(TextModel):
if hparams is None:
with open(dir_model / "config.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
+ if "llm_config" in hparams:
+ hparams["text_config"] = hparams["llm_config"]
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
self.d_model = self.find_hparam(["hidden_size", "d_model", "dim"])
self.d_inner = (
@@ -6428,8 +7732,8 @@ class JambaModel(TextModel):
self.gguf_writer.add_ssm_state_size(d_state)
self.gguf_writer.add_ssm_time_step_rank(dt_rank)
self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)
- self.gguf_writer.add_expert_count(self.hparams["num_experts"])
- self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])
+ self.gguf_writer.add_expert_count(self.find_hparam(["num_local_experts", "num_experts"]))
+ self.gguf_writer.add_expert_used_count(self.find_hparam(["num_experts_per_tok", "num_experts_per_token"]))
self.gguf_writer.add_file_type(self.ftype)
_experts: list[dict[str, Tensor]] | None = None
@@ -6447,7 +7751,7 @@ class JambaModel(TextModel):
# process the experts separately
if ".feed_forward.experts." in name:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
@@ -6533,6 +7837,17 @@ class Cohere2Model(TextModel):
self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size // num_attention_heads)))
self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Cohere2 runtime in llama.cpp expects no bias tensors;
+ # the actual weight only contains 0-value tensors as bias, we can skip them
+ if name.endswith(".bias"):
+ if torch.any(data_torch != 0):
+ raise ValueError(f"Bias tensor {name!r} is not zero.")
+ logger.debug(f"Skipping bias tensor {name!r} for Cohere2 conversion.")
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
@ModelBase.register("OlmoForCausalLM")
@ModelBase.register("OLMoForCausalLM")
@@ -6549,8 +7864,6 @@ class OlmoModel(TextModel):
# Same as super class, but permuting q_proj, k_proj
# Copied from: LlamaModel
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
n_head = self.hparams["num_attention_heads"]
n_kv_head = self.hparams.get("num_key_value_heads")
@@ -6559,7 +7872,7 @@ class OlmoModel(TextModel):
if name.endswith("k_proj.weight"):
data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("SeedOssForCausalLM")
@@ -6575,13 +7888,6 @@ class Olmo2Model(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_attn_factors(rope_scaling["attention_factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
-
if "sliding_window" in self.hparams:
self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
@@ -6604,8 +7910,6 @@ class OlmoeModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_layer_norm_rms_eps(1e-5)
- if (n_experts := self.hparams.get("num_experts")) is not None:
- self.gguf_writer.add_expert_count(n_experts)
_experts: list[dict[str, Tensor]] | None = None
@@ -6613,7 +7917,7 @@ class OlmoeModel(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# process the experts separately
if name.find("experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -6622,8 +7926,6 @@ class OlmoeModel(TextModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["down_proj", "gate_proj", "up_proj"]:
data: list[Tensor] = []
@@ -6637,14 +7939,12 @@ class OlmoeModel(TextModel):
merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
# Copied from: Qwen2MoeModel
def prepare_tensors(self):
@@ -6823,8 +8123,8 @@ class ArcticModel(TextModel):
token_score = 0.0
logger.info(
- f"Setting added token {token_id} to '{token_content}' (type: {token_type}, "
- f"score: {token_score:.2f})"
+ f"Setting added token {token_id} to '{token_content}' (type: {token_type}, score:"
+ f" {token_score:.2f})"
)
tokens[token_id] = token_content.encode("utf-8")
toktypes[token_id] = token_type
@@ -6868,8 +8168,6 @@ class ArcticModel(TextModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for wid in ["w1", "w2", "w3"]:
data: list[Tensor] = []
@@ -6883,14 +8181,12 @@ class ArcticModel(TextModel):
merged_name = f"layers.{bid}.feed_forward.experts.{wid}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -6959,8 +8255,6 @@ class DeepseekModel(TextModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["down_proj", "gate_proj", "up_proj"]:
data: list[Tensor] = []
@@ -6974,14 +8268,12 @@ class DeepseekModel(TextModel):
merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -6997,10 +8289,16 @@ class DeepseekModel(TextModel):
"DeepseekV2ForCausalLM",
"DeepseekV3ForCausalLM",
"KimiVLForConditionalGeneration",
+ "KimiK25ForConditionalGeneration",
+ "YoutuForCausalLM",
+ "YoutuVLForConditionalGeneration",
)
class DeepseekV2Model(TextModel):
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
+ # TODO @ngxson : remove this when we support MTP for deepseek models
+ skip_mtp = True
+
def set_vocab(self):
try:
self._set_vocab_gpt2()
@@ -7064,7 +8362,15 @@ class DeepseekV2Model(TextModel):
super().set_gguf_parameters()
hparams = self.hparams
- self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])
+ # first_k_dense_replace: number of leading layers using dense FFN instead of MoE
+ # For non-MoE models (like Youtu), set to n_layer to use dense FFN for all layers
+ # For MoE models (like DeepSeek-V2), this is the number of leading non-MoE layers
+ has_moe = hparams.get("n_routed_experts") is not None
+ first_k_dense_replace = hparams.get("first_k_dense_replace")
+ if first_k_dense_replace is None:
+ # Default: if no MoE, all layers are dense; if MoE, none are dense
+ first_k_dense_replace = hparams["num_hidden_layers"] if not has_moe else 0
+ self.gguf_writer.add_leading_dense_block_count(first_k_dense_replace)
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
if "q_lora_rank" in hparams and hparams["q_lora_rank"] is not None:
self.gguf_writer.add_q_lora_rank(hparams["q_lora_rank"])
@@ -7076,47 +8382,61 @@ class DeepseekV2Model(TextModel):
self.gguf_writer.add_key_length_mla(hparams["qk_nope_head_dim"] + hparams["qk_rope_head_dim"])
self.gguf_writer.add_value_length_mla(hparams["v_head_dim"])
- self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
- self.gguf_writer.add_expert_count(hparams["n_routed_experts"])
- self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"])
- self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])
- self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])
+ # MoE parameters (required by C++ code for DEEPSEEK2 arch)
+ # For non-MoE models like Youtu, use intermediate_size as expert_feed_forward_length
+ moe_intermediate_size = self.find_hparam(["moe_intermediate_size", "intermediate_size"], optional=False)
+ self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
- if hparams["scoring_func"] == "sigmoid":
- self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
- elif hparams["scoring_func"] == "softmax":
- self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
- else:
- raise ValueError(f"Unsupported scoring_func value: {hparams['scoring_func']}")
+ if (n_routed_experts := hparams.get("n_routed_experts")) is not None:
+ self.gguf_writer.add_expert_count(n_routed_experts)
+
+ # expert_shared_count is required by C++ code, default to 0 for non-MoE models
+ n_shared_experts = hparams.get("n_shared_experts", 0)
+ self.gguf_writer.add_expert_shared_count(n_shared_experts)
+
+ # When not set, C++ code will use scale_w = false to skip the no-op scaling
+ if (routed_scaling_factor := hparams.get("routed_scaling_factor")) is not None:
+ self.gguf_writer.add_expert_weights_scale(routed_scaling_factor)
+
+ if (norm_topk_prob := hparams.get("norm_topk_prob")) is not None and norm_topk_prob:
+ self.gguf_writer.add_expert_weights_norm(norm_topk_prob)
self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
- self.gguf_writer.add_rope_scaling_yarn_log_mul(0.1 * rope_scaling["mscale_all_dim"])
+ if (rope_mscale_all := self.rope_parameters.get("mscale_all_dim")) is not None:
+ # [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX]
+ # note: for legacy reasons, this is not consistent with the other usages
+ # of self.gguf_writer.add_rope_scaling_yarn_log_mul
+ # ref https://github.com/ggml-org/llama.cpp/pull/17945
+ self.gguf_writer.add_rope_scaling_yarn_log_mul(0.1 * rope_mscale_all)
_experts: list[dict[str, Tensor]] | None = None
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- # skip vision tensors and remove "language_model." for Kimi-VL
- if "vision_tower" in name or "multi_modal_projector" in name:
- return []
-
+ # skip vision tensors and remove "language_model." for Kimi-VL and Kimi-K2.5
+ if "vision_tower" in name or "multi_modal_projector" in name or "mm_projector" in name:
+ return
+ if name.startswith("siglip2.") or name.startswith("merger."):
+ return
if name.startswith("language_model."):
name = name.replace("language_model.", "")
+ # skip lm_head.weight if tie_word_embeddings is True
+ if self.hparams.get("tie_word_embeddings", False):
+ if name == "lm_head.weight" or name == "model.lm_head.weight":
+ logger.info("Skipping tied output layer 'lm_head.weight' (will use token_embd.weight)")
+ return
+
# rename e_score_correction_bias tensors
if name.endswith("e_score_correction_bias"):
name = name.replace("e_score_correction_bias", "e_score_correction.bias")
# skip Multi-Token Prediction (MTP) layers
- block_count = self.hparams["num_hidden_layers"]
- match = re.match(r"model.layers.(\d+)", name)
- if match and int(match.group(1)) >= block_count:
- return []
+ if self.skip_mtp:
+ block_count = self.hparams["num_hidden_layers"]
+ match = re.match(r"model.layers.(\d+)", name)
+ if match and int(match.group(1)) >= block_count:
+ return
# process the experts separately
if name.find("mlp.experts") != -1:
@@ -7129,8 +8449,6 @@ class DeepseekV2Model(TextModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["down_proj", "gate_proj", "up_proj"]:
data: list[Tensor] = []
@@ -7144,12 +8462,10 @@ class DeepseekV2Model(TextModel):
merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
# note: MLA with the absorption optimization, needs these two split and k_b_proj transposed
if name.endswith("kv_b_proj.weight"):
@@ -7166,9 +8482,11 @@ class DeepseekV2Model(TextModel):
k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
k_b = k_b.transpose(1, 2)
- return [(self.map_tensor_name(name_kb), k_b), (self.map_tensor_name(name_vb), v_b)]
+ yield from super().modify_tensors(k_b, name_kb, bid)
+ yield from super().modify_tensors(v_b, name_vb, bid)
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -7180,96 +8498,397 @@ class DeepseekV2Model(TextModel):
raise ValueError(f"Unprocessed experts: {experts}")
[email protected]("Dots1ForCausalLM")
-class Dots1Model(Qwen2MoeModel):
- model_arch = gguf.MODEL_ARCH.DOTS1
-
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.hparams["num_experts"] = self.hparams["n_routed_experts"]
[email protected]("MiniMaxM2ForCausalLM")
+class MiniMaxM2Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.MINIMAXM2
+ _experts_cache: dict[int, dict[str, Tensor]] = {}
def set_gguf_parameters(self):
super().set_gguf_parameters()
- self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"])
- self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"])
- self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
- self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
- if self.hparams["scoring_func"] == "noaux_tc":
- self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
- else:
- raise ValueError(f"Unsupported scoring_func value: {self.hparams['scoring_func']}")
+ self.gguf_writer.add_expert_feed_forward_length(self.find_hparam(["intermediate_size"]))
+ self.gguf_writer.add_rope_dimension_count(self.find_hparam(["rotary_dim"]))
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
if name.endswith("e_score_correction_bias"):
name = name.replace("e_score_correction_bias", "e_score_correction.bias")
- if "shared_experts" in name:
- return [(self.map_tensor_name(name), data_torch)]
- return super().modify_tensors(data_torch, name, bid)
+ # merge expert weights
+ if "experts" in name:
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
+ assert bid is not None
[email protected]("PLMForCausalLM")
-class PLMModel(TextModel):
- model_arch = gguf.MODEL_ARCH.PLM
+ expert_cache = self._experts_cache.setdefault(bid, {})
+ expert_cache[name] = data_torch
+ expert_weights = ["w1", "w2", "w3"]
- def set_vocab(self):
- self._set_vocab_gpt2()
+ # not enough expert weights to merge
+ if len(expert_cache) < n_experts * len(expert_weights):
+ return
- def set_gguf_parameters(self):
- super().set_gguf_parameters()
- hparams = self.hparams
- self.gguf_writer.add_vocab_size(hparams["vocab_size"])
- self.gguf_writer.add_kv_lora_rank(hparams["kv_lora_rank"])
- self.gguf_writer.add_key_length(hparams["qk_nope_head_dim"] + hparams["qk_rope_head_dim"])
- self.gguf_writer.add_value_length(hparams["v_head_dim"])
- self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])
+ for w_name in expert_weights:
+ data: list[Tensor] = []
- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- return [(self.map_tensor_name(name), data_torch)]
+ for xid in range(n_experts):
+ ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{w_name}.weight"
+ data.append(expert_cache[ename])
+ del expert_cache[ename]
- def prepare_tensors(self):
- super().prepare_tensors()
+ data_torch = torch.stack(data, dim=0)
+ merged_name = f"model.layers.{bid}.block_sparse_moe.experts.{w_name}.weight"
+ new_name = self.map_tensor_name(merged_name)
+ yield from super().modify_tensors(data_torch, new_name, bid)
+ del self._experts_cache[bid]
+ return
[email protected]("T5WithLMHeadModel")
[email protected]("T5ForConditionalGeneration")
[email protected]("MT5ForConditionalGeneration")
[email protected]("UMT5ForConditionalGeneration")
-class T5Model(TextModel):
- model_arch = gguf.MODEL_ARCH.T5
+ yield from super().modify_tensors(data_torch, name, bid)
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.shared_token_embeddings_found = False
- def set_vocab(self):
- # to avoid TypeError: Descriptors cannot be created directly
- # exception when importing sentencepiece_model_pb2
- os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
- from sentencepiece import SentencePieceProcessor # pylint: disable=E0401
- from sentencepiece import sentencepiece_model_pb2 as model # pylint: disable=E0401
[email protected]("MiMoV2FlashForCausalLM")
+class MimoV2Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.MIMO2
- tokenizer_path = self.dir_model / "tokenizer.model"
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
- # many older models use spiece.model tokenizer model filename
- if not tokenizer_path.is_file():
- tokenizer_path = self.dir_model / "spiece.model"
+ assert self.hparams["swa_head_dim"] == self.hparams["head_dim"]
+ assert self.hparams["swa_num_attention_heads"] == self.hparams["num_attention_heads"]
+ assert self.hparams["swa_v_head_dim"] == self.hparams["v_head_dim"]
+ assert self.hparams["topk_method"] == "noaux_tc"
- if not tokenizer_path.is_file():
- raise FileNotFoundError(f"File not found: {tokenizer_path}")
+ n_head_kv = self.hparams["num_key_value_heads"]
+ n_head_kv_swa = self.hparams["swa_num_key_value_heads"]
+ n_head_kv_arr = [
+ n_head_kv_swa if use_swa == 1 else n_head_kv for use_swa in self.hparams["hybrid_layer_pattern"]
+ ]
+ self.gguf_writer.add_head_count_kv(n_head_kv_arr)
- sentencepiece_model = model.ModelProto() # pylint: disable=E1101
- sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
+ self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+ self.gguf_writer.add_sliding_window_pattern(self.hparams["hybrid_layer_pattern"])
+ self.gguf_writer.add_value_length(self.hparams["v_head_dim"])
+ self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"])
+ self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
- # some models like Pile-T5 family use BPE tokenizer instead of Unigram
- if sentencepiece_model.trainer_spec.model_type == 2: # BPE
- # assure the tokenizer model file name is correct
- assert tokenizer_path.name == "tokenizer.model"
- return self._set_vocab_sentencepiece()
- else:
- assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM
+ rope_dim = int(self.hparams["head_dim"] * self.hparams["partial_rotary_factor"])
+ self.gguf_writer.add_rope_dimension_count(rope_dim)
- add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix
+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("layernorm_epsilon", 1e-5))
+
+ _experts: list[dict[str, Tensor]] | None = None
+
+ def modify_tensors(self, data_torch, name, bid):
+ if name.endswith("e_score_correction_bias"):
+ name = name.replace("e_score_correction_bias", "e_score_correction.bias")
+
+ if "attention_sink" in name and not name.endswith(".weight"):
+ name += ".weight"
+
+ # TODO: mimo v2 does not indicate the number of next-token-prediction layers,
+ # therefore we cannot do the same way as GLM4_MOE
+ if "model.mtp." in name:
+ return
+
+ # process the experts separately
+ if name.find("mlp.experts") != -1:
+ n_experts = self.hparams["n_routed_experts"]
+ assert bid is not None
+
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) >= n_experts * 3:
+ # merge the experts into a single 3d tensor
+ for w_name in ["gate_proj", "up_proj", "down_proj"]:
+ data: list[Tensor] = []
+
+ for xid in range(n_experts):
+ ename_to_retrieve = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"
+ data.append(self._experts[bid][ename_to_retrieve])
+ del self._experts[bid][ename_to_retrieve]
+
+ data_torch = torch.stack(data, dim=0)
+ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
+ else:
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+
+ if self._experts is not None:
+ # flatten `list[dict[str, Tensor]]` into `list[str]`
+ experts = [k for d in self._experts for k in d.keys()]
+ if len(experts) > 0:
+ raise ValueError(f"Unprocessed experts: {experts}")
+
+
[email protected]("Step3p5ForCausalLM")
+class Step35Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.STEP35
+
+ def set_gguf_parameters(self):
+ rope_theta = self.hparams.get("rope_theta")
+ if isinstance(rope_theta, list):
+ self.hparams["rope_theta"] = float(rope_theta[0])
+ self.hparams["local_rope_theta"] = float(rope_theta[1])
+ self.rope_parameters["rope_theta"] = self.hparams["rope_theta"]
+ self.rope_parameters["sliding_attention"] = {"rope_theta": self.hparams["local_rope_theta"]}
+
+ super().set_gguf_parameters()
+
+ layer_types = self.hparams.get("layer_types") or []
+ partial_rotary_factors = self.hparams.get("partial_rotary_factors") or []
+ attn_other = self.hparams.get("attention_other_setting") or {}
+
+ n_head_base = self.hparams["num_attention_heads"]
+ n_kv_base = self.hparams["num_attention_groups"]
+
+ n_head_swa = attn_other.get("num_attention_heads", n_head_base)
+ n_kv_swa = attn_other.get("num_attention_groups", n_kv_base)
+
+ layer_types = layer_types[: self.block_count]
+ partial_rotary_factors = partial_rotary_factors[: self.block_count]
+ assert [1.0 if lt == "sliding_attention" else 0.5 for lt in layer_types] == partial_rotary_factors
+ head_arr = [n_head_swa if lt == "sliding_attention" else n_head_base for lt in layer_types]
+ kv_arr = [n_kv_swa if lt == "sliding_attention" else n_kv_base for lt in layer_types]
+ swa_pat = [lt == "sliding_attention" for lt in layer_types]
+
+ self.gguf_writer.add_head_count(head_arr)
+ self.gguf_writer.add_head_count_kv(kv_arr)
+
+ self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+ self.gguf_writer.add_sliding_window_pattern(swa_pat)
+
+ self.gguf_writer.add_value_length(self.hparams["head_dim"])
+
+ # MoE params
+ self.gguf_writer.add_expert_count(self.hparams["moe_num_experts"])
+ self.gguf_writer.add_expert_used_count(self.hparams["moe_top_k"])
+ self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
+ self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["share_expert_dim"])
+
+ if (moe_router_scaling_factor := self.hparams.get("moe_router_scaling_factor")) is not None:
+ self.gguf_writer.add_expert_weights_scale(moe_router_scaling_factor)
+ if (norm_expert_weight := self.hparams.get("norm_expert_weight")) is not None:
+ self.gguf_writer.add_expert_weights_norm(norm_expert_weight)
+
+ # leading dense blocks
+ leading_dense = 0
+ moe_layers_enum = self.hparams.get("moe_layers_enum")
+ if isinstance(moe_layers_enum, str) and moe_layers_enum.strip():
+ moe_layers = sorted(int(i) for i in moe_layers_enum.strip().split(","))
+ if moe_layers:
+ leading_dense = max(0, moe_layers[0])
+ self.gguf_writer.add_leading_dense_block_count(leading_dense)
+ self.gguf_writer.add_moe_every_n_layers(int(self.hparams.get("moe_every_n_layer", 1)))
+
+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("rms_norm_eps", 1e-5))
+
+ # Optional per-layer SwiGLU clamps.
+ if (limits := self.hparams.get("swiglu_limits")) is not None:
+ limits_f = [0.0 if v is None else float(v) for v in limits[: self.block_count]]
+ self.gguf_writer.add_swiglu_clamp_exp(limits_f)
+ if (limits_shared := self.hparams.get("swiglu_limits_shared")) is not None:
+ limits_shared_f = [0.0 if v is None else float(v) for v in limits_shared[: self.block_count]]
+ self.gguf_writer.add_swiglu_clamp_shexp(limits_shared_f)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
+ # remove mtp layers
+ if (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None:
+ il = int(m.group(1))
+ n_main = int(self.hparams.get("num_hidden_layers", self.block_count))
+ if il >= n_main:
+ return
+ if name.endswith("norm.weight"):
+ data_torch += 1.0
+ # Map router bias (expert selection bias) to a GGUF bias tensor
+ if name.endswith(".moe.router_bias"):
+ name += ".bias"
+
+ if name.endswith(
+ (
+ ".self_attn.g_proj.weight",
+ ".moe.gate.weight",
+ ".moe.up_proj.weight",
+ ".moe.gate_proj.weight",
+ ".moe.down_proj.weight",
+ )
+ ):
+ data_torch = data_torch.squeeze().contiguous()
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # Step35 can optionally use Llama-3 style RoPE scaling (HF: rope_scaling.rope_type == "llama3").
+ # llama.cpp represents this via a single extra tensor: "rope_freqs.weight" (aka MODEL_TENSOR.ROPE_FREQS).
+ rope_params = self.rope_parameters.get("full_attention", self.rope_parameters)
+ rope_type = rope_params.get("rope_type") or ""
+ if rope_type.lower() != "llama3":
+ return
+
+ # Step35 configs can carry per-layer rope_theta as a list; for llama3 rope factors we use the base value.
+ rope_theta = self.hparams.get("rope_theta", 10000.0)
+ if isinstance(rope_theta, list):
+ rope_theta = rope_theta[0]
+ base = float(rope_theta)
+ if (dim := self.hparams.get("head_dim")) is None:
+ dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
+ dim = int(dim)
+
+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
+
+ factor = float(rope_params.get("factor", 8.0))
+ low_freq_factor = float(rope_params.get("low_freq_factor", 1.0))
+ high_freq_factor = float(rope_params.get("high_freq_factor", 4.0))
+ old_context_len = int(
+ rope_params.get(
+ "original_max_position_embeddings", self.hparams.get("original_max_position_embeddings", 8192)
+ )
+ )
+
+ low_freq_wavelen = old_context_len / low_freq_factor
+ high_freq_wavelen = old_context_len / high_freq_factor
+
+ rope_factors: list[float] = []
+ for freq in freqs:
+ wavelen = 2 * math.pi / float(freq)
+ if wavelen < high_freq_wavelen:
+ rope_factors.append(1.0)
+ elif wavelen > low_freq_wavelen:
+ rope_factors.append(factor)
+ else:
+ smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)
+ rope_factors.append(1.0 / ((1.0 - smooth) / factor + smooth))
+
+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32))
+
+
[email protected]("PanguEmbeddedForCausalLM")
+class PanguEmbeddedModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.PANGU_EMBED
+
+ def set_vocab(self):
+ self._set_vocab_sentencepiece()
+
+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"
+ if tokenizer_config_file.is_file():
+ with open(tokenizer_config_file, "r", encoding="utf-8") as f:
+ tokenizer_config_json = json.load(f)
+ if "add_prefix_space" in tokenizer_config_json:
+ self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hparams = self.hparams
+ self.gguf_writer.add_vocab_size(hparams["vocab_size"])
+
+ # PanguEmbedded's hparam loaded from config.json without head_dim
+ if (rope_dim := hparams.get("head_dim")) is None:
+ rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
+ self.gguf_writer.add_rope_dimension_count(rope_dim)
+
+ if hparams.get("head_dim") is None:
+ self.gguf_writer.add_key_length(rope_dim)
+ self.gguf_writer.add_value_length(rope_dim)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name == "lm_head.weight":
+ if self.hparams.get("tie_word_embeddings", False):
+ logger.info("Skipping tied output layer 'lm_head.weight'")
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("Dots1ForCausalLM")
+class Dots1Model(Qwen2MoeModel):
+ model_arch = gguf.MODEL_ARCH.DOTS1
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.hparams["num_experts"] = self.hparams["n_routed_experts"]
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"])
+ self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"])
+ self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
+ self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
+ if name.endswith("e_score_correction_bias"):
+ name = name.replace("e_score_correction_bias", "e_score_correction.bias")
+ if "shared_experts" in name:
+ yield from ModelBase.modify_tensors(self, data_torch, name, bid)
+ else:
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("PLMForCausalLM")
+class PLMModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.PLM
+
+ def set_vocab(self):
+ self._set_vocab_gpt2()
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hparams = self.hparams
+ self.gguf_writer.add_vocab_size(hparams["vocab_size"])
+ self.gguf_writer.add_kv_lora_rank(hparams["kv_lora_rank"])
+ self.gguf_writer.add_key_length(hparams["qk_nope_head_dim"] + hparams["qk_rope_head_dim"])
+ self.gguf_writer.add_value_length(hparams["v_head_dim"])
+ self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+
+
[email protected]("T5WithLMHeadModel")
[email protected]("T5ForConditionalGeneration")
[email protected]("MT5ForConditionalGeneration")
[email protected]("UMT5ForConditionalGeneration")
[email protected]("UMT5Model")
+class T5Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.T5
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.shared_token_embeddings_found = False
+
+ def set_vocab(self):
+ # to avoid TypeError: Descriptors cannot be created directly
+ # exception when importing sentencepiece_model_pb2
+ os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+ from sentencepiece import SentencePieceProcessor # pylint: disable=E0401
+ from sentencepiece import sentencepiece_model_pb2 as model # pylint: disable=E0401
+
+ tokenizer_path = self.dir_model / "tokenizer.model"
+
+ # many older models use spiece.model tokenizer model filename
+ if not tokenizer_path.is_file():
+ tokenizer_path = self.dir_model / "spiece.model"
+
+ if not tokenizer_path.is_file():
+ raise FileNotFoundError(f"File not found: {tokenizer_path}")
+
+ sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue]
+ sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
+
+ # some models like Pile-T5 family use BPE tokenizer instead of Unigram
+ if sentencepiece_model.trainer_spec.model_type == 2: # BPE
+ # assure the tokenizer model file name is correct
+ assert tokenizer_path.name == "tokenizer.model"
+ return self._set_vocab_sentencepiece()
+ else:
+ assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM
+
+ add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix
remove_whitespaces = sentencepiece_model.normalizer_spec.remove_extra_whitespaces
precompiled_charsmap = sentencepiece_model.normalizer_spec.precompiled_charsmap
@@ -7343,7 +8962,7 @@ class T5Model(TextModel):
self.gguf_writer.add_context_length(n_ctx)
self.gguf_writer.add_embedding_length(self.hparams["d_model"])
self.gguf_writer.add_feed_forward_length(self.hparams["d_ff"])
- self.gguf_writer.add_block_count(self.hparams["num_layers"])
+ self.gguf_writer.add_block_count(self.block_count)
if (dec_n_layer := self.hparams.get("num_decoder_layers")) is not None:
self.gguf_writer.add_decoder_block_count(dec_n_layer)
self.gguf_writer.add_head_count(self.hparams["num_heads"])
@@ -7356,17 +8975,20 @@ class T5Model(TextModel):
self.gguf_writer.add_file_type(self.ftype)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
+ # T5 based models contain shared token embeddings tensors saved randomly as either
+ # "encoder.embed_tokens.weight", "decoder.embed_tokens.weight" or "shared.weight" tensor.
+ # In some models there are even multiple of them stored in the safetensors files.
+ # We use the first tensor from these three as the token embeddings for both encoder
+ # and decoder and ignore the remaining ones.
if name in ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight", "shared.weight"]:
if not self.shared_token_embeddings_found:
name = "shared.weight"
self.shared_token_embeddings_found = True
else:
logger.debug(f"Skipping shared tensor {name!r} in safetensors so that convert can end normally.")
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("T5EncoderModel")
@@ -7393,7 +9015,7 @@ class T5EncoderModel(TextModel):
if not tokenizer_path.is_file():
raise FileNotFoundError(f"File not found: {tokenizer_path}")
- sentencepiece_model = model.ModelProto() # pylint: disable=E1101
+ sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue]
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
# some models like Pile-T5 family use BPE tokenizer instead of Unigram
@@ -7478,7 +9100,7 @@ class T5EncoderModel(TextModel):
self.gguf_writer.add_context_length(n_ctx)
self.gguf_writer.add_embedding_length(self.hparams["d_model"])
self.gguf_writer.add_feed_forward_length(self.hparams["d_ff"])
- self.gguf_writer.add_block_count(self.hparams["num_layers"])
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_head_count(self.hparams["num_heads"])
self.gguf_writer.add_key_length(self.hparams["d_kv"])
self.gguf_writer.add_value_length(self.hparams["d_kv"])
@@ -7488,17 +9110,31 @@ class T5EncoderModel(TextModel):
self.gguf_writer.add_file_type(self.ftype)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
+ # T5 based models contain shared token embeddings tensors saved randomly as either
+ # "encoder.embed_tokens.weight", "decoder.embed_tokens.weight" or "shared.weight" tensor.
+ # In some models there are even multiple of them stored in the safetensors files.
+ # We use the first tensor from these three as the token embeddings for both encoder
+ # and decoder and ignore the remaining ones.
if name in ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight", "shared.weight"]:
if not self.shared_token_embeddings_found:
name = "shared.weight"
self.shared_token_embeddings_found = True
else:
logger.debug(f"Skipping shared tensor {name!r} in safetensors so that convert can end normally.")
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("Jais2ForCausalLM")
+class Jais2Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.JAIS2
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hparams = self.hparams
+ head_dim = hparams.get("head_dim", hparams["hidden_size"] // hparams["num_attention_heads"])
+ self.gguf_writer.add_rope_dimension_count(head_dim)
@ModelBase.register("JAISLMHeadModel")
@@ -7537,7 +9173,7 @@ class JaisModel(TextModel):
self._set_vocab_gpt2()
def set_gguf_parameters(self):
- self.gguf_writer.add_block_count(self.hparams["n_layer"])
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_context_length(self.hparams["n_positions"])
self.gguf_writer.add_embedding_length(self.hparams["n_embd"])
self.gguf_writer.add_feed_forward_length(self.hparams["n_inner"])
@@ -7546,15 +9182,11 @@ class JaisModel(TextModel):
self.gguf_writer.add_file_type(self.ftype)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
- tensors: list[tuple[str, Tensor]] = []
-
# we don't need these
- if name.endswith((".attn.bias")):
- return tensors
+ if name.endswith(".attn.bias"):
+ return
- if name.endswith(("relative_pe.slopes")):
+ if name.endswith("relative_pe.slopes"):
# Calculate max ALiBi bias (this is the inverse of the ALiBi calculation)
# Some other models has max_alibi_bias spelled out explicitly in the hyperparams,
# but Jais's PyTorch model simply precalculates the slope values and places them
@@ -7563,7 +9195,7 @@ class JaisModel(TextModel):
first_val = float(data_torch[0].item())
self.max_alibi_bias = -round(math.log2(first_val) * n_head_closest_log2)
- return tensors
+ return
if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight", ".c_fc2.weight")):
data_torch = data_torch.transpose(1, 0)
@@ -7571,13 +9203,11 @@ class JaisModel(TextModel):
new_name = self.map_tensor_name(name)
if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD):
- tensors.append((new_name, data_torch * self.embeddings_scale))
+ yield from super().modify_tensors(data_torch * self.embeddings_scale, new_name, bid)
elif new_name == self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT):
- tensors.append((new_name, data_torch * self.width_scale))
+ yield from super().modify_tensors(data_torch * self.width_scale, new_name, bid)
else:
- tensors.append((new_name, data_torch))
-
- return tensors
+ yield from super().modify_tensors(data_torch, new_name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -7587,6 +9217,15 @@ class JaisModel(TextModel):
@ModelBase.register("Glm4ForCausalLM", "Glm4vForConditionalGeneration")
class Glm4Model(TextModel):
model_arch = gguf.MODEL_ARCH.GLM4
+ use_mrope = False
+ partial_rotary_factor = 0.5
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.partial_rotary_factor = self.rope_parameters.get("partial_rotary_factor", 0.5)
+ if "mrope_section" in self.rope_parameters:
+ self.use_mrope = True
+ logger.info("Q/K weight will need to be permuted for M-RoPE")
def set_vocab(self):
from transformers import AutoTokenizer
@@ -7609,22 +9248,76 @@ class Glm4Model(TextModel):
super().set_gguf_parameters()
if (rope_dim := self.hparams.get("head_dim")) is None:
rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
- self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)))
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
+ self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.partial_rotary_factor))
+
+ @staticmethod
+ def normal_to_neox(
+ weights: Tensor, n_head: int, n_head_kv: int, head_dim: int, partial_rotary_factor: float
+ ) -> Tensor:
+ orig_shape = weights.shape
+ if len(orig_shape) == 1:
+ weights = weights.unsqueeze(1) # [out_dim, 1]
+ if len(weights.shape) != 2:
+ raise ValueError("Only 1D and 2D tensors are supported.")
+ n_effective_heads = weights.shape[0] // head_dim
+ if n_head_kv is not None and n_effective_heads != n_head:
+ if n_effective_heads != n_head_kv:
+ raise AssertionError(
+ f"Mismatch in effective heads: computed {n_effective_heads}, expected {n_head} or {n_head_kv}"
+ )
+ rotary_dim = int(head_dim * partial_rotary_factor)
+ if rotary_dim % 2 != 0:
+ raise ValueError("rotary_dim must be even.")
+ reshaped = weights.reshape(n_effective_heads, head_dim, -1)
+ rot_part = reshaped[:, :rotary_dim, :]
+ non_rot_part = reshaped[:, rotary_dim:, :]
+ permuted_rot = torch.cat((rot_part[:, ::2, :], rot_part[:, 1::2, :]), dim=1)
+ combined = torch.cat((permuted_rot, non_rot_part), dim=1)
+ result = combined.reshape(weights.shape)
+ return result if len(orig_shape) != 1 else result.squeeze(1)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.startswith("model.visual."): # ignore visual part of Glm4v
- return []
+ return
elif name.startswith("model.language_model."):
name = name.replace("language_model.", "") # for Glm4v
- return super().modify_tensors(data_torch, name, bid)
+ if self.use_mrope:
+ n_head = self.hparams["num_attention_heads"]
+ n_kv_head = self.hparams["num_key_value_heads"]
+ n_embd = self.hparams["hidden_size"]
+ head_dim = self.hparams.get("head_dim", n_embd // n_head)
+ # because llama.cpp M-RoPE kernel only supports Neox ordering, we have to permute the weights here
+ if name.endswith(("q_proj.weight", "q_proj.bias")):
+ data_torch = Glm4Model.normal_to_neox(data_torch, n_head, n_head, head_dim, self.partial_rotary_factor)
+ if name.endswith(("k_proj.weight", "k_proj.bias")):
+ data_torch = Glm4Model.normal_to_neox(
+ data_torch, n_head, n_kv_head, head_dim, self.partial_rotary_factor
+ )
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("GlmOcrForConditionalGeneration")
+class GlmOCRModel(Glm4Model):
+ model_arch = gguf.MODEL_ARCH.GLM4
+ use_mrope = False
+ partial_rotary_factor = 0.5
+
+ # Note: GLM-OCR is the same as GLM4, but with an extra NextN/MTP prediction layer
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # GLM-OCR has num_hidden_layers + 1 actual layers (including NextN layer)
+ self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0)
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ # NextN/MTP prediction layers
+ if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
+ self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
[email protected]("Glm4MoeForCausalLM")
[email protected]("Glm4MoeForCausalLM", "Glm4vMoeForConditionalGeneration")
class Glm4MoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.GLM4_MOE
@@ -7635,34 +9328,7 @@ class Glm4MoeModel(TextModel):
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
def set_vocab(self):
- from transformers import AutoTokenizer
-
- tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
- tokens, toktypes, tokpre = self.get_vocab_base()
- self.gguf_writer.add_tokenizer_model("gpt2")
- self.gguf_writer.add_tokenizer_pre(tokpre)
- self.gguf_writer.add_token_list(tokens)
- self.gguf_writer.add_token_types(toktypes)
-
- # Special tokens
- # Note: Using <|endoftext|> (151329) for eot causes endless generation
- special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # 151331
- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # 151336
- special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # 151329
- special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338
-
- # Patch broken chat template
- if (
- isinstance(special_vocab.chat_template, str)
- and "visible_text(m.content).endswith" in special_vocab.chat_template
- ):
- special_vocab.chat_template = special_vocab.chat_template.replace(
- """{{ visible_text(m.content) }}\n{{- '/nothing' if (enable_thinking is defined and not enable_thinking and not visible_text(m.content).endswith("/nothing")) else '' -}}""", # pylint: disable=C0301
- """{% set content = visible_text(m.content) %}{{ content }}\n{{- '/nothing' if (enable_thinking is defined and not enable_thinking and not content.endswith("/nothing")) else '' -}}""", # pylint: disable=C0301
- )
-
- special_vocab.add_to_gguf(self.gguf_writer)
+ return self._set_vocab_glm()
def set_gguf_parameters(self):
super().set_gguf_parameters()
@@ -7697,15 +9363,17 @@ class Glm4MoeModel(TextModel):
_experts: list[dict[str, Tensor]] | None = None
+ # note: unlike GLM4V non-MoE, we don't need to permute Q/K here since GLM4V_MOE uses Neox ordering already
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.startswith("model.visual."): # ignore visual part
- return []
+ return
elif name.startswith("model.language_model."):
name = name.replace("language_model.", "") # for multimodal variants
# Handle main token embedding (but not layer-specific NextN embeddings)
if name == "model.embed_tokens.weight" and ".layers." not in name:
- return [(self.map_tensor_name("token_embd.weight"), data_torch)]
+ yield from super().modify_tensors(data_torch, "token_embd.weight", bid)
+ return
# Handle routed experts
if name.find("mlp.experts") != -1:
@@ -7718,8 +9386,6 @@ class Glm4MoeModel(TextModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["down_proj", "gate_proj", "up_proj"]:
data: list[Tensor] = []
@@ -7733,18 +9399,15 @@ class Glm4MoeModel(TextModel):
merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
if name.endswith("e_score_correction_bias"):
name = name.replace("e_score_correction_bias", "e_score_correction.bias")
- new_name = self.map_tensor_name(name)
-
- return [(new_name, data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -7755,6 +9418,44 @@ class Glm4MoeModel(TextModel):
raise ValueError(f"Unprocessed experts: {experts}")
[email protected]("Glm4MoeLiteForCausalLM")
+class Glm4MoeLiteModel(DeepseekV2Model):
+ model_arch = gguf.MODEL_ARCH.DEEPSEEK2
+
+ def set_vocab(self):
+ return self._set_vocab_glm()
+
+
[email protected]("GlmMoeDsaForCausalLM")
+class GlmMoeDsaModel(DeepseekV2Model):
+ model_arch = gguf.MODEL_ARCH.GLM_DSA
+ skip_mtp = False
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0)
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+ def set_vocab(self):
+ return self._set_vocab_glm()
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ rope_dim = self.hparams["qk_rope_head_dim"]
+ partial_rotary_factor = self.hparams.get("partial_rotary_factor", 1.0)
+ self.gguf_writer.add_rope_dimension_count(int(rope_dim * partial_rotary_factor))
+
+ # NextN/MTP prediction layers
+ if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
+ self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
+
+ # DSA indexer parameters
+ self.gguf_writer.add_indexer_head_count(self.hparams["index_n_heads"])
+ self.gguf_writer.add_indexer_key_length(self.hparams["index_head_dim"])
+ self.gguf_writer.add_indexer_top_k(self.hparams["index_topk"])
+
+
@ModelBase.register("GlmForCausalLM", "ChatGLMModel", "ChatGLMForConditionalGeneration")
class ChatGLMModel(TextModel):
model_arch = gguf.MODEL_ARCH.CHATGLM
@@ -7887,7 +9588,7 @@ class ChatGLMModel(TextModel):
self.gguf_writer.add_feed_forward_length(
self.hparams.get("ffn_hidden_size", self.hparams.get("intermediate_size", 4 * n_embed))
)
- self.gguf_writer.add_block_count(self.hparams.get("num_layers", self.hparams["num_hidden_layers"]))
+ self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_head_count(n_head)
self.gguf_writer.add_head_count_kv(n_head_kv)
self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("layernorm_epsilon", 1e-5))
@@ -7904,13 +9605,11 @@ class ChatGLMModel(TextModel):
self.gguf_writer.add_rope_freq_base(rope_freq)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision."):
- return []
+ return
name = name.removeprefix("transformer.")
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("NemotronForCausalLM")
@@ -7944,10 +9643,14 @@ class NemotronModel(TextModel):
self.gguf_writer.add_rope_scaling_factor(self.hparams["factor"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # * Adding +1 to LayerNorm's weights here to implement layernorm1p w/o changing anything on the GGML engine side
+ # model.layers.{l}.input_layernorm.weight
+ # model.layers.{l}.post_attention_layernorm.weight
+ # model.norm.weight
if name.endswith("norm.weight"):
data_torch = data_torch + 1
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("ExaoneForCausalLM")
@@ -7955,53 +9658,28 @@ class ExaoneModel(TextModel):
model_arch = gguf.MODEL_ARCH.EXAONE
def set_gguf_parameters(self):
+ super().set_gguf_parameters()
hparams = self.hparams
assert hparams["activation_function"] == "silu"
- max_position_embeddings = hparams["max_position_embeddings"]
- embed_dim = hparams["hidden_size"]
- num_heads = hparams["num_attention_heads"]
- num_kv_heads = hparams.get("num_key_value_heads", num_heads)
- layer_norm_eps = hparams["layer_norm_epsilon"]
- intermediate_size = hparams["intermediate_size"] if "intermediate_size" in hparams else 4 * embed_dim
- num_layers = hparams["num_layers"]
- # ignore for now as EXAONE-3.0-7.8B-Instruct attention_dropout is 0.0
- # attention_dropout_rate = hparams["attention_dropout"]
- # ignore for now as EXAONE-3.0-7.8B-Instruct embed_dropout is 0.0
- # embed_dropout_rate = hparams["embed_dropout"]
- self.gguf_writer.add_embedding_length(embed_dim)
- self.gguf_writer.add_head_count(num_heads)
- self.gguf_writer.add_head_count_kv(num_kv_heads)
- self.gguf_writer.add_context_length(max_position_embeddings)
- self.gguf_writer.add_layer_norm_rms_eps(layer_norm_eps)
- self.gguf_writer.add_feed_forward_length(intermediate_size)
- self.gguf_writer.add_block_count(num_layers)
- self.gguf_writer.add_file_type(self.ftype)
-
- if (rope_theta := self.hparams.get("rope_theta")) is not None:
- self.gguf_writer.add_rope_freq_base(rope_theta)
rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"], optional=True)
rotary_factor = rotary_factor if rotary_factor is not None else 1.0
self.gguf_writer.add_rope_dimension_count(
int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))
)
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
- if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):
- if rope_scaling.get("rope_type", "").lower() == "llama3":
- base = self.hparams.get("rope_theta", 10000.0)
+ if rope_params := self.rope_parameters.get("full_attention", self.rope_parameters):
+ if rope_params.get("rope_type", "").lower() == "llama3":
+ base = self.rope_parameters.get("rope_theta", 10000.0)
if (dim := self.hparams.get("head_dim")) is None:
dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
- factor = rope_scaling.get("factor", 8.0)
- low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)
- high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)
+ factor = rope_params.get("factor", 8.0)
+ low_freq_factor = rope_params.get("low_freq_factor", 1.0)
+ high_freq_factor = rope_params.get("high_freq_factor", 4.0)
old_context_len = self.hparams.get("original_max_position_embeddings", 8192)
low_freq_wavelen = old_context_len / low_freq_factor
@@ -8061,22 +9739,17 @@ class Exaone4Model(TextModel):
if len(sliding_window_pattern) == hparams["num_hidden_layers"]:
self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
-
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
- if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):
- if rope_scaling.get("rope_type", "").lower() == "llama3":
- base = self.hparams.get("rope_theta", 10_000.0)
+ if rope_params := self.rope_parameters.get("full_attention", self.rope_parameters):
+ if rope_params.get("rope_type", "").lower() == "llama3":
+ base = rope_params.get("rope_theta", 10_000.0)
if (dim := self.hparams.get("head_dim")) is None:
dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
- factor = rope_scaling.get("factor", 16.0)
- low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)
- high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)
+ factor = rope_params.get("factor", 16.0)
+ low_freq_factor = rope_params.get("low_freq_factor", 1.0)
+ high_freq_factor = rope_params.get("high_freq_factor", 4.0)
old_context_len = self.hparams.get("original_max_position_embeddings", 8192)
low_freq_wavelen = old_context_len / low_freq_factor
@@ -8099,6 +9772,97 @@ class Exaone4Model(TextModel):
)
[email protected]("ExaoneMoEForCausalLM")
+class ExaoneMoEModel(Exaone4Model):
+ model_arch = gguf.MODEL_ARCH.EXAONE_MOE
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0)
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ moe_intermediate_size = self.hparams["moe_intermediate_size"]
+ num_shared_experts = self.hparams["num_shared_experts"]
+ self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
+ self.gguf_writer.add_expert_shared_count(num_shared_experts)
+ self.gguf_writer.add_expert_shared_feed_forward_length(moe_intermediate_size * num_shared_experts)
+ self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
+ self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
+ n_dense_layer = self.hparams.get("first_k_dense_replace", self.hparams.get("first_last_k_dense_replace", 0))
+ self.gguf_writer.add_leading_dense_block_count(n_dense_layer)
+ self.gguf_writer.add_nextn_predict_layers(self.hparams.get("num_nextn_predict_layers", 0))
+
+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+
+ _experts: list[dict[str, Tensor]] | None = None
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.startswith("mtp."):
+ if name.find("layers.") != -1:
+ # `mtp.layers.0.[module_name]` format
+ name = name.replace(f"mtp.layers.{bid}", f"model.layers.{bid + self.hparams['num_hidden_layers']}")
+ else:
+ # mtp fc/norm weights
+ remapper = {
+ "mtp.fc": "model.layers.{bid}.eh_proj",
+ "mtp.pre_fc_norm_embedding": "model.layers.{bid}.enorm",
+ "mtp.pre_fc_norm_hidden": "model.layers.{bid}.hnorm",
+ "mtp.norm": "model.layers.{bid}.shared_head.norm",
+ }
+ _n = Path(name)
+ new_name = remapper[_n.stem] + _n.suffix
+
+ # set shared weights for all NextN/MTP layers
+ for bid in range(self.hparams["num_hidden_layers"], self.block_count):
+ yield from super().modify_tensors(data_torch, new_name.format(bid=bid), bid)
+ return
+
+ if name.endswith("e_score_correction_bias"):
+ name = name.replace("e_score_correction_bias", "e_score_correction.bias")
+
+ if name.find("mlp.experts") != -1:
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
+ assert bid is not None
+
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) >= n_experts * 3:
+ # merge the experts into a single 3d tensor
+ for w_name in ["down_proj", "gate_proj", "up_proj"]:
+ data: list[Tensor] = []
+
+ for xid in range(n_experts):
+ ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"
+ data.append(self._experts[bid][ename])
+ del self._experts[bid][ename]
+
+ data_torch = torch.stack(data, dim=0)
+
+ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+
+ new_name = self.map_tensor_name(merged_name)
+
+ yield from super().modify_tensors(data_torch, new_name, bid)
+ return
+ else:
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+ if self._experts is not None:
+ # flatten `list[dict[str, Tensor]]` into `list[str]`
+ experts = [k for d in self._experts for k in d.keys()]
+ if len(experts) > 0:
+ raise ValueError(f"Unprocessed experts: {experts}")
+
+
@ModelBase.register("GraniteForCausalLM")
class GraniteModel(LlamaModel):
"""Conversion for IBM's GraniteForCausalLM"""
@@ -8160,10 +9924,13 @@ class GraniteMoeModel(GraniteModel):
ffn_dim = self.hparams["intermediate_size"]
assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size"
gate, up = data_torch.split(ffn_dim, dim=-2)
- return [
- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), gate),
- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), up),
- ]
+ yield from ModelBase.modify_tensors(
+ self, gate, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), bid
+ )
+ yield from ModelBase.modify_tensors(
+ self, up, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), bid
+ )
+ return
has_experts = bool(self.hparams.get("num_local_experts"))
@@ -8172,19 +9939,26 @@ class GraniteMoeModel(GraniteModel):
assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size"
gate, up = data_torch.split(ffn_dim, dim=-2)
if has_experts:
- return [
- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_SHEXP, bid), gate),
- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), up),
- ]
- return [
- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), gate),
- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), up),
- ]
+ yield from ModelBase.modify_tensors(
+ self, gate, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_SHEXP, bid), bid
+ )
+ yield from ModelBase.modify_tensors(
+ self, up, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), bid
+ )
+ return
+ yield from ModelBase.modify_tensors(
+ self, gate, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), bid
+ )
+ yield from ModelBase.modify_tensors(self, up, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), bid)
+ return
if not has_experts and name.endswith("shared_mlp.output_linear.weight"):
- return [(self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN, bid), data_torch)]
+ yield from ModelBase.modify_tensors(
+ self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN, bid), bid
+ )
+ return
- return super().modify_tensors(data_torch, name, bid)
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("GraniteMoeHybridForCausalLM", "BambaForCausalLM")
@@ -8212,7 +9986,7 @@ class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
# case, the model architecture needs to be updated to a standard
# "granite" or "granitemoe" model
if not self._ssm_layers:
- has_experts = self.find_hparam(["num_experts_per_tok"], optional=True)
+ has_experts = self.find_hparam(["num_experts_per_tok", "num_experts_per_token"], optional=True)
new_arch = gguf.MODEL_ARCH.GRANITE_MOE if has_experts else gguf.MODEL_ARCH.GRANITE
self.model_arch = new_arch
self.gguf_writer.arch = gguf.MODEL_ARCH_NAMES[new_arch]
@@ -8251,14 +10025,17 @@ class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.endswith("block_sparse_moe.input_linear.weight") or "shared_mlp" in name:
- return GraniteMoeModel.modify_tensors(self, data_torch, name, bid)
+ yield from GraniteMoeModel.modify_tensors(self, data_torch, name, bid)
+ return
# Determine whether this is a mamba layer or an attention layer
if bid in self._ssm_layers:
- return Mamba2Model.modify_tensors(self, data_torch, name, bid)
+ yield from Mamba2Model.modify_tensors(self, data_torch, name, bid)
+ return
elif bid in self._attn_layers:
- return GraniteMoeModel.modify_tensors(self, data_torch, name, bid)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from GraniteMoeModel.modify_tensors(self, data_torch, name, bid)
+ return
+ yield from ModelBase.modify_tensors(self, data_torch, name, bid)
def set_gguf_parameters(self):
"""This method merges params from both parents and some that are
@@ -8307,8 +10084,18 @@ class NemotronHModel(GraniteHybridModel):
"""Hybrid mamba2/attention model from NVIDIA"""
model_arch = gguf.MODEL_ARCH.NEMOTRON_H
+ is_moe: bool = False
def __init__(self, *args, **kwargs):
+ # We have to determine the correct model architecture (MoE vs non-MoE) before
+ # calling the parent __init__. This is because the parent constructor
+ # uses self.model_arch to build the tensor name map, and all MoE-specific
+ # mappings would be missed if it were called with the default non-MoE arch.
+ hparams = ModelBase.load_hparams(args[0], self.is_mistral_format)
+ if "num_experts_per_tok" in hparams:
+ self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE
+ self.is_moe = True
+
super().__init__(*args, **kwargs)
# Save the top-level head_dim for later
@@ -8320,9 +10107,11 @@ class NemotronHModel(GraniteHybridModel):
# Update the ssm / attn / mlp layers
# M: Mamba2, *: Attention, -: MLP
+ # MoE:
+ # M: Mamba2, *: Attention, E: Expert
hybrid_override_pattern = self.hparams["hybrid_override_pattern"]
self._ssm_layers = [i for i, val in enumerate(hybrid_override_pattern) if val == "M"]
- self._mlp_layers = [i for i, val in enumerate(hybrid_override_pattern) if val == "-"]
+ self._mlp_layers = [i for i, val in enumerate(hybrid_override_pattern) if val == ("E" if self.is_moe else "-")]
def get_attn_layers(self):
hybrid_override_pattern = self.hparams["hybrid_override_pattern"]
@@ -8340,10 +10129,28 @@ class NemotronHModel(GraniteHybridModel):
# Set feed_forward_length
# NOTE: This will trigger an override warning. This is preferable to
# duplicating all the parent logic
- n_ff = self.find_hparam(["intermediate_size", "n_inner", "hidden_dim"])
- self.gguf_writer.add_feed_forward_length(
- [n_ff if i in self._mlp_layers else 0 for i in range(self.block_count)]
- )
+ if not self.is_moe:
+ n_ff = self.find_hparam(["intermediate_size", "n_inner", "hidden_dim"])
+ self.gguf_writer.add_feed_forward_length(
+ [n_ff if i in self._mlp_layers else 0 for i in range(self.block_count)]
+ )
+ else:
+ moe_intermediate_size = self.hparams["moe_intermediate_size"]
+ self.gguf_writer.add_feed_forward_length(
+ [moe_intermediate_size if i in self._mlp_layers else 0 for i in range(self.block_count)]
+ )
+ self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])
+ self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
+ self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"])
+ self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"])
+ self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"])
+ self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
+ self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
+ self.gguf_writer.add_expert_group_count(self.hparams["n_group"])
+
+ # number of experts used per token (top-k)
+ if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:
+ self.gguf_writer.add_expert_used_count(n_experts_used)
def set_vocab(self):
super().set_vocab()
@@ -8351,7 +10158,91 @@ class NemotronHModel(GraniteHybridModel):
# The tokenizer _does_ add a BOS token (via post_processor type
# TemplateProcessing) but does not set add_bos_token to true in the
# config, so we need to explicitly override it here.
- self.gguf_writer.add_add_bos_token(True)
+ if not self.is_moe:
+ self.gguf_writer.add_add_bos_token(True)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Skip vision model and projector tensors for VLM models (handled by mmproj) (e.g., Nemotron Nano 12B v2 VL)
+ if name.startswith(("vision_model.", "mlp1.")):
+ return
+
+ # Strip language_model. prefix for VLM models (e.g., Nemotron Nano 12B v2 VL)
+ name = name.removeprefix("language_model.")
+
+ if self.is_moe and bid is not None:
+ if name.endswith("mixer.gate.e_score_correction_bias"):
+ new_name = name.replace("e_score_correction_bias", "e_score_correction.bias")
+ yield from ModelBase.modify_tensors(self, data_torch, new_name, bid)
+ return
+
+ if name.endswith("mixer.dt_bias"):
+ new_name = name.replace("dt_bias", "dt.bias")
+ yield from ModelBase.modify_tensors(self, data_torch, new_name, bid)
+ return
+
+ if name.endswith("mixer.conv1d.weight"):
+ squeezed_data = data_torch.squeeze()
+ yield from ModelBase.modify_tensors(self, squeezed_data, name, bid)
+ return
+
+ if name.endswith("mixer.A_log"):
+ transformed_data = -torch.exp(data_torch)
+ reshaped_data = transformed_data.squeeze().reshape(-1, 1)
+ yield from ModelBase.modify_tensors(self, reshaped_data, name, bid)
+ return
+
+ if name.endswith("mixer.D"):
+ reshaped_data = data_torch.squeeze().reshape(-1, 1)
+ yield from ModelBase.modify_tensors(self, reshaped_data, name, bid)
+ return
+
+ if name.endswith("mixer.norm.weight"):
+ reshaped_data = data_torch.reshape(self.n_group, -1)
+ yield from ModelBase.modify_tensors(self, reshaped_data, name, bid)
+ return
+
+ if name.find("mixer.experts") != -1:
+ n_experts = self.hparams["n_routed_experts"]
+ assert bid is not None
+
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) >= n_experts * 2:
+ # merge the experts into a single tensor
+ for w_name in ["down_proj", "up_proj"]:
+ data: list[Tensor] = []
+
+ for xid in range(n_experts):
+ ename = f"backbone.layers.{bid}.mixer.experts.{xid}.{w_name}.weight"
+ data.append(self._experts[bid][ename])
+ del self._experts[bid][ename]
+
+ data_torch = torch.stack(data, dim=0)
+ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+
+ yield from ModelBase.modify_tensors(self, data_torch, merged_name, bid)
+ return
+ else:
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+
+ if self._experts is not None:
+ # flatten `list[dict[str, Tensor]]` into `list[str]`
+ experts = [k for d in self._experts for k in d.keys()]
+ if len(experts) > 0:
+ raise ValueError(f"Unprocessed experts: {experts}")
+
+
[email protected]("LlamaBidirectionalModel")
+class LlamaEmbedNemotronModel(LlamaModel):
+ model_arch = gguf.MODEL_ARCH.LLAMA_EMBED
@ModelBase.register("BailingMoeForCausalLM")
@@ -8368,18 +10259,10 @@ class BailingMoeModel(TextModel):
rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
self.gguf_writer.add_rope_dimension_count(rope_dim)
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
- else:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
self.gguf_writer.add_expert_weights_scale(1.0)
- self.gguf_writer.add_expert_count(hparams["num_experts"])
self.gguf_writer.add_expert_shared_count(hparams["num_shared_experts"])
self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])
@@ -8405,21 +10288,25 @@ class BailingMoeModel(TextModel):
output_name = self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT)
if name.endswith("attention.dense.weight"):
- return [(self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_OUT, bid), data_torch)]
+ yield from super().modify_tensors(data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_OUT, bid), bid)
+ return
elif name.endswith("query_key_value.weight"):
q, k, v = data_torch.split([n_head * head_dim, n_kv_head * head_dim, n_kv_head * head_dim], dim=-2)
- return [
- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), BailingMoeModel.permute(q, n_head, n_head)),
- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), BailingMoeModel.permute(k, n_head, n_kv_head)),
- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), v),
- ]
+ yield from super().modify_tensors(
+ BailingMoeModel.permute(q, n_head, n_head), self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), bid
+ )
+ yield from super().modify_tensors(
+ BailingMoeModel.permute(k, n_head, n_kv_head),
+ self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid),
+ bid,
+ )
+ yield from super().modify_tensors(v, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), bid)
+ return
elif name.find("mlp.experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
- tensors: list[tuple[str, Tensor]] = []
-
if self._experts is None:
self._experts = [{} for _ in range(self.block_count)]
@@ -8441,9 +10328,9 @@ class BailingMoeModel(TextModel):
new_name = self.map_tensor_name(merged_name)
- tensors.append((new_name, data_torch))
+ yield from super().modify_tensors(data_torch, new_name, bid)
- return tensors
+ return
new_name = self.map_tensor_name(name)
@@ -8451,7 +10338,7 @@ class BailingMoeModel(TextModel):
data_torch = data_torch.float()
data_torch /= torch.norm(data_torch, p=2, dim=0, keepdim=True) + 1e-7
- return [(new_name, data_torch)]
+ yield from super().modify_tensors(data_torch, new_name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -8483,13 +10370,6 @@ class BailingMoeV2Model(TextModel):
rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)))
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
- else:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
@@ -8499,17 +10379,9 @@ class BailingMoeV2Model(TextModel):
)
)
self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])
- self.gguf_writer.add_expert_count(hparams["num_experts"])
self.gguf_writer.add_expert_shared_count(hparams["num_shared_experts"])
self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])
- if hparams["score_function"] == "sigmoid":
- self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
- elif hparams["score_function"] == "softmax":
- self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
- else:
- raise ValueError(f"Unsupported score_function value: {hparams['score_function']}")
-
if (nextn_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
self.gguf_writer.add_nextn_predict_layers(nextn_layers)
@@ -8517,11 +10389,9 @@ class BailingMoeV2Model(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if "mlp.experts" in name:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
- tensors: list[tuple[str, Tensor]] = []
-
if self._experts is None:
self._experts = [{} for _ in range(self.block_count)]
@@ -8541,16 +10411,13 @@ class BailingMoeV2Model(TextModel):
merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
-
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
if name.endswith(".expert_bias"):
name = name.replace(".expert_bias", ".expert_bias.bias")
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -8568,31 +10435,23 @@ class GroveMoeModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- if (n_experts := self.hparams.get("num_experts")) is not None:
- self.gguf_writer.add_expert_count(n_experts)
if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
logger.info(f"gguf: expert feed forward length = {moe_intermediate_size}")
self.gguf_writer.add_expert_chunk_feed_forward_length(self.hparams.get("head_dim") or 128)
self.gguf_writer.add_experts_per_group(2)
self.gguf_writer.add_expert_group_scale(0.05)
- # YaRN is not enabled by default
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
_experts: list[dict[str, Tensor]] | None = None
_chunk_experts: list[dict[str, Tensor]] | None = None
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.endswith(".expert_bias"):
- return []
+ return
# process the experts separately
if name.find("chunk_experts") != -1:
- n_experts = self.hparams["num_experts"] // 2 # see add_experts_per_group
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"]) // 2 # see add_experts_per_group
assert bid is not None
if self._chunk_experts is None:
@@ -8601,8 +10460,6 @@ class GroveMoeModel(TextModel):
self._chunk_experts[bid][name] = data_torch
if len(self._chunk_experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["down_proj", "gate_proj", "up_proj"]:
data: list[Tensor] = []
@@ -8616,14 +10473,12 @@ class GroveMoeModel(TextModel):
merged_name = f"model.layers.{bid}.mlp.chunk_experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
elif name.find("experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -8632,8 +10487,6 @@ class GroveMoeModel(TextModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["down_proj", "gate_proj", "up_proj"]:
data: list[Tensor] = []
@@ -8647,14 +10500,12 @@ class GroveMoeModel(TextModel):
merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -8688,7 +10539,7 @@ class ChameleonModel(TextModel):
# ignore image tokenizer for now
# TODO: remove this once image support is implemented for Chameleon
if name.startswith("model.vqmodel"):
- return []
+ return
n_head = self.hparams["num_attention_heads"]
n_kv_head = self.hparams.get("num_key_value_heads")
@@ -8703,7 +10554,7 @@ class ChameleonModel(TextModel):
if name.endswith(("k_norm.weight", "k_norm.bias")):
data_torch = ChameleonModel._reverse_hf_permute(data_torch, n_kv_head, hidden_dim)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@staticmethod
def _reverse_hf_permute(data_torch, n_heads, hidden_dim):
@@ -8720,11 +10571,68 @@ class UltravoxModel(TextModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
raise NotImplementedError(
- "Ultravox does not have text decoder. Instead, it uses Llama or other models for text."
- " If you want to get the audio encoder, please use --mmproj argument"
+ "Ultravox does not have text decoder. Instead, it uses Llama or other models for text. If you want to get"
+ " the audio encoder, please use --mmproj argument"
)
[email protected]("GlmasrModel")
+class GlmASRWhisperEncoderModel(MmprojModel):
+ has_vision_encoder = False
+ has_audio_encoder = True
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ if "hidden_size" not in self.hparams and "intermediate_size" not in self.hparams:
+ self.hparams["hidden_size"] = self.hparams["d_model"]
+ self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"]
+ self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"]
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GLMA)
+ self.gguf_writer.add_audio_num_mel_bins(self.hparams["num_mel_bins"])
+ self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))
+ self.gguf_writer.add_audio_stack_factor(self.global_config["merge_factor"])
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if ".conv" in name and ".weight" in name:
+ return gguf.GGMLQuantizationType.F16
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.startswith("model.") or name.startswith("lm_head."):
+ # skip language model tensors
+ return
+
+ if name.startswith("audio_encoder.whisper."):
+ name = name.replace("audio_encoder.whisper.", "audio_tower.")
+ if "audio_encoder.layer_norm." in name or "audio_encoder.proj." in name:
+ name = name.replace("audio_encoder.", "audio_encoder.adapting.")
+
+ if name.startswith("audio_encoder.audio_bos_eos_token."):
+ yield from super().modify_tensors(data_torch[0], "model.vision.boi", bid)
+ yield from super().modify_tensors(data_torch[1], "model.vision.eoi", bid)
+ return
+
+ if name.startswith("audio_encoder.adapting."):
+ name = name.replace("audio_encoder.adapting.", "audio.multi_modal_projector.")
+ if ".layer_norm." in name:
+ name = name.replace(".layer_norm.", ".ln_pre.")
+ if ".0." in name:
+ name = name.replace(".0.", ".linear_1.")
+ if ".2." in name:
+ name = name.replace(".2.", ".linear_2.")
+ if ".proj." in name:
+ return
+
+ if "conv1.bias" in name or "conv2.bias" in name:
+ # transpose conv1 and conv2 bias
+ data_torch = data_torch.unsqueeze(-1)
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("Qwen2AudioForConditionalGeneration")
class WhisperEncoderModel(MmprojModel):
has_vision_encoder = False # no vision encoder
@@ -8749,11 +10657,9 @@ class WhisperEncoderModel(MmprojModel):
return super().tensor_force_quant(name, new_name, bid, n_dims)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
if name.startswith("language_model."):
# skip language model tensors
- return []
+ return
# prevent clash naming with vision tensors
if name.startswith("multi_modal_projector"):
@@ -8763,7 +10669,7 @@ class WhisperEncoderModel(MmprojModel):
# transpose conv1 and conv2 bias
data_torch = data_torch.unsqueeze(-1)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("UltravoxModel")
@@ -8788,6 +10694,20 @@ class VoxtralWhisperEncoderModel(WhisperEncoderModel):
self.gguf_writer.add_audio_stack_factor(4) # == intermediate_size // hidden_size
[email protected]("AudioFlamingo3ForConditionalGeneration")
+class AudioFlamingo3WhisperEncoderModel(WhisperEncoderModel):
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MUSIC_FLAMINGO)
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if ".conv" in name and ".weight" in name:
+ # Was trained in BF16, being safe, avoiding quantizing to FP16
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+
@ModelBase.register("FalconH1ForCausalLM")
class FalconH1Model(Mamba2Model):
model_arch = gguf.MODEL_ARCH.FALCON_H1
@@ -8893,7 +10813,7 @@ class FalconH1Model(Mamba2Model):
), f"SSM inner size {self.d_inner} not a multiple of head dim {self.d_head}"
# Add any other Falcon Mamba2 specific configuration
- self.gguf_writer.add_rope_freq_base(self.find_hparam(["rope_theta"]))
+ self.gguf_writer.add_rope_freq_base(self.rope_parameters["rope_theta"])
@ModelBase.register("HunYuanMoEV1ForCausalLM")
@@ -8956,7 +10876,6 @@ class HunYuanMoEModel(TextModel):
super().set_gguf_parameters()
hparams = self.hparams
- self.gguf_writer.add_expert_count(hparams["num_experts"])
self.gguf_writer.add_expert_shared_feed_forward_length(hparams["intermediate_size"])
moe_intermediate_size = hparams["moe_intermediate_size"]
@@ -8972,10 +10891,9 @@ class HunYuanMoEModel(TextModel):
self.gguf_writer.add_expert_shared_count(moe_shared_expert[0])
# Rope
- rope_scaling = hparams.get("rope_scaling", {})
- if rope_scaling.get("type") == "dynamic":
- alpha = rope_scaling.get("alpha", 1000)
- base = hparams.get("rope_theta", 10000.0)
+ if self.rope_parameters.get("rope_type") == "dynamic":
+ alpha = self.rope_parameters.get("alpha", 1000)
+ base = self.rope_parameters.get("rope_theta", 10000.0)
dim = hparams["hidden_size"] // hparams["num_attention_heads"] # 128
scaled_base = base * (alpha ** (dim / (dim - 2))) # 10000 * (1000 ** (128 / 126)) = 11158839.9251
self.gguf_writer.add_rope_freq_base(scaled_base)
@@ -8998,10 +10916,10 @@ class HunYuanMoEModel(TextModel):
if name == "lm_head.weight":
if self.hparams.get("tie_word_embeddings", False):
logger.info("Skipping tied output layer 'lm_head.weight'")
- return []
+ return
if name.find("mlp.experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -9011,7 +10929,6 @@ class HunYuanMoEModel(TextModel):
if len(self._experts[bid]) >= n_experts * 3:
# merge the experts into a single 3d tensor
- tensors: list[tuple[str, Tensor]] = []
for w_name in ["down_proj", "gate_proj", "up_proj"]:
data: list[Tensor] = []
@@ -9022,14 +10939,13 @@ class HunYuanMoEModel(TextModel):
data_torch = torch.stack(data, dim=0)
merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -9045,16 +10961,9 @@ class LLaDAMoEModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- if (n_experts := self.hparams.get("num_experts")) is not None:
- self.gguf_writer.add_expert_count(n_experts)
-
if (expert_intermediate_size := self.hparams.get("expert_intermediate_size")) is not None:
self.gguf_writer.add_expert_feed_forward_length(expert_intermediate_size)
- # number of experts used per token (top-k)
- if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:
- self.gguf_writer.add_expert_used_count(n_experts_used)
-
self.gguf_writer.add_mask_token_id(156895)
self.gguf_writer.add_causal_attention(False)
self.gguf_writer.add_diffusion_shift_logits(False)
@@ -9065,7 +10974,7 @@ class LLaDAMoEModel(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# process the experts separately
if name.find("experts") != -1:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
if self._experts is None:
@@ -9074,8 +10983,6 @@ class LLaDAMoEModel(TextModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["down_proj", "gate_proj", "up_proj"]:
data: list[Tensor] = []
@@ -9089,14 +10996,12 @@ class LLaDAMoEModel(TextModel):
merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
# Copied from: Qwen2MoeModel
def prepare_tensors(self):
@@ -9174,10 +11079,9 @@ class HunYuanModel(TextModel):
hparams = self.hparams
# Rope
- rope_scaling = hparams.get("rope_scaling", {})
- if rope_scaling.get("type") == "dynamic":
- alpha = rope_scaling.get("alpha", 50)
- base = hparams.get("rope_theta", 10000.0)
+ if self.rope_parameters.get("rope_type") == "dynamic":
+ alpha = self.rope_parameters.get("alpha", 50)
+ base = self.rope_parameters.get("rope_theta", 10000.0)
dim = hparams["head_dim"]
scaled_base = base * (alpha ** (dim / (dim - 2)))
self.gguf_writer.add_rope_freq_base(scaled_base)
@@ -9196,26 +11100,15 @@ class HunYuanModel(TextModel):
if name == "lm_head.weight":
if self.hparams.get("tie_word_embeddings", False):
logger.info("Skipping tied output layer 'lm_head.weight'")
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("SmolLM3ForCausalLM")
class SmolLM3Model(LlamaModel):
model_arch = gguf.MODEL_ARCH.SMOLLM3
- def set_vocab(self):
- super().set_vocab()
- # remove unsupported array slicing in chat template
- # ref: https://huggingface.co/ggml-org/SmolLM3-3B-GGUF/discussions/1
- from transformers import AutoTokenizer
-
- tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
- if tokenizer.chat_template is not None:
- chat_template = tokenizer.chat_template.replace("[:]", "")
- self.gguf_writer.add_chat_template(chat_template)
-
@ModelBase.register("GptOssForCausalLM")
class GptOssModel(TextModel):
@@ -9289,8 +11182,6 @@ class GptOssModel(TextModel):
return []
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
-
if "sinks" in name:
name += ".weight"
@@ -9304,7 +11195,7 @@ class GptOssModel(TextModel):
data_torch = data_torch.transpose(-1, -2)
else:
# otherwise, it should already be repacked to ggml MXFP4 format
- return []
+ return
# split the gate_up into gate and up
if "gate_up_proj" in name:
@@ -9312,25 +11203,18 @@ class GptOssModel(TextModel):
name_up = name.replace("gate_up_proj_bias", "up_proj.bias")
name_gate = name.replace("gate_up_proj_bias", "gate_proj.bias")
gate_proj_bias, up_proj_bias = data_torch[..., ::2], data_torch[..., 1::2]
- return [
- (self.map_tensor_name(name_gate), gate_proj_bias),
- (self.map_tensor_name(name_up), up_proj_bias),
- ]
+ yield from super().modify_tensors(gate_proj_bias, name_gate, bid)
+ yield from super().modify_tensors(up_proj_bias, name_up, bid)
elif "_blocks" not in name and "_scales" not in name:
logger.warning(f"{name} is not in MXFP4, performance may be degraded")
name_up = name.replace("gate_up_proj", "up_proj.weight")
name_gate = name.replace("gate_up_proj", "gate_proj.weight")
data_torch = data_torch.transpose(-1, -2)
gate_proj_weight, up_proj_weight = data_torch[:, ::2, :], data_torch[:, 1::2, :]
- return [
- (self.map_tensor_name(name_gate), gate_proj_weight),
- (self.map_tensor_name(name_up), up_proj_weight),
- ]
- else:
- # otherwise, it should already be repacked to ggml MXFP4 format
- return []
-
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(gate_proj_weight, name_gate, bid)
+ yield from super().modify_tensors(up_proj_weight, name_up, bid)
+ else:
+ yield from super().modify_tensors(data_torch, name, bid)
def set_vocab(self):
self._set_vocab_gpt2()
@@ -9340,13 +11224,6 @@ class GptOssModel(TextModel):
self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
self.gguf_writer.add_expert_feed_forward_length(self.hparams["intermediate_size"])
- rope_scaling = self.hparams.get("rope_scaling") or {}
- rope_type = rope_scaling.get("rope_type", rope_scaling.get("type"))
- assert rope_type == "yarn", f"GPT-OSS only supports yarn rope scaling, got {rope_type}"
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling.get("original_max_position_embeddings", 4096))
-
@ModelBase.register("Lfm2ForCausalLM", "LFM2ForCausalLM")
class LFM2Model(TextModel):
@@ -9372,7 +11249,7 @@ class LFM2Model(TextModel):
def set_gguf_parameters(self):
# set num_key_value_heads only for attention layers
self.hparams["num_key_value_heads"] = [
- self.hparams["num_key_value_heads"] if layer_type == "full_attention" else 0
+ self.hparams["num_key_value_heads"] if layer_type != "conv" else 0
for layer_type in self.hparams["layer_types"]
]
@@ -9383,18 +11260,43 @@ class LFM2Model(TextModel):
self._add_feed_forward_length()
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- is_vision_tensor = "vision_tower" in name or "multi_modal_projector" in name
- if is_vision_tensor:
- # skip vision tensors
- return []
+ if self._is_vision_tensor(name) or ConformerAudioModel.is_audio_tensor(name):
+ # skip multimodal tensors
+ return
- name = name.replace("language_model.", "")
+ name = name.replace("language_model.", "") # vision
+ name = name.replace("lfm.", "model.") # audio
# conv op requires 2d tensor
if "conv.conv" in name:
data_torch = data_torch.squeeze(1)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
+
+ def _is_vision_tensor(self, name: str) -> bool:
+ return "vision_tower" in name or "multi_modal_projector" in name
+
+
[email protected]("Lfm2Model")
+class LFM2ColBertModel(LFM2Model):
+ model_arch = gguf.MODEL_ARCH.LFM2
+ dense_tensor_name = "dense_2"
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if not name.startswith(self.dense_tensor_name):
+ name = "model." + name
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # dense tensor is stored in a separate safetensors file
+ from safetensors.torch import load_file
+
+ tensors_file = self.dir_model / "1_Dense" / "model.safetensors"
+ assert tensors_file.is_file()
+ tensor = load_file(tensors_file)["linear.weight"]
+ self.gguf_writer.add_embedding_length_out(tensor.shape[0])
+ yield f"{self.dense_tensor_name}.weight", tensor.clone()
@ModelBase.register("Lfm2MoeForCausalLM")
@@ -9410,7 +11312,6 @@ class LFM2MoeModel(TextModel):
super().set_gguf_parameters()
- self.gguf_writer.add_expert_count(self.hparams["num_experts"])
self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
self.gguf_writer.add_leading_dense_block_count(self.hparams["num_dense_layers"])
self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
@@ -9431,7 +11332,7 @@ class LFM2MoeModel(TextModel):
# merge expert weights
if "experts" in name:
- n_experts = self.hparams["num_experts"]
+ n_experts = self.find_hparam(["num_local_experts", "num_experts"])
assert bid is not None
expert_cache = self._experts_cache.setdefault(bid, {})
@@ -9440,9 +11341,8 @@ class LFM2MoeModel(TextModel):
# not enough expert weights to merge
if len(expert_cache) < n_experts * len(expert_weights):
- return []
+ return
- tensors: list[tuple[str, Tensor]] = []
for w_name in expert_weights:
data: list[Tensor] = []
@@ -9453,13 +11353,13 @@ class LFM2MoeModel(TextModel):
data_torch = torch.stack(data, dim=0)
merged_name = f"layers.{bid}.feed_forward.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
- tensors.append((new_name, data_torch))
+
+ yield from super().modify_tensors(data_torch, merged_name, bid)
del self._experts_cache[bid]
- return tensors
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -9468,6 +11368,7 @@ class LFM2MoeModel(TextModel):
@ModelBase.register("Lfm2VlForConditionalGeneration")
class LFM2VLModel(MmprojModel):
+
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert self.hparams_vision is not None
@@ -9480,11 +11381,11 @@ class LFM2VLModel(MmprojModel):
self.gguf_writer.add_vision_attention_layernorm_eps(self.find_vparam(["layer_norm_eps"]))
self.gguf_writer.add_vision_projector_scale_factor(self.global_config.get("downsample_factor", 2))
self.gguf_writer.add_vision_use_gelu(True)
+ # python notation, e.g. for vision_feature_layer == -1, we pick last layer -> vision_feature_layers_to_drop = 0
vision_feature_layers_to_drop = -(self.global_config.get("vision_feature_layer", -1) + 1)
self.gguf_writer.add_vision_block_count(self.find_vparam(self.n_block_keys) - vision_feature_layers_to_drop)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
is_vision_tensor = "vision_tower" in name or "multi_modal_projector" in name
if is_vision_tensor:
@@ -9495,9 +11396,67 @@ class LFM2VLModel(MmprojModel):
if "patch_embedding.weight" in name:
data_torch = data_torch.view(data_torch.shape[0], 16, 16, 3).permute(0, 3, 1, 2)
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
+
+ return # skip other tensors
+
+
[email protected]("Lfm2AudioForConditionalGeneration")
+class LFM2AudioModel(ConformerAudioModel):
+ has_vision_encoder = False
+ has_audio_encoder = True
+ model_name = "Lfm2AudioEncoder"
+
+ def get_audio_config(self) -> dict[str, Any] | None:
+ return self.global_config.get("encoder")
+
+ def set_gguf_parameters(self):
+ assert self.hparams_audio is not None
+ self.hparams_audio["hidden_size"] = self.hparams_audio["d_model"]
+ self.hparams_audio["intermediate_size"] = self.hparams_audio["d_model"]
+ self.hparams_audio["num_attention_heads"] = self.hparams_audio["n_heads"]
+ super().set_gguf_parameters()
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.LFM2A)
+ self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["feat_in"])
+ self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+
+ def modify_tensors(self, data_torch, name, bid):
+ # skip language model tensors
+ if name.startswith("lfm."):
+ return
- return [] # skip other tensors
+ # for training only
+ if any(p in name for p in ["audio_loss_weight"]):
+ return
+
+ # for audio output
+ if any(p in name for p in ["codebook_offsets", "depth_embeddings", "depth_linear", "depthformer"]):
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("Lfm25AudioTokenizer")
+class LFM25AudioTokenizer(LFM2Model):
+ model_arch = gguf.MODEL_ARCH.LFM2
+
+ def set_vocab(self):
+ self._set_vocab_none()
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+ self.gguf_writer.add_embedding_length_out(self.hparams["output_size"])
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name == "istft.window" or name.startswith("emb.emb"):
+ return
+
+ if name.startswith("lin"):
+ name = name.replace("lin", "dense_2_out")
+
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("SmallThinkerForCausalLM")
@@ -9506,13 +11465,9 @@ class SmallThinkerModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- if (n_experts := self.hparams.get("num_experts", self.hparams.get("moe_num_primary_experts"))) is not None:
+ if (n_experts := self.hparams.get("moe_num_primary_experts")) is not None:
self.gguf_writer.add_expert_count(n_experts)
- if (
- n_experts_used := self.hparams.get(
- "num_experts_per_tok", self.hparams.get("moe_num_active_primary_experts")
- )
- ) is not None:
+ if (n_experts_used := self.hparams.get("moe_num_active_primary_experts")) is not None:
self.gguf_writer.add_expert_used_count(n_experts_used)
if (moe_intermediate_size := self.hparams.get("moe_ffn_hidden_size")) is not None:
self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
@@ -9522,11 +11477,6 @@ class SmallThinkerModel(TextModel):
self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
else:
self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
- rope_scaling = self.hparams.get("rope_scaling") or {}
- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
sliding_window_layout = self.hparams.get("sliding_window_layout")
if sliding_window_layout:
@@ -9542,7 +11492,9 @@ class SmallThinkerModel(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# process the experts separately
if name.find("experts") != -1:
- n_experts = self.hparams.get("num_experts", self.hparams.get("moe_num_primary_experts"))
+ n_experts = self.hparams.get("moe_num_primary_experts") or self.find_hparam(
+ ["num_local_experts", "num_experts"]
+ )
assert bid is not None
if self._experts is None:
@@ -9551,8 +11503,6 @@ class SmallThinkerModel(TextModel):
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
- tensors: list[tuple[str, Tensor]] = []
-
# merge the experts into a single 3d tensor
for w_name in ["down", "gate", "up"]:
data: list[Tensor] = []
@@ -9566,14 +11516,12 @@ class SmallThinkerModel(TextModel):
merged_name = f"model.layers.{bid}.block_sparse_moe.experts.{w_name}.weight"
- new_name = self.map_tensor_name(merged_name)
-
- tensors.append((new_name, data_torch))
- return tensors
+ yield from super().modify_tensors(data_torch, merged_name, bid)
+ return
else:
- return []
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
@@ -9585,6 +11533,38 @@ class SmallThinkerModel(TextModel):
raise ValueError(f"Unprocessed experts: {experts}")
[email protected]("ModernBertModel", "ModernBertForMaskedLM", "ModernBertForSequenceClassification")
+class ModernBertModel(BertModel):
+ model_arch = gguf.MODEL_ARCH.MODERN_BERT
+
+ def set_vocab(self):
+ self.gguf_writer.add_add_bos_token(True)
+ self.gguf_writer.add_add_eos_token(True)
+ self.gguf_writer.add_add_sep_token(True)
+ self._set_vocab_gpt2()
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_sliding_window(self.hparams["local_attention"])
+ if (sliding_window_pattern := self.hparams.get("global_attn_every_n_layers")) is not None:
+ self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)
+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ name = name.removeprefix("model.")
+
+ if self.cls_out_labels:
+ # For BertForSequenceClassification (direct projection layer)
+ if name == "classifier.weight":
+ name = "classifier.out_proj.weight"
+
+ if name == "classifier.bias":
+ name = "classifier.out_proj.bias"
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("ApertusForCausalLM")
class ApertusModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.APERTUS
@@ -9602,33 +11582,55 @@ class ApertusModel(LlamaModel):
self._alpha_n[bid] = data_torch.to("cpu").float().item()
if len(self._alpha_n) == n_layers:
self.gguf_writer.add_xielu_alpha_n([self._alpha_n[k] for k in sorted(self._alpha_n)])
- return []
+ return
if name.endswith(".act_fn.alpha_p"):
self._alpha_p[bid] = data_torch.to("cpu").float().item()
if len(self._alpha_p) == n_layers:
self.gguf_writer.add_xielu_alpha_p([self._alpha_p[k] for k in sorted(self._alpha_p)])
- return []
+ return
if name.endswith(".act_fn.beta"):
self._beta[bid] = data_torch.to("cpu").float().item()
if len(self._beta) == n_layers:
self.gguf_writer.add_xielu_beta([self._beta[k] for k in sorted(self._beta)])
- return []
+ return
if name.endswith(".act_fn.eps"):
self._eps[bid] = data_torch.to("cpu").float().item()
if len(self._eps) == n_layers:
self.gguf_writer.add_xielu_eps([self._eps[k] for k in sorted(self._eps)])
- return []
+ return
- return super().modify_tensors(data_torch, name, bid)
+ yield from super().modify_tensors(data_torch, name, bid)
class MistralModel(LlamaModel):
- model_arch = gguf.MODEL_ARCH.LLAMA
+ model_arch = gguf.MODEL_ARCH.MISTRAL3
model_name = "Mistral"
hf_arch = ""
is_mistral_format = True
undo_permute = False
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # for compatibility, we use LLAMA arch for older models
+ # TODO: remove this once everyone migrates to newer version of llama.cpp
+ if "llama_4_scaling" not in self.hparams:
+ self.model_arch = gguf.MODEL_ARCH.LLAMA
+ self.gguf_writer.arch = gguf.MODEL_ARCH_NAMES[self.model_arch]
+ self.gguf_writer.add_architecture()
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+ def dequant_model(self):
+ # transform quantization config into HF format
+ quant_config = self.hparams.get("quantization")
+ if quant_config is not None:
+ assert quant_config["qformat_weight"] == "fp8_e4m3"
+ self.hparams["quantization_config"] = {
+ "activation_scheme": "static",
+ "quant_method": "fp8",
+ "weight_block_size": None,
+ }
+ return super().dequant_model()
+
@staticmethod
def get_community_chat_template(vocab: MistralVocab, templates_dir: Path, is_mistral_format: bool):
assert (
@@ -9656,9 +11658,8 @@ class MistralModel(LlamaModel):
err_message = f"Unknown tokenizer type: {vocab.tokenizer_type} and version {vocab.tokenizer.version}"
if is_mistral_format:
err_message += (
- " . Please pass --disable-mistral-community-chat-template argument to the CLI "
- "if you want to skip this error and use the Mistral official `mistral-common` "
- "pre-processing library."
+ " . Please pass --disable-mistral-community-chat-template argument to the CLI if you want to skip"
+ " this error and use the Mistral official `mistral-common` pre-processing library."
)
raise ValueError(err_message)
@@ -9671,6 +11672,110 @@ class MistralModel(LlamaModel):
return template
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ MistralModel.set_mistral_config(self.gguf_writer, self.hparams)
+
+ @staticmethod
+ def set_mistral_config(gguf_writer: gguf.GGUFWriter, hparams: dict):
+ if "yarn" in hparams:
+ yarn_params = hparams["yarn"]
+ gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
+ gguf_writer.add_rope_scaling_factor(yarn_params["factor"])
+ gguf_writer.add_rope_scaling_yarn_beta_fast(yarn_params["beta"])
+ gguf_writer.add_rope_scaling_yarn_beta_slow(yarn_params["alpha"])
+ gguf_writer.add_rope_scaling_yarn_log_mul(1.0) # mscale_all_dim
+ gguf_writer.add_rope_scaling_orig_ctx_len(yarn_params["original_max_position_embeddings"])
+
+ if "llama_4_scaling" in hparams:
+ gguf_writer.add_attn_temperature_scale(hparams["llama_4_scaling"]["beta"])
+
+
+class MistralMoeModel(DeepseekV2Model):
+ model_arch = gguf.MODEL_ARCH.DEEPSEEK2
+ model_name = "Mistral"
+ hf_arch = ""
+ is_mistral_format = True
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ logger.info("Using MistralMoeModel")
+ config = self.hparams
+ # Mistral key -> HF key
+ config_mapping = {
+ "dim": "hidden_size",
+ "norm_eps": "rms_norm_eps",
+ "n_kv_heads": "num_key_value_heads",
+ "n_layers": "num_hidden_layers",
+ "n_heads": "num_attention_heads",
+ "hidden_dim": "intermediate_size",
+ }
+ # HF key -> (Mistral key, default value)
+ top_level_mapping_with_default = {
+ "model_type": ("model_type", "transformer"),
+ "hidden_act": ("activation", "silu"),
+ "tie_word_embeddings": ("tied_embeddings", False),
+ "max_seq_len": ("max_seq_len", config.get("max_position_embeddings", 128_000)),
+ "max_position_embeddings": ("max_position_embeddings", 128_000),
+ }
+ # mapping top-level keys
+ for key, new_key in config_mapping.items():
+ if key in config:
+ config[new_key] = config[key]
+ for new_key, (key, default_value) in top_level_mapping_with_default.items():
+ config[new_key] = config.get(key, default_value)
+ # mapping MoE-specific keys
+ moe_config_map = {
+ "route_every_n": "moe_layer_freq",
+ "first_k_dense_replace": "first_k_dense_replace",
+ "num_experts_per_tok": "num_experts_per_tok",
+ "num_experts": "n_routed_experts",
+ "expert_hidden_dim": "moe_intermediate_size",
+ "routed_scale": "routed_scaling_factor",
+ "num_shared_experts": "n_shared_experts",
+ "num_expert_groups": "n_group",
+ "num_expert_groups_per_tok": "topk_group",
+ }
+ moe = config["moe"]
+ for key, new_key in moe_config_map.items():
+ if key in moe:
+ config[new_key] = moe[key]
+ # provide missing values
+ config["topk_method"] = None
+ config["norm_topk_prob"] = True
+ config["scoring_func"] = "softmax"
+
+ def set_vocab(self):
+ self._set_vocab_mistral()
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ MistralModel.set_mistral_config(self.gguf_writer, self.hparams)
+ yarn_params = self.hparams["yarn"]
+ self.gguf_writer.add_attn_temperature_length(yarn_params["original_max_position_embeddings"])
+
+ self.gguf_writer.add_rope_scaling_yarn_log_mul(0.1) # mscale_all_dim * 0.1
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
+ if name.startswith("vision_") or name.startswith("patch_merger.") or "mm_projector" in name:
+ return
+
+ # rename certain tensors so that we can reuse DeepseekV2Model modify_tensors logic
+ if name.endswith(".qscale_act"):
+ name = name.replace(".qscale_act", ".input_scale")
+ if name.endswith(".qscale_weight"):
+ name = name.replace(".qscale_weight", ".weight_scale")
+ if ".wkv_b." in name:
+ name = name.replace(".wkv_b.", ".kv_b_proj.")
+ if ".experts." in name:
+ name = name.replace(".experts.", ".mlp.experts.")
+ name = name.replace(".w1.", ".gate_proj.")
+ name = name.replace(".w2.", ".down_proj.")
+ name = name.replace(".w3.", ".up_proj.")
+ name = "model." + name
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
class PixtralModel(LlavaVisionModel):
model_name = "Pixtral"
@@ -9710,11 +11815,12 @@ class LightOnOCRVisionModel(LlavaVisionModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
name = name.replace("model.vision_encoder.", "vision_tower.")
name = name.replace("model.vision_projection.", "multi_modal_projector.")
- return super().modify_tensors(data_torch, name, bid)
+ yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("KimiVLForConditionalGeneration")
class KimiVLModel(MmprojModel):
+
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert self.hparams_vision is not None
@@ -9730,24 +11836,323 @@ class KimiVLModel(MmprojModel):
self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("layer_norm_eps", 1e-5))
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- del bid # unused
is_vision_tensor = "vision_tower" in name or "multi_modal_projector" in name
if is_vision_tensor:
if "pos_emb.weight" in name:
data_torch = data_torch.view(data_torch.shape[0] * data_torch.shape[1], data_torch.shape[2])
- elif "wqkv" in name:
+
+ if "wqkv" in name:
split_dim = 0 if "weight" in name else -1
wq, wk, wv = data_torch.chunk(3, dim=split_dim)
- return [
- (self.map_tensor_name(name.replace("wqkv", "wq")), wq),
- (self.map_tensor_name(name.replace("wqkv", "wk")), wk),
- (self.map_tensor_name(name.replace("wqkv", "wv")), wv),
- ]
+ yield from super().modify_tensors(wq, name.replace("wqkv", "wq"), bid)
+ yield from super().modify_tensors(wk, name.replace("wqkv", "wk"), bid)
+ yield from super().modify_tensors(wv, name.replace("wqkv", "wv"), bid)
+ else:
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("KimiK25ForConditionalGeneration")
+class KimiK25Model(MmprojModel):
+ """Kimi-K2.5 with MoonViT3d vision encoder"""
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ assert self.hparams_vision is not None, "Kimi-K2.5 requires vision_config in model config"
+
+ self.merge_kernel_size = tuple(self.hparams_vision.get("merge_kernel_size", [2, 2]))
+ self.patch_size = self.hparams_vision.get("patch_size", 14)
+
+ # Set image_size for compatibility with base class
+ # Use position embedding dimensions as image_size reference
+ pos_emb_h = self.hparams_vision.get("init_pos_emb_height", 64)
+ self.hparams_vision["image_size"] = pos_emb_h * self.patch_size
+
+ def set_gguf_parameters(self):
+ # Base class MmprojModel.set_gguf_parameters() already writes:
+ # - vision_block_count, vision_head_count, vision_embedding_length
+ # - vision_feed_forward_length, vision_patch_size, image_mean, image_std
+ # via find_vparam() which handles the vt_* prefixed keys in Kimi-K2.5's config
+ super().set_gguf_parameters()
+ assert self.hparams_vision is not None
+
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.KIMIK25)
+
+ # Position embedding parameters (for interpolation)
+ self.gguf_writer.add_uint32("vision.pos_emb_height", self.hparams_vision.get("init_pos_emb_height", 64))
+ self.gguf_writer.add_uint32("vision.pos_emb_width", self.hparams_vision.get("init_pos_emb_width", 64))
+ self.gguf_writer.add_uint32("vision.pos_emb_time", self.hparams_vision.get("init_pos_emb_time", 4))
+
+ # Projector parameters
+ self.gguf_writer.add_vision_use_gelu(self.hparams_vision.get("projector_hidden_act", "gelu") == "gelu")
+ self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("projector_ln_eps", 1e-5))
+ self.gguf_writer.add_vision_projector_scale_factor(self.merge_kernel_size[0])
+
+ # Image size limits
+ # Note: in_patch_limit is for images, in_patch_limit_each_frame is for video (not supported yet)
+ in_patch_limit = self.preprocessor_config.get("in_patch_limit", 16384)
+ min_patches = 8 # reasonable minimum
+ pixels_per_patch = self.patch_size**2
+ self.gguf_writer.add_vision_min_pixels(min_patches * pixels_per_patch)
+ self.gguf_writer.add_vision_max_pixels(in_patch_limit * pixels_per_patch)
+
+ @staticmethod
+ def permute(weights: Tensor, n_head: int) -> Tensor:
+ out_dim, in_dim = weights.shape
+ head_dim = out_dim // n_head
+ w = weights.reshape(n_head, head_dim // 4, 2, 2, in_dim)
+ w = w.permute(0, 2, 1, 3, 4)
+ return w.reshape(out_dim, in_dim)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Only process vision and projector tensors
+ is_vision = any(x in name for x in ["vision_tower", "mm_projector"])
+
+ if not is_vision:
+ return
+
+ assert self.hparams_vision is not None
+ n_head = self.hparams_vision.get("num_attention_heads", 16)
+
+ # Permute Q/K weights/biases from interleaved to split RoPE format
+ # This allows using build_rope_2d at runtime without post-permutation.
+ if "wqkv" in name:
+ out_dim = data_torch.shape[0]
+ qkv_dim = out_dim // 3
+ head_dim = qkv_dim // n_head
+
+ if "weight" in name:
+ wq, wk, wv = data_torch[:qkv_dim, :], data_torch[qkv_dim : 2 * qkv_dim, :], data_torch[2 * qkv_dim :, :]
+ wq = self.permute(wq, n_head)
+ wk = self.permute(wk, n_head)
+ data_torch = torch.cat([wq, wk, wv], dim=0)
+ elif "bias" in name:
+ bq, bk, bv = data_torch[:qkv_dim], data_torch[qkv_dim : 2 * qkv_dim], data_torch[2 * qkv_dim :]
+ bq = bq.reshape(n_head, head_dim // 4, 2, 2).permute(0, 2, 1, 3).reshape(-1)
+ bk = bk.reshape(n_head, head_dim // 4, 2, 2).permute(0, 2, 1, 3).reshape(-1)
+ data_torch = torch.cat([bq, bk, bv], dim=0)
+
+ # Temporal embeddings: (T, 1, C) → (T, C)
+ if "pos_emb.time_weight" in name:
+ T, _, C = data_torch.shape
+ data_torch = data_torch.reshape(T, C)
+
+ # PatchMergerMLP tensor name mapping
+ # proj.0.weight → proj.linear_1.weight
+ # proj.2.weight → proj.linear_2.weight
+ if "mm_projector.proj.0." in name:
+ name = name.replace(".proj.0.", ".proj.linear_1.")
+ elif "mm_projector.proj.2." in name:
+ name = name.replace(".proj.2.", ".proj.linear_2.")
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("CogVLMForCausalLM")
+class CogVLMVisionModel(MmprojModel):
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-6))
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.COGVLM)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if not name.startswith("model.vision."):
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("CogVLMForCausalLM")
+class CogVLMModel(LlamaModel):
+ model_arch = gguf.MODEL_ARCH.COGVLM
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # block vision tensors
+ if name.startswith("model.vision."):
+ return
+
+ yield from ModelBase.modify_tensors(self, data_torch, name, bid)
+
+
[email protected]("JanusForConditionalGeneration")
+class JanusProModel(LlamaModel):
+ model_arch = gguf.MODEL_ARCH.LLAMA # reuse Llama arch
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Skip vision, aligner, and generation tensors
+ skip_prefixes = (
+ "model.vision_model.",
+ "model.aligner.",
+ "model.vqmodel.",
+ "model.generation_embeddings.",
+ "model.generation_aligner.",
+ "model.generation_head.",
+ )
+ if name.startswith(skip_prefixes):
+ return
+
+ if name.startswith("model.language_model."):
+ name = name.replace("model.language_model.", "model.")
+ elif name.startswith("language_model."):
+ name = name.replace("language_model.", "")
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("JanusForConditionalGeneration")
+class JanusProVisionModel(MmprojModel):
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ assert self.hparams_vision is not None
+ if "intermediate_size" not in self.hparams_vision:
+ mlp_ratio = self.hparams_vision.get("mlp_ratio")
+ hidden_size = self.hparams_vision.get("hidden_size")
+ if mlp_ratio is not None and hidden_size is not None:
+ self.hparams_vision["intermediate_size"] = int(round(hidden_size * mlp_ratio))
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ assert self.hparams_vision is not None
+
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.JANUS_PRO)
+
+ self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("layer_norm_eps", 1e-6))
+
+ hidden_act = str(self.hparams_vision.get("hidden_act", "")).lower()
+ if hidden_act == "gelu":
+ self.gguf_writer.add_vision_use_gelu(True)
+ elif hidden_act == "silu":
+ self.gguf_writer.add_vision_use_silu(True)
+
+ def _map_aligner_tensor(self, data_torch: Tensor, name: str) -> Iterable[tuple[str, Tensor]]:
+ """Map aligner tensors to projector format"""
+ suffix = ".bias" if name.endswith(".bias") else ".weight"
+
+ if name.startswith("model.aligner."):
+ local_name = name[len("model.aligner.") :]
+ elif name.startswith("aligner."):
+ local_name = name[len("aligner.") :]
+ else:
+ raise ValueError(f"Unsupported Janus aligner prefix: {name}")
+
+ if local_name.startswith("fc1."):
+ mm_index = 0
+ elif local_name.startswith("hidden_layers."):
+ parts = local_name.split(".", 2)
+ if len(parts) < 3:
+ raise ValueError(f"Unexpected Janus aligner tensor name: {name}")
+ mm_index = int(parts[1]) + 1
+ else:
+ raise ValueError(f"Unsupported Janus aligner tensor: {name}")
+
+ tensor_name = self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, mm_index, suffix=suffix)
+ return [(tensor_name, data_torch)]
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Skip language model tensors as they will be handled by `JanusProModel`
+ if name.startswith(("model.language_model.", "language_model.")):
+ return
- return [(self.map_tensor_name(name), data_torch)]
+ # Skip generation-related components
+ skip_generation_prefixes = (
+ "model.vqmodel.",
+ "vqmodel.",
+ "model.generation_embeddings.",
+ "generation_embeddings.",
+ "model.generation_aligner.",
+ "generation_aligner.",
+ "model.generation_head.",
+ "generation_head.",
+ )
+ if name.startswith(skip_generation_prefixes):
+ return
+
+ # Handle aligner tensors
+ if name.startswith(("model.aligner.", "aligner.")):
+ yield from self._map_aligner_tensor(data_torch, name)
+ return
+
+ # Handle vision tensors
+ if name.startswith(("model.vision_model.", "vision_model.")):
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
+
+ return
+
+
[email protected]("YoutuVLForConditionalGeneration")
+class YoutuVLVisionModel(MmprojModel):
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ assert self.hparams_vision is not None
+ self.hparams_vision["image_size"] = self.hparams_vision.get("image_size", 560)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
- return [] # skip other tensors
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.YOUTUVL)
+ self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-6))
+
+ # Handle activation function
+ hidden_act = str(self.hparams.get("hidden_act", "gelu_pytorch_tanh")).lower()
+ if hidden_act in ("gelu", "gelu_pytorch_tanh", "gelu_fast", "gelu_new", "gelu_accurate"):
+ self.gguf_writer.add_vision_use_gelu(True)
+ elif hidden_act == "silu":
+ self.gguf_writer.add_vision_use_silu(True)
+ else:
+ raise ValueError(f"Unsupported activation function for YOUTUVL: {hidden_act}")
+
+ self.gguf_writer.add_vision_spatial_merge_size(self.hparams.get("spatial_merge_size", 2))
+
+ window_size = self.hparams.get("window_size")
+ if window_size is not None:
+ self.gguf_writer.add_vision_window_size(window_size)
+ # fullatt_block_indexes contains explicit layer indices that use full attention
+ # e.g., [2, 5, 8, 11] means layers 2, 5, 8, 11 use full attention
+ # All other layers use window attention
+ fullatt_block_indexes = self.hparams.get("fullatt_block_indexes")
+ assert fullatt_block_indexes is not None, "fullatt_block_indexes is required for youtuvl"
+ # Store the explicit layer indices for YoutuVL (irregular pattern approach)
+ self.gguf_writer.add_vision_wa_layer_indexes(layers=fullatt_block_indexes)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Skip language model tensors
+ skip_prefixes = ("lm_head.", "model.layers.", "model.embed_tokens.", "model.norm.")
+ if name.startswith(skip_prefixes):
+ return
+
+ # Try to map the tensor using TensorNameMap (handles vision encoder and projector)
+ try:
+ yield from super().modify_tensors(data_torch, name, bid)
+ except ValueError:
+ # If mapping fails, log warning and skip
+ logger.warning(f"Cannot map tensor: {name}")
+ return
+
+
[email protected]("SolarOpenForCausalLM")
+class SolarOpenModel(Glm4MoeModel):
+ model_arch = gguf.MODEL_ARCH.GLM4_MOE
+
+ def set_vocab(self):
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ tokens, toktypes, tokpre = self.get_vocab_base()
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+ special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<unk>"])
+ special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|startoftext|>"])
+ special_vocab.add_to_gguf(self.gguf_writer)
###### CONVERSION LOGIC ######
@@ -9767,6 +12172,25 @@ class LazyTorchTensor(gguf.LazyBase):
torch.uint8: np.uint8,
}
+ # only used when byteswapping data. Only correct size is needed
+ _dtype_byteswap_map: dict[torch.dtype, type] = {
+ torch.float64: np.float64,
+ torch.float32: np.float32,
+ torch.bfloat16: np.float16,
+ torch.float16: np.float16,
+ torch.int64: np.int64,
+ torch.uint64: np.uint64,
+ torch.int32: np.int32,
+ torch.uint32: np.uint32,
+ torch.int16: np.int16,
+ torch.uint16: np.uint16,
+ torch.int8: np.int8,
+ torch.uint8: np.uint8,
+ torch.bool: np.uint8,
+ torch.float8_e4m3fn: np.uint8,
+ torch.float8_e5m2: np.uint8,
+ }
+
_dtype_str_map: dict[str, torch.dtype] = {
"F64": torch.float64,
"F32": torch.float32,
@@ -9808,13 +12232,45 @@ class LazyTorchTensor(gguf.LazyBase):
)
return cast(torch.Tensor, lazy)
+ @classmethod
+ def from_local_tensor(cls, t: gguf.utility.LocalTensor) -> Tensor:
+
+ def load_tensor(tensor: gguf.utility.LocalTensor) -> Tensor:
+
+ def byteswap_tensor(tensor: np.ndarray, dtype: type) -> np.ndarray:
+ if sys.byteorder == "big":
+ # switch data back to big endian
+ tensor = tensor.view(dtype).byteswap(inplace=False)
+ return tensor
+
+ dtype = cls._dtype_str_map[tensor.dtype]
+ numpy_dtype = cls._dtype_byteswap_map[dtype]
+ return torch.from_numpy(byteswap_tensor(tensor.mmap_bytes(), numpy_dtype)).view(dtype).reshape(tensor.shape)
+
+ dtype = cls._dtype_str_map[t.dtype]
+ shape = t.shape
+ lazy = cls(meta=cls.meta_with_dtype_and_shape(dtype, shape), args=(t,), func=lambda r: load_tensor(r))
+ return cast(torch.Tensor, lazy)
+
@classmethod
def from_remote_tensor(cls, remote_tensor: gguf.utility.RemoteTensor):
+
+ def byteswap_tensor(tensor: np.ndarray, dtype: type) -> np.ndarray:
+ if sys.byteorder == "big":
+ # switch data back to big endian
+ tensor = tensor.view(dtype).byteswap(inplace=False)
+ return tensor
+
dtype = cls._dtype_str_map[remote_tensor.dtype]
+ numpy_dtype = cls._dtype_byteswap_map[dtype]
shape = remote_tensor.shape
meta = cls.meta_with_dtype_and_shape(dtype, shape)
lazy = cls(
- meta=meta, args=(remote_tensor,), func=lambda r: torch.frombuffer(r.data(), dtype=dtype).reshape(shape)
+ meta=meta,
+ args=(remote_tensor,),
+ func=lambda r: torch.from_numpy(byteswap_tensor(np.frombuffer(r.data(), dtype=numpy_dtype), numpy_dtype))
+ .view(dtype)
+ .reshape(shape),
)
return cast(torch.Tensor, lazy)
diff --git a/test/helpers.py b/test/helpers.py
index 207864ccf..240791a6a 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -1,6 +1,7 @@
import copy
import os
import re
+import shutil
import pytest
import torch
@@ -165,6 +166,15 @@ def save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, is_mllm=F
tiny_model_path = os.path.join(test_path, tiny_model_path.removeprefix("./"))
model.save_pretrained(tiny_model_path)
tokenizer.save_pretrained(tiny_model_path)
+ # copy tokenizer.model
+ if not os.path.isdir(model_name_or_path):
+ from auto_round.utils import download_hf_model
+
+ model_path = download_hf_model(model_name_or_path)
+ else:
+ model_path = model_name_or_path
+ if os.path.isfile(os.path.join(model_path, "tokenizer.model")):
+ shutil.copy(os.path.join(model_path, "tokenizer.model"), os.path.join(tiny_model_path, "tokenizer.model"))
if is_mllm:
processor = transformers.AutoProcessor.from_pretrained(model_name_or_path, **kwargs)
image_processor = transformers.AutoImageProcessor.from_pretrained(model_name_or_path, **kwargs)
diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.py
index f1980c455..aae5384eb 100644
--- a/test/test_cpu/export/test_gguf_format.py
+++ b/test/test_cpu/export/test_gguf_format.py
@@ -15,11 +15,6 @@ AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])
- transformers_version >= version.parse("5.0.0"),
- reason="GGUF format saving and loading failed in transformers v5, \
- https://github.com/huggingface/transformers/issues/43482",
-)
class TestGGUF:
@classmethod
diff --git a/test/test_cuda/advanced/test_fp8_input.py b/test/test_cuda/advanced/test_fp8_input.py
index adce4dced..209ff8973 100644
--- a/test/test_cuda/advanced/test_fp8_input.py
+++ b/test/test_cuda/advanced/test_fp8_input.py
@@ -48,11 +48,6 @@ class TestAutoRound:
generate_prompt(model, tokenizer)
shutil.rmtree(self.save_dir, ignore_errors=True)
- @pytest.mark.skipif(
- transformers_version >= version.parse("5.0.0"),
- reason="GGUF format saving and loading failed in transformers v5, \
- https://github.com/huggingface/transformers/issues/43482",
- )
def test_gguf_imatrix(self, mock_fp8_capable_device):
model, tokenizer = self.tiny_fp8_model()
ar = AutoRound(model=model, tokenizer=tokenizer, iters=0)
@@ -99,11 +94,6 @@ class TestAutoRound:
evaluate_accuracy(self.save_dir, threshold=0.33)
shutil.rmtree(self.save_dir, ignore_errors=True)
- @pytest.mark.skipif(
- transformers_version >= version.parse("5.0.0"),
- reason="GGUF format saving and loading failed in transformers v5, \
- https://github.com/huggingface/transformers/issues/43482",
- )
def test_fp8_model_gguf(self, mock_fp8_capable_device):
from llama_cpp import Llama
diff --git a/test/test_cuda/export/test_gguf.py b/test/test_cuda/export/test_gguf.py
index e81a0e5b0..824bbbeb9 100644
--- a/test/test_cuda/export/test_gguf.py
+++ b/test/test_cuda/export/test_gguf.py
@@ -17,11 +17,6 @@ AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])
- transformers_version >= version.parse("5.0.0"),
- reason="GGUF format saving and loading failed in transformers v5, \
- https://github.com/huggingface/transformers/issues/43482",
-)
class TestAutoRound:
save_dir = "./saved"