PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Integrate RTN quantization into GGUF packing to enhance robustness (#644)
Heng Guo06e0f520ee69
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex bc439ede5..ed38fc5fb 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1016,20 +1016,13 @@ class AutoRound(object):self.layer_config[name]["data_type"] = m.data_type# Step 2: Try quantization on GPU first, fall back to CPU if OOM- try:- m.to(self.device)- m = WrapperLinear(- m,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- )- m = m.unwrapper({})- m.to("cpu")- except RuntimeError as e:- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):- logger.warning("Out of VRAM, falling back to CPU.")- m.to("cpu")+ # if only export gguf, using gguf-packing instead of rtn+ if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0]:+ m.scale = None+ m.zp = None+ else:+ try:+ m.to(self.device)m = WrapperLinear(m,enable_minmax_tuning=False,@@ -1037,11 +1030,23 @@ class AutoRound(object):enable_round_tuning=False,)m = m.unwrapper({})- else:- raise+ m.to("cpu")+ except RuntimeError as e:+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ logger.warning("Out of VRAM, falling back to CPU.")+ m.to("cpu")+ m = WrapperLinear(+ m,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ )+ m = m.unwrapper({})+ else:+ raise- if self.low_gpu_mem_usage:- clear_memory()+ if self.low_gpu_mem_usage:+ clear_memory()# Step 3: Optional immediate packing/exportif self.is_packing_immediate:diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 96d8bf7da..2d26c97ca 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -271,7 +271,6 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_def quant_tensor_gguf_asym_dq(tensor,bits=4,- group_size=-1,v=0,min_scale=1.0,max_scale=1.0,@@ -279,8 +278,6 @@ def quant_tensor_gguf_asym_dq(tensor_min=None,tensor_max=None,q_scale_thresh=1e-5,- super_group_size=8,- super_bits=6,imatrix=None,**kwargs):"""Quantizes and dequantizes a tensor using asymmetric integer quantization for formats like Q2_K, Q4_K, and Q5_K.@@ -305,12 +302,14 @@ def quant_tensor_gguf_asym_dq(Tuple: (Quantized-dequantized tensor, scale dictionary, zero-point dictionary)"""orig_dtype = tensor.dtype+ maxq = 2 ** bits - 1+ group_size = 16 if bits ==2 else 32+ super_bits = 4 if bits == 2 else 6+ super_group_size = 16 if bits == 2 else 8tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)tensor =tensor.to(torch.float32)if bits not in [2, 4, 5]:raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")- maxq = 2 ** bits - 1- QK_K = 256quant_weights = Noneif imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):search_kwargs = {@@ -457,8 +456,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u@register_dtype("rtn_int_sym_dq")def quant_tensor_gguf_sym_dq(tensor,- bits=4,- group_size=-1,+ bits=3,v=0,min_scale=1.0,max_scale=1.0,@@ -466,8 +464,6 @@ def quant_tensor_gguf_sym_dq(tensor_min=None,tensor_max=None,q_scale_thresh=1e-5,- super_group_size=16,- super_bits=6,imatrix=None,**kwargs):"""Quantize and de-quantize tensor asymmetrically. For Q3_K, Q6_K.@@ -492,9 +488,13 @@ def quant_tensor_gguf_sym_dq(if bits not in [3, 6]:raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")++ maxq = 2 ** (bits - 1)+ group_size = 16+ super_bits = 6 if bits == 3 else 8+ super_group_size = 16tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** (bits - 1)ggml_type = f"q{bits}_k"block_size, type_size = GGML_QUANT_SIZES[ggml_type]orig_dtype = tensor.dtypediff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 27ea57aba..4b074d762 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -53,10 +53,10 @@ from itertools import chainimport mathimport numpy as npimport torch+from transformers import AutoConfig-from auto_round.utils import logger, LazyImport, clear_memory, get_module, clean_module_parameter-from auto_round.export.export_to_gguf.packing import ggml_quant_gpu-+from auto_round.utils import logger, LazyImport, get_module, clean_module_parameter+from auto_round.export.export_to_gguf.packing import ggml_quantgguf = LazyImport("gguf")@@ -66,9 +66,9 @@ if TYPE_CHECKING:if 'NO_LOCAL_GGUF' not in os.environ:sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))-###### MODEL DEFINITIONS ######+class SentencePieceTokenTypes(IntEnum):NORMAL = 1UNKNOWN = 2@@ -78,10 +78,19 @@ class SentencePieceTokenTypes(IntEnum):BYTE = 6-AnyModel = TypeVar("AnyModel", bound="type[Model]")+class ModelType(IntEnum):+ TEXT = 1+ MMPROJ = 2+++AnyModel = TypeVar("AnyModel", bound="type[ModelBase]")+class OriModel:- _model_classes: dict[str, type[Model]] = {}+ _model_classes: dict[ModelType, dict[str, type[ModelBase]]] = {+ ModelType.TEXT: {},+ ModelType.MMPROJ: {},+ }dir_model: Pathftype: gguf.LlamaFileType@@ -93,8 +102,6 @@ class OriModel:part_names: list[str]is_safetensors: boolhparams: dict[str, Any]- block_count: int- tensor_map: gguf.TensorNameMaptensor_names: set[str] | Nonegguf_writer: gguf.GGUFWritermodel_name: str | None@@ -105,11 +112,16 @@ class OriModel:# subclasses should define this!model_arch: gguf.MODEL_ARCH+ # subclasses should initialize this!+ block_count: int+ tensor_map: gguf.TensorNameMap+def __init__(self,dir_model: Path,ftype: gguf.LlamaFileType,fname_out: Path,+ *,is_big_endian: bool = False,use_temp_file: bool = False,eager: bool = False,@@ -121,7 +133,9 @@ class OriModel:small_first_shard: bool = False,hparams: dict[str, Any] | None = None,remote_hf_model_id: str | None = None):- if type(self) is Model:+ if type(self) is ModelBase or \+ type(self) is TextModel or \+ type(self) is MmprojModel:raise TypeError(f"{type(self).__name__!r} should not be directly instantiated")self.dir_model = dir_model@@ -145,13 +159,11 @@ class OriModel:self.get_tensors = get_remote_tensorselse:- self.part_names = Model.get_model_part_names(self.dir_model, "model", ".safetensors")+ self.part_names = ModelBase.get_model_part_names(self.dir_model, "model", ".safetensors")self.is_safetensors = len(self.part_names) > 0if not self.is_safetensors:- self.part_names = Model.get_model_part_names(self.dir_model, "pytorch_model", ".bin")- self.hparams = Model.load_hparams(self.dir_model) if hparams is None else hparams- self.block_count = self.find_hparam(["n_layers", "num_hidden_layers", "n_layer", "num_layers"])- self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)+ self.part_names = ModelBase.get_model_part_names(self.dir_model, "pytorch_model", ".bin")+ self.hparams = ModelBase.load_hparams(self.dir_model) if hparams is None else hparamsself.tensor_names = Noneself.metadata_override = metadata_overrideself.model_name = model_name@@ -179,6 +191,12 @@ class OriModel:dry_run=dry_run,small_first_shard=small_first_shard)+ @classmethod+ def add_prefix_to_filename(cls, path: Path, prefix: str) -> Path:+ stem, suffix = path.stem, path.suffix+ new_name = f"{prefix}{stem}{suffix}"+ return path.with_name(new_name)+def find_hparam(self, keys: Iterable[str], optional: bool = False) -> Any:key = next((k for k in keys if k in self.hparams), None)if key is not None:@@ -187,9 +205,6 @@ class OriModel:return Noneraise KeyError(f"could not find any of: {keys}")- def set_vocab(self):- self._set_vocab_gpt2()-def get_tensors(self) -> Iterator[tuple[str, Tensor]]: # pylint: disable=E0202tensor_names_from_parts: set[str] = set()@@ -280,53 +295,7 @@ class OriModel:return new_namedef set_gguf_parameters(self):- self.gguf_writer.add_block_count(self.block_count)-- if (n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx"], optional=True)) is not None:- self.gguf_writer.add_context_length(n_ctx)- logger.info(f"gguf: context length = {n_ctx}")-- if (n_embd := self.find_hparam(["hidden_size", "n_embd"], optional=True)) is not None:- self.gguf_writer.add_embedding_length(n_embd)- logger.info(f"gguf: embedding length = {n_embd}")-- if (n_ff := self.find_hparam(["intermediate_size", "n_inner"], optional=True)) is not None:- self.gguf_writer.add_feed_forward_length(n_ff)- logger.info(f"gguf: feed forward length = {n_ff}")-- if (n_head := self.find_hparam(["num_attention_heads", "n_head"], optional=True)) is not None:- self.gguf_writer.add_head_count(n_head)- logger.info(f"gguf: head count = {n_head}")-- if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None:- self.gguf_writer.add_head_count_kv(n_head_kv)- logger.info(f"gguf: key-value head count = {n_head_kv}")-- if (rope_theta := self.hparams.get("rope_theta")) is not None:- self.gguf_writer.add_rope_freq_base(rope_theta)- logger.info(f"gguf: rope theta = {rope_theta}")- if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None:- self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)- logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")- if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"],- optional=True)) is not None:- self.gguf_writer.add_layer_norm_eps(f_norm_eps)- logger.info(f"gguf: layer norm epsilon = {f_norm_eps}")- if (n_experts := self.hparams.get("num_local_experts")) is not None:- self.gguf_writer.add_expert_count(n_experts)- logger.info(f"gguf: expert count = {n_experts}")- if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:- self.gguf_writer.add_expert_used_count(n_experts_used)- logger.info(f"gguf: experts used count = {n_experts_used}")-- if (head_dim := self.hparams.get("head_dim")) is not None:- # Workaround for incorrect AutoConfig value for DeepSeekV3 (is set correctly in DeepSeekV2Model class)- if self.hparams.get("model_type") != "deepseek_v3":- self.gguf_writer.add_key_length(head_dim)- self.gguf_writer.add_value_length(head_dim)-- self.gguf_writer.add_file_type(self.ftype)- logger.info(f"gguf: file type = {self.ftype}")+ raise NotImplementedError("set_gguf_parameters() must be implemented in subclasses")def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused@@ -395,12 +364,20 @@ class OriModel:gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,gguf.MODEL_TENSOR.POSNET_NORM1,gguf.MODEL_TENSOR.POSNET_NORM2,+ gguf.MODEL_TENSOR.V_ENC_EMBD_POS,+ gguf.MODEL_TENSOR.A_ENC_EMBD_POS,+ gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,+ gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,)) or not new_name.endswith(".weight")):data_qtype = gguf.GGMLQuantizationType.F32if data_qtype is False and any(self.match_model_tensor_name(new_name, key, bid) for key in (gguf.MODEL_TENSOR.TOKEN_EMBD,+ gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,gguf.MODEL_TENSOR.OUTPUT,+ gguf.MODEL_TENSOR.ALTUP_ROUTER,+ gguf.MODEL_TENSOR.LAUREL_L,+ gguf.MODEL_TENSOR.LAUREL_R,)):if self.ftype in (gguf.LlamaFileType.MOSTLY_TQ1_0,@@ -466,41 +443,6 @@ class OriModel:if self.metadata.size_label is None and total_params > 0:self.metadata.size_label = gguf.size_label(total_params, shared_params, expert_params, expert_count)- # Extract the encoding scheme from the file type name. e.g. 'gguf.LlamaFileType.MOSTLY_Q8_0' --> 'Q8_0'- output_type: str = self.ftype.name.partition("_")[2]-- # Filename Output- if self.fname_out.is_dir():- # Generate default filename based on model specification and available metadata- if not vocab_only:- fname_default: str = gguf.naming_convention(- self.metadata.name,- self.metadata.basename,- self.metadata.finetune,- self.metadata.version,- self.metadata.size_label,- output_type,- model_type="LoRA" if total_params < 0 else None)- else:- fname_default: str = gguf.naming_convention(- self.metadata.name,- self.metadata.basename,- self.metadata.finetune,- self.metadata.version,- size_label=None,- output_type=None,- model_type="vocab")-- # Use the default filename- self.fname_out = self.fname_out / f"{fname_default}.gguf"- else:- # Output path is a custom defined templated filename- # Note: `not is_dir()` is used because `.is_file()` will not detect- # file template strings as it doesn't actually exist as a file-- # Process templated file name with the output ftype, useful with the "auto" ftype- self.fname_out = self.fname_out.parent / gguf.fill_templated_filename(self.fname_out.name, output_type)-self.set_type()logger.info("Set meta model")@@ -509,12 +451,12 @@ class OriModel:logger.info("Set model parameters")self.set_gguf_parameters()- logger.info("Set model tokenizer")- self.set_vocab()-logger.info("Set model quantization version")self.gguf_writer.add_quantization_version(gguf.GGML_QUANT_VERSION)+ def write_vocab(self):+ raise NotImplementedError("write_vocab() must be implemented in subclasses")+def write(self):self.prepare_tensors()self.prepare_metadata(vocab_only=False)@@ -523,6 +465,576 @@ class OriModel:self.gguf_writer.write_tensors_to_file(progress=True)self.gguf_writer.close()+ @staticmethod+ def get_model_part_names(dir_model: Path, prefix: str, suffix: str) -> list[str]:+ part_names: list[str] = []+ for filename in os.listdir(dir_model):+ if filename.startswith(prefix) and filename.endswith(suffix):+ part_names.append(filename)++ part_names.sort()++ return part_names++ @staticmethod+ def load_hparams(dir_model: Path):+ try:+ # for security reason, we don't allow loading remote code by default+ # if a model need remote code, we will fallback to config.json+ config = AutoConfig.from_pretrained(dir_model, trust_remote_code=False).to_dict()+ except Exception as e:+ logger.warning(f"Failed to load model config from {dir_model}: {e}")+ logger.warning("Trying to load config.json instead")+ with open(dir_model / "config.json", "r", encoding="utf-8") as f:+ config = json.load(f)+ if "llm_config" in config:+ # rename for InternVL+ config["text_config"] = config["llm_config"]+ if "thinker_config" in config:+ # rename for Qwen2.5-Omni+ config["text_config"] = config["thinker_config"]["text_config"]+ return config++ @classmethod+ def register(cls, *names: str) -> Callable[[AnyModel], AnyModel]:+ assert names++ def func(modelcls: AnyModel) -> AnyModel:+ model_type = ModelType.MMPROJ if modelcls.model_arch == gguf.MODEL_ARCH.MMPROJ else ModelType.TEXT+ for name in names:+ cls._model_classes[model_type][name] = modelcls+ return modelcls++ return func++ @classmethod+ def print_registered_models(cls):+ for model_type, model_classes in cls._model_classes.items():+ logger.error(f"{model_type.name} models:")+ for name in sorted(model_classes.keys()):+ logger.error(f" - {name}")++ @classmethod+ def from_model_architecture(cls, arch: str, model_type=ModelType.TEXT) -> type[ModelBase]:+ try:+ return cls._model_classes[model_type][arch]+ except KeyError:+ raise NotImplementedError(f'Architecture {arch!r} not supported!') from None+++class ModelBase(OriModel):++ def __init__(+ self,+ model,+ layer_config: dict,+ dir_model: Path,+ ftype: gguf.LlamaFileType,+ fname_out: Path,+ low_cpu_mem_usage: bool = False,+ is_big_endian: bool = False,+ use_temp_file: bool = False,+ eager: bool = False,+ metadata_override: Path | None = None,+ model_name: str | None = None,+ split_max_tensors: int = 0,+ split_max_size: int = 0,+ dry_run: bool = False,+ small_first_shard: bool = False,+ hparams: dict[str, Any] | None = None):+ self.model = model+ self.layer_config = layer_config+ self.low_cpu_mem_usage = self._need_low_cpu_mem(low_cpu_mem_usage)+ super().__init__(+ dir_model=dir_model,+ ftype=ftype,+ fname_out=fname_out,+ is_big_endian=is_big_endian,+ use_temp_file=use_temp_file,+ eager=eager,+ metadata_override=metadata_override,+ model_name=model_name,+ split_max_tensors=split_max_tensors,+ split_max_size=split_max_size,+ dry_run=dry_run,+ small_first_shard=small_first_shard,+ hparams=hparams)++ def _need_low_cpu_mem(self, low_cpu_mem_usage):+ if not low_cpu_mem_usage:+ return False++ # process = psutil.Process(os.getpid())+ # mem_usage = process.memory_info().rss+ # memory_info = psutil.virtual_memory()+ # if memory_info.available > mem_usage / 3:+ # return False+ # else:+ # logger.info("use low cpu memory mode.")+ return True++ def get_moe_name(self, name, new_name):+ type_mapping = {+ "FFN_GATE_EXP": ["gate_proj", "w1", "linear"],+ "FFN_DOWN_EXP": ["down_proj", "w2", "linear_1"],+ "FFN_UP_EXP": ["up_proj", "w3", "linear_v"],+ }+ nums = re.findall('\d+', name)+ if len(nums) != 2:+ return name+ name_tmp = name[:-len(".weight")].replace(f'.{nums[1]}', '')+ new_name_tmp = new_name[:-len(".weight")]+ if self.tensor_map.get_type(name_tmp) == self.tensor_map.get_type(new_name_tmp):+ return name++ tensor_type = self.tensor_map.get_type(new_name_tmp).name+ experts_name = name_tmp.split(".")[-1]+ for k, v in type_mapping.items():+ if experts_name in v:+ idx = v.index(experts_name)+ name = name.replace(experts_name, type_mapping[tensor_type][idx])+ return name+ return name++ def get_tensors(self) -> Iterator[tuple[str, Tensor]]:+ for name, tensor in self.model.named_parameters():+ yield name, tensor++ def _quant_data_with_args(self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None):+ if torch.cuda.is_available():+ device = "cuda"+ elif torch.xpu.is_available():+ device = "xpu"+ else:+ device = "cpu"+ data_torch = data_torch.to(torch.float32)+ scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale+ zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp+ d_scale = d_scale.to(torch.float32) if isinstance(d_scale, torch.Tensor) else d_scale+ d_wmin = d_wmin.to(torch.float32) if isinstance(d_wmin, torch.Tensor) else d_wmin+ wmin = wmin.to(torch.float32) if isinstance(wmin, torch.Tensor) else wmin++ data = ggml_quant(+ data_torch, data_qtype.name.lower(), scale, zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, device=device)+ return data++ def _quant_data(self, data_torch, data_qtype, name, bid):+ suffix = '.weight'+ if suffix in name:+ layer_name = name[:-len(suffix)]+ module = get_module(self.model, layer_name)+ if hasattr(module, "scale"):+ if hasattr(self, "permute"):+ bs = module.weight.shape[0]+ for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:+ if hasattr(module, attr) and getattr(module, attr) is not None:+ attr_tensor = getattr(module, attr)+ ori_shape = attr_tensor.shape+ attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), name, bid)[0][1]+ attr_tensor = attr_tensor.reshape(ori_shape)+ setattr(module, attr, attr_tensor)+ scale = module.scale if hasattr(module, "scale") else None+ zp = module.zp if hasattr(module, "zp") else None+ if torch.cuda.is_available():+ device = "cuda"+ elif torch.xpu.is_available():+ device = "xpu"+ else:+ device = "cpu"+ data_torch = data_torch.to(torch.float32)+ scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale+ zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp+ if data_qtype.name.lower().endswith("_k"):+ d_scale = module.w_d_scale.to(torch.float32) if hasattr(module, "w_d_scale") else None+ d_wmin = module.w_d_wmin.to(torch.float32) if hasattr(module, "w_d_wmin") else None+ wmin = module.w_wmin.to(torch.float32) if hasattr(module, "w_wmin") else None+ imatrix = module.imatrix.to(torch.float32) if hasattr(module, "imatrix") else None++ data = ggml_quant(+ data_torch,+ data_qtype.name.lower(),+ scale,+ zp,+ wmin=wmin,+ d_scale=d_scale,+ d_wmin=d_wmin,+ imatrix=imatrix,+ device=device)+ else:+ data = ggml_quant(data_torch, data_qtype.name.lower(), scale, zp, device=device)+ else:+ # if data_torch.dtype ==torch.float32:+ # data_qtype = gguf.GGMLQuantizationType.F32+ # else:+ # data_qtype = gguf.GGMLQuantizationType.F16+ data_qtype = gguf.GGMLQuantizationType.F32 ##FP16 has issues at inference+ data = data_torch.to(torch.float32).squeeze().cpu().numpy()+ else:+ # if data_torch.dtype == torch.float32:+ # data_qtype = gguf.GGMLQuantizationType.F32+ # else:+ # data_qtype = gguf.GGMLQuantizationType.F16+ # data = data_torch.squeeze().cpu().numpy()+ data_qtype = gguf.GGMLQuantizationType.F32+ data = data_torch.to(torch.float32).squeeze().cpu().numpy()+ return data, data_qtype++ def get_qtype_by_layer_config(self, layer_config, name, data_qtype):+ name = name[:-len('.weight')]+ if name not in layer_config or layer_config[name]['bits'] >= 16:+ if isinstance(data_qtype, bool):+ return gguf.GGMLQuantizationType.F32+ return data_qtype+ bits = layer_config[name]['bits']+ super_bits = layer_config[name]["super_bits"]+ sym = layer_config[name]["sym"]+ if bits == 2:+ return gguf.GGMLQuantizationType.Q2_K+ if bits == 3:+ return gguf.GGMLQuantizationType.Q3_K+ if bits == 4:+ if super_bits is not None:+ return gguf.GGMLQuantizationType.Q4_K+ if super_bits is None and sym:+ return gguf.GGMLQuantizationType.Q4_0+ if super_bits is None and not sym:+ return gguf.GGMLQuantizationType.Q4_1+ if bits == 5:+ if super_bits is not None:+ return gguf.GGMLQuantizationType.Q5_K+ if super_bits is None and sym:+ return gguf.GGMLQuantizationType.Q5_0+ if super_bits is None and not sym:+ return gguf.GGMLQuantizationType.Q5_1+ if bits == 6:+ return gguf.GGMLQuantizationType.Q6_K+ if bits == 8:+ return gguf.GGMLQuantizationType.Q8_0+ raise ValueError(f"Unknown file type: {data_qtype}")++ # data_qtype = re.sub(r"Q\d", f"Q{bits}", self.ftype.name)+ #+ # if data_qtype == "MOSTLY_Q8_0":+ # data_qtype = gguf.GGMLQuantizationType.Q8_0+ # elif data_qtype == "MOSTLY_Q4_0":+ # data_qtype = gguf.GGMLQuantizationType.Q4_0+ # elif data_qtype == "MOSTLY_Q4_1":+ # data_qtype = gguf.GGMLQuantizationType.Q4_1+ # elif data_qtype == "MOSTLY_Q5_0":+ # data_qtype = gguf.GGMLQuantizationType.Q5_0+ # elif data_qtype == "MOSTLY_Q5_1":+ # data_qtype = gguf.GGMLQuantizationType.Q5_1+ # elif data_qtype.startswith("MOSTLY_Q2_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q2_K+ # elif data_qtype.startswith("MOSTLY_Q3_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q3_K+ # elif data_qtype.startswith("MOSTLY_Q4_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q4_K+ # elif data_qtype.startswith("MOSTLY_Q5_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q5_K+ # elif data_qtype.startswith("MOSTLY_Q6_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q6_K+ # else:+ # raise ValueError(f"Unknown file type: {data_qtype}")+ # return data_qtype++ def prepare_tensors(self):+ max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")++ for name, data_torch in chain(self.generate_extra_tensors(), self.get_tensors()):+ if data_torch is None:+ continue+ # we don't need these+ if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):+ continue+ if hasattr(self, "current_packing_block") and self.current_packing_block is not None: # pylint: disable=E1101+ current_packing_block_split = self.current_packing_block.split('.') # pylint: disable=E1101+ name_split = name.split('.')+ if (len(name_split) < len(current_packing_block_split) or+ name_split[:len(current_packing_block_split)] != current_packing_block_split):+ continue++ old_dtype = data_torch.dtype++ # convert any unsupported data types to float32+ if data_torch.dtype not in (torch.float16, torch.float32):+ data_torch = data_torch.to(torch.float32)++ # use the first number-like part of the tensor name as the block id+ bid = None+ for part in name.split("."):+ if part.isdecimal():+ bid = int(part)+ break++ clean_weight_list = []+ for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):+ data = data_torch.squeeze().cpu().numpy()++ # if data ends up empty, it means data_torch was a scalar tensor -> restore+ if len(data.shape) == 0:+ data = data_torch.numpy()++ n_dims = len(data.shape)+ data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)++ # Most of the codebase that takes in 1D tensors or norms only handles F32 tensors+ if n_dims <= 1 or new_name.endswith("_norm.weight"):+ data_qtype = gguf.GGMLQuantizationType.F32++ # Conditions should closely match those in llama_model_quantize_internal in llama.cpp+ # Some tensor types are always in float32+ if data_qtype is False and (any(self.match_model_tensor_name(new_name, key, bid) for key in (+ gguf.MODEL_TENSOR.FFN_GATE_INP,+ gguf.MODEL_TENSOR.POS_EMBD,+ gguf.MODEL_TENSOR.TOKEN_TYPES,+ gguf.MODEL_TENSOR.SSM_CONV1D,+ gguf.MODEL_TENSOR.TIME_MIX_FIRST,+ gguf.MODEL_TENSOR.TIME_MIX_W1,+ gguf.MODEL_TENSOR.TIME_MIX_W2,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,+ )) or not new_name.endswith(".weight")):+ data_qtype = gguf.GGMLQuantizationType.F32++ if data_qtype is False and any(self.match_model_tensor_name(new_name, key, bid) for key in (+ gguf.MODEL_TENSOR.TOKEN_EMBD,+ gguf.MODEL_TENSOR.OUTPUT,+ )):+ if self.ftype in (+ gguf.LlamaFileType.MOSTLY_TQ1_0,+ gguf.LlamaFileType.MOSTLY_TQ2_0,+ ):+ # TODO: use Q4_K and Q6_K+ data_qtype = gguf.GGMLQuantizationType.F16+ # data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type++ # get name by new_name (for experts),+ name = self.get_moe_name(name, new_name)+ clean_weight_list.append(name)+ # get data_qtype by layer_config+ data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)++ # # No override (data_qtype is False), or wants to be quantized (data_qtype is True)+ # if isinstance(data_qtype, bool):+ # if self.ftype == gguf.LlamaFileType.ALL_F32:+ # data_qtype = gguf.GGMLQuantizationType.F32+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:+ # data_qtype = gguf.GGMLQuantizationType.F16+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:+ # data_qtype = gguf.GGMLQuantizationType.BF16+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:+ # data_qtype = gguf.GGMLQuantizationType.Q8_0+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:+ # data_qtype = gguf.GGMLQuantizationType.Q4_0+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:+ # data_qtype = gguf.GGMLQuantizationType.Q4_1+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:+ # data_qtype = gguf.GGMLQuantizationType.Q5_0+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:+ # data_qtype = gguf.GGMLQuantizationType.Q5_1+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:+ # data_qtype = gguf.GGMLQuantizationType.Q2_K+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S:+ # data_qtype = gguf.GGMLQuantizationType.Q3_K+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:+ # data_qtype = gguf.GGMLQuantizationType.Q4_K+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:+ # data_qtype = gguf.GGMLQuantizationType.Q4_K+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S:+ # data_qtype = gguf.GGMLQuantizationType.Q5_K+ # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:+ # data_qtype = gguf.GGMLQuantizationType.Q6_K+ # else:+ # raise ValueError(f"Unknown file type: {self.ftype.name}")++ if isinstance(data_qtype, bool) or data_qtype in [+ gguf.GGMLQuantizationType.F16, gguf.GGMLQuantizationType.BF16, gguf.GGMLQuantizationType.F32+ ]:+ try:+ data = gguf.quants.quantize(data, data_qtype)+ except gguf.QuantError as e:+ logger.warning("%s, %s", e, "falling back to F16")+ data_qtype = gguf.GGMLQuantizationType.F16+ data = gguf.quants.quantize(data, data_qtype)+ else:+ # for deepseek v2+ if name.endswith("kv_b_proj.weight") and self.model_arch.name == 'DEEPSEEK2':+ layer_name = name[:-len('.weight')]+ module = get_module(self.model, layer_name)+ n_head_kv = self.hparams["num_key_value_heads"]+ v_head_dim = self.hparams["v_head_dim"]+ qk_nope_head_dim = self.hparams["qk_nope_head_dim"]++ attr_list = {"scale": None, "zp": None, "d_scale": None, "wmin": None, "d_wmin": None}+ for attr in attr_list:+ if hasattr(module, attr) or hasattr(module, "w_" + attr):+ if attr in ['scale', "zp"]:+ attr_tensor = getattr(module, attr)+ else:+ attr_tensor = getattr(module, "w_" + attr)+ kv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)+ k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)+ k_b = k_b.transpose(1, 2)++ name_kb = name.replace("kv_b_proj", "k_b_proj")+ if new_name == self.map_tensor_name(name_kb):+ attr_list[attr] = k_b+ else:+ attr_list[attr] = v_b+ data = self._quant_data_with_args(data_torch, data_qtype, **attr_list)++ # for MOE model+ elif len(data_torch.shape) == 3:+ new_data = []+ for idx, arr in enumerate(data_torch):+ arr_name = name.split('.')+ for i in range(len(arr_name) - 1, -1, -1):+ if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):+ arr_name[i] = str(idx)+ arr_name = ".".join(arr_name)+ arr, data_qtype = self._quant_data(arr, data_qtype, arr_name, bid)+ new_data.append(arr)+ data = np.array(new_data)+ del new_data+ else:+ data, data_qtype = self._quant_data(data_torch, data_qtype, name, bid)++ shape = gguf.quant_shape_from_byte_shape(+ data.shape, data_qtype) if data.dtype == np.uint8 else data.shape++ # reverse shape to make it similar to the internal ggml dimension order+ shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"++ # n_dims is implicit in the shape+ logger.info(+ f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}"+ f" --> {data_qtype.name}, shape = {shape_str}")++ self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)++ # # save cpu memory, but slow+ if self.low_cpu_mem_usage:+ for weight_name in clean_weight_list:+ module = get_module(self.model, ".".join(weight_name.split(".")[:-1]))+ clean_module_parameter(module, weight_name.split(".")[-1])+++class TextModel(ModelBase):+ model_type = ModelType.TEXT+ hf_arch: str++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.hf_arch = get_model_architecture(self.hparams, self.model_type)++ if "text_config" in self.hparams:+ # move the text_config to the root level+ self.hparams = {**self.hparams, **self.hparams["text_config"]}++ self.block_count = self.find_hparam(["n_layers", "num_hidden_layers", "n_layer", "num_layers"])+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)++ @classmethod+ def __init_subclass__(cls):+ # can't use an abstract property, because overriding it without type errors+ # would require using decorated functions instead of simply defining the property+ if "model_arch" not in cls.__dict__:+ raise TypeError(f"Missing property 'model_arch' for {cls.__name__!r}")++ def set_vocab(self):+ self._set_vocab_gpt2()++ def prepare_metadata(self, vocab_only: bool):+ super().prepare_metadata(vocab_only=vocab_only)++ total_params = self.gguf_writer.get_total_parameter_count()[0]+ # Extract the encoding scheme from the file type name. e.g. 'gguf.LlamaFileType.MOSTLY_Q8_0' --> 'Q8_0'+ output_type: str = self.ftype.name.partition("_")[2]++ # Filename Output+ if self.fname_out.is_dir():+ # Generate default filename based on model specification and available metadata+ if not vocab_only:+ fname_default: str = gguf.naming_convention(+ self.metadata.name,+ self.metadata.basename,+ self.metadata.finetune,+ self.metadata.version,+ self.metadata.size_label,+ output_type,+ model_type="LoRA" if total_params < 0 else None)+ else:+ fname_default: str = gguf.naming_convention(+ self.metadata.name,+ self.metadata.basename,+ self.metadata.finetune,+ self.metadata.version,+ size_label=None,+ output_type=None,+ model_type="vocab")++ # Use the default filename+ self.fname_out = self.fname_out / f"{fname_default}.gguf"+ else:+ # Output path is a custom defined templated filename+ # Note: `not is_dir()` is used because `.is_file()` will not detect+ # file template strings as it doesn't actually exist as a file++ # Process templated file name with the output ftype, useful with the "auto" ftype+ self.fname_out = self.fname_out.parent / gguf.fill_templated_filename(self.fname_out.name, output_type)++ logger.info("Set model tokenizer")+ self.set_vocab()++ def set_gguf_parameters(self):+ self.gguf_writer.add_block_count(self.block_count)++ if (n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx", "n_positions", "max_length"],+ optional=True)) is not None:+ self.gguf_writer.add_context_length(n_ctx)+ logger.info(f"gguf: context length = {n_ctx}")++ if (n_embd := self.find_hparam(["hidden_size", "n_embd", "dim"], optional=True)) is not None:+ self.gguf_writer.add_embedding_length(n_embd)+ logger.info(f"gguf: embedding length = {n_embd}")++ if (n_ff := self.find_hparam(["intermediate_size", "n_inner", "hidden_dim"], optional=True)) is not None:+ self.gguf_writer.add_feed_forward_length(n_ff)+ logger.info(f"gguf: feed forward length = {n_ff}")++ if (n_head := self.find_hparam(["num_attention_heads", "n_head", "n_heads"], optional=True)) is not None:+ self.gguf_writer.add_head_count(n_head)+ logger.info(f"gguf: head count = {n_head}")++ if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None:+ self.gguf_writer.add_head_count_kv(n_head_kv)+ logger.info(f"gguf: key-value head count = {n_head_kv}")++ if (rope_theta := self.hparams.get("rope_theta")) is not None:+ self.gguf_writer.add_rope_freq_base(rope_theta)+ logger.info(f"gguf: rope theta = {rope_theta}")+ if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None:+ self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)+ logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")+ if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"],+ optional=True)) is not None:+ self.gguf_writer.add_layer_norm_eps(f_norm_eps)+ logger.info(f"gguf: layer norm epsilon = {f_norm_eps}")+ if (n_experts := self.hparams.get("num_local_experts")) is not None:+ self.gguf_writer.add_expert_count(n_experts)+ logger.info(f"gguf: expert count = {n_experts}")+ if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:+ self.gguf_writer.add_expert_used_count(n_experts_used)+ logger.info(f"gguf: experts used count = {n_experts_used}")++ if (head_dim := self.hparams.get("head_dim")) is not None:+ self.gguf_writer.add_key_length(head_dim)+ self.gguf_writer.add_value_length(head_dim)++ self.gguf_writer.add_file_type(self.ftype)+ logger.info(f"gguf: file type = {self.ftype}")+def write_vocab(self):if len(self.gguf_writer.tensors) != 1:raise ValueError('Splitting the vocabulary is not supported')@@ -532,45 +1044,6 @@ class OriModel:self.gguf_writer.write_kv_data_to_file()self.gguf_writer.close()- @staticmethod- def get_model_part_names(dir_model: Path, prefix: str, suffix: str) -> list[str]:- part_names: list[str] = []- for filename in os.listdir(dir_model):- if filename.startswith(prefix) and filename.endswith(suffix):- part_names.append(filename)-- part_names.sort()-- return part_names-- @staticmethod- def load_hparams(dir_model: Path):- with open(dir_model / "config.json", "r", encoding="utf-8") as f:- return json.load(f)-- @classmethod- def register(cls, *names: str) -> Callable[[AnyModel], AnyModel]:- assert names-- def func(modelcls: AnyModel) -> AnyModel:- for name in names:- cls._model_classes[name] = modelcls- return modelcls-- return func-- @classmethod- def print_registered_models(cls):- for name in sorted(cls._model_classes.keys()):- logger.error(f"- {name}")-- @classmethod- def from_model_architecture(cls, arch: str) -> type[Model]:- try:- return cls._model_classes[arch]- except KeyError:- raise NotImplementedError(f'Architecture {arch!r} not supported!') from None-def does_token_look_special(self, token: str | bytes) -> bool:if isinstance(token, (bytes, bytearray)):token_text = token.decode(encoding="utf-8")@@ -589,10 +1062,10 @@ class OriModel:)seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))- seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>")) # deepseek-coder+ seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))# TODO: should these be marked as UNUSED instead? (maybe not)- seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">")) # gemma{,-2}+ seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">"))return seems_special@@ -627,15 +1100,16 @@ class OriModel:token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))if previous_token != token:logger.info(- f"{repr(previous_token)} is encoded and decoded back"- f" to {repr(token)} using AutoTokenizer")+ f"{repr(previous_token)} is encoded and decoded back to"+ f" {repr(token)} using AutoTokenizer"+ )if added_tokens_decoder[i].special or self.does_token_look_special(token):toktypes.append(gguf.TokenType.CONTROL)else:# NOTE: this was added for Gemma.# Encoding and decoding the tokens above isn't sufficient for this case.- token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spaces+ token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")toktypes.append(gguf.TokenType.USER_DEFINED)else:toktypes.append(gguf.TokenType.NORMAL)@@ -653,7 +1127,7 @@ class OriModel:# we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can# use in llama.cpp to implement the same pre-tokenizer- chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български \'\'\'\'\'\'```````""""......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL' # pylint: disable=C0301+ chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български \'\'\'\'\'\'```````""""......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL' # pylint: disable=C0301chktok = tokenizer.encode(chktxt)chkhsh = sha256(str(chktok).encode()).hexdigest()@@ -678,12 +1152,12 @@ class OriModel:if chkhsh == "8aeee3860c56296a157a1fe2fad249ec40aa59b1bb5709f4ade11c4e6fe652ed":# ref: https://huggingface.co/tiiuae/falcon-7bres = "falcon"- if chkhsh == "9d032fcbd5501f4a38150912590928bfb36091efb5df11b8e2124b0390e3fb1e":- # ref: https://huggingface.co/tiiuae/Falcon3-7B-Base- res = "falcon3"if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f":# ref: https://huggingface.co/BAAI/bge-small-en-v1.5res = "bert-bge"+ if chkhsh == "9d032fcbd5501f4a38150912590928bfb36091efb5df11b8e2124b0390e3fb1e":+ # ref: https://huggingface.co/tiiuae/Falcon3-7B-Base+ res = "falcon3"if chkhsh == "8e62295832751ca1e8f92f2226f403dea30dc5165e448b5bfa05af5340c64ec7":# ref: https://huggingface.co/BAAI/bge-large-zh-v1.5res = "bert-bge-large"@@ -735,10 +1209,6 @@ class OriModel:if chkhsh == "7967bfa498ade6b757b064f31e964dddbb80f8f9a4d68d4ba7998fcf281c531a":# ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-coderes = "jina-v2-code"- if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b" \- or chkhsh == "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516":- # ref: https://huggingface.co/THUDM/glm-4-9b-chat- res = "chatglm-bpe"if chkhsh == "7fc505bd3104ca1083b150b17d088b59534ede9bde81f0dd2090967d7fe52cee":# ref: https://huggingface.co/LumiOpen/Viking-7Bres = "viking"@@ -769,9 +1239,6 @@ class OriModel:if chkhsh == "60824e3c0d9401f89943cbb2fff727f0e2d4c545ba4df2d6e4f09a6db0f5b450":# ref: https://huggingface.co/facebook/chameleon-7bres = "chameleon"- if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":- # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0- res = "minerva-7b"if chkhsh == "8b5a93ed704057481f240da0be7e7dca721d7f8f4755263b6807227a2cbeae65":# ref: https://huggingface.co/sentence-transformers/stsb-roberta-baseres = "roberta-bpe"@@ -802,9 +1269,24 @@ class OriModel:if chkhsh == "d353350c764d8c3b39c763113960e4fb4919bea5fbf208a0e3b22e8469dc7406":# ref: https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instructres = "llama4"+ if chkhsh == "0e9433cbbb161f89e264eb32e8e64bfe69e834973ffca5d41d3948a604a3e2a3":+ # ref: https://huggingface.co/mistral-community/pixtral-12b+ res = "pixtral"+ if chkhsh == "d5f1dd6f980fec569fb218a81a7658ac45fc56b38c5a0adeb1c232fbe04ef5ec":+ # ref: https://huggingface.co/ByteDance-Seed/Seed-Coder-8B-Base+ res = "seed-coder"+ if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b":+ # ref: https://huggingface.co/THUDM/glm-4-9b-chat+ res = "chatglm-bpe"+ if chkhsh == "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516":+ # ref: https://huggingface.co/THUDM/glm-4-9b-chat+ res = "chatglm-bpe"if chkhsh == "a1336059768a55c99a734006ffb02203cd450fed003e9a71886c88acf24fdbc2":# ref: https://huggingface.co/THUDM/glm-4-9b-hfres = "glm4"+ if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":+ # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0+ res = "minerva-7b"if res is None:logger.warning("\n")@@ -918,13 +1400,22 @@ class OriModel:tokenizer = SentencePieceProcessor()tokenizer.LoadFromFile(str(tokenizer_path))- vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())+ vocab_size = self.find_hparam(+ [+ "vocab_size_per_layer_input", # gemma3n+ "vocab_size",+ ],+ optional=True) or tokenizer.vocab_size()tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_sizetoktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_sizefor token_id in range(tokenizer.vocab_size()):+ if token_id >= vocab_size:+ logger.warning(f'ignore tokens from {token_id}: id is out of range, max={vocab_size - 1}')+ break+piece = tokenizer.IdToPiece(token_id)text = piece.encode("utf-8")score = tokenizer.GetScore(token_id)@@ -1045,6 +1536,10 @@ class OriModel:special_vocab.chat_template = "rwkv-world"# hack: Add '\n\n' as the EOT token to make it chat normallyspecial_vocab._set_special_token("eot", 261)+ # hack: Override these as they have already been set (incorrectly)+ special_vocab.special_token_ids["bos"] = 0+ special_vocab.special_token_ids["eos"] = 0+special_vocab.add_to_gguf(self.gguf_writer)def _set_vocab_builtin(self, model_name: Literal["gpt-neox", "llama-spm"], vocab_size: int):@@ -1092,427 +1587,147 @@ class OriModel:if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.ADD_EOS)) is not None:self.gguf_writer.add_add_eos_token(field.parts[-1].tolist()[0])+ def _try_set_pooling_type(self) -> None:+ # get pooling path+ pooling_path = None+ module_path = self.dir_model / "modules.json"+ if module_path.is_file():+ with open(module_path, encoding="utf-8") as f:+ modules = json.load(f)+ for mod in modules:+ if mod["type"] == "sentence_transformers.models.Pooling":+ pooling_path = mod["path"]+ break-class Model(OriModel):- def __init__(- self,- model,- layer_config: dict,- dir_model: Path,- ftype: gguf.LlamaFileType,- fname_out: Path,- low_cpu_mem_usage: bool = False,- is_big_endian: bool = False,- use_temp_file: bool = False,- eager: bool = False,- metadata_override: Path | None = None,- model_name: str | None = None,- split_max_tensors: int = 0,- split_max_size: int = 0,- dry_run: bool = False,- small_first_shard: bool = False,- hparams: dict[str, Any] | None = None):- self.model = model- self.layer_config = layer_config- self.low_cpu_mem_usage = self._need_low_cpu_mem(low_cpu_mem_usage)- super().__init__(- dir_model=dir_model,- ftype=ftype,- fname_out=fname_out,- is_big_endian=is_big_endian,- use_temp_file=use_temp_file,- eager=eager,- metadata_override=metadata_override,- model_name=model_name,- split_max_tensors=split_max_tensors,- split_max_size=split_max_size,- dry_run=dry_run,- small_first_shard=small_first_shard,- hparams=hparams)-- @classmethod- def __init_subclass__(cls):- # can't use an abstract property, because overriding it without type errors- # would require using decorated functions instead of simply defining the property- if "model_arch" not in cls.__dict__:- raise TypeError(f"Missing property 'model_arch' for {cls.__name__!r}")-- def _need_low_cpu_mem(self, low_cpu_mem_usage):- if not low_cpu_mem_usage:- return False-- # process = psutil.Process(os.getpid())- # mem_usage = process.memory_info().rss- # memory_info = psutil.virtual_memory()- # if memory_info.available > mem_usage / 3:- # return False- # else:- # logger.info("use low cpu memory mode.")- return True-- def get_moe_name(self, name, new_name):- type_mapping = {- "FFN_GATE_EXP": ["gate_proj", "w1", "linear"],- "FFN_DOWN_EXP": ["down_proj", "w2", "linear_1"],- "FFN_UP_EXP": ["up_proj", "w3", "linear_v"],- }- nums = re.findall('\d+', name)- if len(nums) != 2:- return name- name_tmp = name[:-len(".weight")].replace(f'.{nums[1]}', '')- new_name_tmp = new_name[:-len(".weight")]- if self.tensor_map.get_type(name_tmp) == self.tensor_map.get_type(new_name_tmp):- return name-- tensor_type = self.tensor_map.get_type(new_name_tmp).name- experts_name = name_tmp.split(".")[-1]- for k, v in type_mapping.items():- if experts_name in v:- idx = v.index(experts_name)- name = name.replace(experts_name, type_mapping[tensor_type][idx])- return name- return name--- def get_tensors(self) -> Iterator[tuple[str, Tensor]]:- for name, tensor in self.model.named_parameters():- yield name, tensor-- def _quant_data_with_args(self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None):- if torch.cuda.is_available():- device = "cuda"- elif torch.xpu.is_available():- device = "xpu"- else:- device = "cpu"- data_torch = data_torch.to(torch.float32)- scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale- zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp- d_scale = d_scale.to(torch.float32) if isinstance(d_scale, torch.Tensor) else d_scale- d_wmin = d_wmin.to(torch.float32) if isinstance(d_wmin, torch.Tensor) else d_wmin- wmin = wmin.to(torch.float32) if isinstance(wmin, torch.Tensor) else wmin-- data = ggml_quant_gpu(- data_torch,- data_qtype.name.lower(),- scale,- zp,- wmin=wmin,- d_scale=d_scale,- d_wmin=d_wmin,- device=device- )- return data-- def _quant_data(self, data_torch, data_qtype, name, bid):- suffix = '.weight'- if suffix in name:- layer_name = name[:-len(suffix)]- module = get_module(self.model, layer_name)- if hasattr(module, "scale"):- if hasattr(self, "permute"):- bs = module.weight.shape[0]- for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:- if hasattr(module, attr) and getattr(module, attr) is not None:- attr_tensor = getattr(module, attr)- ori_shape = attr_tensor.shape- attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), name, bid)[0][1]- attr_tensor = attr_tensor.reshape(ori_shape)- setattr(module, attr, attr_tensor)- scale = module.scale if hasattr(module, "scale") else None- zp = module.zp if hasattr(module, "zp") else None- if torch.cuda.is_available():- device = "cuda"- elif torch.xpu.is_available():- device = "xpu"- else:- device = "cpu"- data_torch = data_torch.to(torch.float32)- scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale- zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp- if data_qtype.name.lower().endswith("_k"):- d_scale = module.w_d_scale.to(torch.float32) if hasattr(module, "w_d_scale") else None- d_wmin = module.w_d_wmin.to(torch.float32) if hasattr(module, "w_d_wmin") else None- wmin = module.w_wmin.to(torch.float32) if hasattr(module, "w_wmin") else None-- data = ggml_quant_gpu(- data_torch,- data_qtype.name.lower(),- scale,- zp,- wmin=wmin,- d_scale=d_scale,- d_wmin=d_wmin,- device=device- )- else:- data = ggml_quant_gpu(data_torch, data_qtype.name.lower(), scale, zp, device=device)- else:- # if data_torch.dtype ==torch.float32:- # data_qtype = gguf.GGMLQuantizationType.F32- # else:- # data_qtype = gguf.GGMLQuantizationType.F16- data_qtype = gguf.GGMLQuantizationType.F32 ##FP16 has issues at inference- data = data_torch.to(torch.float32).squeeze().cpu().numpy()- else:- # if data_torch.dtype == torch.float32:- # data_qtype = gguf.GGMLQuantizationType.F32- # else:- # data_qtype = gguf.GGMLQuantizationType.F16- # data = data_torch.squeeze().cpu().numpy()- data_qtype = gguf.GGMLQuantizationType.F32- data = data_torch.to(torch.float32).squeeze().cpu().numpy()- return data, data_qtype-- def get_qtype_by_layer_config(self, layer_config, name, data_qtype):- name = name[:-len('.weight')]- if name not in layer_config or layer_config[name]['bits'] >= 16:- if isinstance(data_qtype, bool):- return gguf.GGMLQuantizationType.F32- return data_qtype- bits = layer_config[name]['bits']- super_bits = layer_config[name]["super_bits"]- sym = layer_config[name]["sym"]- if bits == 2:- return gguf.GGMLQuantizationType.Q2_K- if bits == 3:- return gguf.GGMLQuantizationType.Q3_K- if bits == 4:- if super_bits is not None:- return gguf.GGMLQuantizationType.Q4_K- if super_bits is None and sym:- return gguf.GGMLQuantizationType.Q4_0- if super_bits is None and not sym:- return gguf.GGMLQuantizationType.Q4_1- if bits == 5:- if super_bits is not None:- return gguf.GGMLQuantizationType.Q5_K- if super_bits is None and sym:- return gguf.GGMLQuantizationType.Q5_0- if super_bits is None and not sym:- return gguf.GGMLQuantizationType.Q5_1- if bits == 6:- return gguf.GGMLQuantizationType.Q6_K- if bits == 8:- return gguf.GGMLQuantizationType.Q8_0- raise ValueError(f"Unknown file type: {data_qtype}")+ # get pooling type+ if pooling_path is not None:+ with open(self.dir_model / pooling_path / "config.json", encoding="utf-8") as f:+ pooling = json.load(f)+ if pooling["pooling_mode_mean_tokens"]:+ pooling_type = gguf.PoolingType.MEAN+ elif pooling["pooling_mode_cls_token"]:+ pooling_type = gguf.PoolingType.CLS+ elif pooling["pooling_mode_lasttoken"]:+ pooling_type = gguf.PoolingType.LAST+ else:+ raise NotImplementedError("Only MEAN, CLS, and LAST pooling types supported")+ self.gguf_writer.add_pooling_type(pooling_type)- # data_qtype = re.sub(r"Q\d", f"Q{bits}", self.ftype.name)- #- # if data_qtype == "MOSTLY_Q8_0":- # data_qtype = gguf.GGMLQuantizationType.Q8_0- # elif data_qtype == "MOSTLY_Q4_0":- # data_qtype = gguf.GGMLQuantizationType.Q4_0- # elif data_qtype == "MOSTLY_Q4_1":- # data_qtype = gguf.GGMLQuantizationType.Q4_1- # elif data_qtype == "MOSTLY_Q5_0":- # data_qtype = gguf.GGMLQuantizationType.Q5_0- # elif data_qtype == "MOSTLY_Q5_1":- # data_qtype = gguf.GGMLQuantizationType.Q5_1- # elif data_qtype.startswith("MOSTLY_Q2_K"):- # data_qtype = gguf.GGMLQuantizationType.Q2_K- # elif data_qtype.startswith("MOSTLY_Q3_K"):- # data_qtype = gguf.GGMLQuantizationType.Q3_K- # elif data_qtype.startswith("MOSTLY_Q4_K"):- # data_qtype = gguf.GGMLQuantizationType.Q4_K- # elif data_qtype.startswith("MOSTLY_Q5_K"):- # data_qtype = gguf.GGMLQuantizationType.Q5_K- # elif data_qtype.startswith("MOSTLY_Q6_K"):- # data_qtype = gguf.GGMLQuantizationType.Q6_K- # else:- # raise ValueError(f"Unknown file type: {data_qtype}")- # return data_qtype- def prepare_tensors(self):- max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")+class MmprojModel(ModelBase):+ model_type = ModelType.MMPROJ+ model_arch = gguf.MODEL_ARCH.MMPROJ+ preprocessor_config: dict[str, Any]+ global_config: dict[str, Any]- for name, data_torch in chain(self.generate_extra_tensors(), self.get_tensors()):- if data_torch is None:- continue- # we don't need these- if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):- continue- if hasattr(self,"current_packing_block") and self.current_packing_block is not None: # pylint: disable=E1101- current_packing_block_split = self.current_packing_block.split('.') # pylint: disable=E1101- name_split = name.split('.')- if (len(name_split)<len(current_packing_block_split) or- name_split[:len(current_packing_block_split)]!=current_packing_block_split):- continue+ n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]- old_dtype = data_torch.dtype+ has_vision_encoder: bool = True # by default+ has_audio_encoder: bool = False- # convert any unsupported data types to float32- if data_torch.dtype not in (torch.float16, torch.float32):- data_torch = data_torch.to(torch.float32)+ # for models having multiple encoders, we need to separate their hparams+ hparams_vision: dict[str, Any] | None = None+ hparams_audio: dict[str, Any] | None = None- # use the first number-like part of the tensor name as the block id- bid = None- for part in name.split("."):- if part.isdecimal():- bid = int(part)- break-- clean_weight_list = []- for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):- data = data_torch.squeeze().cpu().numpy()+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)- # if data ends up empty, it means data_torch was a scalar tensor -> restore- if len(data.shape) == 0:- data = data_torch.numpy()+ if self.model_arch != gguf.MODEL_ARCH.MMPROJ:+ raise TypeError("MmprojModel must be subclassed with model_arch = gguf.MODEL_ARCH.MMPROJ")- n_dims = len(data.shape)- data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)+ # get n_embd of the text model+ if "text_config" not in self.hparams:+ self.hparams["text_config"] = {}+ if "audio_config" not in self.hparams:+ self.hparams["audio_config"] = {}+ text_config = {**self.hparams, **self.hparams["text_config"]}+ self.n_embd_text = text_config.get("hidden_size", text_config.get("n_embd", 0))+ assert self.n_embd_text > 0, "n_embd not found in hparams"- # Most of the codebase that takes in 1D tensors or norms only handles F32 tensors- if n_dims <= 1 or new_name.endswith("_norm.weight"):- data_qtype = gguf.GGMLQuantizationType.F32+ # move vision config to the top level, while preserving the original hparams in global_config+ import copy+ self.global_config = copy.deepcopy(self.hparams)+ self.hparams_vision = self.get_vision_config()+ self.hparams_audio = self.get_audio_config()- # Conditions should closely match those in llama_model_quantize_internal in llama.cpp- # Some tensor types are always in float32- if data_qtype is False and (any(self.match_model_tensor_name(new_name, key, bid) for key in (- gguf.MODEL_TENSOR.FFN_GATE_INP,- gguf.MODEL_TENSOR.POS_EMBD,- gguf.MODEL_TENSOR.TOKEN_TYPES,- gguf.MODEL_TENSOR.SSM_CONV1D,- gguf.MODEL_TENSOR.TIME_MIX_FIRST,- gguf.MODEL_TENSOR.TIME_MIX_W1,- gguf.MODEL_TENSOR.TIME_MIX_W2,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,- )) or not new_name.endswith(".weight")):- data_qtype = gguf.GGMLQuantizationType.F32+ if self.hparams_vision is None and self.hparams_audio is None:+ raise ValueError("vision_config / audio_config not found in hparams")- if data_qtype is False and any(self.match_model_tensor_name(new_name, key, bid) for key in (- gguf.MODEL_TENSOR.TOKEN_EMBD,- gguf.MODEL_TENSOR.OUTPUT,- )):- if self.ftype in (- gguf.LlamaFileType.MOSTLY_TQ1_0,- gguf.LlamaFileType.MOSTLY_TQ2_0,- ):- # TODO: use Q4_K and Q6_K- data_qtype = gguf.GGMLQuantizationType.F16- # data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type+ # for compat with vision-only models+ self.hparams = self.hparams_vision or self.hparams_audio or self.hparams- # get name by new_name (for experts),- name = self.get_moe_name(name, new_name)- clean_weight_list.append(name)- # get data_qtype by layer_config- data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)+ # TODO @ngxson : this is a hack to support both vision and audio encoders+ have_multiple_encoders = self.has_audio_encoder and self.has_vision_encoder+ self.block_count = 128 if have_multiple_encoders else self.find_hparam(self.n_block_keys, True)+ self.tensor_map = gguf.get_tensor_name_map(gguf.MODEL_ARCH.MMPROJ, self.block_count)- # # No override (data_qtype is False), or wants to be quantized (data_qtype is True)- # if isinstance(data_qtype, bool):- # if self.ftype == gguf.LlamaFileType.ALL_F32:- # data_qtype = gguf.GGMLQuantizationType.F32- # elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:- # data_qtype = gguf.GGMLQuantizationType.F16- # elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:- # data_qtype = gguf.GGMLQuantizationType.BF16- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:- # data_qtype = gguf.GGMLQuantizationType.Q8_0- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:- # data_qtype = gguf.GGMLQuantizationType.Q4_0- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:- # data_qtype = gguf.GGMLQuantizationType.Q4_1- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:- # data_qtype = gguf.GGMLQuantizationType.Q5_0- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:- # data_qtype = gguf.GGMLQuantizationType.Q5_1- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:- # data_qtype = gguf.GGMLQuantizationType.Q2_K- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S:- # data_qtype = gguf.GGMLQuantizationType.Q3_K- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:- # data_qtype = gguf.GGMLQuantizationType.Q4_K- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:- # data_qtype = gguf.GGMLQuantizationType.Q4_K- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S:- # data_qtype = gguf.GGMLQuantizationType.Q5_K- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:- # data_qtype = gguf.GGMLQuantizationType.Q6_K- # else:- # raise ValueError(f"Unknown file type: {self.ftype.name}")+ # load preprocessor config+ with open(self.dir_model / "preprocessor_config.json", "r", encoding="utf-8") as f:+ self.preprocessor_config = json.load(f)- if isinstance(data_qtype, bool) or data_qtype in [gguf.GGMLQuantizationType.F16,- gguf.GGMLQuantizationType.BF16,- gguf.GGMLQuantizationType.F32]:- try:- data = gguf.quants.quantize(data, data_qtype)- except gguf.QuantError as e:- logger.warning("%s, %s", e, "falling back to F16")- data_qtype = gguf.GGMLQuantizationType.F16- data = gguf.quants.quantize(data, data_qtype)- else:- # for deepseek v2- if name.endswith("kv_b_proj.weight") and self.model_arch.name == 'DEEPSEEK2':- layer_name = name[:-len('.weight')]- module = get_module(self.model, layer_name)- n_head_kv = self.hparams["num_key_value_heads"]- v_head_dim = self.hparams["v_head_dim"]- qk_nope_head_dim = self.hparams["qk_nope_head_dim"]+ def get_vision_config(self) -> dict[str, Any] | None:+ return self.global_config.get("vision_config")- attr_list = {- "scale": None,- "zp": None,- "d_scale": None,- "wmin": None,- "d_wmin": None- }- for attr in attr_list:- if hasattr(module, attr) or hasattr(module, "w_" + attr):- if attr in ['scale', "zp"]:- attr_tensor = getattr(module, attr)- else:- attr_tensor = getattr(module, "w_" + attr)- kv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)- k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)- k_b = k_b.transpose(1, 2)+ def get_audio_config(self) -> dict[str, Any] | None:+ return self.global_config.get("audio_config")- name_kb = name.replace("kv_b_proj", "k_b_proj")- if new_name == self.map_tensor_name(name_kb):- attr_list[attr] = k_b- else:- attr_list[attr] = v_b- data = self._quant_data_with_args(data_torch, data_qtype, **attr_list)+ def set_type(self):+ self.gguf_writer.add_type(gguf.GGUFType.MMPROJ)- # for MOE model- elif len(data_torch.shape) == 3:- new_data = []- for idx, arr in enumerate(data_torch):- arr_name = name.split('.')- for i in range(len(arr_name) - 1, -1, -1):- if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):- arr_name[i] = str(idx)- arr_name = ".".join(arr_name)- arr, data_qtype = self._quant_data(arr, data_qtype, arr_name, bid)- new_data.append(arr)- data = np.array(new_data)- del new_data- else:- data, data_qtype = self._quant_data(data_torch, data_qtype, name, bid)+ def set_gguf_parameters(self):+ self.gguf_writer.add_file_type(self.ftype)- shape = gguf.quant_shape_from_byte_shape(- data.shape, data_qtype) if data.dtype == np.uint8 else data.shape+ if self.has_vision_encoder:+ self.gguf_writer.add_clip_has_vision_encoder(True)+ self.gguf_writer.add_vision_projection_dim(self.n_embd_text)- # reverse shape to make it similar to the internal ggml dimension order- shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"+ # vision config+ self.gguf_writer.add_vision_image_size(self.find_vparam(["image_size"]))+ self.gguf_writer.add_vision_patch_size(self.find_vparam(["patch_size"]))+ self.gguf_writer.add_vision_embedding_length(self.find_vparam(["hidden_size"]))+ self.gguf_writer.add_vision_feed_forward_length(self.find_vparam(["intermediate_size"]))+ self.gguf_writer.add_vision_block_count(self.find_vparam(self.n_block_keys))+ self.gguf_writer.add_vision_head_count(self.find_vparam(["num_attention_heads"]))- # n_dims is implicit in the shape- logger.info(- f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}"- f" --> {data_qtype.name}, shape = {shape_str}")+ # preprocessor config+ self.gguf_writer.add_vision_image_mean(self.preprocessor_config["image_mean"])+ self.gguf_writer.add_vision_image_std(self.preprocessor_config["image_std"])- self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)+ if self.has_audio_encoder:+ self.gguf_writer.add_clip_has_audio_encoder(True)+ self.gguf_writer.add_audio_projection_dim(self.n_embd_text)- # # save cpu memory, but slow- if self.low_cpu_mem_usage:- for weight_name in clean_weight_list:- module = get_module(self.model, ".".join(weight_name.split(".")[:-1]))- clean_module_parameter(module, weight_name.split(".")[-1])+ # audio config+ self.gguf_writer.add_audio_embedding_length(self.find_aparam(["hidden_size"]))+ self.gguf_writer.add_audio_feed_forward_length(self.find_aparam(["intermediate_size"]))+ self.gguf_writer.add_audio_block_count(self.find_aparam(self.n_block_keys))+ self.gguf_writer.add_audio_head_count(self.find_aparam(["num_attention_heads"]))++ if not self.has_vision_encoder and not self.has_audio_encoder:+ raise ValueError("MmprojModel must have either vision or audio encoder")++ def write_vocab(self):+ raise ValueError("MmprojModel does not support vocab writing")++ def find_vparam(self, keys: Iterable[str], optional: bool = False) -> Any:+ assert self.hparams_vision is not None+ return self._find_param(self.hparams_vision, keys, optional)++ def find_aparam(self, keys: Iterable[str], optional: bool = False) -> Any:+ assert self.hparams_audio is not None+ return self._find_param(self.hparams_audio, keys, optional)++ def _find_param(self, obj: dict[str, Any], keys: Iterable[str], optional: bool = False) -> Any:+ key = next((k for k in keys if k in obj), None)+ if key is not None:+ return obj[key]+ if optional:+ return None+ raise KeyError(f"could not find any of: {keys}")[email protected]("GPTNeoXForCausalLM")-class GPTNeoXModel(Model):[email protected]("GPTNeoXForCausalLM")+class GPTNeoXModel(TextModel):model_arch = gguf.MODEL_ARCH.GPTNEOXdef set_gguf_parameters(self):@@ -1523,7 +1738,7 @@ class GPTNeoXModel(Model):self.gguf_writer.add_block_count(block_count)self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])self.gguf_writer.add_rope_dimension_count(- int(self.hparams["rotary_pct"] * (self.hparams["hidden_size"] // self.hparams["num_attention_heads"])), )+ int(self.hparams["rotary_pct"] * (self.hparams["hidden_size"] // self.hparams["num_attention_heads"])),)self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True))self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"])@@ -1564,8 +1779,8 @@ class GPTNeoXModel(Model):return tensors[email protected]("BloomForCausalLM", "BloomModel")-class BloomModel(Model):[email protected]("BloomForCausalLM", "BloomModel")+class BloomModel(TextModel):model_arch = gguf.MODEL_ARCH.BLOOMdef set_gguf_parameters(self):@@ -1618,8 +1833,8 @@ class BloomModel(Model):return tensors[email protected]("MPTForCausalLM")-class MPTModel(Model):[email protected]("MPTForCausalLM")+class MPTModel(TextModel):model_arch = gguf.MODEL_ARCH.MPTdef set_vocab(self):@@ -1662,8 +1877,8 @@ class MPTModel(Model):return [(new_name, data_torch)][email protected]("OrionForCausalLM")-class OrionModel(Model):[email protected]("OrionForCausalLM")+class OrionModel(TextModel):model_arch = gguf.MODEL_ARCH.ORIONdef set_vocab(self):@@ -1695,8 +1910,8 @@ class OrionModel(Model):self.gguf_writer.add_layer_norm_eps(self.hparams["rms_norm_eps"])[email protected]("BaichuanForCausalLM", "BaiChuanForCausalLM")-class BaichuanModel(Model):[email protected]("BaichuanForCausalLM", "BaiChuanForCausalLM")+class BaichuanModel(TextModel):model_arch = gguf.MODEL_ARCH.BAICHUANdef set_vocab(self):@@ -1728,10 +1943,10 @@ class BaichuanModel(Model):self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_file_type(self.ftype)- if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:- if self.hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:head_count = self.hparams["num_attention_heads"]@@ -1764,22 +1979,22 @@ class BaichuanModel(Model):*weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))def _reverse_hf_permute_part(- self,- weights: Tensor,- n_part: int,- n_head: int,- n_head_kv: int | None = None,+ self,+ weights: Tensor,+ n_part: int,+ n_head: int,+ n_head_kv: int | None = None,) -> Tensor:r = weights.shape[0] // 3- return self._reverse_hf_permute(weights[r * n_part:r * n_part + r, ...], n_head, n_head_kv)+ return self._reverse_hf_permute(weights[r * n_part:r*n_part + r, ...], n_head, n_head_kv)def _reverse_hf_part(self, weights: Tensor, n_part: int) -> Tensor:r = weights.shape[0] // 3- return weights[r * n_part:r * n_part + r, ...]+ return weights[r * n_part:r*n_part + r, ...][email protected]("XverseForCausalLM")-class XverseModel(Model):[email protected]("XverseForCausalLM")+class XverseModel(TextModel):model_arch = gguf.MODEL_ARCH.XVERSEdef set_vocab(self):@@ -1855,10 +2070,10 @@ class XverseModel(Model):self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_file_type(self.ftype)- if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:- if self.hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused@@ -1883,8 +2098,8 @@ class XverseModel(Model):*weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))[email protected]("FalconForCausalLM", "RWForCausalLM")-class FalconModel(Model):[email protected]("FalconForCausalLM", "RWForCausalLM")+class FalconModel(TextModel):model_arch = gguf.MODEL_ARCH.FALCONdef set_gguf_parameters(self):@@ -1928,7 +2143,7 @@ class FalconModel(Model):n_head_kv = self.find_hparam(["num_kv_heads", "n_head_kv"], optional=True) or 1head_dim = self.hparams["hidden_size"] // n_head- qkv = data_torch.view(n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)+ qkv = data_torch.view(n_head_kv, n_head//n_head_kv + 2, head_dim, head_dim * n_head)q = qkv[:, :-2].reshape(n_head * head_dim, head_dim * n_head)k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)@@ -1937,8 +2152,8 @@ class FalconModel(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("GPTBigCodeForCausalLM")-class StarCoderModel(Model):[email protected]("GPTBigCodeForCausalLM")+class StarCoderModel(TextModel):model_arch = gguf.MODEL_ARCH.STARCODERdef set_gguf_parameters(self):@@ -1954,8 +2169,8 @@ class StarCoderModel(Model):self.gguf_writer.add_file_type(self.ftype)[email protected]("GPTRefactForCausalLM")-class RefactModel(Model):[email protected]("GPTRefactForCausalLM")+class RefactModel(TextModel):model_arch = gguf.MODEL_ARCH.REFACTdef set_vocab(self):@@ -1975,7 +2190,7 @@ class RefactModel(Model):inner_dim = 4 * hidden_dimhidden_dim = int(2 * inner_dim / 3)multiple_of = 256- ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)+ ff_dim = multiple_of * ((hidden_dim+multiple_of-1) // multiple_of)block_count = self.hparams["n_layer"]@@ -1995,7 +2210,7 @@ class RefactModel(Model):inner_dim = 4 * hidden_dimhidden_dim = int(2 * inner_dim / 3)multiple_of = 256- ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)+ ff_dim = multiple_of * ((hidden_dim+multiple_of-1) // multiple_of)n_head = self.hparams["n_head"]n_head_kv = 1head_dim = self.hparams["n_embd"] // n_head@@ -2020,8 +2235,8 @@ class RefactModel(Model):return tensors[email protected]("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM")-class StableLMModel(Model):[email protected]("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM")+class StableLMModel(TextModel):model_arch = gguf.MODEL_ARCH.STABLELMdef set_vocab(self):@@ -2109,11 +2324,19 @@ class StableLMModel(Model):raise ValueError(f"Unprocessed norms: {norms}")[email protected]("LLaMAForCausalLM", "LlamaForCausalLM", "MistralForCausalLM", "MixtralForCausalLM")-class LlamaModel(Model):+ "LLaMAForCausalLM", "LlamaForCausalLM", "MistralForCausalLM", "MixtralForCausalLM", "VLlama3ForCausalLM",+ "LlavaForConditionalGeneration", "LlamaModel")+class LlamaModel(TextModel):model_arch = gguf.MODEL_ARCH.LLAMAundo_permute = True+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ # fix for SmolVLM2, missing `num_attention_heads` in config.json+ if self.hf_arch == "VLlama3ForCausalLM":+ self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 32)+def set_vocab(self):try:self._set_vocab_sentencepiece()@@ -2150,16 +2373,14 @@ class LlamaModel(Model):hparams = self.hparamsself.gguf_writer.add_vocab_size(hparams["vocab_size"])- if "head_dim" in hparams:- rope_dim = hparams["head_dim"]- else:+ if (rope_dim := hparams.get("head_dim")) is None:rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]self.gguf_writer.add_rope_dimension_count(rope_dim)- if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:- if self.hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])@staticmethoddef permute(weights: Tensor, n_head: int, n_head_kv: int | None):@@ -2174,6 +2395,19 @@ class LlamaModel(Model):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]n_kv_head = self.hparams.get("num_key_value_heads")+ is_vision_tensor = "vision_tower" in name \+ or "vision_model" in name \+ or "model.connector" in name \+ or "multi_modal_projector" in name++ if is_vision_tensor:+ return [] # skip vision tensors+ elif self.hf_arch == "LlamaModel":+ name = "model." + name+ elif name.startswith("model.text_model"):+ name = name.replace("text_model.", "") # for SmolVLM+ elif name.startswith("language_model."):+ name = name.replace("language_model.", "") # for the restif self.undo_permute:if name.endswith(("q_proj.weight", "q_proj.bias")):@@ -2221,8 +2455,9 @@ class LlamaModel(Model):if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):if rope_scaling.get("rope_type", '').lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)- dim = self.hparams.get("head_dim", self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ if (dim := self.hparams.get("head_dim")) is None:+ dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]+ freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)@@ -2241,47 +2476,153 @@ class LlamaModel(Model):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)- rope_factors.append(1 / ((1 - smooth) / factor + smooth))+ smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)+ rope_factors.append(1 / ((1-smooth) / factor + smooth))++ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),+ torch.tensor(rope_factors, dtype=torch.float32))++ def prepare_tensors(self):+ super().prepare_tensors()++ if self._experts is not None:+ # flatten `list[dict[str, Tensor]]` into `list[str]`+ experts = [k for d in self._experts for k in d.keys()]+ if len(experts) > 0:+ raise ValueError(f"Unprocessed experts: {experts}")++[email protected]("ArceeForCausalLM")+class ArceeModel(LlamaModel):+ model_arch = gguf.MODEL_ARCH.ARCEE++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self._try_set_pooling_type()+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])+ self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])+++ "LlavaForConditionalGeneration", # pixtral+ "Mistral3ForConditionalGeneration", # mistral small 3.1+)+class LlavaVisionModel(MmprojModel):+ img_break_tok_id = -1++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ if self.hparams["model_type"] == "pixtral":+ # layer_norm_eps is not in config.json, it is hard-coded in modeling_pixtral.py+ self.hparams["layer_norm_eps"] = self.hparams.get("layer_norm_eps", 1e-5)+ self.img_break_tok_id = self.get_token_id("[IMG_BREAK]")+ logger.info(f"Image break token id: {self.img_break_tok_id}")+ else:+ raise ValueError(f"Unsupported model type: {self.hparams['model_type']}")++ def get_token_id(self, token: str) -> int:+ tokenizer_config_file = self.dir_model / 'tokenizer_config.json'+ with open(tokenizer_config_file, "r", encoding="utf-8") as f:+ added_tokens_decoder = json.load(f)['added_tokens_decoder']+ for id_, token_data in added_tokens_decoder.items():+ if token_data["content"] == token:+ return int(id_)+ raise ValueError(f"Token '{token}' not found in tokenizer config.")++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ hparams = self.hparams+ if hparams["model_type"] == "pixtral":+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.PIXTRAL)+ self.gguf_writer.add_vision_attention_layernorm_eps(hparams["layer_norm_eps"])++ # hidden_act+ if hparams["hidden_act"] == "silu":+ self.gguf_writer.add_vision_use_silu(True)+ elif hparams["hidden_act"] == "gelu":+ self.gguf_writer.add_vision_use_gelu(True)+ else:+ raise ValueError(f"Unsupported hidden_act: {hparams['hidden_act']}")++ # spatial_merge_size+ if "spatial_merge_size" in self.global_config:+ self.gguf_writer.add_vision_spatial_merge_size(self.global_config["spatial_merge_size"])++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused+ n_head = self.hparams["num_attention_heads"]+ n_kv_head = n_head++ if name.startswith("multi_modal_projector.") or name.startswith("vision_tower."):+ # process vision tensors+ if name.endswith(("q_proj.weight", "q_proj.bias")):+ data_torch = LlamaModel.permute(data_torch, n_head, n_head)+ if name.endswith(("k_proj.weight", "k_proj.bias")):+ data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)+ return [(self.map_tensor_name(name), data_torch)]++ if self.img_break_tok_id > 0 and "embed_tokens.weight" in name:+ logger.info(f"Extracting [IMG_BREAK] token embedding from {name}")+ # for pixtral model, we need to extract the [IMG_BREAK] token embedding+ img_break_embd = data_torch[self.img_break_tok_id]+ name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK]+ return [(self.map_tensor_name(name), img_break_embd)]++ return [] # skip other tensors++[email protected]("Idefics3ForConditionalGeneration", "SmolVLMForConditionalGeneration")+class SmolVLMModel(MmprojModel):++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ if self.hparams["model_type"] == "smolvlm_vision":+ # fix for SmolVLM2, missing some keys in config.json+ # default values are taken from transformers code+ self.hparams["hidden_size"] = self.hparams.get("hidden_size", 1152)+ self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 16)+ self.hparams["intermediate_size"] = self.hparams.get("intermediate_size", 3072)- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32))+ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.IDEFICS3)+ self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))+ self.gguf_writer.add_vision_projector_scale_factor(self.global_config.get("scale_factor", 2))+ self.gguf_writer.add_vision_use_gelu(True)++ def tensor_force_quant(self, name, new_name, bid, n_dims):+ del bid, new_name, n_dims # unused+ if ".embeddings." in name:+ return gguf.GGMLQuantizationType.F32+ return False- def prepare_tensors(self):- super().prepare_tensors()+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused+ is_vision_tensor = "vision_tower" in name or "vision_model" in name or "model.connector" in name- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")+ if is_vision_tensor:+ return [(self.map_tensor_name(name), data_torch)]++ return [] # skip other tensors[email protected]("Llama4ForConditionalGeneration")[email protected]("Llama4ForConditionalGeneration")class Llama4Model(LlamaModel):model_arch = gguf.MODEL_ARCH.LLAMA4- has_vision: bool = Falseundo_permute = False- # TODO @ngxson : avoid duplicate this code everywhere by at least support "text_config"- # same with llama, but we need to merge the text_config into the root level of hparamsdef __init__(self, *args, **kwargs):- hparams = kwargs["hparams"] if "hparams" in kwargs else Model.load_hparams(args[0])- if "text_config" in hparams:- hparams = {**hparams, **hparams["text_config"]}- kwargs["hparams"] = hparamssuper().__init__(*args, **kwargs)- if "vision_config" in hparams:- logger.info("Has vision encoder, but it will be ignored")- self.has_vision = True# IMPORTANT: the normal "intermediate_size" is renamed to "intermediate_size_mlp", we need to undo thisself.hparams["intermediate_size_moe"] = self.hparams["intermediate_size"]self.hparams["intermediate_size"] = self.hparams["intermediate_size_mlp"]def set_vocab(self):self._set_vocab_gpt2()- self.gguf_writer.add_add_bos_token(True)def set_gguf_parameters(self):super().set_gguf_parameters()@@ -2289,6 +2630,9 @@ class Llama4Model(LlamaModel):self.gguf_writer.add_expert_feed_forward_length(self.hparams["intermediate_size_moe"])def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):+ if name.startswith("language_model."):+ name = name.replace("language_model.", "")+# split the gate_up into gate and upif "gate_up_proj" in name:name_up = name.replace("gate_up_proj", "up_proj.weight")@@ -2307,18 +2651,34 @@ class Llama4Model(LlamaModel):return super().modify_tensors(data_torch, name, bid)[email protected]("Mistral3ForConditionalGeneration")[email protected]("Llama4ForConditionalGeneration")+class Llama4VisionModel(MmprojModel):++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.LLAMA4)+ self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams["norm_eps"])+ self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / self.hparams["pixel_shuffle_ratio"]))+ assert self.hparams["hidden_act"] == "gelu"+ self.gguf_writer.add_vision_use_gelu(True)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused+ if "multi_modal_projector" in name or "vision_model" in name:+ # process vision tensors+ if "positional_embedding_vlm" in name and ".weight" not in name:+ name += ".weight"+ if "multi_modal_projector.linear_1" in name:+ # despite the name with number postfix, this is a single fully connected layer+ return [(gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_MMPROJ_FC] + '.weight', data_torch)]+ return [(self.map_tensor_name(name), data_torch)]+ return []++[email protected]("Mistral3ForConditionalGeneration")class Mistral3Model(LlamaModel):model_arch = gguf.MODEL_ARCH.LLAMA- # we need to merge the text_config into the root level of hparams- def __init__(self, *args, **kwargs):- hparams = kwargs["hparams"] if "hparams" in kwargs else Model.load_hparams(args[0])- if "text_config" in hparams:- hparams = {**hparams, **hparams["text_config"]}- kwargs["hparams"] = hparams- super().__init__(*args, **kwargs)-def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):name = name.replace("language_model.", "")if "multi_modal_projector" in name or "vision_tower" in name:@@ -2326,8 +2686,8 @@ class Mistral3Model(LlamaModel):return super().modify_tensors(data_torch, name, bid)[email protected]("DeciLMForCausalLM")-class DeciModel(Model):[email protected]("DeciLMForCausalLM")+class DeciModel(TextModel):model_arch = gguf.MODEL_ARCH.DECI@staticmethod@@ -2341,7 +2701,7 @@ class DeciModel(Model):# DeciLM-specific codeif n % k == 0:return n- return n + k - (n % k)+ return n + k - (n%k)def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)@@ -2362,6 +2722,9 @@ class DeciModel(Model):# if n_heads_in_group is not None, then# _num_kv_heads[il] is num_attention_head // n_heads_in_group and# _num_heads[il] is num_attention_head+ # ***dummy layer*** for nemotron 253B+ # if n_heads_in_group is None and ffn_mult is None+ # then _num_kv_heads[il] is 0 and _num_heads[il] is 0 and _ffn_dims is 0for il in range(len(_block_configs)):if _block_configs[il]["attention"]["n_heads_in_group"] is None:if _block_configs[il]["attention"]["replace_with_linear"] is True:@@ -2374,7 +2737,10 @@ class DeciModel(Model):self._num_kv_heads.append(self.hparams["num_attention_heads"] // _block_configs[il]["attention"]["n_heads_in_group"])self._num_heads.append(self.hparams["num_attention_heads"])- _ffn_multipliers.append(_block_configs[il]["ffn"]["ffn_mult"])+ if _block_configs[il]["ffn"]["ffn_mult"] is None: # dummy layer+ _ffn_multipliers.append(0.0)+ else:+ _ffn_multipliers.append(_block_configs[il]["ffn"]["ffn_mult"])assert self.block_count == len(self._num_kv_heads)assert self.block_count == len(self._num_heads)assert self.block_count == len(_ffn_multipliers)@@ -2428,16 +2794,14 @@ class DeciModel(Model):hparams = self.hparamsself.gguf_writer.add_vocab_size(hparams["vocab_size"])- if "head_dim" in hparams:- rope_dim = hparams["head_dim"]- else:+ if (rope_dim := hparams.get("head_dim")) is None:rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]self.gguf_writer.add_rope_dimension_count(rope_dim)- if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:- if self.hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])@staticmethoddef permute(weights: Tensor, n_head: int, n_head_kv: int | None):@@ -2470,8 +2834,9 @@ class DeciModel(Model):if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):if rope_scaling.get("rope_type", '').lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)- dim = self.hparams.get("head_dim", self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ if (dim := self.hparams.get("head_dim")) is None:+ dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]+ freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)@@ -2490,8 +2855,8 @@ class DeciModel(Model):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)- rope_factors.append(1 / ((1 - smooth) / factor + smooth))+ smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)+ rope_factors.append(1 / ((1-smooth) / factor + smooth))yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),@@ -2501,8 +2866,8 @@ class DeciModel(Model):super().prepare_tensors()[email protected]("BitnetForCausalLM")-class BitnetModel(Model):[email protected]("BitnetForCausalLM")+class BitnetModel(TextModel):model_arch = gguf.MODEL_ARCH.BITNETdef set_vocab(self):@@ -2525,13 +2890,13 @@ class BitnetModel(Model):new_name = self.map_tensor_name(name)if any(self.match_model_tensor_name(new_name, key, bid) for key in [- gguf.MODEL_TENSOR.ATTN_Q,- gguf.MODEL_TENSOR.ATTN_K,- gguf.MODEL_TENSOR.ATTN_V,- gguf.MODEL_TENSOR.ATTN_OUT,- gguf.MODEL_TENSOR.FFN_UP,- gguf.MODEL_TENSOR.FFN_DOWN,- gguf.MODEL_TENSOR.FFN_GATE,+ gguf.MODEL_TENSOR.ATTN_Q,+ gguf.MODEL_TENSOR.ATTN_K,+ gguf.MODEL_TENSOR.ATTN_V,+ gguf.MODEL_TENSOR.ATTN_OUT,+ gguf.MODEL_TENSOR.FFN_UP,+ gguf.MODEL_TENSOR.FFN_DOWN,+ gguf.MODEL_TENSOR.FFN_GATE,]):# transform weight into 1/0/-1 (in fp32)data_torch = self.weight_quant(data_torch)@@ -2539,8 +2904,8 @@ class BitnetModel(Model):yield (new_name, data_torch)[email protected]("GrokForCausalLM")-class GrokModel(Model):[email protected]("GrokForCausalLM")+class GrokModel(TextModel):model_arch = gguf.MODEL_ARCH.GROKdef set_vocab(self):@@ -2592,8 +2957,8 @@ class GrokModel(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("DbrxForCausalLM")-class DbrxModel(Model):[email protected]("DbrxForCausalLM")+class DbrxModel(TextModel):model_arch = gguf.MODEL_ARCH.DBRXdef set_gguf_parameters(self):@@ -2632,7 +2997,11 @@ class DbrxModel(Model):# But llama.cpp moe graph works differently# AND the dimensions in ggml are typically in the reverse order of the pytorch dimensions# so (n_expert, n_ff, n_embd) in pytorch is {n_embd, n_ff, n_expert} in ggml_tensor- exp_tensor_names = {"ffn.experts.mlp.w1": None, "ffn.experts.mlp.w2": (0, 2, 1), "ffn.experts.mlp.v1": None}+ exp_tensor_names = {+ "ffn.experts.mlp.w1": None, # LLM_TENSOR_FFN_GATE_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}+ "ffn.experts.mlp.w2": (0, 2, 1), # LLM_TENSOR_FFN_DOWN_EXPS ggml_tensor->ne{n_ff, n_embd, n_expert}+ "ffn.experts.mlp.v1": None+ } # LLM_TENSOR_FFN_UP_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}experts = Falsefor exp_tensor_name in exp_tensor_names.keys():@@ -2653,15 +3022,15 @@ class DbrxModel(Model):return [(new_name, data_torch)]- def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) \- -> gguf.GGMLQuantizationType | bool:+ def tensor_force_quant(+ self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:del name, new_name, bid # unusedreturn n_dims > 1[email protected]("MiniCPMForCausalLM")-class MiniCPMModel(Model):[email protected]("MiniCPMForCausalLM")+class MiniCPMModel(TextModel):model_arch = gguf.MODEL_ARCH.MINICPMdef set_gguf_parameters(self):@@ -2669,16 +3038,16 @@ class MiniCPMModel(Model):embedding_scale = float(self.hparams["scale_emb"])self.gguf_writer.add_embedding_scale(embedding_scale)logger.info(f"gguf: (minicpm) embedding_scale = {embedding_scale}")- residual_scale = self.hparams["scale_depth"] / self.hparams["num_hidden_layers"] ** 0.5+ residual_scale = self.hparams["scale_depth"] / self.hparams["num_hidden_layers"]**0.5self.gguf_writer.add_residual_scale(residual_scale)logger.info(f"gguf: (minicpm) residual_scale = {residual_scale}")logit_scale = self.hparams["hidden_size"] / self.hparams["dim_model_base"]self.gguf_writer.add_logit_scale(logit_scale)logger.info(f"gguf: (minicpm) logit_scale = {logit_scale}")- if self.hparams.get("rope_scaling") is not None:- if self.hparams["rope_scaling"].get("type") == "longrope":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LONGROPE)- logger.info(f"gguf: (minicpm) rope_scaling_type = {gguf.RopeScalingType.LONGROPE}")+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "longrope":+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LONGROPE)+ logger.info(f"gguf: (minicpm) rope_scaling_type = {gguf.RopeScalingType.LONGROPE}")def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:rope_dims = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]@@ -2719,8 +3088,8 @@ class MiniCPMModel(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("MiniCPM3ForCausalLM")-class MiniCPM3Model(Model):[email protected]("MiniCPM3ForCausalLM")+class MiniCPM3Model(TextModel):model_arch = gguf.MODEL_ARCH.MINICPM3def set_gguf_parameters(self):@@ -2774,8 +3143,8 @@ class MiniCPM3Model(Model):*weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))[email protected]("QWenLMHeadModel")-class QwenModel(Model):[email protected]("QWenLMHeadModel")+class QwenModel(TextModel):model_arch = gguf.MODEL_ARCH.QWEN@staticmethod@@ -2816,8 +3185,8 @@ class QwenModel(Model):self.gguf_writer.add_file_type(self.ftype)[email protected]("Qwen2ForCausalLM")-class Qwen2Model(Model):[email protected]("Qwen2Model", "Qwen2ForCausalLM", "Qwen2AudioForConditionalGeneration")+class Qwen2Model(TextModel):model_arch = gguf.MODEL_ARCH.QWEN2def set_vocab(self):@@ -2828,16 +3197,74 @@ class Qwen2Model(Model):def set_gguf_parameters(self):super().set_gguf_parameters()- if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:- if self.hparams["rope_scaling"].get("type") == "yarn":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])- self.gguf_writer.add_rope_scaling_orig_ctx_len(- self.hparams["rope_scaling"]["original_max_position_embeddings"])+ self._try_set_pooling_type()+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])+ self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if self.hf_arch == "Qwen2Model":+ name = f"model.{name}" # map to Qwen2ForCausalLM tensors+ if "language_model." in name:+ name = name.replace("language_model.", "") # for InternVL+ if name.startswith("mlp") or name.startswith("multi_modal_projector") \+ or name.startswith("vision_model") or name.startswith("audio_tower"):+ # skip vision and audio tensors+ return []+ yield from super().modify_tensors(data_torch, name, bid)++[email protected]("Ernie4_5_ForCausalLM")+class Ernie4_5Model(TextModel):+ model_arch = gguf.MODEL_ARCH.ERNIE4_5++ def set_vocab(self):+ self._set_vocab_sentencepiece()++ def set_gguf_parameters(self):+ super().set_gguf_parameters()++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ num_heads = self.hparams["num_attention_heads"]+ num_kv_heads = self.hparams["num_key_value_heads"]+ head_dim = self.hparams["head_dim"]++ if "ernie." in name:+ name = name.replace("ernie.", "model.")+ # split the qkv weights+ # qkv_proj shape: [(num_heads + 2 * num_kv_heads) * head_dim, hidden_size]+ if "qkv_proj" in name:+ name_q = name.replace("qkv_proj.weight", "q_proj.weight")+ name_k = name.replace("qkv_proj.weight", "k_proj.weight")+ name_v = name.replace("qkv_proj.weight", "v_proj.weight")+ total_q_dim = num_heads * head_dim+ total_k_dim = num_kv_heads * head_dim+ total_v_dim = num_kv_heads * head_dim+ q_proj_weight, k_proj_weight, v_proj_weight = data_torch.split([total_q_dim, total_k_dim, total_v_dim],+ dim=0)+ return [(self.map_tensor_name(name_q), q_proj_weight), (self.map_tensor_name(name_k), k_proj_weight),+ (self.map_tensor_name(name_v), v_proj_weight)]+ # split the up_gate_proj into gate and up+ # up_gate_proj shape: [2 * intermediate_size, hidden_size]+ if "up_gate_proj" in name:+ name_up = name.replace("up_gate_proj.weight", "up_proj.weight")+ name_gate = name.replace("up_gate_proj.weight", "gate_proj.weight")+ dim_half = data_torch.shape[0] // 2+ gate_proj_weight, up_proj_weight = data_torch.split(dim_half, dim=0)+ return [(self.map_tensor_name(name_gate), gate_proj_weight),+ (self.map_tensor_name(name_up), up_proj_weight)]+ return [(self.map_tensor_name(name), data_torch)][email protected]("Qwen2VLForConditionalGeneration", "Qwen2_5_VLForConditionalGeneration")-class Qwen2VLModel(Model):+ "Qwen2VLModel",+ "Qwen2VLForConditionalGeneration",+ "Qwen2_5_VLForConditionalGeneration",+ "Qwen2_5OmniModel",+)+class Qwen2VLModel(TextModel):model_arch = gguf.MODEL_ARCH.QWEN2VLdef set_gguf_parameters(self):@@ -2852,15 +3279,219 @@ class Qwen2VLModel(Model):except FileNotFoundError:self._set_vocab_gpt2()- def get_tensors(self) -> Iterator[tuple[str, Tensor]]:- for name, data in super().get_tensors():- if name.startswith("visual."):- continue- yield name, data+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused+ if name.startswith("thinker."):+ name = name.replace("thinker.", "")+ if name.startswith("visual") or name.startswith("audio") or \+ name.startswith("talker") or name.startswith("token2wav"):+ # skip multimodal tensors+ return []+ return [(self.map_tensor_name(name), data_torch)]++[email protected]("Qwen2VLModel", "Qwen2VLForConditionalGeneration", "Qwen2_5_VLForConditionalGeneration")+class Qwen2VLVisionModel(MmprojModel):++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ assert self.hparams_vision is not None+ self.hparams_vision["image_size"] = self.hparams_vision.get("image_size", 560)+ # rename config.json values+ self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_heads")+ self.hparams_vision["num_hidden_layers"] = self.hparams_vision.get("depth")+ if "embed_dim" in self.hparams_vision: # qwen2vl+ self.hparams_vision["intermediate_size"] = self.hparams_vision.get("hidden_size")+ self.hparams_vision["hidden_size"] = self.hparams_vision.get("embed_dim")++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ assert self.hparams_vision is not None+ hparams = self.hparams_vision+ model_type = self.global_config['model_type']+ if model_type == 'qwen2_vl':+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN2VL)+ elif model_type == 'qwen2_5_vl' or model_type == 'qwen2_5_omni':+ if model_type == 'qwen2_5_omni':+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN25O)+ else:+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN25VL)+ self.gguf_writer.add_vision_use_silu(True)+ # find n_wa_pattern (window attention pattern)+ fullatt_block_indexes = hparams.get("fullatt_block_indexes")+ assert fullatt_block_indexes is not None, "fullatt_block_indexes is required for qwen2_5_vl"+ n_wa_pattern = fullatt_block_indexes[0] + 1+ # validate n_wa_pattern+ for i in range(1, len(fullatt_block_indexes)):+ if fullatt_block_indexes[i] - fullatt_block_indexes[i - 1] != n_wa_pattern:+ raise ValueError(f"Invalid fullatt_block_indexes: {fullatt_block_indexes}")+ self.gguf_writer.add_vision_n_wa_pattern(n_wa_pattern)+ else:+ raise ValueError(f"Unknown QwenVL model type: {self.global_config['model_type']}")+ # default values below are taken from HF transformers code+ self.gguf_writer.add_vision_attention_layernorm_eps(self.global_config.get("rms_norm_eps", 1e-6))++ def tensor_force_quant(self, name, new_name, bid, n_dims):+ del bid, name, n_dims # unused+ if ".patch_embd." in new_name:+ return gguf.GGMLQuantizationType.F16+ if ".position_embd." in new_name:+ return gguf.GGMLQuantizationType.F32+ return False++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused+ if name.startswith("visual."):+ # process visual tensors+ # split QKV tensors if needed+ if ".qkv." in name:+ if data_torch.ndim == 2: # weight+ c3, _ = data_torch.shape+ else: # bias+ c3 = data_torch.shape[0]+ assert c3 % 3 == 0+ c = c3 // 3+ wq = data_torch[:c]+ wk = data_torch[c:c * 2]+ wv = data_torch[c * 2:]+ return [+ (self.map_tensor_name(name.replace("qkv", "q")), wq),+ (self.map_tensor_name(name.replace("qkv", "k")), wk),+ (self.map_tensor_name(name.replace("qkv", "v")), wv),+ ]+ elif 'patch_embed.proj.weight' in name:+ # split Conv3D into Conv2Ds+ c1, c2, kt, kh, kw = data_torch.shape+ del c1, c2, kh, kw # unused+ assert kt == 2, "Current implementation only support temporal_patch_size of 2"+ return [+ (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight", data_torch[:, :, 0, ...]),+ (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch[:, :, 1, ...]),+ ]+ else:+ return [(self.map_tensor_name(name), data_torch)]+ return [] # skip other tensors++[email protected]("Qwen2_5OmniModel")+class Qwen25OmniModel(Qwen2VLVisionModel):+ has_vision_encoder = True+ has_audio_encoder = True++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ assert self.hparams_audio is not None+ self.hparams_audio["hidden_size"] = self.hparams_audio["d_model"]+ self.hparams_audio["intermediate_size"] = self.hparams_audio["encoder_ffn_dim"]+ self.hparams_audio["num_attention_heads"] = self.hparams_audio["encoder_attention_heads"]++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ assert self.hparams_audio is not None+ self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["num_mel_bins"])+ self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio.get("layer_norm_eps", 1e-5))++ def get_vision_config(self) -> dict[str, Any] | None:+ return self.global_config["thinker_config"].get("vision_config")++ def get_audio_config(self) -> dict[str, Any] | None:+ return self.global_config["thinker_config"].get("audio_config")++ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:+ # SinusoidsPositionEmbedding+ assert self.hparams_audio is not None+ max_timescale = 10000+ length = 1500+ channels = self.hparams_audio["hidden_size"]+ log_timescale_increment = np.log(max_timescale) / (channels//2 - 1)+ inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2).float())+ scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]+ pos_embd = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1).to(dtype=torch.float32)+ yield ("audio_tower.embed_positions.weight", pos_embd)++ def tensor_force_quant(self, name, new_name, bid, n_dims):+ del bid, new_name, n_dims # unused+ if ".conv" in name and ".weight" in name:+ return gguf.GGMLQuantizationType.F16+ return False++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if name.startswith("thinker."):+ name = name.replace("thinker.", "")++ if name.startswith("audio_tower"):+ # process audio tensors+ if "conv1.bias" in name or "conv2.bias" in name:+ # transpose conv1 and conv2 bias+ data_torch = data_torch.unsqueeze(-1)+ if "audio_bos_eos_token" in name:+ # this tensor is left unused in transformers code+ # https://github.com/huggingface/transformers/blob/6e3063422c4b1c014aa60c32b9254fd2902f0f28/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py#L1809+ return []+ return [(self.map_tensor_name(name), data_torch)]++ return super().modify_tensors(data_torch, name, bid)++[email protected]("InternVisionModel")+class InternVisionModel(MmprojModel):++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ hparams = self.hparams+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.INTERNVL)+ self.gguf_writer.add_vision_attention_layernorm_eps(hparams["layer_norm_eps"])+ # hidden_act+ if hparams["hidden_act"] == "silu":+ self.gguf_writer.add_vision_use_silu(True)+ elif hparams["hidden_act"] == "gelu":+ self.gguf_writer.add_vision_use_gelu(True)+ else:+ raise ValueError(f"Unsupported hidden_act: {hparams['hidden_act']}")+ # downsample_ratio+ downsample_ratio = self.global_config.get("downsample_ratio")+ assert downsample_ratio is not None+ self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / downsample_ratio))++ def tensor_force_quant(self, name, new_name, bid, n_dims):+ del bid, name, n_dims # unused+ if ".patch_embd." in new_name:+ return gguf.GGMLQuantizationType.F16+ if ".position_embd." in new_name:+ return gguf.GGMLQuantizationType.F32+ return False++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused+ if name.startswith("vision_model") or name.startswith("mlp"):+ # process visual tensors+ # correct name+ if name.startswith("vision_model"):+ name = "vision_tower." + name+ if (".ls" in name or "position_embedding" in name) and not name.endswith(".weight"):+ name += ".weight"+ # split QKV tensors if needed+ if ".qkv." in name:+ if data_torch.ndim == 2: # weight+ c3, _ = data_torch.shape+ else: # bias+ c3 = data_torch.shape[0]+ assert c3 % 3 == 0+ c = c3 // 3+ wq = data_torch[:c]+ wk = data_torch[c:c * 2]+ wv = data_torch[c * 2:]+ return [+ (self.map_tensor_name(name.replace("attn.qkv", "self_attn.q_proj")), wq),+ (self.map_tensor_name(name.replace("attn.qkv", "self_attn.k_proj")), wk),+ (self.map_tensor_name(name.replace("attn.qkv", "self_attn.v_proj")), wv),+ ]+ return [(self.map_tensor_name(name), data_torch)]+ return [] # skip other tensors[email protected]("WavTokenizerDec")-class WavTokenizerDecModel(Model):[email protected]("WavTokenizerDec")+class WavTokenizerDecModel(TextModel):model_arch = gguf.MODEL_ARCH.WAVTOKENIZER_DECdef modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:@@ -2868,8 +3499,8 @@ class WavTokenizerDecModel(Model):if \name.endswith("codebook.cluster_size") or \- name.endswith("codebook.embed_avg") or \- name.endswith("codebook.inited"):+ name.endswith("codebook.embed_avg") or \+ name.endswith("codebook.inited"):logger.debug(f"Skipping {name!r}")return []@@ -2897,8 +3528,8 @@ class WavTokenizerDecModel(Model):self.gguf_writer.add_causal_attention(False)[email protected]("Qwen2MoeForCausalLM")-class Qwen2MoeModel(Model):[email protected]("Qwen2MoeForCausalLM")+class Qwen2MoeModel(TextModel):model_arch = gguf.MODEL_ARCH.QWEN2MOEdef set_gguf_parameters(self):@@ -2911,6 +3542,13 @@ class Qwen2MoeModel(Model):if (shared_expert_intermediate_size := self.hparams.get('shared_expert_intermediate_size')) is not None:self.gguf_writer.add_expert_shared_feed_forward_length(shared_expert_intermediate_size)logger.info(f"gguf: expert shared feed forward length = {shared_expert_intermediate_size}")+ # YaRN is not enabled by default+ # To enable it, please refer to this guide: https://huggingface.co/Qwen/Qwen3-30B-A3B#processing-long-texts+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])+ self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])_experts: list[dict[str, Tensor]] | None = None@@ -2960,18 +3598,18 @@ class Qwen2MoeModel(Model):raise ValueError(f"Unprocessed experts: {experts}")[email protected]("Qwen3ForCausalLM")[email protected]("Qwen3ForCausalLM")class Qwen3Model(Qwen2Model):model_arch = gguf.MODEL_ARCH.QWEN3[email protected]("Qwen3MoeForCausalLM")[email protected]("Qwen3MoeForCausalLM")class Qwen3MoeModel(Qwen2MoeModel):model_arch = gguf.MODEL_ARCH.QWEN3MOE[email protected]("GPT2LMHeadModel")-class GPT2Model(Model):[email protected]("GPT2LMHeadModel")+class GPT2Model(TextModel):model_arch = gguf.MODEL_ARCH.GPT2def set_gguf_parameters(self):@@ -3002,8 +3640,8 @@ class GPT2Model(Model):return tensors[email protected]("PhiForCausalLM")-class Phi2Model(Model):[email protected]("PhiForCausalLM")+class Phi2Model(TextModel):model_arch = gguf.MODEL_ARCH.PHI2def set_gguf_parameters(self):@@ -3026,8 +3664,8 @@ class Phi2Model(Model):self.gguf_writer.add_add_bos_token(False)[email protected]("Phi3ForCausalLM")-class Phi3MiniModel(Model):[email protected]("Phi3ForCausalLM")+class Phi3MiniModel(TextModel):model_arch = gguf.MODEL_ARCH.PHI3def set_vocab(self):@@ -3102,8 +3740,9 @@ class Phi3MiniModel(Model):if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:if tokens[token_id] != token:logger.warning(- f"replacing token {token_id}: {tokens[token_id].decode('utf-8')!r}"- f" -> {token.decode('utf-8')!r}")+ f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '+ f'-> {token.decode("utf-8")!r}'+ )tokens[token_id] = tokenscores[token_id] = -1000.0toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED@@ -3121,8 +3760,9 @@ class Phi3MiniModel(Model):if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:if tokens[token_id] != token:logger.warning(- f"replacing token {token_id}: {tokens[token_id].decode('utf-8')!r}"- f" -> {token.decode('utf-8')!r}")+ f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '+ f'-> {token.decode("utf-8")!r}'+ )tokens[token_id] = tokenscores[token_id] = -1000.0toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED@@ -3182,7 +3822,7 @@ class Phi3MiniModel(Model):scale = max_pos_embds / orig_max_pos_embds- rope_scaling_type = rope_scaling.get('type', '').lower()+ rope_scaling_type = rope_scaling.get('rope_type', rope_scaling.get('type', '')).lower()if len(rope_scaling_type) == 0:raise KeyError('Missing the required key rope_scaling.type')@@ -3203,8 +3843,9 @@ class Phi3MiniModel(Model):if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:raise ValueError(- f"The length of rope long and short factors must be {rope_dims / 2}."- f" long_factors = {len(long_factors)}, short_factors = {len(short_factors)}.")+ f'The length of rope long and short factors must be {rope_dims / 2}.'+ f' long_factors = {len(long_factors)}, short_factors = {len(short_factors)}.'+ )yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),@@ -3214,7 +3855,7 @@ class Phi3MiniModel(Model):torch.tensor(short_factors, dtype=torch.float32))[email protected]("PhiMoEForCausalLM")[email protected]("PhiMoEForCausalLM")class PhiMoeModel(Phi3MiniModel):model_arch = gguf.MODEL_ARCH.PHIMOE@@ -3271,8 +3912,8 @@ class PhiMoeModel(Phi3MiniModel):raise ValueError(f"Unprocessed experts: {experts}")[email protected]("PlamoForCausalLM")-class PlamoModel(Model):[email protected]("PlamoForCausalLM")+class PlamoModel(TextModel):model_arch = gguf.MODEL_ARCH.PLAMOdef set_vocab(self):@@ -3319,8 +3960,8 @@ class PlamoModel(Model):return [(new_name, data_torch)][email protected]("CodeShellForCausalLM")-class CodeShellModel(Model):[email protected]("CodeShellForCausalLM")+class CodeShellModel(TextModel):model_arch = gguf.MODEL_ARCH.CODESHELLdef set_gguf_parameters(self):@@ -3360,8 +4001,8 @@ class CodeShellModel(Model):return [(new_name, data_torch)][email protected]("InternLM2ForCausalLM")-class InternLM2Model(Model):[email protected]("InternLM2ForCausalLM")+class InternLM2Model(TextModel):model_arch = gguf.MODEL_ARCH.INTERNLM2def set_vocab(self):@@ -3445,8 +4086,9 @@ class InternLM2Model(Model):if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:if tokens[token_id] != token:logger.warning(- f"replacing token {token_id}: {tokens[token_id].decode('utf-8')!r} "- f"-> {token.decode('utf-8')!r}")+ f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '+ f'-> {token.decode("utf-8")!r}'+ )tokens[token_id] = tokenscores[token_id] = -1000.0toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED@@ -3467,8 +4109,9 @@ class InternLM2Model(Model):if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:if tokens[token_id] != token:logger.warning(- f"replacing token {token_id}: {tokens[token_id].decode('utf-8')!r} "- f"-> {token.decode('utf-8')!r}")+ f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '+ f'-> {token.decode("utf-8")!r}'+ )tokens[token_id] = tokenscores[token_id] = -1000.0toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED@@ -3505,10 +4148,10 @@ class InternLM2Model(Model):self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"])self.gguf_writer.add_file_type(self.ftype)- if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:- if self.hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:num_heads = self.hparams["num_attention_heads"]@@ -3518,6 +4161,11 @@ class InternLM2Model(Model):head_dim = n_embd // num_headsnum_groups = num_heads // q_per_kv+ name = name.replace("language_model.", "") # InternVL+ if name.startswith("mlp") or name.startswith("vision_model"):+ # skip visual tensors+ return []+if bid is not None and f"model.layers.{bid}.attention.wqkv" in name:qkv = data_torch@@ -3538,8 +4186,8 @@ class InternLM2Model(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("InternLM3ForCausalLM")-class InternLM3Model(Model):[email protected]("InternLM3ForCausalLM")+class InternLM3Model(TextModel):model_arch = gguf.MODEL_ARCH.LLAMAdef set_vocab(self):@@ -3577,21 +4225,22 @@ class InternLM3Model(Model):hparams = self.hparamsself.gguf_writer.add_vocab_size(hparams["vocab_size"])- if "head_dim" in hparams:- rope_dim = hparams["head_dim"]- else:+ if (rope_dim := hparams.get("head_dim")) is None:rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]self.gguf_writer.add_rope_dimension_count(rope_dim)- if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:- if self.hparams["rope_scaling"].get("type") == "linear" or self.hparams["rope_scaling"].get(- "rope_type") == "linear":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]n_kv_head = self.hparams.get("num_key_value_heads")+ name = name.replace("language_model.", "") # InternVL+ if name.startswith("mlp") or name.startswith("vision_model"):+ # skip visual tensors+ return []if name.endswith(("q_proj.weight", "q_proj.bias")):data_torch = LlamaModel.permute(data_torch, n_head, n_head)if name.endswith(("k_proj.weight", "k_proj.bias")):@@ -3599,40 +4248,27 @@ class InternLM3Model(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("BertModel", "BertForMaskedLM", "CamembertModel")-class BertModel(Model):[email protected]("BertModel", "BertForMaskedLM", "CamembertModel", "BertForSequenceClassification")+class BertModel(TextModel):model_arch = gguf.MODEL_ARCH.BERTdef __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)self.vocab_size = None+ if cls_out_labels := self.hparams.get("id2label"):+ if len(cls_out_labels) == 2 and cls_out_labels[0] == "LABEL_0":+ # Remove dummy labels added by AutoConfig+ cls_out_labels = None+ self.cls_out_labels = cls_out_labels+def set_gguf_parameters(self):super().set_gguf_parameters()self.gguf_writer.add_causal_attention(False)+ self._try_set_pooling_type()- # get pooling path- pooling_path = None- module_path = self.dir_model / "modules.json"- if module_path.is_file():- with open(module_path, encoding="utf-8") as f:- modules = json.load(f)- for mod in modules:- if mod["type"] == "sentence_transformers.models.Pooling":- pooling_path = mod["path"]- break-- # get pooling type- if pooling_path is not None:- with open(self.dir_model / pooling_path / "config.json", encoding="utf-8") as f:- pooling = json.load(f)- if pooling["pooling_mode_mean_tokens"]:- pooling_type = gguf.PoolingType.MEAN- elif pooling["pooling_mode_cls_token"]:- pooling_type = gguf.PoolingType.CLS- else:- raise NotImplementedError("Only MEAN and CLS pooling types supported")- self.gguf_writer.add_pooling_type(pooling_type)+ if self.cls_out_labels:+ self.gguf_writer.add_classifier_output_labels([v for k, v in sorted(self.cls_out_labels.items())])def set_vocab(self):tokens, toktypes, tokpre = self.get_vocab_base()@@ -3685,10 +4321,175 @@ class BertModel(Model):if name.startswith("cls.seq_relationship"):return []+ if self.cls_out_labels:+ # For BertForSequenceClassification (direct projection layer)+ if name == "classifier.weight":+ name = "classifier.out_proj.weight"++ if name == "classifier.bias":+ name = "classifier.out_proj.bias"+return [(self.map_tensor_name(name), data_torch)]+ def _xlmroberta_tokenizer_init(self) -> None:+ # we need the pad_token_id to know how to chop down position_embd matrix+ if (pad_token_id := self.hparams.get("pad_token_id")) is not None:+ self._position_offset = 1 + pad_token_id+ if "max_position_embeddings" in self.hparams:+ self.hparams["max_position_embeddings"] -= self._position_offset+ else:+ self._position_offset = None++ def _xlmroberta_set_vocab(self) -> None:+ # to avoid TypeError: Descriptors cannot be created directly+ # exception when importing sentencepiece_model_pb2+ os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"+ from sentencepiece import SentencePieceProcessor+ from sentencepiece import sentencepiece_model_pb2 as model++ tokenizer_path = self.dir_model / 'sentencepiece.bpe.model'++ tokenizer_json = {}+ tokenizer_config_json = {}+ if not tokenizer_path.is_file():+ tokenizer_path = self.dir_model / 'tokenizer.json'+ tokenizer_config_path = self.dir_model / 'tokenizer_config.json'++ if not tokenizer_path.is_file():+ raise FileNotFoundError(f"File not found: {tokenizer_path}")++ from base64 import b64decode+ from transformers import AutoTokenizer+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model)++ with open(tokenizer_path, "r", encoding="utf-8") as fp:+ tokenizer_json = json.load(fp)++ if tokenizer_config_path.is_file():+ with open(tokenizer_config_path, "r", encoding="utf-8") as fp:+ tokenizer_config_json = json.load(fp)++ add_prefix = tokenizer.add_prefix_space+ remove_whitespaces = tokenizer.clean_up_tokenization_spaces+ precompiled_charsmap = b64decode(tokenizer_json["normalizer"]["precompiled_charsmap"])++ vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size)+ else:+ sentencepiece_model = model.ModelProto() # pylint: disable=E1101+ sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())+ assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM++ add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix+ remove_whitespaces = sentencepiece_model.normalizer_spec.remove_extra_whitespaces+ precompiled_charsmap = sentencepiece_model.normalizer_spec.precompiled_charsmap++ tokenizer = SentencePieceProcessor()+ tokenizer.LoadFromFile(str(tokenizer_path))++ vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size())++ tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]+ scores: list[float] = [-10000.0] * vocab_size+ toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size++ if isinstance(tokenizer, SentencePieceProcessor):+ for token_id in range(tokenizer.vocab_size()):+ piece = tokenizer.IdToPiece(token_id)+ text = piece.encode("utf-8")+ score = tokenizer.GetScore(token_id)++ toktype = SentencePieceTokenTypes.NORMAL+ if tokenizer.IsUnknown(token_id):+ toktype = SentencePieceTokenTypes.UNKNOWN+ elif tokenizer.IsControl(token_id):+ toktype = SentencePieceTokenTypes.CONTROL+ elif tokenizer.IsUnused(token_id):+ toktype = SentencePieceTokenTypes.UNUSED+ elif tokenizer.IsByte(token_id):+ toktype = SentencePieceTokenTypes.BYTE++ tokens[token_id] = text+ scores[token_id] = score+ toktypes[token_id] = toktype+ else:+ added_vocab = tokenizer.get_added_vocab()+ unk_token = tokenizer_config_json.get("unk_token")+ unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3))++ for token_id in range(tokenizer.vocab_size):+ piece = tokenizer._convert_id_to_token(token_id)+ if (piece := tokenizer._convert_id_to_token(token_id)) is not None:+ text = piece.encode("utf-8")+ score = tokenizer_json["model"]["vocab"][token_id][1]++ toktype = SentencePieceTokenTypes.NORMAL+ if token_id == unk_token_id:+ toktype = SentencePieceTokenTypes.UNKNOWN+ elif token_id in tokenizer.all_special_ids:+ toktype = SentencePieceTokenTypes.CONTROL+ elif token_id in added_vocab.values():+ toktype = SentencePieceTokenTypes.USER_DEFINED+ # No reliable way to detect this, but jina doesn't have any+ # elif tokenizer.IsByte(token_id):+ # toktype = SentencePieceTokenTypes.BYTE++ tokens[token_id] = text+ scores[token_id] = score+ toktypes[token_id] = toktype++ if isinstance(tokenizer, SentencePieceProcessor):+ # realign tokens (see HF tokenizer code)+ tokens = [b'<s>', b'<pad>', b'</s>', b'<unk>'] + tokens[3:-1]+ scores = [0.0, 0.0, 0.0, 0.0] + scores[3:-1]+ toktypes = [+ SentencePieceTokenTypes.CONTROL,+ SentencePieceTokenTypes.CONTROL,+ SentencePieceTokenTypes.CONTROL,+ SentencePieceTokenTypes.UNKNOWN,+ ] + toktypes[3:-1]++ if self.model_arch == gguf.MODEL_ARCH.NOMIC_BERT_MOE:+ # Add mask token missing from sentencepiece.bpe.model+ tokens[250001] = b'<mask>'+ scores[250001] = 0.0+ toktypes[250001] = SentencePieceTokenTypes.CONTROL++ self.gguf_writer.add_tokenizer_model("t5")+ self.gguf_writer.add_tokenizer_pre("default")+ self.gguf_writer.add_token_list(tokens)+ self.gguf_writer.add_token_scores(scores)+ self.gguf_writer.add_token_types(toktypes)+ self.gguf_writer.add_add_space_prefix(add_prefix)+ self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))+ self.gguf_writer.add_remove_extra_whitespaces(remove_whitespaces)+ if precompiled_charsmap:+ self.gguf_writer.add_precompiled_charsmap(precompiled_charsmap)++ special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))+ special_vocab.add_to_gguf(self.gguf_writer)++[email protected]("DistilBertModel", "DistilBertForMaskedLM", "DistilBertForSequenceClassification")+class DistilBertModel(BertModel):+ model_arch = gguf.MODEL_ARCH.BERT++ def set_gguf_parameters(self):+ self.gguf_writer.add_layer_norm_eps(1e-12)+ logger.info("gguf: layer norm epsilon = 1e-12")+ super().set_gguf_parameters()[email protected]("RobertaModel")+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if name.startswith("distilbert."):+ name = name[11:]++ # These layers act as MLM head, so we don't need them+ if name.startswith("vocab_"):+ return []++ return super().modify_tensors(data_torch, name, bid)++[email protected]("RobertaModel", "RobertaForSequenceClassification")class RobertaModel(BertModel):model_arch = gguf.MODEL_ARCH.BERT@@ -3708,8 +4509,6 @@ class RobertaModel(BertModel):bpe_tok_path = self.dir_model / "tokenizer.json"if bpe_tok_path.exists():self._set_vocab_gpt2()- self.gguf_writer.add_add_bos_token(True)- self.gguf_writer.add_add_eos_token(True)# we need this to validate the size of the token_type embeddings# though currently we are passing all zeros to the token_type embeddings@@ -3733,24 +4532,41 @@ class RobertaModel(BertModel):return super().modify_tensors(data_torch, name, bid)[email protected]("NomicBertModel")[email protected]("NomicBertModel")class NomicBertModel(BertModel):- model_arch = gguf.MODEL_ARCH.NOMIC_BERT+ model_arch = gguf.MODEL_ARCH.BERT- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)+ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, **kwargs: Any):+ hparams = kwargs.pop("hparams", None)+ if hparams is None:+ hparams = ModelBase.load_hparams(dir_model)- # the HF config claims n_ctx=8192, but it uses RoPE scaling- self.hparams["n_ctx"] = 2048+ self.is_moe = bool(hparams.get("moe_every_n_layers"))+ self.model_arch = gguf.MODEL_ARCH.NOMIC_BERT_MOE if self.is_moe else gguf.MODEL_ARCH.NOMIC_BERT++ super().__init__(dir_model, ftype, fname_out, hparams=hparams, **kwargs)++ self._tokenizer_is_xlmroberta = self._is_tokenizer_xlmroberta()+ if self._tokenizer_is_xlmroberta:+ self._xlmroberta_tokenizer_init()++ npos, mtp = self.hparams["n_positions"], self.hparams.get("max_trained_positions", 2048)+ if npos == 8192 and mtp == 2048:+ self.hparams["n_positions"] = 2048 # nomic-embed-text v1 and v1.5 are trained for 2048 tokens.+ elif npos == 2048 and mtp == 2048:+ self.hparams["n_positions"] = 512 # nomic-embed-text-v2-moe is trained for 512 tokens.+ else:+ raise ValueError(f"unrecognized parameters: n_positions={npos}, max_trained_positions={mtp}")++ assert self.hparams["activation_function"] == "gelu" if self.is_moe else "swiglu"- # SwigLU activation- assert self.hparams["activation_function"] == "swiglu"# this doesn't do anything in the HF versionassert self.hparams["causal"] is False- # no bias tensors- assert self.hparams["qkv_proj_bias"] is False- assert self.hparams["mlp_fc1_bias"] is False- assert self.hparams["mlp_fc2_bias"] is False+ # no bias tensors unless MoE+ assert self.hparams["qkv_proj_bias"] == self.is_moe+ assert self.hparams["mlp_fc1_bias"] == self.is_moe+ assert self.hparams["mlp_fc2_bias"] == self.is_moe+# norm at end of layerassert self.hparams["prenorm"] is False# standard RoPE@@ -3758,107 +4574,85 @@ class NomicBertModel(BertModel):assert self.hparams["rotary_emb_interleaved"] is Falseassert self.hparams["rotary_emb_scale_base"] is None- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_rope_freq_base(self.hparams["rotary_emb_base"])--[email protected]("XLMRobertaModel", "XLMRobertaForSequenceClassification")-class XLMRobertaModel(BertModel):- model_arch = gguf.MODEL_ARCH.BERT-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)+ def set_vocab(self) -> None:+ if self._tokenizer_is_xlmroberta:+ return self._xlmroberta_set_vocab()+ return super().set_vocab()- # we need the pad_token_id to know how to chop down position_embd matrix- if (pad_token_id := self.hparams.get("pad_token_id")) is not None:- self._position_offset = 1 + pad_token_id- if "max_position_embeddings" in self.hparams:- self.hparams["max_position_embeddings"] -= self._position_offset- else:- self._position_offset = None+ def modify_tensors(self, data_torch: torch.Tensor, name: str,+ bid: int | None) -> Iterable[tuple[str, torch.Tensor]]:+ # If the tensor is an experts bias tensor, skip it by returning an empty list.+ if "mlp.experts.bias" in name:+ return [] # Explicitly return an empty list.- def set_vocab(self):- # to avoid TypeError: Descriptors cannot be created directly- # exception when importing sentencepiece_model_pb2- os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"- from sentencepiece import SentencePieceProcessor- from sentencepiece import sentencepiece_model_pb2 as model+ if "mlp.experts.mlp.w1" in name:+ data_torch = data_torch.view(self.hparams["num_experts"], self.hparams["n_inner"], self.hparams["n_embd"])+ name += ".weight"- tokenizer_path = self.dir_model / 'sentencepiece.bpe.model'- if not tokenizer_path.is_file():- raise FileNotFoundError(f"File not found: {tokenizer_path}")+ if "mlp.experts.mlp.w2" in name:+ data_torch = data_torch.view(self.hparams["num_experts"], self.hparams["n_inner"], self.hparams["n_embd"])+ data_torch = data_torch.transpose(1, 2)+ name += ".weight"- sentencepiece_model = model.ModelProto() # pylint: disable=E1101- sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())- assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM+ return [(self.map_tensor_name(name), data_torch)]- add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix- remove_whitespaces = sentencepiece_model.normalizer_spec.remove_extra_whitespaces- precompiled_charsmap = sentencepiece_model.normalizer_spec.precompiled_charsmap+ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_rope_freq_base(self.hparams["rotary_emb_base"])+ if self.is_moe:+ self.gguf_writer.add_moe_every_n_layers(self.hparams["moe_every_n_layers"])+ self.gguf_writer.add_expert_count(self.hparams["num_experts"])+ self.gguf_writer.add_expert_used_count(self.hparams["moe_top_k"])++ def _is_tokenizer_xlmroberta(self) -> bool:+ with open(self.dir_model / "tokenizer.json") as f:+ tokenizer_json = json.load(f)+ toktyp = tokenizer_json["model"]["type"]+ if toktyp == "Unigram":+ return True+ if toktyp == "WordPiece":+ return False+ raise ValueError(f"unknown tokenizer: {toktyp}")- tokenizer = SentencePieceProcessor()- tokenizer.LoadFromFile(str(tokenizer_path))- vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())[email protected]("NeoBERT", "NeoBERTLMHead", "NeoBERTForSequenceClassification")+class NeoBert(BertModel):+ model_arch = gguf.MODEL_ARCH.NEO_BERT- tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]- scores: list[float] = [-10000.0] * vocab_size- toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size+ def set_gguf_parameters(self):+ super().set_gguf_parameters()- for token_id in range(tokenizer.vocab_size()):- piece = tokenizer.IdToPiece(token_id)- text = piece.encode("utf-8")- score = tokenizer.GetScore(token_id)+ # NeoBERT uses 2/3 of the intermediate size as feed forward length+ self.gguf_writer.add_feed_forward_length(int(2 * self.hparams["intermediate_size"] / 3))+ self.gguf_writer.add_rope_freq_base(10000.0) # default value for NeoBERT+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)- toktype = SentencePieceTokenTypes.NORMAL- if tokenizer.IsUnknown(token_id):- toktype = SentencePieceTokenTypes.UNKNOWN- elif tokenizer.IsControl(token_id):- toktype = SentencePieceTokenTypes.CONTROL- elif tokenizer.IsUnused(token_id):- toktype = SentencePieceTokenTypes.UNUSED- elif tokenizer.IsByte(token_id):- toktype = SentencePieceTokenTypes.BYTE+ f_rms_eps = self.hparams.get("norm_eps", 1e-6) # default value for NeoBERT+ self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)+ logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")- tokens[token_id] = text- scores[token_id] = score- toktypes[token_id] = toktype+ self.gguf_writer.add_pooling_type(gguf.PoolingType.CLS) # https://huggingface.co/chandar-lab/NeoBERT#how-to-use- if vocab_size > len(tokens):- pad_count = vocab_size - len(tokens)- logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")- for i in range(1, pad_count + 1):- tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))- scores.append(-1000.0)- toktypes.append(SentencePieceTokenTypes.UNUSED)+ def modify_tensors(self, data_torch, name, bid):+ if name.startswith("decoder."):+ return []- # realign tokens (see HF tokenizer code)- tokens = [b'<s>', b'<pad>', b'</s>', b'<unk>'] + tokens[3:-1]- scores = [0.0, 0.0, 0.0, 0.0] + scores[3:-1]- toktypes = [- SentencePieceTokenTypes.CONTROL,- SentencePieceTokenTypes.CONTROL,- SentencePieceTokenTypes.CONTROL,- SentencePieceTokenTypes.UNKNOWN,- ] + toktypes[3:-1]+ if name.startswith("model."):+ name = name[6:]- self.gguf_writer.add_tokenizer_model("t5")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)- self.gguf_writer.add_add_space_prefix(add_prefix)- self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))- self.gguf_writer.add_remove_extra_whitespaces(remove_whitespaces)- if precompiled_charsmap:- self.gguf_writer.add_precompiled_charsmap(precompiled_charsmap)+ return super().modify_tensors(data_torch, name, bid)- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)- self.gguf_writer.add_add_bos_token(True)- self.gguf_writer.add_add_eos_token(True)[email protected]("XLMRobertaModel", "XLMRobertaForSequenceClassification")+class XLMRobertaModel(BertModel):+ model_arch = gguf.MODEL_ARCH.BERT++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self._xlmroberta_tokenizer_init()++ def set_vocab(self):+ self._xlmroberta_set_vocab()def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# if name starts with "roberta.", remove the prefix@@ -3874,8 +4668,8 @@ class XLMRobertaModel(BertModel):return super().modify_tensors(data_torch, name, bid)[email protected]("GemmaForCausalLM")-class GemmaModel(Model):[email protected]("GemmaForCausalLM")+class GemmaModel(TextModel):model_arch = gguf.MODEL_ARCH.GEMMAdef set_vocab(self):@@ -3903,8 +4697,8 @@ class GemmaModel(Model):self.gguf_writer.add_block_count(block_count)self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])self.gguf_writer.add_head_count(hparams["num_attention_heads"])- self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"] \- if "num_key_value_heads" in hparams else hparams["num_attention_heads"])+ self.gguf_writer.add_head_count_kv(+ self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"])self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_key_length(hparams["head_dim"])self.gguf_writer.add_value_length(hparams["head_dim"])@@ -3925,8 +4719,8 @@ class GemmaModel(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("Gemma2ForCausalLM")-class Gemma2Model(Model):[email protected]("Gemma2ForCausalLM")+class Gemma2Model(TextModel):model_arch = gguf.MODEL_ARCH.GEMMA2def set_vocab(self):@@ -3943,8 +4737,8 @@ class Gemma2Model(Model):self.gguf_writer.add_block_count(block_count)self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])self.gguf_writer.add_head_count(hparams["num_attention_heads"])- self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"] \- if "num_key_value_heads" in hparams else hparams["num_attention_heads"])+ self.gguf_writer.add_head_count_kv(+ self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"])self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_key_length(hparams["head_dim"])self.gguf_writer.add_value_length(hparams["head_dim"])@@ -3968,27 +4762,10 @@ class Gemma2Model(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("Gemma3ForCausalLM", "Gemma3ForConditionalGeneration")-class Gemma3Model(Model):[email protected]("Gemma3ForCausalLM", "Gemma3ForConditionalGeneration")+class Gemma3Model(TextModel):model_arch = gguf.MODEL_ARCH.GEMMA3- has_vision: bool = False-- # we need to merge the text_config into the root level of hparams- def __init__(self, *args, **kwargs):- hparams = kwargs["hparams"] if "hparams" in kwargs else Model.load_hparams(args[0])- if "text_config" in hparams:- hparams = {**hparams, **hparams["text_config"]}- kwargs["hparams"] = hparams- super().__init__(*args, **kwargs)- if "vision_config" in hparams:- logger.info("Has vision encoder, but it will be ignored")- self.has_vision = True-- def write(self):- super().write()- if self.has_vision:- logger.info("NOTE: this script only convert the language model to GGUF")- logger.info(" for the vision model, please use gemma3_convert_encoder_to_gguf.py")+ norm_shift = 1.0 # Gemma3RMSNorm adds 1.0 to the norm valuedef set_vocab(self):self._set_vocab_sentencepiece()@@ -4010,9 +4787,8 @@ class Gemma3Model(Model):self.gguf_writer.add_value_length(hparams.get("head_dim", 256))self.gguf_writer.add_file_type(self.ftype)self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 1_000_000.0)) # for global layers- # both attn_logit_softcapping and final_logit_softcapping are removed in Gemma3+ # attn_logit_softcapping is removed in Gemma3assert hparams.get("attn_logit_softcapping") is None- assert hparams.get("final_logit_softcapping") is Noneself.gguf_writer.add_sliding_window(hparams["sliding_window"])self.gguf_writer.add_head_count_kv(hparams.get("num_key_value_heads", 4))if hparams.get("rope_scaling") is not None:@@ -4024,12 +4800,12 @@ class Gemma3Model(Model):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused- if name.startswith("language_model."):+ if "language_model." in name:name = name.replace("language_model.", "")+elif name.startswith("multi_modal_projector.") or name.startswith("vision_tower.") \or name.startswith("multimodal_projector.") or name.startswith("vision_model."):- # ignore vision tensors- return []+ return [] # skip vision tensors# remove OOV (out-of-vocabulary) rows in token_embdif "embed_tokens.weight" in name:@@ -4039,19 +4815,168 @@ class Gemma3Model(Model):# ref code in Gemma3RMSNorm# output = output * (1.0 + self.weight.float())+ # note: this is not the case on gemma3nif name.endswith("norm.weight"):- data_torch = data_torch + 1+ data_torch = data_torch + self.norm_shiftreturn [(self.map_tensor_name(name), data_torch)][email protected]("Starcoder2ForCausalLM")-class StarCoder2Model(Model):[email protected]("Gemma3ForConditionalGeneration")+class Gemma3VisionModel(MmprojModel):++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ hparams = self.hparams+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GEMMA3)+ # default values below are taken from HF transformers code+ self.gguf_writer.add_vision_attention_layernorm_eps(hparams.get("layer_norm_eps", 1e-6))+ self.gguf_writer.add_vision_use_gelu(True)+ # calculate proj_scale_factor (used by tinygemma3 test model)+ image_seq_length = self.preprocessor_config.get("image_seq_length", 256)+ n_per_side = int(image_seq_length**0.5)+ image_size = self.hparams["image_size"]+ patch_size = self.hparams["patch_size"]+ proj_scale_factor = (image_size//patch_size) // n_per_side+ if proj_scale_factor > 0 and proj_scale_factor != 4:+ # we only need to write this if it's not the default value+ # in this case, we are converting a test model+ self.gguf_writer.add_vision_projector_scale_factor(proj_scale_factor)++ def tensor_force_quant(self, name, new_name, bid, n_dims):+ del bid, new_name, n_dims # unused+ # related to https://github.com/ggml-org/llama.cpp/issues/13025+ if "input_projection" in name:+ return gguf.GGMLQuantizationType.F16+ if ".embeddings." in name:+ return gguf.GGMLQuantizationType.F32+ return False++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused++ if "vision_model.head." in name:+ return [] # skip redundant tensors for tinygemma3++ if name.startswith("multi_modal_projector.") or name.startswith("vision_tower.") \+ or name.startswith("multimodal_projector.") or name.startswith("vision_model."):+ # process vision tensors+ name = name.replace("_weight", ".weight")++ # correct norm value ; only this "soft_emb_norm" need to be corrected as it's part of Gemma projector+ # the other norm values are part of SigLIP model, and they are already correct+ # ref code: Gemma3RMSNorm+ if "soft_emb_norm.weight" in name:+ logger.info(f"Correcting norm value for '{name}'")+ data_torch = data_torch + 1++ return [(self.map_tensor_name(name), data_torch)]++ return [] # skip other tensors++[email protected]("Gemma3nForConditionalGeneration")+class Gemma3NModel(Gemma3Model):+ model_arch = gguf.MODEL_ARCH.GEMMA3N+ norm_shift = 0.0 # same value with Gemma3p5RMSNorm scale_shift on python code++ _altup_proj: list[Tensor] = []+ _altup_unembd: list[Tensor] = []++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ assert self.hparams["altup_num_inputs"] == 4, "Current conversion only supports 4 altup inputs"+ self._altup_proj = [+ torch.Tensor(), # to be replaced+ torch.Tensor(), # to be replaced+ torch.Tensor(), # to be replaced+ ]+ self._altup_unembd = [+ torch.Tensor(), # to be replaced+ torch.Tensor(), # to be replaced+ torch.Tensor(), # to be replaced+ ]++ def set_vocab(self):+ super().set_vocab()++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_altup_active_idx(self.hparams["altup_active_idx"])+ self.gguf_writer.add_altup_num_inputs(self.hparams["altup_num_inputs"])+ self.gguf_writer.add_embedding_length_per_layer_input(self.hparams["hidden_size_per_layer_input"])+ self.gguf_writer.add_shared_kv_layers(self.hparams["num_kv_shared_layers"])++ activation_sparsity_scale = []+ for s in self.hparams["activation_sparsity_pattern"]:+ normal_dist = torch.distributions.normal.Normal(0, 1)+ std_multiplier = normal_dist.icdf(torch.tensor(s, dtype=torch.float32))+ activation_sparsity_scale.append(std_multiplier.item())+ self.gguf_writer.add_activation_sparsity_scale(activation_sparsity_scale)++ sliding_window_pattern = []+ for t in self.hparams["layer_types"]:+ sliding_window_pattern.append(t == "sliding_attention")+ self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)++ def _stack_matrices(self, matrices: list[Tensor]) -> Tensor | None:+ has_all = all(m.numel() > 0 for m in matrices)+ if not has_all:+ return None+ else:+ return torch.stack(matrices, dim=0)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if name.endswith("_scale"):+ name = name + ".weight"++ # TODO: implement self.prediction_coefs.weight.clamp_(...)++ if "language_model." not in name:+ return [] # skip non-language model tensors++ if "altup_unembed_projections" in name:+ data_torch = data_torch.to(device="cpu")+ if ".0." in name:+ self._altup_unembd[0] = data_torch+ elif ".1." in name:+ self._altup_unembd[1] = data_torch+ elif ".2." in name:+ self._altup_unembd[2] = data_torch+ else:+ raise ValueError(f"Unknown name: {name}")+ out = self._stack_matrices(self._altup_unembd)+ if out is not None:+ return [(self.map_tensor_name("model.altup_unembed_projections.weight"), out)]+ else:+ return []++ if "altup_projections" in name:+ data_torch = data_torch.to(device="cpu")+ if ".0." in name:+ self._altup_proj[0] = data_torch+ elif ".1." in name:+ self._altup_proj[1] = data_torch+ elif ".2." in name:+ self._altup_proj[2] = data_torch+ else:+ raise ValueError(f"Unknown name: {name}")+ out = self._stack_matrices(self._altup_proj)+ if out is not None:+ return [(self.map_tensor_name("model.altup_projections.weight"), out)]+ else:+ return []++ return super().modify_tensors(data_torch, name, bid)++[email protected]("Starcoder2ForCausalLM")+class StarCoder2Model(TextModel):model_arch = gguf.MODEL_ARCH.STARCODER2[email protected]("Rwkv6ForCausalLM")-class Rwkv6Model(Model):[email protected]("Rwkv6ForCausalLM")+class Rwkv6Model(TextModel):model_arch = gguf.MODEL_ARCH.RWKV6def set_vocab(self):@@ -4063,8 +4988,8 @@ class Rwkv6Model(Model):hidden_size = self.hparams["hidden_size"]layer_norm_eps = self.hparams["layer_norm_epsilon"]rescale_every_n_layers = self.hparams["rescale_every"]- intermediate_size = self.hparams["intermediate_size"] \- if self.hparams["intermediate_size"] is not None else int((hidden_size * 3.5) // 32 * 32)+ intermediate_size = self.hparams["intermediate_size"] if self.hparams["intermediate_size"] is not None else int(+ (hidden_size*3.5) // 32 * 32)time_mix_extra_dim = 64 if hidden_size == 4096 else 32time_decay_extra_dim = 128 if hidden_size == 4096 else 64@@ -4091,8 +5016,8 @@ class Rwkv6Model(Model):if not (new_name.endswith(".weight") or new_name.endswith(".bias")):new_name += ".weight"- if new_name.endswith("time_mix_w1.weight") \- or new_name.endswith("time_mix_decay_w1.weight") or new_name.endswith("time_mix_decay_w2.weight"):+ if new_name.endswith("time_mix_w1.weight") or new_name.endswith(+ "time_mix_decay_w1.weight") or new_name.endswith("time_mix_decay_w2.weight"):data_torch = data_torch.transpose(0, 1)if new_name.endswith("time_mix_w2.weight"):@@ -4105,7 +5030,7 @@ class Rwkv6Model(Model):rescale_every_n_layers = self.hparams["rescale_every"]if rescale_every_n_layers > 0:if new_name.endswith("time_mix_output.weight") or new_name.endswith("channel_mix_value.weight"):- data_torch = data_torch.div_(2 ** int(bid // rescale_every_n_layers))+ data_torch = data_torch.div_(2**int(bid // rescale_every_n_layers))except KeyError:pass@@ -4119,15 +5044,18 @@ class Rwkv6Model(Model):if all(f"blk.{bid}.time_mix_lerp_{i}.weight" in self.lerp_weights[bid].keys()for i in ["w", "k", "v", "r", "g"]):new_name = f"blk.{bid}.time_mix_lerp_fused.weight"- data = torch.stack([self.lerp_weights[bid][f"blk.{bid}.time_mix_lerp_{i}.weight"].unsqueeze(0) \- for i in ["w", "k", "v", "r", "g"]], dim=0).unsqueeze(1)+ data = torch.stack([+ self.lerp_weights[bid][f"blk.{bid}.time_mix_lerp_{i}.weight"].unsqueeze(0)+ for i in ["w", "k", "v", "r", "g"]+ ],+ dim=0).unsqueeze(1)yield (new_name, data)returnyield (new_name, data_torch)[email protected]("RWKV6Qwen2ForCausalLM")[email protected]("RWKV6Qwen2ForCausalLM")class RWKV6Qwen2Model(Rwkv6Model):model_arch = gguf.MODEL_ARCH.RWKV6QWEN2@@ -4181,15 +5109,15 @@ class RWKV6Qwen2Model(Rwkv6Model):yield (new_name, data)[email protected]("Rwkv7ForCausalLM", "RWKV7ForCausalLM")-class Rwkv7Model(Model):[email protected]("Rwkv7ForCausalLM", "RWKV7ForCausalLM")+class Rwkv7Model(TextModel):model_arch = gguf.MODEL_ARCH.RWKV7def set_vocab(self):self._set_vocab_rwkv_world()def calc_lora_rank(self, hidden_size, exponent, multiplier):- return max(1, round(hidden_size ** exponent * multiplier / 32)) * 32+ return max(1, round(hidden_size**exponent * multiplier / 32)) * 32def set_gguf_parameters(self):block_count = self.hparams["num_hidden_layers"]@@ -4200,36 +5128,28 @@ class Rwkv7Model(Model):head_size = self.hparams["head_dim"]layer_norm_eps = self.hparams["norm_eps"]hidden_size = self.hparams["hidden_size"]- intermediate_size = self.hparams["intermediate_size"] \- if self.hparams["intermediate_size"] is not None else (hidden_size * 4)+ intermediate_size = self.hparams["intermediate_size"] if self.hparams["intermediate_size"] is not None else (+ hidden_size * 4)# ICLR: In-Context-Learning-Ratetry:- lora_rank_decay = self.hparams["lora_rank_decay"] if self.hparams["lora_rank_decay"] \- is not None else self.calc_lora_rank(hidden_size, 0.5,- 1.8)- lora_rank_iclr = self.hparams["lora_rank_iclr"] if self.hparams["lora_rank_iclr"] \- is not None else self.calc_lora_rank(hidden_size, 0.5,- 1.8)- lora_rank_value_residual_mix = self.hparams["lora_rank_value_residual_mix"] \- if self.hparams["lora_rank_value_residual_mix"] is not None \- else self.calc_lora_rank(hidden_size, 0.5, 1.3)- lora_rank_gate = self.hparams["lora_rank_gate"] if self.hparams["lora_rank_gate"] \- is not None else self.calc_lora_rank(hidden_size, 0.8,- 0.6)+ lora_rank_decay = self.hparams["lora_rank_decay"] if self.hparams[+ "lora_rank_decay"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ lora_rank_iclr = self.hparams["lora_rank_iclr"] if self.hparams[+ "lora_rank_iclr"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ lora_rank_value_residual_mix = self.hparams["lora_rank_value_residual_mix"] if self.hparams[+ "lora_rank_value_residual_mix"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.3)+ lora_rank_gate = self.hparams["lora_rank_gate"] if self.hparams[+ "lora_rank_gate"] is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6)except KeyError:- lora_rank_decay = self.hparams["decay_low_rank_dim"] if self.hparams["decay_low_rank_dim"] \- is not None else self.calc_lora_rank(hidden_size,- 0.5, 1.8)- lora_rank_iclr = self.hparams["a_low_rank_dim"] if self.hparams["a_low_rank_dim"] \- is not None else self.calc_lora_rank(hidden_size, 0.5,- 1.8)- lora_rank_value_residual_mix = self.hparams["v_low_rank_dim"] if self.hparams["v_low_rank_dim"] \- is not None else self.calc_lora_rank(- hidden_size, 0.5, 1.3)- lora_rank_gate = self.hparams["gate_low_rank_dim"] if self.hparams["gate_low_rank_dim"] \- is not None else self.calc_lora_rank(hidden_size, 0.8,- 0.6)+ lora_rank_decay = self.hparams["decay_low_rank_dim"] if self.hparams[+ "decay_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ lora_rank_iclr = self.hparams["a_low_rank_dim"] if self.hparams[+ "a_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ lora_rank_value_residual_mix = self.hparams["v_low_rank_dim"] if self.hparams[+ "v_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.3)+ lora_rank_gate = self.hparams["gate_low_rank_dim"] if self.hparams[+ "gate_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6)# RWKV isn't context limitedself.gguf_writer.add_context_length(1048576)@@ -4298,14 +5218,14 @@ class Rwkv7Model(Model):new_name += ".weight"if self.lora_needs_transpose and any(new_name.endswith(t) for t in [- "time_mix_w1.weight",- "time_mix_w2.weight",- "time_mix_a1.weight",- "time_mix_a2.weight",- "time_mix_v1.weight",- "time_mix_v2.weight",- "time_mix_g1.weight",- "time_mix_g2.weight",+ "time_mix_w1.weight",+ "time_mix_w2.weight",+ "time_mix_a1.weight",+ "time_mix_a2.weight",+ "time_mix_v1.weight",+ "time_mix_v2.weight",+ "time_mix_g1.weight",+ "time_mix_g2.weight",]):data_torch = data_torch.transpose(0, 1)@@ -4320,7 +5240,7 @@ class Rwkv7Model(Model):yield (new_name, data_torch)[email protected]("RwkvHybridForCausalLM")[email protected]("RwkvHybridForCausalLM")class ARwkv7Model(Rwkv7Model):model_arch = gguf.MODEL_ARCH.ARWKV7@@ -4363,10 +5283,18 @@ class ARwkv7Model(Rwkv7Model):self.gguf_writer.add_head_count(0)[email protected]("MambaForCausalLM", "MambaLMHeadModel", "FalconMambaForCausalLM")-class MambaModel(Model):[email protected]("MambaForCausalLM", "MambaLMHeadModel", "FalconMambaForCausalLM")+class MambaModel(TextModel):model_arch = gguf.MODEL_ARCH.MAMBA+ def __init__(self, dir_model: Path, *args, **kwargs):+ # Avoid using AutoConfig for hparams+ hparams = kwargs.pop("hparams", None)+ if hparams is None:+ with open(dir_model / "config.json", "r", encoding="utf-8") as f:+ hparams = json.load(f)+ super().__init__(dir_model, *args, hparams=hparams, **kwargs)+def set_vocab(self):vocab_size = self.hparams["vocab_size"]# Round vocab size to next multiple of 8@@ -4398,7 +5326,7 @@ class MambaModel(Model):# Fail early for models which don't have a block expansion factor of 2assert d_inner == 2 * d_model- self.gguf_writer.add_context_length(2 ** 20) # arbitrary value; for those who use the default+ self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the defaultself.gguf_writer.add_embedding_length(d_model)self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loadingself.gguf_writer.add_head_count(0) # unused, but seemingly required when loading@@ -4439,8 +5367,103 @@ class MambaModel(Model):return [(new_name, data_torch)][email protected]("CohereForCausalLM")-class CommandR2Model(Model):[email protected]("Mamba2ForCausalLM")+class Mamba2Model(TextModel):+ model_arch = gguf.MODEL_ARCH.MAMBA2++ def __init__(self, dir_model: Path, *args, **kwargs):+ # Avoid using AutoConfig for hparams+ # It wrongly assumes all Mamba2 models are Mamba-Codestral-7B-v0.1+ hparams = kwargs.pop("hparams", None)+ if hparams is None:+ with open(dir_model / "config.json", "r", encoding="utf-8") as f:+ hparams = json.load(f)+ super().__init__(dir_model, *args, hparams=hparams, **kwargs)++ def set_vocab(self):+ vocab_size = self.hparams["vocab_size"]+ # Round vocab size to next multiple of 16+ pad_vocab = self.hparams.get("pad_vocab_size_multiple", 16)+ # pad using ceiling division+ # ref: https://stackoverflow.com/a/17511341/22827863+ vocab_size = -(vocab_size // -pad_vocab) * pad_vocab+ self.hparams["vocab_size"] = vocab_size++ if (self.dir_model / "tokenizer.model").is_file():+ self._set_vocab_sentencepiece()+ elif (self.dir_model / "tokenizer.model.v3").is_file():+ # mamba-codestral+ raise NotImplementedError(+ f"Please rename {self.dir_model / 'tokenizer.model.v3'} to {self.dir_model / 'tokenizer.model'}")+ elif (self.dir_model / "tokenizer.json").is_file():+ self._set_vocab_gpt2()+ else:+ # Use the GPT-NeoX tokenizer when no tokenizer files are present+ self._set_vocab_builtin("gpt-neox", vocab_size)++ def set_gguf_parameters(self):+ d_model = self.find_hparam(["hidden_size", "d_model", "dim"])+ d_conv = self.find_hparam(["conv_kernel", "d_conv"], optional=True) or 4+ d_inner = self.find_hparam(["intermediate_size", "d_inner"], optional=True) or 2 * d_model+ d_state = self.find_hparam(["state_size", "d_state"], optional=True) or 128+ head_dim = self.find_hparam(["head_dim"], optional=True) or 64+ n_group = self.find_hparam(["n_groups"], optional=True) or 1++ rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-5++ # Fail early for models which don't have a block expansion factor of 2+ # TODO: does this really matter?+ assert d_inner == 2 * d_model+ assert d_inner % head_dim == 0++ self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default+ self.gguf_writer.add_embedding_length(d_model)+ self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loading+ self.gguf_writer.add_head_count(0) # unused, but seemingly required when loading+ self.gguf_writer.add_block_count(self.block_count)+ self.gguf_writer.add_ssm_conv_kernel(d_conv)+ self.gguf_writer.add_ssm_inner_size(d_inner)+ self.gguf_writer.add_ssm_state_size(d_state)+ self.gguf_writer.add_ssm_time_step_rank(d_inner // head_dim)+ self.gguf_writer.add_ssm_group_count(n_group)+ self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)+ self.gguf_writer.add_file_type(self.ftype)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:++ if name.startswith("model.backbone") or name.startswith("model.lm_head"):+ # map Mamba-Codestral-7B-v0.1 tensor names to the names used by Mamba-2+ name = name.removeprefix("model.")++ if name.endswith(".dt_bias"):+ name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"++ new_name = self.map_tensor_name(name)++ if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_CONV1D, bid):+ data_torch = data_torch.squeeze()+ elif any(self.match_model_tensor_name(new_name, t, bid, suffix="") for t in [+ gguf.MODEL_TENSOR.SSM_A,+ gguf.MODEL_TENSOR.SSM_D,+ ]):+ # unsqueeze A to use similar shape semantics as Mamba-1+ # (D is also unsqueezed, but for more straightforward broadcast internally)+ data_torch = data_torch.reshape((*data_torch.shape, 1))+ elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_NORM, bid):+ d_model = self.find_hparam(["hidden_size", "d_model", "dim"])+ d_inner = self.find_hparam(["intermediate_size", "d_inner"], optional=True) or 2 * d_model+ n_group = self.hparams.get("n_groups", 1)+ data_torch = data_torch.reshape((n_group, d_inner // n_group))++ if name.endswith(".A_log"):+ logger.debug("A_log --> A ==> " + new_name)+ data_torch = -torch.exp(data_torch)++ yield (new_name, data_torch)++[email protected]("CohereForCausalLM")+class CommandR2Model(TextModel):model_arch = gguf.MODEL_ARCH.COMMAND_Rdef __init__(self, *args, **kwargs):@@ -4457,8 +5480,8 @@ class CommandR2Model(Model):self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)[email protected]("Cohere2ForCausalLM")-class Cohere2Model(Model):[email protected]("Cohere2ForCausalLM")+class Cohere2Model(TextModel):model_arch = gguf.MODEL_ARCH.COHERE2def set_gguf_parameters(self):@@ -4471,13 +5494,13 @@ class Cohere2Model(Model):rotary_pct = self.hparams["rotary_pct"]hidden_size = self.hparams["hidden_size"]num_attention_heads = self.hparams["num_attention_heads"]- self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size // num_attention_heads)))+ self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size//num_attention_heads)))self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)[email protected]("OlmoForCausalLM")[email protected]("OLMoForCausalLM")-class OlmoModel(Model):[email protected]("OlmoForCausalLM")[email protected]("OLMoForCausalLM")+class OlmoModel(TextModel):model_arch = gguf.MODEL_ARCH.OLMOdef set_gguf_parameters(self):@@ -4503,13 +5526,13 @@ class OlmoModel(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("Olmo2ForCausalLM")-class Olmo2Model(Model):[email protected]("Olmo2ForCausalLM")+class Olmo2Model(TextModel):model_arch = gguf.MODEL_ARCH.OLMO2[email protected]("OlmoeForCausalLM")-class OlmoeModel(Model):[email protected]("OlmoeForCausalLM")+class OlmoeModel(TextModel):model_arch = gguf.MODEL_ARCH.OLMOEdef set_gguf_parameters(self):@@ -4568,29 +5591,10 @@ class OlmoeModel(Model):raise ValueError(f"Unprocessed experts: {experts}")[email protected]("JinaBertModel", "JinaBertForMaskedLM")[email protected]("JinaBertModel", "JinaBertForMaskedLM")class JinaBertV2Model(BertModel):model_arch = gguf.MODEL_ARCH.JINA_BERT_V2- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.intermediate_size = self.hparams["intermediate_size"]-- def get_tensors(self):- for name, data in super().get_tensors():- if 'gated_layer' in name:- d1 = data[:self.intermediate_size, :]- name1 = name.replace('gated_layers', 'gated_layers_w')- name1 = name1.replace('up_gated_layer', 'gated_layers_v')- d2 = data[self.intermediate_size:, :]- name2 = name.replace('gated_layers', 'gated_layers_v')- name2 = name2.replace('up_gated_layer', 'gated_layers_w')- yield name1, d1- yield name2, d2- continue-- yield name, data-def set_vocab(self):tokenizer_class = 'BertTokenizer'with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f:@@ -4603,25 +5607,15 @@ class JinaBertV2Model(BertModel):self.gguf_writer.add_token_type_count(2)else:raise NotImplementedError(f'Tokenizer {tokenizer_class} is not supported for JinaBertModel')- self.gguf_writer.add_add_bos_token(True)- self.gguf_writer.add_add_eos_token(True)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # if name starts with "bert.", remove the prefix- # e.g. https://huggingface.co/jinaai/jina-reranker-v1-tiny-en- if name.startswith("bert."):- name = name[5:]-- return super().modify_tensors(data_torch, name, bid)[email protected]("OpenELMForCausalLM")-class OpenELMModel(Model):[email protected]("OpenELMForCausalLM")+class OpenELMModel(TextModel):model_arch = gguf.MODEL_ARCH.OPENELM@staticmethoddef _make_divisible(v: float | int, divisor: int) -> int:- new_v = max(divisor, int(v + divisor / 2) // divisor * divisor)+ new_v = max(divisor, int(v + divisor/2) // divisor * divisor)# Make sure that round down does not go down by more than 10%.if new_v < 0.9 * v:new_v += divisor@@ -4688,8 +5682,8 @@ class OpenELMModel(Model):yield (self.map_tensor_name(name), data_torch)[email protected]("ArcticForCausalLM")-class ArcticModel(Model):[email protected]("ArcticForCausalLM")+class ArcticModel(TextModel):model_arch = gguf.MODEL_ARCH.ARCTICdef set_vocab(self):@@ -4763,8 +5757,9 @@ class ArcticModel(Model):token_score = 0.0logger.info(- f"Setting added token {token_id} to '{token_content}' "- f"(type: {token_type}, score: {token_score:.2f})")+ f"Setting added token {token_id} to '{token_content}' (type: {token_type},"+ f" score: {token_score:.2f})"+ )tokens[token_id] = token_content.encode("utf-8")toktypes[token_id] = token_typescores[token_id] = token_score@@ -4841,8 +5836,8 @@ class ArcticModel(Model):raise ValueError(f"Unprocessed experts: {experts}")[email protected]("DeepseekForCausalLM")-class DeepseekModel(Model):[email protected]("DeepseekForCausalLM")+class DeepseekModel(TextModel):model_arch = gguf.MODEL_ARCH.DEEPSEEKdef set_vocab(self):@@ -4854,9 +5849,7 @@ class DeepseekModel(Model):def set_gguf_parameters(self):super().set_gguf_parameters()hparams = self.hparams- if "head_dim" in hparams:- rope_dim = hparams["head_dim"]- else:+ if (rope_dim := hparams.get("head_dim")) is None:rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]self.gguf_writer.add_rope_dimension_count(rope_dim)@@ -4932,9 +5925,9 @@ class DeepseekModel(Model):raise ValueError(f"Unprocessed experts: {experts}")[email protected]("DeepseekV2ForCausalLM")[email protected]("DeepseekV3ForCausalLM")-class DeepseekV2Model(Model):[email protected]("DeepseekV2ForCausalLM")[email protected]("DeepseekV3ForCausalLM")+class DeepseekV2Model(TextModel):model_arch = gguf.MODEL_ARCH.DEEPSEEK2def set_vocab(self):@@ -4975,13 +5968,12 @@ class DeepseekV2Model(Model):self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])- if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:- if self.hparams["rope_scaling"].get("type") == "yarn":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])- self.gguf_writer.add_rope_scaling_orig_ctx_len(- self.hparams["rope_scaling"]["original_max_position_embeddings"])- self.gguf_writer.add_rope_scaling_yarn_log_mul(0.1 * hparams["rope_scaling"]["mscale_all_dim"])+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])+ self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])+ self.gguf_writer.add_rope_scaling_yarn_log_mul(0.1 * rope_scaling["mscale_all_dim"])_experts: list[dict[str, Tensor]] | None = None@@ -5038,7 +6030,7 @@ class DeepseekV2Model(Model):v_head_dim = self.hparams["v_head_dim"]qk_nope_head_dim = self.hparams["qk_nope_head_dim"]- assert data_torch.shape[0] == n_head_kv * (v_head_dim + qk_nope_head_dim)+ assert data_torch.shape[0] == n_head_kv * (v_head_dim+qk_nope_head_dim)kv_b = data_torch.view(n_head_kv, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)@@ -5058,8 +6050,36 @@ class DeepseekV2Model(Model):raise ValueError(f"Unprocessed experts: {experts}")[email protected]("PLMForCausalLM")-class PLMModel(Model):[email protected]("Dots1ForCausalLM")+class Dots1Model(Qwen2MoeModel):+ model_arch = gguf.MODEL_ARCH.DOTS1++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.hparams["num_experts"] = self.hparams["n_routed_experts"]++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"])+ self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"])+ self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])+ self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])++ if self.hparams["scoring_func"] == "noaux_tc":+ self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)+ else:+ raise ValueError(f"Unsupported scoring_func value: {self.hparams['scoring_func']}")++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):+ if name.endswith("e_score_correction_bias"):+ name = name.replace("e_score_correction_bias", "e_score_correction.bias")+ if "shared_experts" in name:+ return [(self.map_tensor_name(name), data_torch)]+ return super().modify_tensors(data_torch, name, bid)++[email protected]("PLMForCausalLM")+class PLMModel(TextModel):model_arch = gguf.MODEL_ARCH.PLMdef set_vocab(self):@@ -5081,11 +6101,11 @@ class PLMModel(Model):super().prepare_tensors()[email protected]("T5WithLMHeadModel")[email protected]("T5ForConditionalGeneration")[email protected]("MT5ForConditionalGeneration")[email protected]("UMT5ForConditionalGeneration")-class T5Model(Model):[email protected]("T5WithLMHeadModel")[email protected]("T5ForConditionalGeneration")[email protected]("MT5ForConditionalGeneration")[email protected]("UMT5ForConditionalGeneration")+class T5Model(TextModel):model_arch = gguf.MODEL_ARCH.T5def __init__(self, *args, **kwargs):@@ -5186,9 +6206,6 @@ class T5Model(Model):special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))special_vocab.add_to_gguf(self.gguf_writer)- self.gguf_writer.add_add_bos_token(False)- self.gguf_writer.add_add_eos_token(True)-def set_gguf_parameters(self):if (n_ctx := self.find_hparam(["n_positions"], optional=True)) is None:logger.warning("Couldn't find context length in config.json, assuming default value of 512")@@ -5220,8 +6237,8 @@ class T5Model(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("T5EncoderModel")-class T5EncoderModel(Model):[email protected]("T5EncoderModel")+class T5EncoderModel(TextModel):model_arch = gguf.MODEL_ARCH.T5ENCODERdef __init__(self, *args, **kwargs):@@ -5322,9 +6339,6 @@ class T5EncoderModel(Model):special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))special_vocab.add_to_gguf(self.gguf_writer)- self.gguf_writer.add_add_bos_token(False)- self.gguf_writer.add_add_eos_token(True)-def set_gguf_parameters(self):if (n_ctx := self.find_hparam(["n_positions"], optional=True)) is None:logger.warning("Couldn't find context length in config.json, assuming default value of 512")@@ -5355,8 +6369,8 @@ class T5EncoderModel(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("JAISLMHeadModel")-class JaisModel(Model):[email protected]("JAISLMHeadModel")+class JaisModel(TextModel):model_arch = gguf.MODEL_ARCH.JAISdef __init__(self, *args, **kwargs):@@ -5413,7 +6427,7 @@ class JaisModel(Model):# Some other models has max_alibi_bias spelled out explicitly in the hyperparams,# but Jais's PyTorch model simply precalculates the slope values and places them# in relative_pes.slopes- n_head_closest_log2 = 2 ** math.floor(math.log2(self.hparams["n_head"]))+ n_head_closest_log2 = 2**math.floor(math.log2(self.hparams["n_head"]))first_val = float(data_torch[0].item())self.max_alibi_bias = -round(math.log2(first_val) * n_head_closest_log2)@@ -5438,25 +6452,39 @@ class JaisModel(Model):self.gguf_writer.add_max_alibi_bias(self.max_alibi_bias)[email protected]("Glm4ForCausalLM")-class Glm4Model(Model):[email protected]("Glm4ForCausalLM")+class Glm4Model(TextModel):model_arch = gguf.MODEL_ARCH.GLM4def set_vocab(self):- self._set_vocab_gpt2()+ from transformers import AutoTokenizer+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)+ tokens, toktypes, tokpre = self.get_vocab_base()+ self.gguf_writer.add_tokenizer_model("gpt2")+ self.gguf_writer.add_tokenizer_pre(tokpre)+ self.gguf_writer.add_token_list(tokens)+ self.gguf_writer.add_token_types(toktypes)+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)+ special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])+ special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])+ special_vocab.add_to_gguf(self.gguf_writer)def set_gguf_parameters(self):super().set_gguf_parameters()- if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:- if self.hparams["rope_scaling"].get("type") == "yarn":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])- self.gguf_writer.add_rope_scaling_orig_ctx_len(- self.hparams["rope_scaling"]["original_max_position_embeddings"])+ rope_dim = self.hparams["head_dim"]+ self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)))+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])+ self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])[email protected]("GlmForCausalLM", "ChatGLMModel", "ChatGLMForConditionalGeneration")-class ChatGLMModel(Model):[email protected]("GlmForCausalLM", "ChatGLMModel", "ChatGLMForConditionalGeneration")+class ChatGLMModel(TextModel):model_arch = gguf.MODEL_ARCH.CHATGLMdef set_vocab_chatglm3(self):@@ -5609,8 +6637,8 @@ class ChatGLMModel(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("NemotronForCausalLM")-class NemotronModel(Model):[email protected]("NemotronForCausalLM")+class NemotronModel(TextModel):model_arch = gguf.MODEL_ARCH.NEMOTRONdef set_vocab(self):@@ -5646,8 +6674,8 @@ class NemotronModel(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("ExaoneForCausalLM")-class ExaoneModel(Model):[email protected]("ExaoneForCausalLM")+class ExaoneModel(TextModel):model_arch = gguf.MODEL_ARCH.EXAONEdef set_gguf_parameters(self):@@ -5681,17 +6709,18 @@ class ExaoneModel(Model):rotary_factor = rotary_factor if rotary_factor is not None else 1.0self.gguf_writer.add_rope_dimension_count(int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"])))- if hparams.get("rope_scaling") is not None and "factor" in hparams["rope_scaling"]:- if hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(hparams["rope_scaling"]["factor"])+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):if rope_scaling.get("rope_type", '').lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)- dim = self.hparams.get("head_dim", self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ if (dim := self.hparams.get("head_dim")) is None:+ dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]+ freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)@@ -5710,15 +6739,15 @@ class ExaoneModel(Model):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)- rope_factors.append(1 / ((1 - smooth) / factor + smooth))+ smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)+ rope_factors.append(1 / ((1-smooth) / factor + smooth))yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),torch.tensor(rope_factors, dtype=torch.float32))[email protected]("GraniteForCausalLM")[email protected]("GraniteForCausalLM")class GraniteModel(LlamaModel):"""Conversion for IBM's GraniteForCausalLM"""model_arch = gguf.MODEL_ARCH.GRANITE@@ -5752,11 +6781,20 @@ class GraniteModel(LlamaModel):logger.info("gguf: (granite) logits_scale = %s", logits_scale)[email protected]("GraniteMoeForCausalLM")[email protected]("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM")class GraniteMoeModel(GraniteModel):"""Conversion for IBM's GraniteMoeForCausalLM"""model_arch = gguf.MODEL_ARCH.GRANITE_MOE+ def set_gguf_parameters(self):+ """GraniteMoeShared uses GraniteMoe parameters plus the following:+ - shared_intermediate_size+ """+ super().set_gguf_parameters()+ if shared_feed_forward_length := self.hparams.get("shared_intermediate_size"):+ self.gguf_writer.add_expert_shared_feed_forward_length(shared_feed_forward_length)+ logger.info("gguf: (granitemoeshared) shared_feed_forward_length = %s", shared_feed_forward_length)+def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:"""In modeling_granitemoe, the JetMoe implementation of parallel expertsis used. This essentially merges w1 and w3 into a single tensor with 2x@@ -5767,17 +6805,26 @@ class GraniteMoeModel(GraniteModel):if name.endswith("block_sparse_moe.input_linear.weight"):ffn_dim = self.hparams["intermediate_size"]assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size"- gate, up = data_torch[..., :ffn_dim, :], data_torch[..., ffn_dim:, :]+ gate, up = data_torch.split(ffn_dim, dim=-2)return [(self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), gate),(self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), up),]+ if name.endswith("shared_mlp.input_linear.weight"):+ ffn_dim = self.hparams["shared_intermediate_size"]+ assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size"+ gate, up = data_torch.split(ffn_dim, dim=-2)+ return [+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_SHEXP, bid), gate),+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), up),+ ]+return super().modify_tensors(data_torch, name, bid)[email protected]("BailingMoeForCausalLM")-class BailingMoeModel(Model):[email protected]("BailingMoeForCausalLM")+class BailingMoeModel(TextModel):model_arch = gguf.MODEL_ARCH.BAILINGMOEdef set_vocab(self):@@ -5786,10 +6833,17 @@ class BailingMoeModel(Model):def set_gguf_parameters(self):super().set_gguf_parameters()hparams = self.hparams- rope_dim = hparams.get("head_dim") or hparams["hidden_size"] // hparams["num_attention_heads"]+ if (rope_dim := hparams.get("head_dim")) is None:+ rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]self.gguf_writer.add_rope_dimension_count(rope_dim)- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])+ self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])+ else:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])self.gguf_writer.add_vocab_size(hparams["vocab_size"])self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])@@ -5812,7 +6866,8 @@ class BailingMoeModel(Model):n_head = self.hparams["num_attention_heads"]n_kv_head = self.hparams.get("num_key_value_heads")n_embd = self.hparams["hidden_size"]- head_dim = self.hparams.get("head_dim") or n_embd // n_head+ if (head_dim := self.hparams.get("head_dim")) is None:+ head_dim = n_embd // n_headoutput_name = self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT)@@ -5875,9 +6930,9 @@ class BailingMoeModel(Model):raise ValueError(f"Unprocessed experts: {experts}")[email protected]("ChameleonForConditionalGeneration")[email protected]("ChameleonForCausalLM") # obsolete-class ChameleonModel(Model):[email protected]("ChameleonForConditionalGeneration")[email protected]("ChameleonForCausalLM") # obsolete+class ChameleonModel(TextModel):model_arch = gguf.MODEL_ARCH.CHAMELEONdef set_gguf_parameters(self):@@ -5916,6 +6971,69 @@ class ChameleonModel(Model):return data_torch[email protected]("UltravoxModel")+class UltravoxModel(TextModel):+ model_arch = gguf.MODEL_ARCH.LLAMA # dummy++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ raise NotImplementedError(+ "Ultravox does not have text decoder. Instead, it uses Llama or other models for text. If you want to"+ " get the audio encoder, please use --mmproj argument"+ )++[email protected]("Qwen2AudioForConditionalGeneration")+class WhisperEncoderModel(MmprojModel):+ has_vision_encoder = False # no vision encoder+ has_audio_encoder = True++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.hparams["hidden_size"] = self.hparams["d_model"]+ self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"]+ self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"]++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN2A)+ self.gguf_writer.add_audio_num_mel_bins(self.hparams["num_mel_bins"])+ self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))++ def tensor_force_quant(self, name, new_name, bid, n_dims):+ del bid, new_name, n_dims # unused+ if ".conv" in name and ".weight" in name:+ return gguf.GGMLQuantizationType.F16+ return False++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused++ if name.startswith("language_model."):+ # skip language model tensors+ return []++ # prevent clash naming with vision tensors+ if name.startswith("multi_modal_projector"):+ name = "audio." + name++ if "conv1.bias" in name or "conv2.bias" in name:+ # transpose conv1 and conv2 bias+ data_torch = data_torch.unsqueeze(-1)++ return [(self.map_tensor_name(name), data_torch)]++[email protected]("UltravoxModel")+class UltravoxWhisperEncoderModel(WhisperEncoderModel):+ has_vision_encoder = False # no vision encoder+ has_audio_encoder = True++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_audio_stack_factor(self.global_config["stack_factor"])++###### CONVERSION LOGIC ######@@ -6007,9 +7125,9 @@ def parse_args() -> argparse.Namespace:type=str,choices=["f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"],default="f16",- help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or"- " tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type depending on the first loaded"- " tensor type",+ help=+ "output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0"+ " for ternary, and auto for the highest-fidelity 16-bit float type depending on the first loaded tensor type",)parser.add_argument("--bigendian",@@ -6018,8 +7136,8 @@ def parse_args() -> argparse.Namespace:)parser.add_argument("model",- type=Path,- help="directory containing model file",+ type=str,+ help="directory containing model file or huggingface repository ID (if --remote)",nargs="?",)parser.add_argument(@@ -6069,11 +7187,18 @@ def parse_args() -> argparse.Namespace:parser.add_argument("--remote",action="store_true",- help="(Experimental) Read safetensors file remotely without downloading to disk."- " Config and tokenizer files will still be downloaded. To use this feature, you need"- " to specify Hugging Face model repo name instead of a local directory. For example:"- " 'HuggingFaceTB/SmolLM2-1.7B-Instruct'. Note: To access gated repo, set HF_TOKEN "- "environment variable to your Hugging Face token.",+ help=+ "(Experimental) Read safetensors file remotely without downloading to disk. Config and tokenizer files will"+ " still be downloaded. To use this feature, you need to specify Hugging Face model repo name instead of a"+ " local directory. For example: 'HuggingFaceTB/SmolLM2-1.7B-Instruct'. Note: To access gated repo, set"+ " HF_TOKEN environment variable to your Hugging Face token.",+ )+ parser.add_argument(+ "--mmproj",+ action="store_true",+ help=+ "(Experimental) Export multimodal projector (mmproj) for vision models. This will only work on some"+ " vision models. A prefix 'mmproj-' will be added to the output file name.",)args = parser.parse_args()@@ -6100,12 +7225,34 @@ def split_str_to_n_bytes(split_str: str) -> int:return n+def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> str:+ # TODO @ngxson : this won't work correctly if the model has both audio & vision encoders+ # maybe we should fallback to text model's arch in that case, since not many models have both+ text_config = hparams.get("text_config", {})+ vision_config = hparams.get("vision_config", {})+ arch = None+ if (arches := hparams.get("architectures")) is not None and len(arches) > 0:+ arch = arches[0]+ elif "ssm_cfg" in hparams:+ # For non-hf Mamba and Mamba2 models+ arch = hparams["ssm_cfg"].get("layer", "Mamba") + "ForCausalLM"++ # if "architectures" is found in the sub-config, use that instead+ if model_type == ModelType.TEXT and text_config.get("architectures") is not None:+ arch = text_config["architectures"][0]+ elif model_type == ModelType.MMPROJ and vision_config.get("architectures") is not None:+ arch = vision_config["architectures"][0]+ if arch is None:+ raise ValueError("Failed to detect model architecture")+ return arch++def main() -> None:args = parse_args()if args.print_supported_models:logger.error("Supported models:")- Model.print_registered_models()+ ModelBase.print_registered_models()sys.exit(0)if args.verbose:@@ -6113,17 +7260,19 @@ def main() -> None:else:logging.basicConfig(level=logging.INFO)- dir_model = args.model-if args.remote:+ hf_repo_id = args.modelfrom huggingface_hub import snapshot_downloadlocal_dir = snapshot_download(- repo_id=str(dir_model), allow_patterns=["LICENSE", "*.json", "*.md", "*.txt", "tokenizer.model"])+ repo_id=hf_repo_id, allow_patterns=["LICENSE", "*.json", "*.md", "*.txt", "tokenizer.model"])dir_model = Path(local_dir)logger.info(f"Downloaded config and tokenizer to {local_dir}")+ else:+ hf_repo_id = None+ dir_model = Path(args.model)if not dir_model.is_dir():- logger.error(f'Error: {args.model} is not a directory')+ logger.error(f'Error: {dir_model} is not a directory')sys.exit(1)ftype_map: dict[str, gguf.LlamaFileType] = {@@ -6143,21 +7292,26 @@ def main() -> None:if args.outfile is not None:fname_out = args.outfile- elif args.remote:+ elif hf_repo_id:# if remote, use the model ID as the output file name- fname_out = Path("./" + str(args.model).replace("/", "-") + "-{ftype}.gguf")+ fname_out = Path("./" + hf_repo_id.replace("/", "-") + "-{ftype}.gguf")else:fname_out = dir_modellogger.info(f"Loading model: {dir_model.name}")- hparams = Model.load_hparams(dir_model)+ if args.mmproj:+ if "mmproj" not in fname_out.name:+ fname_out = ModelBase.add_prefix_to_filename(fname_out, "mmproj-")with torch.inference_mode():output_type = ftype_map[args.outtype]- model_architecture = hparams["architectures"][0]+ model_type = ModelType.MMPROJ if args.mmproj else ModelType.TEXT+ hparams = ModelBase.load_hparams(dir_model)+ model_architecture = get_model_architecture(hparams, model_type)+ logger.info(f"Model architecture: {model_architecture}")try:- model_class = Model.from_model_architecture(model_architecture)+ model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)except NotImplementedError:logger.error(f"Model {model_architecture} is not supported")sys.exit(1)@@ -6175,7 +7329,7 @@ def main() -> None:split_max_size=split_str_to_n_bytes(args.split_max_size),dry_run=args.dry_run,small_first_shard=args.no_tensor_first_split,- remote_hf_model_id=str(args.model) if args.remote else None)+ remote_hf_model_id=hf_repo_id)if args.vocab_only:logger.info("Exporting model vocab...")diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex a950c7b11..b75fe51f9 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -18,7 +18,7 @@ import shutilimport torchfrom pathlib import Pathimport time-from auto_round.export.export_to_gguf.convert import Model+from auto_round.export.export_to_gguf.convert import ModelBasefrom auto_round.utils import logger, LazyImport, get_block_names, flatten_list, check_to_quantized, get_modulegguf = LazyImport("gguf")@@ -49,14 +49,22 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {def create_model_class(output_dir, model, layer_config, backend="gguf:q4_0", low_cpu_mem_usage=False):tmp_work_dir = Path(os.path.join(output_dir, 'tmp_dir'))with torch.inference_mode():- hparams = Model.load_hparams(tmp_work_dir)- model_architecture = hparams["architectures"][0]+ hparams = ModelBase.load_hparams(tmp_work_dir)+ if "architectures" in hparams:+ model_architecture = hparams["architectures"][0]+ else:+ model_architecture = type(model).__name__+ if model_architecture not in ModelBase._model_classes:+ if model_architecture.replace("CausalLM", "ConditionalGeneration") in ModelBase._model_classes:+ model_architecture = model_architecture.replace("CausalLM", "ConditionalGeneration")+ elif model_architecture.replace("ConditionalGeneration", "CausalLM") in ModelBase._model_classes:+ model_architecture = model_architecture.replace("ConditionalGeneration", "CausalLM")try:- model_class = Model.from_model_architecture(model_architecture)+ model_class = ModelBase.from_model_architecture(model_architecture)except NotImplementedError:logger.error(f"Model {model_architecture} is not supported")sys.exit(1)- model_class = Model.from_model_architecture(model_architecture)+ model_class = ModelBase.from_model_architecture(model_architecture)model_name = model.name_or_path.split('/')if len(model_name[-1]) == 0:model_name = model_name[-2]diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex 3fcc3f842..98a5223fe 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -28,8 +28,17 @@ def register_qtype(name):return register-def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, imatrix=None,- device="cuda"):+def ggml_quant(+ data,+ ggml_type,+ scale=None,+ zp=None,+ wmin=None,+ d_scale=None,+ d_wmin=None,+ imatrix=None,+ device="cuda",+ original=False):block_size, type_size = GGML_QUANT_SIZES[ggml_type]data = data.to(torch.float32).to(device)@@ -43,7 +52,8 @@ def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin=None, d_scale=Nonen_blocks = data.nelement() // block_sizeblocks = data.reshape((n_blocks, block_size))quant_func = GGML_QUANT_TYPE[ggml_type]- new_data = quant_func(blocks, scale, zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix)+ new_data = quant_func(+ blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)assert new_data.shape[-1] == type_sizenew_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)@@ -122,27 +132,24 @@ def make_q3_quants(data, bits, do_rmse=False):if do_rmse:L = torch.round(iscale * data).clip(-nmax, nmax - 1)w = torch.pow(data, 2)- sumlx = torch.sum(w * data * L, dim=-1)[0]- suml2 = torch.sum(w * L * L, dim=-1)[0]+ sumlx = torch.sum(w * data * L, dim=-1)+ suml2 = torch.sum(w * L * L, dim=-1)for itry in range(5):- # w = np.power(data, 2)- if len(sumlx.shape) == 2:- sumlx = sumlx.unsqueeze(-1)- if len(suml2.shape) == 2:- suml2 = suml2.unsqueeze(-1)- slx = sumlx - w * data * L- replace_idx = slx > 0- sl2 = suml2 - w * L * L- new_L = torch.round(data * sl2 / slx).clip(-nmax, nmax - 1)- tmp_replace_idx = replace_idx & (new_L != L)- slx[tmp_replace_idx] += w[tmp_replace_idx] * data[tmp_replace_idx] * new_L[tmp_replace_idx]- sl2[tmp_replace_idx] += w[tmp_replace_idx] * new_L[tmp_replace_idx] * new_L[tmp_replace_idx]- replace_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)-- L[replace_idx] = new_L[replace_idx]- sumlx = slx- suml2 = sl2+ for i in range(sumlx.shape[-1]):+ w_tmp, data_tmp, L_tmp = w[:,:, i], data[:,:, i], L[:,:, i]+ slx = sumlx - w_tmp * data_tmp * L_tmp+ replace_idx = slx > 0+ sl2 = suml2 - w_tmp * L_tmp ** 2+ new_L = torch.round(data_tmp * sl2 / slx).clip(-nmax, nmax - 1)+ tmp_replace_idx = replace_idx & (new_L != L_tmp)+ slx[tmp_replace_idx] += w_tmp[tmp_replace_idx] * data_tmp[tmp_replace_idx] * new_L[tmp_replace_idx]+ sl2[tmp_replace_idx] += w_tmp[tmp_replace_idx] * new_L[tmp_replace_idx] * new_L[tmp_replace_idx]+ replace_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)+ L[:,:, i][replace_idx] = new_L[replace_idx]+ sumlx = slx+ suml2 = sl2+ return sumlx / suml2, L.to(torch.uint8)L = torch.round(iscale * data).clip(-nmax, nmax - 1) + nmaxscales = torch.where(iscale != 0, 1 / iscale, 0).reshape(iscale.shape[:2])@@ -404,7 +411,7 @@ def q8_0_quant_block(blocks, scale=None, zp=None, **kwargs) -> np.ndarray:@register_qtype("q2_k")-def q2_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):+def q2_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, imatrix=None, original=False, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)@@ -414,36 +421,49 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmimins = wmin.reshape((-1, QK_K // 16))output_d = d_scale.reshape(-1, 1).to(torch.float32)output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)- inv_scales = torch.where(output_d == 0, 0, 1 / output_d)- inv_mins = torch.where(d_wmin == 0, 0, 1 / output_dmin)- max_scales = torch.max(scales, dim=-1, keepdim=True)[0]- max_mins = torch.max(mins, dim=-1, keepdim=True)[0]- all_L = torch.zeros_like(blocks, dtype=torch.uint8)- else:+ output_scale = torch.round(scales / output_d).clip(0, 15).to(torch.uint8)+ output_scale |= torch.round(mins / output_dmin).clip(0, 15).to(torch.uint8) << 4+ all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 3).to(torch.uint8)+ elif original:scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]max_mins = torch.max(mins, dim=-1, keepdim=True)[0]output_d = torch.where(max_scales > 0, max_scales / 15, 0)output_dmin = torch.where(max_mins > 0, max_mins / 15., 0)- inv_scales = torch.where(max_scales > 0, 15. / max_scales, 0)- inv_mins = torch.where(max_mins > 0, 15. / max_mins, 0)-- replace_ids = (max_scales > 0).squeeze()- output_scale = torch.zeros_like(scales).to(torch.uint8)- output_scale[replace_ids] = torch.round(- inv_scales[replace_ids] * scales[replace_ids]).clip(0, 15).to(torch.uint8)-- replace_ids = (max_mins > 0).squeeze()- output_scale[replace_ids] |= torch.round(- inv_mins[replace_ids] * mins[replace_ids]).clip(0, 15).to(torch.uint8) << 4-- d_tmp = output_d * (output_scale & 0xF)- dm_tmp = output_dmin * (output_scale >> 4)-- replace_ids = d_tmp != 0- all_L[replace_ids] = torch.round(- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 3).to(- torch.uint8)+ id_scales = torch.where(max_scales > 0, 15. / max_scales, 0)+ id_mins = torch.where(max_mins > 0, 15. / max_mins, 0)++ replace_ids = (max_scales > 0).squeeze()+ output_scale = torch.zeros_like(scales).to(torch.uint8)+ output_scale[replace_ids] = torch.round(+ id_scales[replace_ids] * scales[replace_ids]).clip(0, 15).to(torch.uint8)++ replace_ids = (max_mins > 0).squeeze()+ output_scale[replace_ids] |= torch.round(+ id_mins[replace_ids] * mins[replace_ids]).clip(0, 15).to(torch.uint8) << 4++ d_tmp = output_d * (output_scale & 0xF)+ dm_tmp = output_dmin * (output_scale >> 4)++ replace_ids = d_tmp != 0+ all_L[replace_ids] = torch.round(+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / \+ d_tmp[replace_ids].unsqueeze(-1)).clip(0, 3).to(torch.uint8)+ else:+ from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq+ blocks.reshape(blocks.shape[0], -1)+ blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=2, scale_dtype=torch.float32, imatrix=imatrix)+ scales, d_scale = scales['scale'], scales['d_scale']+ mins, d_wmin = mins['wmin'], mins['d_wmin']++ blocks = blocks.reshape((nb, QK_K // 16, 16))+ scales = scales.reshape((-1, QK_K // 16))+ mins = mins.reshape((-1, QK_K // 16))+ output_d = d_scale.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)+ output_scale = torch.round(scales / output_d).clip(0, 15).to(torch.uint8)+ output_scale |= torch.round(mins / output_dmin).clip(0, 15).to(torch.uint8) << 4+ all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 3).to(torch.uint8)output_scale = output_scale.cpu().numpy()all_L = all_L.reshape(-1, 4, 32)@@ -458,7 +478,7 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmireturn np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)@register_qtype("q3_k")-def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):+def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False, imatrix=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape(nb, QK_K // 16, 16)@@ -466,14 +486,10 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin=None, d_scale=Nif scale is not None:qdq_scale = scale.reshape(-1, QK_K // 16).to(torch.float32)- dq_scale=d_scale.reshape(-1, 1).to(torch.float32)- inverse_dq_scale = torch.where(dq_scale==0,0,1/dq_scale)- # output_d = d_scale.reshape(-1, 1).to(torch.float32)- # iscales = torch.where(output_d == 0, 0, 1 / output_d)- # inverse_scale = torch.where(scales==0, 0, 1.0 / scales)- # all_L = torch.round(blocks*inverse_scale.unsqueeze(-1)).clip(-4, 3) + 4- # all_L = all_L.to(torch.uint8)- else: ## this is correct+ dq_scale = d_scale.reshape(-1, 1).to(torch.float32)+ all_L = (torch.round(blocks / qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)+ q_scales_offset = torch.round(qdq_scale / dq_scale).clip(-32, 31) + 32+ elif original: ## this is correctscales, _ = make_q3_quants(blocks, bits=3, do_rmse=True)scales_abs_max = abs(scales).argmax(dim=-1, keepdim=True)max_scales_mag = torch.take_along_dim(scales, scales_abs_max, dim=-1)@@ -481,18 +497,26 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin=None, d_scale=Ndq_scale = torch.where(inverse_dq_scale != 0, 1 / inverse_dq_scale, 0)qscale = torch.round(inverse_dq_scale * scales).clip(-32, 31)qdq_scale = dq_scale.to(torch.float32) * qscale+ reverse_qdq_scale = torch.where(qdq_scale == 0, 0, 1 / qdq_scale)+ all_L = (torch.round(blocks * reverse_qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)+ q_scales_offset = torch.round(qdq_scale * inverse_dq_scale).clip(-32, 31) + 32+ else:+ from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq+ blocks = blocks.reshape(blocks.shape[0], -1)+ blocks, scales, _ = quant_tensor_gguf_sym_dq(blocks, bits=3, scale_dtype=torch.float32, imatrix=imatrix)+ scales, d_scale = scales['scale'], scales['d_scale']+ blocks = blocks.reshape((nb, QK_K // 16, 16))+ qdq_scale = scales.reshape((-1, QK_K // 16)).to(torch.float32)+ dq_scale = d_scale.reshape(-1, 1).to(torch.float32)+ all_L = (torch.round(blocks / qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)+ q_scales_offset = torch.round(qdq_scale / dq_scale).clip(-32, 31) + 32- # q_scales_offset = torch.round(inverse_dq_scale * scales).clip(-32, 31) + 32- q_scales_offset = torch.round(qdq_scale*inverse_dq_scale).clip(-32, 31) + 32q_scales_offset = q_scales_offset.cpu().numpy().astype(np.uint8)output_scale[:, :8] = (q_scales_offset[:, :8] & 0xF) | ((q_scales_offset[:, 8:] & 0xF) << 4)hmask = q_scales_offset >> 4output_scale[:, 8:] = hmask[:, :4] | hmask[:, 4:8] << 2 | hmask[:, 8:12] << 4 | hmask[:, 12:] << 6- reverse_qdq_scale = torch.where(qdq_scale == 0, 0, 1 / qdq_scale)- all_L = (torch.round(blocks * reverse_qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(- torch.uint8)output_hmask = all_L.reshape(nb, 8, 32) >> 2 << torch.arange(8, device=all_L.device).reshape(1, 8, 1)output_hmask = np.bitwise_or.reduce(output_hmask.cpu().numpy(), axis=1, dtype=np.uint8) # pylint: disable=E1121@@ -509,7 +533,7 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin=None, d_scale=N@register_qtype("q4_k")-def q4_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):+def q4_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, imatrix=None, original=False, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 32, 32))@@ -520,34 +544,41 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmimins = wmin.reshape(-1, QK_K // 32)output_d = d_scale.reshape(-1, 1).to(torch.float32)output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)- inv_scales = torch.where(output_d == 0, 0, 1 / output_d)- inv_mins = torch.where(d_wmin == 0, 0, 1 / output_dmin)- max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)- max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)- all_L = torch.zeros_like(blocks, dtype=torch.uint8)- # inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)- # all_L = torch.round(- # (blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(- # 0,15).to(torch.uint8)- else:+ q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)+ all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 15).to(torch.uint8)+ elif original:scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]max_mins = torch.max(mins, dim=-1, keepdim=True)[0]- inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)- inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)-- q_scales = torch.round(inv_scales * scales).clip(0, 63).to(torch.uint8)- q_mins = torch.round(inv_mins * mins).clip(0, 63).to(torch.uint8)-- if d_scale is None:+ id_scales = torch.where(max_scales > 0, 63. / max_scales, 0)+ id_mins = torch.where(max_mins > 0, 63. / max_mins, 0)output_d = max_scales / 63- if d_wmin is None:output_dmin = max_mins / 63+ q_scales = torch.round(id_scales * scales).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(id_mins * mins).clip(0, 63).to(torch.uint8)++ d_tmp = output_d * q_scales+ dm_tmp = output_dmin * q_mins+ replace_ids = d_tmp != 0+ all_L[replace_ids] = torch.round((blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) /+ d_tmp[replace_ids].unsqueeze(-1)).clip(0, 15).to(torch.uint8)+ else:+ from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq+ blocks.reshape(blocks.shape[0], -1)+ blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix)+ scales, d_scale = scales['scale'], scales['d_scale']+ mins, d_wmin = mins['wmin'], mins['d_wmin']++ blocks = blocks.reshape((nb, QK_K // 32, 32))+ scales = scales.reshape((-1, QK_K // 32))+ mins = mins.reshape((-1, QK_K // 32))+ output_d = d_scale.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)+ q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)+ all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 15).to(torch.uint8)- d_tmp = output_d * q_scales- dm_tmp = output_dmin * q_mins- # q_scales = q_scales.cpu().numpy().astype(np.uint8)- # q_mins = q_mins.cpu().numpy().astype(np.uint8)output_scale[:, :4] = q_scales[:, :4]output_scale[:, 4:8] = q_mins[:, :4]@@ -555,10 +586,6 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmioutput_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)- replace_ids = d_tmp != 0- all_L[replace_ids] = torch.round(- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(- 0, 15).to(torch.uint8)output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)output_d = output_d.cpu().numpy()@@ -573,41 +600,54 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmi@register_qtype("q5_k")-def q5_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):+def q5_k_quant_block(+ blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, imatrix=None, original=False, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 32, 32))output_scale = torch.empty((nb, K_SCALE_SIZE), dtype=torch.uint8, device=blocks.device)+if scale is not None:scales = scale.reshape(-1, QK_K // 32)mins = wmin.reshape(-1, QK_K // 32)output_d = d_scale.reshape(-1, 1).to(torch.float32)output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)- inv_scales = torch.where(output_d == 0, 0, 1 / output_d)- inv_mins = torch.where(d_wmin == 0, 0, 1 / output_dmin)- max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)- max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)- all_L = torch.zeros_like(blocks, dtype=torch.uint8)- else:+ q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)+ all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 31).to(torch.uint8)+ elif original:scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]max_mins = torch.max(mins, dim=-1, keepdim=True)[0]- inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)- inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)-- q_scales = torch.round(inv_scales * scales).clip(0, 63).to(torch.uint8)- q_mins = torch.round(inv_mins * mins).clip(0, 63).to(torch.uint8)-- if d_scale is None:+ id_scales = torch.where(max_scales > 0, 63. / max_scales, 0)+ id_mins = torch.where(max_mins > 0, 63. / max_mins, 0)output_d = max_scales / 63- if d_wmin is None:output_dmin = max_mins / 63+ q_scales = torch.round(id_scales * scales).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(id_mins * mins).clip(0, 63).to(torch.uint8)++ d_tmp = output_d * q_scales+ dm_tmp = output_dmin * q_mins+ replace_ids = d_tmp != 0+ all_L[replace_ids] = torch.round(+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) \+ / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 31).to(torch.uint8)+ else:+ from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq+ blocks.reshape(blocks.shape[0], -1)+ blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix)+ scales, d_scale = scales['scale'], scales['d_scale']+ mins, d_wmin = mins['wmin'], mins['d_wmin']++ blocks = blocks.reshape((nb, QK_K // 32, 32))+ scales = scales.reshape((-1, QK_K // 32))+ mins = mins.reshape((-1, QK_K // 32))+ output_d = d_scale.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)+ q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)+ all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 31).to(torch.uint8)- d_tmp = output_d * q_scales- dm_tmp = output_dmin * q_mins-- # q_scales = q_scales.cpu().numpy().astype(np.uint8)- # q_mins = q_mins.cpu().numpy().astype(np.uint8)output_scale[:, :4] = q_scales[:, :4]output_scale[:, 4:8] = q_mins[:, :4]@@ -615,13 +655,6 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmioutput_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)- replace_ids = d_tmp != 0- all_L[replace_ids] = torch.round(- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) \- / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 31).to(torch.uint8)- # all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 31) # (nb, 8, 32)- all_L = torch.clamp(all_L, 0, 31)-output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)output_qh = all_L >> 4 << torch.arange(8, device=all_L.device).reshape(1, 8, 1)output_qh = np.bitwise_or.reduce(@@ -639,30 +672,37 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmi@register_qtype("q6_k")-def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):+def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False, imatrix=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 16, 16))- scale = Noneif scale is not None:scales = scale.reshape(-1, QK_K // 16)output_d = d_scale.reshape(-1, 1).to(torch.float32)- iscales = torch.where(output_d == 0, 0, 1 / output_d)- all_L = torch.zeros_like(blocks, dtype=torch.uint8) + 32- else:+ output_scale = torch.round(scales / output_d).clip(max=127).to(torch.int8)+ all_L = torch.round(blocks / scales.unsqueeze(-1) + 32).clip(0, 63).to(torch.uint8)+ elif original:scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)imax = abs(scales).argmax(dim=-1, keepdim=True)max_scales = torch.take_along_dim(scales, imax, dim=-1)iscales = torch.where(max_scales != 0, -128 / max_scales, 0)output_d = torch.where(iscales != 0, 1 / iscales, -1)+ output_scale = torch.round(iscales * scales).clip(max=127).to(torch.int8)+ d_tmp = output_d * output_scale.to(torch.float32)+ replace_ids = d_tmp != 0+ all_L[replace_ids] = torch.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32) \+ .clip(0, 63).to(torch.uint8)+ else:+ from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq+ blocks = blocks.reshape(blocks.shape[0], -1)+ blocks, scales, _ = quant_tensor_gguf_sym_dq(blocks, bits=6, scale_dtype=torch.float32, imatrix=imatrix)+ scales, d_scale = scales['scale'], scales['d_scale']+ blocks = blocks.reshape((nb, QK_K // 16, 16))+ scales = scales.reshape((-1, QK_K // 16))+ output_d = d_scale.reshape(-1, 1).to(torch.float32)+ output_scale = torch.round(scales / output_d).clip(max=127).to(torch.int8)+ all_L = torch.round(blocks / scales.unsqueeze(-1) + 32).clip(0, 63).to(torch.uint8)- output_scale = torch.round(iscales * scales).clip(max=127).to(torch.int8)- d_tmp = output_d * output_scale.to(torch.float32)- replace_ids = d_tmp != 0- all_L[replace_ids] = torch.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32) \- .clip(0, 63).to(torch.uint8)-- # all_L = all_L.cpu().numpy()tmp_L = all_L.reshape(nb, 4, 64) & 0xFoutput_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)ouptut_qh = (all_L >> 4).reshape(nb, 2, 4, 32) << torch.tensor(diff --git a/auto_round/utils.py b/auto_round/utils.pyindex ce2345161..45b989980 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1193,7 +1193,7 @@ def _gguf_args_check(args_or_ar, format_str=None):if format == "q6_k_s":logger.warning("Please not that q6_k_s is q6_k.")try:- from auto_round.export.export_to_gguf.convert import Model+ from auto_round.export.export_to_gguf.convert import ModelBaseexcept:raise ImportError(f"Please use the latest gguf-py for {format}, you can use the following command to install it:\n"@@ -1207,11 +1207,11 @@ def _gguf_args_check(args_or_ar, format_str=None):if isinstance(args_or_ar.model, str) and os.path.isdir(args_or_ar.model):from pathlib import Path- from auto_round.export.export_to_gguf.convert import Model- hparams = Model.load_hparams(Path(args_or_ar.model))+ from auto_round.export.export_to_gguf.convert import ModelBase+ hparams = ModelBase.load_hparams(Path(args_or_ar.model))model_architecture = hparams["architectures"][0]try:- model_class = Model.from_model_architecture(model_architecture)+ model_class = ModelBase.from_model_architecture(model_architecture)except NotImplementedError:logger.error(f"Model {model_architecture} is not supported to export GGUF format")sys.exit(1)@@ -1611,10 +1611,18 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)import gguf # pylint: disable=E0401- from auto_round.export.export_to_gguf.convert import Model- model_architecture = model.config.architectures[0]+ from auto_round.export.export_to_gguf.convert import ModelBase+ if model.config.architectures is not None:+ model_architecture = model.config.architectures[0]+ else:+ model_architecture = type(model).__name__+ if model_architecture not in ModelBase._model_classes:+ if model_architecture.replace("CausalLM", "ConditionalGeneration") in ModelBase._model_classes:+ model_architecture = model_architecture.replace("CausalLM", "ConditionalGeneration")+ elif model_architecture.replace("ConditionalGeneration", "CausalLM") in ModelBase._model_classes:+ model_architecture = model_architecture.replace("ConditionalGeneration", "CausalLM")try:- model_class = Model.from_model_architecture(model_architecture)+ model_class = ModelBase.from_model_architecture(model_architecture)except NotImplementedError:return layer_config, {}