PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
update gguf convert.py and support for gpt-oss (#721)
Heng Guo73ecd7f83d52
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex cec587d33..6c6273f32 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -34,24 +34,16 @@from __future__ import annotations-import argparse-import ast-import contextlib-import gcimport json-import logging-import mathimport osimport re-import sys-from enum import IntEnum-from hashlib import sha256+from functools import partialfrom itertools import chainfrom pathlib import Path-from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterable, Iterator, Literal, Sequence, TypeVar, cast+from typing import TYPE_CHECKING, Any, Iteratorimport numpy as np-import psutil+import requestsimport torchfrom transformers import AutoConfig@@ -63,8386 +55,512 @@ gguf = LazyImport("gguf")if TYPE_CHECKING:from torch import Tensor-if "NO_LOCAL_GGUF" not in os.environ:- sys.path.insert(1, str(Path(__file__).parent / "gguf-py"))-###### MODEL DEFINITIONS ######+def download_convert_file(redownload=False):+ CONVERT_URL = "https://raw.githubusercontent.com/ggml-org/llama.cpp/refs/heads/master/convert_hf_to_gguf.py"+ FILE_NAME = "convert_hf_to_gguf.py"+ gguf_export_dir = os.path.dirname(__file__)+ if redownload is False and FILE_NAME in os.listdir(gguf_export_dir):+ return+ response = requests.get(CONVERT_URL)+ with open(os.path.join(gguf_export_dir, FILE_NAME), "w") as f:+ f.write(response.text)-class SentencePieceTokenTypes(IntEnum):- NORMAL = 1- UNKNOWN = 2- CONTROL = 3- USER_DEFINED = 4- UNUSED = 5- BYTE = 6+def wrapper_model_instance(model_instance, model, layer_config, low_cpu_mem_usage=False):+ if model_instance.model_arch == gguf.MODEL_ARCH.MMPROJ and model_instance.fname_out.is_dir():+ model_instance.fname_out = model_instance.fname_out / "mmproj-model.gguf"+ model_instance.model = model+ model_instance.layer_config = layer_config+ model_instance.low_cpu_mem_usage = _need_low_cpu_mem(low_cpu_mem_usage)+ model_instance.load_hparams = load_hparams+ model_instance.get_tensors = partial(get_tensors, model_instance)+ model_instance.prepare_tensors = partial(prepare_tensors, model_instance)-class ModelType(IntEnum):- TEXT = 1- MMPROJ = 2+ return model_instance-AnyModel = TypeVar("AnyModel", bound="type[ModelBase]")---class OriModel:- _model_classes: dict[ModelType, dict[str, type[ModelBase]]] = {- ModelType.TEXT: {},- ModelType.MMPROJ: {},- }-- dir_model: Path- ftype: gguf.LlamaFileType- fname_out: Path- is_big_endian: bool- endianess: gguf.GGUFEndian- use_temp_file: bool- lazy: bool- part_names: list[str]- is_safetensors: bool- hparams: dict[str, Any]- tensor_names: set[str] | None- gguf_writer: gguf.GGUFWriter- model_name: str | None- metadata_override: Path | None- dir_model_card: Path- remote_hf_model_id: str | None-- # subclasses should define this!- model_arch: gguf.MODEL_ARCH-- # subclasses should initialize this!- block_count: int- tensor_map: gguf.TensorNameMap-- def __init__(- self,- dir_model: Path,- ftype: gguf.LlamaFileType,- fname_out: Path,- *,- is_big_endian: bool = False,- use_temp_file: bool = False,- eager: bool = False,- metadata_override: Path | None = None,- model_name: str | None = None,- split_max_tensors: int = 0,- split_max_size: int = 0,- dry_run: bool = False,- small_first_shard: bool = False,- hparams: dict[str, Any] | None = None,- remote_hf_model_id: str | None = None,- ):- if type(self) is ModelBase or type(self) is TextModel or type(self) is MmprojModel:- raise TypeError(f"{type(self).__name__!r} should not be directly instantiated")-- self.dir_model = dir_model- self.ftype = ftype- self.fname_out = fname_out- self.is_big_endian = is_big_endian- self.endianess = gguf.GGUFEndian.BIG if is_big_endian else gguf.GGUFEndian.LITTLE- self.use_temp_file = use_temp_file- self.lazy = not eager or (remote_hf_model_id is not None)- self.remote_hf_model_id = remote_hf_model_id- if remote_hf_model_id is not None:- self.is_safetensors = True-- def get_remote_tensors() -> Iterator[tuple[str, Tensor]]:- logger.info(f"Using remote model with HuggingFace id: {remote_hf_model_id}")- remote_tensors = gguf.utility.SafetensorRemote.get_list_tensors_hf_model(remote_hf_model_id)- self.tensor_names = set(name for name in remote_tensors.keys())- for name, remote_tensor in gguf.utility.SafetensorRemote.get_list_tensors_hf_model(- remote_hf_model_id- ).items():- yield (name, LazyTorchTensor.from_remote_tensor(remote_tensor))-- self.get_tensors = get_remote_tensors- else:- self.part_names = ModelBase.get_model_part_names(self.dir_model, "model", ".safetensors")- self.is_safetensors = len(self.part_names) > 0- if not self.is_safetensors:- self.part_names = ModelBase.get_model_part_names(self.dir_model, "pytorch_model", ".bin")- self.hparams = ModelBase.load_hparams(self.dir_model) if hparams is None else hparams- self.tensor_names = None- self.metadata_override = metadata_override- self.model_name = model_name- self.dir_model_card = dir_model # overridden in convert_lora_to_gguf.py-- # Apply heuristics to figure out typical tensor encoding based on first layer tensor encoding type- if self.ftype == gguf.LlamaFileType.GUESSED:- # NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie.- _, first_tensor = next(self.get_tensors())- if first_tensor.dtype == torch.float16:- logger.info(f"choosing --outtype f16 from first tensor type ({first_tensor.dtype})")- self.ftype = gguf.LlamaFileType.MOSTLY_F16- else:- logger.info(f"choosing --outtype bf16 from first tensor type ({first_tensor.dtype})")- self.ftype = gguf.LlamaFileType.MOSTLY_BF16-- # Configure GGUF Writer- self.gguf_writer = gguf.GGUFWriter(- path=None,- arch=gguf.MODEL_ARCH_NAMES[self.model_arch],- endianess=self.endianess,- use_temp_file=self.use_temp_file,- split_max_tensors=split_max_tensors,- split_max_size=split_max_size,- dry_run=dry_run,- small_first_shard=small_first_shard,- )-- @classmethod- def add_prefix_to_filename(cls, path: Path, prefix: str) -> Path:- stem, suffix = path.stem, path.suffix- new_name = f"{prefix}{stem}{suffix}"- return path.with_name(new_name)-- def find_hparam(self, keys: Iterable[str], optional: bool = False) -> Any:- key = next((k for k in keys if k in self.hparams), None)- if key is not None:- return self.hparams[key]- if optional:- return None- raise KeyError(f"could not find any of: {keys}")-- def get_tensors(self) -> Iterator[tuple[str, Tensor]]: # pylint: disable=E0202- tensor_names_from_parts: set[str] = set()-- index_name = "model.safetensors" if self.is_safetensors else "pytorch_model.bin"- index_name += ".index.json"- index_file = self.dir_model / index_name-- if index_file.is_file():- self.tensor_names = set()- logger.info(f"gguf: loading model weight map from '{index_name}'")- with open(index_file, "r", encoding="utf-8") as f:- index: dict[str, Any] = json.load(f)- weight_map = index.get("weight_map")- if weight_map is None or not isinstance(weight_map, dict):- raise ValueError(f"Can't load 'weight_map' from {index_name!r}")- self.tensor_names.update(weight_map.keys())- else:- self.tensor_names = tensor_names_from_parts- weight_map = {}-- for part_name in self.part_names:- logger.info(f"gguf: loading model part '{part_name}'")- ctx: ContextManager[Any]- if self.is_safetensors:- from safetensors import safe_open-- ctx = cast(ContextManager[Any], safe_open(self.dir_model / part_name, framework="pt", device="cpu"))- else:- ctx = contextlib.nullcontext(- torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True)- )-- with ctx as model_part:- tensor_names_from_parts.update(model_part.keys())-- for name in model_part.keys():- if self.is_safetensors:- if self.lazy:- data = model_part.get_slice(name)- data = LazyTorchTensor.from_safetensors_slice(data)- else:- data = model_part.get_tensor(name)- else:- data = model_part[name]- if self.lazy:- data = LazyTorchTensor.from_eager(data)- yield name, data-- # verify tensor name presence and identify potentially missing files- if len(tensor_names_from_parts.symmetric_difference(self.tensor_names)) > 0:- missing = sorted(self.tensor_names.difference(tensor_names_from_parts))- extra = sorted(tensor_names_from_parts.difference(self.tensor_names))- missing_files = sorted(set(weight_map[n] for n in missing if n in weight_map))- if len(extra) == 0 and len(missing_files) > 0:- raise ValueError(f"Missing or incomplete model files: {missing_files}\n" f"Missing tensors: {missing}")- else:- raise ValueError(- "Mismatch between weight map and model parts for tensor names:\n"- f"Missing tensors: {missing}\n"- f"Extra tensors: {extra}"- )-- def format_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str:- if key not in gguf.MODEL_TENSORS[self.model_arch]:- raise ValueError(f"Missing {key!r} for MODEL_TENSORS of {self.model_arch!r}")- name: str = gguf.TENSOR_NAMES[key]- if "{bid}" in name:- assert bid is not None- name = name.format(bid=bid)- return name + suffix-- def match_model_tensor_name(- self, name: str, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight"- ) -> bool:- if key not in gguf.MODEL_TENSORS[self.model_arch]:- return False- key_name: str = gguf.TENSOR_NAMES[key]- if "{bid}" in key_name:- if bid is None:- return False- key_name = key_name.format(bid=bid)- else:- if bid is not None:- return False- return name == (key_name + suffix)-- def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:- new_name = self.tensor_map.get_name(key=name, try_suffixes=try_suffixes)- if new_name is None:- raise ValueError(f"Can not map tensor {name!r}")- return new_name-- def set_gguf_parameters(self):- raise NotImplementedError("set_gguf_parameters() must be implemented in subclasses")-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- return [(self.map_tensor_name(name), data_torch)]-- def tensor_force_quant(- self, name: str, new_name: str, bid: int | None, n_dims: int- ) -> gguf.GGMLQuantizationType | bool:- del name, new_name, bid, n_dims # unused-+def _need_low_cpu_mem(low_cpu_mem_usage):+ if not low_cpu_mem_usage:return False- # some models need extra generated tensors (like rope_freqs)- def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- return ()-- def prepare_tensors(self):- max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")-- for name, data_torch in chain(self.generate_extra_tensors(), self.get_tensors()):- # we don't need these- if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):- continue-- old_dtype = data_torch.dtype-- # convert any unsupported data types to float32- if data_torch.dtype not in (torch.float16, torch.float32):- data_torch = data_torch.to(torch.float32)-- # use the first number-like part of the tensor name as the block id- bid = None- for part in name.split("."):- if part.isdecimal():- bid = int(part)- break-- for new_name, data_torch in self.modify_tensors(data_torch, name, bid):- # TODO: why do we squeeze here?- # data = data_torch.squeeze().numpy()- data = data_torch.numpy()-- # if data ends up empty, it means data_torch was a scalar tensor -> restore- if len(data.shape) == 0:- data = data_torch.numpy()-- n_dims = len(data.shape)- data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)-- # Most of the codebase that takes in 1D tensors or norms only handles F32 tensors- if n_dims <= 1 or new_name.endswith("_norm.weight"):- data_qtype = gguf.GGMLQuantizationType.F32-- # Conditions should closely match those in llama_model_quantize_internal in llama.cpp- # Some tensor types are always in float32- if data_qtype is False and (- any(- self.match_model_tensor_name(new_name, key, bid)- for key in (- gguf.MODEL_TENSOR.FFN_GATE_INP,- gguf.MODEL_TENSOR.POS_EMBD,- gguf.MODEL_TENSOR.TOKEN_TYPES,- gguf.MODEL_TENSOR.SSM_CONV1D,- gguf.MODEL_TENSOR.SHORTCONV_CONV,- gguf.MODEL_TENSOR.TIME_MIX_FIRST,- gguf.MODEL_TENSOR.TIME_MIX_W1,- gguf.MODEL_TENSOR.TIME_MIX_W2,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,- gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,- gguf.MODEL_TENSOR.POSNET_NORM1,- gguf.MODEL_TENSOR.POSNET_NORM2,- gguf.MODEL_TENSOR.V_ENC_EMBD_POS,- gguf.MODEL_TENSOR.A_ENC_EMBD_POS,- gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,- gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,- )- )- or not new_name.endswith(".weight")- ):- data_qtype = gguf.GGMLQuantizationType.F32-- if data_qtype is False and any(- self.match_model_tensor_name(new_name, key, bid)- for key in (- gguf.MODEL_TENSOR.TOKEN_EMBD,- gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,- gguf.MODEL_TENSOR.OUTPUT,- gguf.MODEL_TENSOR.ALTUP_ROUTER,- gguf.MODEL_TENSOR.LAUREL_L,- gguf.MODEL_TENSOR.LAUREL_R,- )- ):- if self.ftype in (- gguf.LlamaFileType.MOSTLY_TQ1_0,- gguf.LlamaFileType.MOSTLY_TQ2_0,- ):- # TODO: use Q4_K and Q6_K- data_qtype = gguf.GGMLQuantizationType.F16-- # No override (data_qtype is False), or wants to be quantized (data_qtype is True)- if isinstance(data_qtype, bool):- if self.ftype == gguf.LlamaFileType.ALL_F32:- data_qtype = gguf.GGMLQuantizationType.F32- elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:- data_qtype = gguf.GGMLQuantizationType.F16- elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:- data_qtype = gguf.GGMLQuantizationType.BF16- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:- data_qtype = gguf.GGMLQuantizationType.Q8_0- elif self.ftype == gguf.LlamaFileType.MOSTLY_TQ1_0:- data_qtype = gguf.GGMLQuantizationType.TQ1_0- elif self.ftype == gguf.LlamaFileType.MOSTLY_TQ2_0:- data_qtype = gguf.GGMLQuantizationType.TQ2_0- else:- raise ValueError(f"Unknown file type: {self.ftype.name}")-- try:- data = gguf.quants.quantize(data, data_qtype)- except gguf.QuantError as e:- logger.warning("%s, %s", e, "falling back to F16")- data_qtype = gguf.GGMLQuantizationType.F16- data = gguf.quants.quantize(data, data_qtype)-- shape = (- gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape- )-- # reverse shape to make it similar to the internal ggml dimension order- shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"-- # n_dims is implicit in the shape- logger.info(- f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"- )-- self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)-- def set_type(self):- self.gguf_writer.add_type(gguf.GGUFType.MODEL)-- def prepare_metadata(self, vocab_only: bool):-- total_params, shared_params, expert_params, expert_count = self.gguf_writer.get_total_parameter_count()-- self.metadata = gguf.Metadata.load(self.metadata_override, self.dir_model_card, self.model_name, total_params)-- # If we are using HF model id, set the metadata name to the model id- if self.remote_hf_model_id:- self.metadata.name = self.remote_hf_model_id-- # Fallback to model directory name if metadata name is still missing- if self.metadata.name is None:- self.metadata.name = self.dir_model.name-- # Generate parameter weight class (useful for leader boards) if not yet determined- if self.metadata.size_label is None and total_params > 0:- self.metadata.size_label = gguf.size_label(total_params, shared_params, expert_params, expert_count)-- self.set_type()-- logger.info("Set meta model")- self.metadata.set_gguf_meta_model(self.gguf_writer)-- logger.info("Set model parameters")- self.set_gguf_parameters()-- logger.info("Set model quantization version")- self.gguf_writer.add_quantization_version(gguf.GGML_QUANT_VERSION)-- def write_vocab(self):- raise NotImplementedError("write_vocab() must be implemented in subclasses")-- def write(self):- self.prepare_tensors()- self.prepare_metadata(vocab_only=False)- self.gguf_writer.write_header_to_file(path=self.fname_out)- self.gguf_writer.write_kv_data_to_file()- self.gguf_writer.write_tensors_to_file(progress=True)- self.gguf_writer.close()-- @staticmethod- def get_model_part_names(dir_model: Path, prefix: str, suffix: str) -> list[str]:- part_names: list[str] = []- for filename in os.listdir(dir_model):- if filename.startswith(prefix) and filename.endswith(suffix):- part_names.append(filename)-- part_names.sort()-- return part_names-- @staticmethod- def load_hparams(dir_model: Path):- try:- # for security reason, we don't allow loading remote code by default- # if a model need remote code, we will fallback to config.json- config = AutoConfig.from_pretrained(dir_model, trust_remote_code=False).to_dict()- except Exception as e:- logger.warning(f"Failed to load model config from {dir_model}: {e}")- logger.warning("Trying to load config.json instead")- with open(dir_model / "config.json", "r", encoding="utf-8") as f:- config = json.load(f)- if "llm_config" in config:- # rename for InternVL- config["text_config"] = config["llm_config"]- if "thinker_config" in config:- # rename for Qwen2.5-Omni- config["text_config"] = config["thinker_config"]["text_config"]- return config-- @classmethod- def register(cls, *names: str) -> Callable[[AnyModel], AnyModel]:- assert names-- def func(modelcls: AnyModel) -> AnyModel:- model_type = ModelType.MMPROJ if modelcls.model_arch == gguf.MODEL_ARCH.MMPROJ else ModelType.TEXT- for name in names:- cls._model_classes[model_type][name] = modelcls- return modelcls-- return func-- @classmethod- def print_registered_models(cls):- for model_type, model_classes in cls._model_classes.items():- logger.error(f"{model_type.name} models:")- for name in sorted(model_classes.keys()):- logger.error(f" - {name}")-- @classmethod- def from_model_architecture(cls, arch: str, model_type=ModelType.TEXT) -> type[ModelBase]:- try:- return cls._model_classes[model_type][arch]- except KeyError:- raise NotImplementedError(f"Architecture {arch!r} not supported!") from None---class ModelBase(OriModel):-- def __init__(- self,- model,- layer_config: dict,- dir_model: Path,- ftype: gguf.LlamaFileType,- fname_out: Path,- low_cpu_mem_usage: bool = False,- is_big_endian: bool = False,- use_temp_file: bool = False,- eager: bool = False,- metadata_override: Path | None = None,- model_name: str | None = None,- split_max_tensors: int = 0,- split_max_size: int = 0,- dry_run: bool = False,- small_first_shard: bool = False,- hparams: dict[str, Any] | None = None,- ):- if self.model_arch == gguf.MODEL_ARCH.MMPROJ and fname_out.is_dir():- fname_out = fname_out / "mmproj-model.gguf"- self.model = model- self.layer_config = layer_config- self.low_cpu_mem_usage = self._need_low_cpu_mem(low_cpu_mem_usage)- super().__init__(- dir_model=dir_model,- ftype=ftype,- fname_out=fname_out,- is_big_endian=is_big_endian,- use_temp_file=use_temp_file,- eager=eager,- metadata_override=metadata_override,- model_name=model_name,- split_max_tensors=split_max_tensors,- split_max_size=split_max_size,- dry_run=dry_run,- small_first_shard=small_first_shard,- hparams=hparams,- )-- def _need_low_cpu_mem(self, low_cpu_mem_usage):- if not low_cpu_mem_usage:- return False-- # process = psutil.Process(os.getpid())- # mem_usage = process.memory_info().rss- # memory_info = psutil.virtual_memory()- # if memory_info.available > mem_usage / 3:- # return False- # else:- # logger.info("use low cpu memory mode.")- return True-- def get_moe_name(self, name, new_name):- type_mapping = {- "FFN_GATE_EXP": ["gate_proj", "w1", "linear"],- "FFN_DOWN_EXP": ["down_proj", "w2", "linear_1"],- "FFN_UP_EXP": ["up_proj", "w3", "linear_v"],- }- nums = re.findall("\d+", name)- if len(nums) != 2:- return name- name_tmp = name[: -len(".weight")].replace(f".{nums[1]}", "")- new_name_tmp = new_name[: -len(".weight")]- if self.tensor_map.get_type(name_tmp) == self.tensor_map.get_type(new_name_tmp):- return name-- tensor_type = self.tensor_map.get_type(new_name_tmp).name- experts_name = name_tmp.split(".")[-1]- for k, v in type_mapping.items():- if experts_name in v:- idx = v.index(experts_name)- name = name.replace(experts_name, type_mapping[tensor_type][idx])- return name+ # process = psutil.Process(os.getpid())+ # mem_usage = process.memory_info().rss+ # memory_info = psutil.virtual_memory()+ # if memory_info.available > mem_usage / 3:+ # return False+ # else:+ # logger.info("use low cpu memory mode.")+ return True+++def get_moe_name(cls, name, new_name):+ type_mapping = {+ "FFN_GATE_EXP": ["gate_proj", "w1", "linear"],+ "FFN_DOWN_EXP": ["down_proj", "w2", "linear_1"],+ "FFN_UP_EXP": ["up_proj", "w3", "linear_v"],+ }+ nums = re.findall("\d+", name)+ if len(nums) != 2:+ return name+ name_tmp = name[: -len(".weight")].replace(f".{nums[1]}", "")+ new_name_tmp = new_name[: -len(".weight")]+ if cls.tensor_map.get_type(name_tmp) == cls.tensor_map.get_type(new_name_tmp):return name- def get_tensors(self) -> Iterator[tuple[str, Tensor]]:- for name, tensor in self.model.named_parameters():- yield name, tensor+ tensor_type = cls.tensor_map.get_type(new_name_tmp).name+ experts_name = name_tmp.split(".")[-1]+ for k, v in type_mapping.items():+ if experts_name in v:+ idx = v.index(experts_name)+ name = name.replace(experts_name, type_mapping[tensor_type][idx])+ return name+ return name- def _quant_data_with_args(- self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None- ):- if torch.cuda.is_available():- device = "cuda"- elif torch.xpu.is_available():- device = "xpu"- else:- device = "cpu"- data_torch = data_torch.to(torch.float32)- scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale- zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp- d_scale = d_scale.to(torch.float32) if isinstance(d_scale, torch.Tensor) else d_scale- d_wmin = d_wmin.to(torch.float32) if isinstance(d_wmin, torch.Tensor) else d_wmin- wmin = wmin.to(torch.float32) if isinstance(wmin, torch.Tensor) else wmin- imatrix = imatrix.to(torch.float32) if isinstance(imatrix, torch.Tensor) else imatrix- data = ggml_quant(- data_torch,- data_qtype.name.lower(),- scale,- zp,- wmin=wmin,- d_scale=d_scale,- d_wmin=d_wmin,- imatrix=imatrix,- device=device,- )- return data+def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:+ for name, tensor in cls.model.named_parameters():+ yield name, tensor- def _quant_data(self, data_torch, data_qtype, name, modify_name, bid):- suffix = ".weight"- if torch.cuda.is_available():- device = "cuda"- elif torch.xpu.is_available():- device = "xpu"- else:- device = "cpu"- if suffix in name:- layer_name = name[: -len(suffix)]- module = get_module(self.model, layer_name)- if hasattr(module, "scale"):- if hasattr(self, "permute"):- bs = module.weight.shape[0]- for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:- if hasattr(module, attr) and getattr(module, attr) is not None:- attr_tensor = getattr(module, attr)- ori_shape = attr_tensor.shape- attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)[0][1]- attr_tensor = attr_tensor.reshape(ori_shape)- setattr(module, attr, attr_tensor)- scale = module.scale if hasattr(module, "scale") else None- zp = module.zp if hasattr(module, "zp") else None- data_torch = data_torch.to(torch.float32)- scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale- zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp- if data_qtype.name.lower().endswith("_k"):- d_scale = module.w_d_scale.to(torch.float32) if hasattr(module, "w_d_scale") else None- d_wmin = module.w_d_wmin.to(torch.float32) if hasattr(module, "w_d_wmin") else None- wmin = module.w_wmin.to(torch.float32) if hasattr(module, "w_wmin") else None- imatrix = module.imatrix.to(torch.float32) if hasattr(module, "imatrix") else None- data = ggml_quant(- data_torch,- data_qtype.name.lower(),- scale,- zp,- wmin=wmin,- d_scale=d_scale,- d_wmin=d_wmin,- imatrix=imatrix,- device=device,- )- else:- data = ggml_quant(data_torch, data_qtype.name.lower(), scale, zp, device=device)+def _quant_data_with_args(data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None):+ if torch.cuda.is_available():+ device = "cuda"+ elif torch.xpu.is_available():+ device = "xpu"+ else:+ device = "cpu"+ data_torch = data_torch.to(torch.float32)+ scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale+ zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp+ d_scale = d_scale.to(torch.float32) if isinstance(d_scale, torch.Tensor) else d_scale+ d_wmin = d_wmin.to(torch.float32) if isinstance(d_wmin, torch.Tensor) else d_wmin+ wmin = wmin.to(torch.float32) if isinstance(wmin, torch.Tensor) else wmin+ imatrix = imatrix.to(torch.float32) if isinstance(imatrix, torch.Tensor) else imatrix++ data = ggml_quant(+ data_torch,+ data_qtype.name.lower(),+ scale,+ zp,+ wmin=wmin,+ d_scale=d_scale,+ d_wmin=d_wmin,+ imatrix=imatrix,+ device=device,+ )+ return data+++def _quant_data(cls, data_torch, data_qtype, name, modify_name, bid):+ suffix = ".weight"+ if torch.cuda.is_available():+ device = "cuda"+ elif torch.xpu.is_available():+ device = "xpu"+ else:+ device = "cpu"+ if suffix in name:+ layer_name = name[: -len(suffix)]+ module = get_module(cls.model, layer_name)+ if hasattr(module, "scale"):+ if hasattr(cls, "permute"):+ bs = module.weight.shape[0]+ for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:+ if hasattr(module, attr) and getattr(module, attr) is not None:+ attr_tensor = getattr(module, attr)+ ori_shape = attr_tensor.shape+ attr_tensor = cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)[0][1]+ attr_tensor = attr_tensor.reshape(ori_shape)+ setattr(module, attr, attr_tensor)+ scale = module.scale if hasattr(module, "scale") else None+ zp = module.zp if hasattr(module, "zp") else None+ data_torch = data_torch.to(torch.float32)+ scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale+ zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp+ if data_qtype.name.lower().endswith("_k"):+ d_scale = module.w_d_scale.to(torch.float32) if hasattr(module, "w_d_scale") else None+ d_wmin = module.w_d_wmin.to(torch.float32) if hasattr(module, "w_d_wmin") else None+ wmin = module.w_wmin.to(torch.float32) if hasattr(module, "w_wmin") else None+ imatrix = module.imatrix.to(torch.float32) if hasattr(module, "imatrix") else None++ data = ggml_quant(+ data_torch,+ data_qtype.name.lower(),+ scale,+ zp,+ wmin=wmin,+ d_scale=d_scale,+ d_wmin=d_wmin,+ imatrix=imatrix,+ device=device,+ )else:- # if data_torch.dtype ==torch.float32:- # data_qtype = gguf.GGMLQuantizationType.F32- # else:- # data_qtype = gguf.GGMLQuantizationType.F16- data_qtype = gguf.GGMLQuantizationType.F32 ##FP16 has issues at inference- data = data_torch.to(torch.float32).squeeze().cpu().numpy()+ data = ggml_quant(data_torch, data_qtype.name.lower(), scale, zp, device=device)else:- # for Llama-4- # if data_torch.dtype == torch.float32:+ # if data_torch.dtype ==torch.float32:# data_qtype = gguf.GGMLQuantizationType.F32# else:# data_qtype = gguf.GGMLQuantizationType.F16- # data = data_torch.squeeze().cpu().numpy()- # data_qtype = gguf.GGMLQuantizationType.F32- # data = data_torch.to(torch.float32).squeeze().cpu().numpy()- data = ggml_quant(data_torch, data_qtype.name.lower(), device=device)- return data, data_qtype-- def get_qtype_by_layer_config(self, layer_config, name, data_qtype):- name = name[: -len(".weight")]- if name not in layer_config or layer_config[name]["bits"] >= 16:- return data_qtype- bits = layer_config[name]["bits"]- super_bits = layer_config[name]["super_bits"]- sym = layer_config[name]["sym"]- if bits == 2:- return gguf.GGMLQuantizationType.Q2_K- if bits == 3:- return gguf.GGMLQuantizationType.Q3_K- if bits == 4:- if super_bits is not None:- return gguf.GGMLQuantizationType.Q4_K- if super_bits is None and sym:- return gguf.GGMLQuantizationType.Q4_0- if super_bits is None and not sym:- return gguf.GGMLQuantizationType.Q4_1- if bits == 5:- if super_bits is not None:- return gguf.GGMLQuantizationType.Q5_K- if super_bits is None and sym:- return gguf.GGMLQuantizationType.Q5_0- if super_bits is None and not sym:- return gguf.GGMLQuantizationType.Q5_1- if bits == 6:- return gguf.GGMLQuantizationType.Q6_K- if bits == 8:- return gguf.GGMLQuantizationType.Q8_0- raise ValueError(f"Unknown file type: {data_qtype}")-- # data_qtype = re.sub(r"Q\d", f"Q{bits}", self.ftype.name)- #- # if data_qtype == "MOSTLY_Q8_0":- # data_qtype = gguf.GGMLQuantizationType.Q8_0- # elif data_qtype == "MOSTLY_Q4_0":- # data_qtype = gguf.GGMLQuantizationType.Q4_0- # elif data_qtype == "MOSTLY_Q4_1":- # data_qtype = gguf.GGMLQuantizationType.Q4_1- # elif data_qtype == "MOSTLY_Q5_0":- # data_qtype = gguf.GGMLQuantizationType.Q5_0- # elif data_qtype == "MOSTLY_Q5_1":- # data_qtype = gguf.GGMLQuantizationType.Q5_1- # elif data_qtype.startswith("MOSTLY_Q2_K"):- # data_qtype = gguf.GGMLQuantizationType.Q2_K- # elif data_qtype.startswith("MOSTLY_Q3_K"):- # data_qtype = gguf.GGMLQuantizationType.Q3_K- # elif data_qtype.startswith("MOSTLY_Q4_K"):- # data_qtype = gguf.GGMLQuantizationType.Q4_K- # elif data_qtype.startswith("MOSTLY_Q5_K"):- # data_qtype = gguf.GGMLQuantizationType.Q5_K- # elif data_qtype.startswith("MOSTLY_Q6_K"):- # data_qtype = gguf.GGMLQuantizationType.Q6_K+ data_qtype = gguf.GGMLQuantizationType.F32 ##FP16 has issues at inference+ data = data_torch.to(torch.float32).squeeze().cpu().numpy()+ else:+ # for Llama-4+ # if data_torch.dtype == torch.float32:+ # data_qtype = gguf.GGMLQuantizationType.F32# else:- # raise ValueError(f"Unknown file type: {data_qtype}")- # return data_qtype-- def _special_name_handle(self, name):- # for Qwen2VL- def remove_prefix(name, key_list):- for key in key_list:- if key in name and not name.startswith(key):- name = ".".join(name.split(".")[1:])- break- return name-- if self.model_arch == gguf.MODEL_ARCH.QWEN2VL:- name = name.replace("language_model.", "")- visual_keys = ["thinker", "visual", "audio", "talker", "token2wav"]- name = remove_prefix(name, visual_keys)- if isinstance(self, Qwen2VLVisionModel):- if "visual." in name and not name.startswith("visual."):+ # data_qtype = gguf.GGMLQuantizationType.F16+ # data = data_torch.squeeze().cpu().numpy()+ # data_qtype = gguf.GGMLQuantizationType.F32+ # data = data_torch.to(torch.float32).squeeze().cpu().numpy()+ data = ggml_quant(data_torch, data_qtype.name.lower(), device=device)+ return data, data_qtype+++def get_qtype_by_layer_config(layer_config, name, data_qtype):+ name = name[: -len(".weight")]+ if name not in layer_config or layer_config[name]["bits"] >= 16:+ return data_qtype+ bits = layer_config[name]["bits"]+ super_bits = layer_config[name]["super_bits"]+ sym = layer_config[name]["sym"]+ if bits == 2:+ return gguf.GGMLQuantizationType.Q2_K+ if bits == 3:+ return gguf.GGMLQuantizationType.Q3_K+ if bits == 4:+ if super_bits is not None:+ return gguf.GGMLQuantizationType.Q4_K+ if super_bits is None and sym:+ return gguf.GGMLQuantizationType.Q4_0+ if super_bits is None and not sym:+ return gguf.GGMLQuantizationType.Q4_1+ if bits == 5:+ if super_bits is not None:+ return gguf.GGMLQuantizationType.Q5_K+ if super_bits is None and sym:+ return gguf.GGMLQuantizationType.Q5_0+ if super_bits is None and not sym:+ return gguf.GGMLQuantizationType.Q5_1+ if bits == 6:+ return gguf.GGMLQuantizationType.Q6_K+ if bits == 8:+ return gguf.GGMLQuantizationType.Q8_0+ raise ValueError(f"Unknown file type: {data_qtype}")++ # data_qtype = re.sub(r"Q\d", f"Q{bits}", cls.ftype.name)+ #+ # if data_qtype == "MOSTLY_Q8_0":+ # data_qtype = gguf.GGMLQuantizationType.Q8_0+ # elif data_qtype == "MOSTLY_Q4_0":+ # data_qtype = gguf.GGMLQuantizationType.Q4_0+ # elif data_qtype == "MOSTLY_Q4_1":+ # data_qtype = gguf.GGMLQuantizationType.Q4_1+ # elif data_qtype == "MOSTLY_Q5_0":+ # data_qtype = gguf.GGMLQuantizationType.Q5_0+ # elif data_qtype == "MOSTLY_Q5_1":+ # data_qtype = gguf.GGMLQuantizationType.Q5_1+ # elif data_qtype.startswith("MOSTLY_Q2_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q2_K+ # elif data_qtype.startswith("MOSTLY_Q3_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q3_K+ # elif data_qtype.startswith("MOSTLY_Q4_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q4_K+ # elif data_qtype.startswith("MOSTLY_Q5_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q5_K+ # elif data_qtype.startswith("MOSTLY_Q6_K"):+ # data_qtype = gguf.GGMLQuantizationType.Q6_K+ # else:+ # raise ValueError(f"Unknown file type: {data_qtype}")+ # return data_qtype+++def _special_name_handle(cls, name):+ # for Qwen2VL+ def remove_prefix(name, key_list):+ for key in key_list:+ if key in name and not name.startswith(key):name = ".".join(name.split(".")[1:])+ break+ return name- # # for gemma3- if self.model_arch == gguf.MODEL_ARCH.GEMMA3 or isinstance(self, Gemma3VisionModel):- visual_keys = ["multi_modal_projector", "vision_tower", "multimodal_projector"]- name = remove_prefix(name, visual_keys)+ if cls.model_arch == gguf.MODEL_ARCH.QWEN2VL:+ name = name.replace("language_model.", "")+ visual_keys = ["thinker", "visual", "audio", "talker", "token2wav"]+ name = remove_prefix(name, visual_keys)+ if cls.__class__.__name__ == "Qwen2VLVisionModel":+ if "visual." in name and not name.startswith("visual."):+ name = ".".join(name.split(".")[1:])++ # # for gemma3+ if cls.model_arch == gguf.MODEL_ARCH.GEMMA3 or cls.__class__.__name__ == "Gemma3VisionModel":+ visual_keys = ["multi_modal_projector", "vision_tower", "multimodal_projector"]+ name = remove_prefix(name, visual_keys)++ # for LlavaForConditionalGeneration+ if cls.model_arch == gguf.MODEL_ARCH.LLAMA:+ name = name.replace("language_model.", "")+ # name = name.replace("model.", "")+ if cls.__class__.__name__ == "LlavaVisionModel":+ visual_keys = ["multi_modal_projector", "vision_tower"]+ name = remove_prefix(name, visual_keys)- # for LlavaForConditionalGeneration- if self.model_arch == gguf.MODEL_ARCH.LLAMA:- name = name.replace("language_model.", "")- # name = name.replace("model.", "")- if isinstance(self, LlavaVisionModel):- visual_keys = ["multi_modal_projector", "vision_tower"]- name = remove_prefix(name, visual_keys)+ # for InternVisionModel+ if cls.__class__.__name__ == "InternVisionModel":+ visual_keys = ["vision_model"]+ name = remove_prefix(name, visual_keys)- # for InternVisionModel- if isinstance(self, InternVisionModel):- visual_keys = ["vision_model"]- name = remove_prefix(name, visual_keys)+ return name- return name- def prepare_tensors(self):- max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")+def prepare_tensors(cls):+ max_name_len = max(len(s) for _, s in cls.tensor_map.mapping.values()) + len(".weight,")- for name, data_torch in chain(self.generate_extra_tensors(), self.get_tensors()):- if data_torch is None:- continue- # we don't need these- if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):- continue+ for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):+ if data_torch is None:+ continue+ # we don't need these+ if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):+ continue+ if hasattr(cls, "current_packing_block") and cls.current_packing_block is not None: # pylint: disable=E1101+ current_packing_block_split = cls.current_packing_block.split(".") # pylint: disable=E1101+ name_split = name.split(".")if (- hasattr(self, "current_packing_block")- and self.current_packing_block is not None # pylint: disable=E1101+ len(name_split) < len(current_packing_block_split)+ or name_split[: len(current_packing_block_split)] != current_packing_block_split):- current_packing_block_split = self.current_packing_block.split(".") # pylint: disable=E1101- name_split = name.split(".")- if (- len(name_split) < len(current_packing_block_split)- or name_split[: len(current_packing_block_split)] != current_packing_block_split- ):- continue+ continue- old_dtype = data_torch.dtype+ old_dtype = data_torch.dtype- # convert any unsupported data types to float32- if data_torch.dtype not in (torch.float16, torch.float32):- data_torch = data_torch.to(torch.float32)+ # convert any unsupported data types to float32+ if data_torch.dtype not in (torch.float16, torch.float32):+ data_torch = data_torch.to(torch.float32)- # use the first number-like part of the tensor name as the block id- bid = None- for part in name.split("."):- if part.isdecimal():- bid = int(part)- break+ # use the first number-like part of the tensor name as the block id+ bid = None+ for part in name.split("."):+ if part.isdecimal():+ bid = int(part)+ break- clean_weight_list = []+ clean_weight_list = []- modify_name = self._special_name_handle(name)- for new_name, data_torch in self.modify_tensors(data_torch, modify_name, bid):- data = data_torch.squeeze().cpu().numpy()+ modify_name = _special_name_handle(cls, name)+ for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):+ data = data_torch.squeeze().cpu().numpy()- # if data ends up empty, it means data_torch was a scalar tensor -> restore- if len(data.shape) == 0:- data = data_torch.numpy()+ # if data ends up empty, it means data_torch was a scalar tensor -> restore+ if len(data.shape) == 0:+ data = data_torch.numpy()- n_dims = len(data.shape)- data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)+ n_dims = len(data.shape)+ data_qtype: gguf.GGMLQuantizationType | bool = cls.tensor_force_quant(name, new_name, bid, n_dims)- # Most of the codebase that takes in 1D tensors or norms only handles F32 tensors- if n_dims <= 1 or new_name.endswith("_norm.weight"):- data_qtype = gguf.GGMLQuantizationType.F32+ # Most of the codebase that takes in 1D tensors or norms only handles F32 tensors+ if n_dims <= 1 or new_name.endswith("_norm.weight"):+ data_qtype = gguf.GGMLQuantizationType.F32- # Conditions should closely match those in llama_model_quantize_internal in llama.cpp- # Some tensor types are always in float32- if data_qtype is False and (- any(- self.match_model_tensor_name(new_name, key, bid)- for key in (- gguf.MODEL_TENSOR.FFN_GATE_INP,- gguf.MODEL_TENSOR.POS_EMBD,- gguf.MODEL_TENSOR.TOKEN_TYPES,- gguf.MODEL_TENSOR.SSM_CONV1D,- gguf.MODEL_TENSOR.TIME_MIX_FIRST,- gguf.MODEL_TENSOR.TIME_MIX_W1,- gguf.MODEL_TENSOR.TIME_MIX_W2,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,- )+ # Conditions should closely match those in llama_model_quantize_internal in llama.cpp+ # Some tensor types are always in float32+ if data_qtype is False and (+ any(+ cls.match_model_tensor_name(new_name, key, bid)+ for key in (+ gguf.MODEL_TENSOR.FFN_GATE_INP,+ gguf.MODEL_TENSOR.POS_EMBD,+ gguf.MODEL_TENSOR.TOKEN_TYPES,+ gguf.MODEL_TENSOR.SSM_CONV1D,+ gguf.MODEL_TENSOR.SHORTCONV_CONV,+ gguf.MODEL_TENSOR.TIME_MIX_FIRST,+ gguf.MODEL_TENSOR.TIME_MIX_W1,+ gguf.MODEL_TENSOR.TIME_MIX_W2,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,+ gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,+ gguf.MODEL_TENSOR.POSNET_NORM1,+ gguf.MODEL_TENSOR.POSNET_NORM2,+ gguf.MODEL_TENSOR.V_ENC_EMBD_POS,+ gguf.MODEL_TENSOR.A_ENC_EMBD_POS,+ gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,+ gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,)- or not new_name.endswith(".weight")+ )+ or not new_name.endswith(".weight")+ ):+ data_qtype = gguf.GGMLQuantizationType.F32++ if data_qtype is False and any(+ cls.match_model_tensor_name(new_name, key, bid)+ for key in (+ gguf.MODEL_TENSOR.TOKEN_EMBD,+ gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,+ gguf.MODEL_TENSOR.OUTPUT,+ gguf.MODEL_TENSOR.ALTUP_ROUTER,+ gguf.MODEL_TENSOR.LAUREL_L,+ gguf.MODEL_TENSOR.LAUREL_R,+ )+ ):+ if cls.ftype in (+ gguf.LlamaFileType.MOSTLY_TQ1_0,+ gguf.LlamaFileType.MOSTLY_TQ2_0,):+ # TODO: use Q4_K and Q6_K+ data_qtype = gguf.GGMLQuantizationType.F16+ # data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type++ # get name by new_name (for experts),+ name = get_moe_name(cls, name, new_name)+ clean_weight_list.append(name)+ # get data_qtype by layer_config+ if isinstance(data_qtype, bool):+ data_qtype = get_qtype_by_layer_config(cls.layer_config, name, data_qtype)++ # # No override (data_qtype is False), or wants to be quantized (data_qtype is True)+ if isinstance(data_qtype, bool):+ if cls.ftype == gguf.LlamaFileType.ALL_F32:data_qtype = gguf.GGMLQuantizationType.F32-- if data_qtype is False and any(- self.match_model_tensor_name(new_name, key, bid)- for key in (- gguf.MODEL_TENSOR.TOKEN_EMBD,- gguf.MODEL_TENSOR.OUTPUT,- )+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_F16:+ data_qtype = gguf.GGMLQuantizationType.F16+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_BF16:+ data_qtype = gguf.GGMLQuantizationType.BF16+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:+ data_qtype = gguf.GGMLQuantizationType.Q8_0+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:+ data_qtype = gguf.GGMLQuantizationType.Q4_0+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:+ data_qtype = gguf.GGMLQuantizationType.Q4_1+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:+ data_qtype = gguf.GGMLQuantizationType.Q5_0+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:+ data_qtype = gguf.GGMLQuantizationType.Q5_1+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S or cls.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:+ data_qtype = gguf.GGMLQuantizationType.Q2_K+ elif (+ cls.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S+ or cls.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M+ or cls.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L):- if self.ftype in (- gguf.LlamaFileType.MOSTLY_TQ1_0,- gguf.LlamaFileType.MOSTLY_TQ2_0,- ):- # TODO: use Q4_K and Q6_K- data_qtype = gguf.GGMLQuantizationType.F16- # data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type-- # get name by new_name (for experts),- name = self.get_moe_name(name, new_name)- clean_weight_list.append(name)- # get data_qtype by layer_config- if isinstance(data_qtype, bool):- data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)-- # # No override (data_qtype is False), or wants to be quantized (data_qtype is True)- if isinstance(data_qtype, bool):- if self.ftype == gguf.LlamaFileType.ALL_F32:- data_qtype = gguf.GGMLQuantizationType.F32- elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:- data_qtype = gguf.GGMLQuantizationType.F16- elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:- data_qtype = gguf.GGMLQuantizationType.BF16- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:- data_qtype = gguf.GGMLQuantizationType.Q8_0- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:- data_qtype = gguf.GGMLQuantizationType.Q4_0- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:- data_qtype = gguf.GGMLQuantizationType.Q4_1- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:- data_qtype = gguf.GGMLQuantizationType.Q5_0- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:- data_qtype = gguf.GGMLQuantizationType.Q5_1- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S or self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:- data_qtype = gguf.GGMLQuantizationType.Q2_K- elif (- self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S- or self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M- or self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L- ):- data_qtype = gguf.GGMLQuantizationType.Q3_K- elif (- self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S or self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M- ):- data_qtype = gguf.GGMLQuantizationType.Q4_K- elif (- self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S or self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_M- ):- data_qtype = gguf.GGMLQuantizationType.Q5_K- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:- data_qtype = gguf.GGMLQuantizationType.Q6_K- else:- raise ValueError(f"Unknown file type: {self.ftype.name}")-- if data_qtype.name.endswith("_K") and data_torch.shape[-1] % 256 != 0:- if data_qtype in [- gguf.GGMLQuantizationType.Q2_K,- gguf.GGMLQuantizationType.Q3_K,- gguf.GGMLQuantizationType.Q4_K,- ]:- data_qtype = gguf.GGMLQuantizationType.Q5_0- elif data_qtype == gguf.GGMLQuantizationType.Q5_K:- data_qtype = gguf.GGMLQuantizationType.Q5_0- elif data_qtype == gguf.GGMLQuantizationType.Q6_K:- data_qtype = gguf.GGMLQuantizationType.Q8_0-- if isinstance(data_qtype, bool) or data_qtype in [- gguf.GGMLQuantizationType.F16,- gguf.GGMLQuantizationType.BF16,- gguf.GGMLQuantizationType.F32,- ]:- try:- data = gguf.quants.quantize(data, data_qtype)- except gguf.QuantError as e:- logger.warning("%s, %s", e, "falling back to F16")- data_qtype = gguf.GGMLQuantizationType.F16- data = gguf.quants.quantize(data, data_qtype)+ data_qtype = gguf.GGMLQuantizationType.Q3_K+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S or cls.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:+ data_qtype = gguf.GGMLQuantizationType.Q4_K+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S or cls.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_M:+ data_qtype = gguf.GGMLQuantizationType.Q5_K+ elif cls.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:+ data_qtype = gguf.GGMLQuantizationType.Q6_Kelse:- # for deepseek v2- if name.endswith("kv_b_proj.weight") and self.model_arch.name == "DEEPSEEK2":- layer_name = name[: -len(".weight")]- module = get_module(self.model, layer_name)- n_head_kv = self.hparams["num_key_value_heads"]- v_head_dim = self.hparams["v_head_dim"]- qk_nope_head_dim = self.hparams["qk_nope_head_dim"]-- attr_list = {"scale": None, "zp": None, "d_scale": None, "wmin": None, "d_wmin": None}- for attr in attr_list:- if hasattr(module, attr) or hasattr(module, "w_" + attr):- if attr in ["scale", "zp"]:- attr_tensor = getattr(module, attr)- else:- attr_tensor = getattr(module, "w_" + attr)- if attr_tensor is None:- continue- kv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)- k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)- k_b = k_b.transpose(1, 2)-- name_kb = modify_name.replace("kv_b_proj", "k_b_proj")- if new_name == self.map_tensor_name(name_kb):- attr_list[attr] = k_b- else:- attr_list[attr] = v_b- attr_list["imatrix"] = module.imatrix if hasattr(module, "imatrix") else None- data = self._quant_data_with_args(data_torch, data_qtype, **attr_list)-- # for MOE model- elif len(data_torch.shape) == 3 and len(re.findall("\d+", name)) == 2:- new_data = []- for idx, arr in enumerate(data_torch):- arr_name = name.split(".")- for i in range(len(arr_name) - 1, -1, -1):- if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):- arr_name[i] = str(idx)- arr_name = ".".join(arr_name)- arr, data_qtype = self._quant_data(arr, data_qtype, arr_name, modify_name, bid)- new_data.append(arr)- data = np.array(new_data)- del new_data- else:- data, data_qtype = self._quant_data(data_torch, data_qtype, name, modify_name, bid)-- shape = (- gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape- )-- # reverse shape to make it similar to the internal ggml dimension order- shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"-- # n_dims is implicit in the shape- logger.info(- f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}"- f" --> {data_qtype.name}, shape = {shape_str}"- )-- self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)-- # # save cpu memory, but slow- if self.low_cpu_mem_usage:- for weight_name in clean_weight_list:- module = get_module(self.model, ".".join(weight_name.split(".")[:-1]))- for key in ["scale", "zp", "d_scale", "wmin", "d_wmin", "imatrix"]:- if hasattr(module, key):- setattr(module, key, None)- if hasattr(module, "w_" + key):- setattr(module, "w_" + key, None)- if self.model_arch == gguf.MODEL_ARCH.LLAMA and "embed_tokens.weight" in weight_name:- continue- clean_module_parameter(module, weight_name.split(".")[-1])---class TextModel(ModelBase):- model_type = ModelType.TEXT- hf_arch: str-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.hf_arch = get_model_architecture(self.hparams, self.model_type)-- if "text_config" in self.hparams:- # move the text_config to the root level- self.hparams = {**self.hparams, **self.hparams["text_config"]}-- self.block_count = self.find_hparam(["n_layers", "num_hidden_layers", "n_layer", "num_layers"])- self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)-- @classmethod- def __init_subclass__(cls):- # can't use an abstract property, because overriding it without type errors- # would require using decorated functions instead of simply defining the property- if "model_arch" not in cls.__dict__:- raise TypeError(f"Missing property 'model_arch' for {cls.__name__!r}")-- def set_vocab(self):- self._set_vocab_gpt2()-- def prepare_metadata(self, vocab_only: bool):- super().prepare_metadata(vocab_only=vocab_only)-- total_params = self.gguf_writer.get_total_parameter_count()[0]- # Extract the encoding scheme from the file type name. e.g. 'gguf.LlamaFileType.MOSTLY_Q8_0' --> 'Q8_0'- output_type: str = self.ftype.name.partition("_")[2]-- # Filename Output- if self.fname_out.is_dir():- # Generate default filename based on model specification and available metadata- if not vocab_only:- fname_default: str = gguf.naming_convention(- self.metadata.name,- self.metadata.basename,- self.metadata.finetune,- self.metadata.version,- self.metadata.size_label,- output_type,- model_type="LoRA" if total_params < 0 else None,- )- else:- fname_default: str = gguf.naming_convention(- self.metadata.name,- self.metadata.basename,- self.metadata.finetune,- self.metadata.version,- size_label=None,- output_type=None,- model_type="vocab",- )-- # Use the default filename- self.fname_out = self.fname_out / f"{fname_default}.gguf"- else:- # Output path is a custom defined templated filename- # Note: `not is_dir()` is used because `.is_file()` will not detect- # file template strings as it doesn't actually exist as a file-- # Process templated file name with the output ftype, useful with the "auto" ftype- self.fname_out = self.fname_out.parent / gguf.fill_templated_filename(self.fname_out.name, output_type)-- logger.info("Set model tokenizer")- self.set_vocab()-- def set_gguf_parameters(self):- self.gguf_writer.add_block_count(self.block_count)-- if (- n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx", "n_positions", "max_length"], optional=True)- ) is not None:- self.gguf_writer.add_context_length(n_ctx)- logger.info(f"gguf: context length = {n_ctx}")-- if (n_embd := self.find_hparam(["hidden_size", "n_embd", "dim"], optional=True)) is not None:- self.gguf_writer.add_embedding_length(n_embd)- logger.info(f"gguf: embedding length = {n_embd}")-- if (n_ff := self.find_hparam(["intermediate_size", "n_inner", "hidden_dim"], optional=True)) is not None:- self.gguf_writer.add_feed_forward_length(n_ff)- logger.info(f"gguf: feed forward length = {n_ff}")-- if (n_head := self.find_hparam(["num_attention_heads", "n_head", "n_heads"], optional=True)) is not None:- self.gguf_writer.add_head_count(n_head)- logger.info(f"gguf: head count = {n_head}")-- if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None:- self.gguf_writer.add_head_count_kv(n_head_kv)- logger.info(f"gguf: key-value head count = {n_head_kv}")-- if (rope_theta := self.hparams.get("rope_theta")) is not None:- self.gguf_writer.add_rope_freq_base(rope_theta)- logger.info(f"gguf: rope theta = {rope_theta}")- if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None:- self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)- logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")- if (- f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)- ) is not None:- self.gguf_writer.add_layer_norm_eps(f_norm_eps)- logger.info(f"gguf: layer norm epsilon = {f_norm_eps}")- if (n_experts := self.hparams.get("num_local_experts")) is not None:- self.gguf_writer.add_expert_count(n_experts)- logger.info(f"gguf: expert count = {n_experts}")- if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:- self.gguf_writer.add_expert_used_count(n_experts_used)- logger.info(f"gguf: experts used count = {n_experts_used}")-- if (head_dim := self.hparams.get("head_dim")) is not None:- self.gguf_writer.add_key_length(head_dim)- self.gguf_writer.add_value_length(head_dim)-- self.gguf_writer.add_file_type(self.ftype)- logger.info(f"gguf: file type = {self.ftype}")-- def write_vocab(self):- if len(self.gguf_writer.tensors) != 1:- raise ValueError("Splitting the vocabulary is not supported")-- self.prepare_metadata(vocab_only=True)- self.gguf_writer.write_header_to_file(path=self.fname_out)- self.gguf_writer.write_kv_data_to_file()- self.gguf_writer.close()-- def does_token_look_special(self, token: str | bytes) -> bool:- if isinstance(token, (bytes, bytearray)):- token_text = token.decode(encoding="utf-8")- elif isinstance(token, memoryview):- token_text = token.tobytes().decode(encoding="utf-8")- else:- token_text = token-- # Some models mark some added tokens which ought to be control tokens as not special.- # (e.g. command-r, command-r-plus, deepseek-coder, gemma{,-2})- seems_special = token_text in (- "<pad>", # deepseek-coder- "<mask>",- "<2mass>",- "[@BOS@]", # gemma{,-2}- )-- seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))- seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>")) # deepseek-coder-- # TODO: should these be marked as UNUSED instead? (maybe not)- seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">")) # gemma{,-2}+ raise ValueError(f"Unknown file type: {cls.ftype.name}")- return seems_special-- # used for GPT-2 BPE and WordPiece vocabs- def get_vocab_base(self) -> tuple[list[str], list[int], str]:- tokens: list[str] = []- toktypes: list[int] = []-- from transformers import AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(self.dir_model)- vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))- assert max(tokenizer.vocab.values()) < vocab_size-- tokpre = self.get_vocab_base_pre(tokenizer)-- reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}- added_vocab = tokenizer.get_added_vocab()-- added_tokens_decoder = tokenizer.added_tokens_decoder-- for i in range(vocab_size):- if i not in reverse_vocab:- tokens.append(f"[PAD{i}]")- toktypes.append(gguf.TokenType.UNUSED)+ if data_qtype.name.endswith("_K") and data_torch.shape[-1] % 256 != 0:+ if data_qtype in [+ gguf.GGMLQuantizationType.Q2_K,+ gguf.GGMLQuantizationType.Q3_K,+ gguf.GGMLQuantizationType.Q4_K,+ ]:+ data_qtype = gguf.GGMLQuantizationType.Q5_0+ elif data_qtype == gguf.GGMLQuantizationType.Q5_K:+ data_qtype = gguf.GGMLQuantizationType.Q5_0+ elif data_qtype == gguf.GGMLQuantizationType.Q6_K:+ data_qtype = gguf.GGMLQuantizationType.Q8_0++ if isinstance(data_qtype, bool) or data_qtype in [+ gguf.GGMLQuantizationType.F16,+ gguf.GGMLQuantizationType.BF16,+ gguf.GGMLQuantizationType.F32,+ ]:+ try:+ data = gguf.quants.quantize(data, data_qtype)+ except gguf.QuantError as e:+ logger.warning("%s, %s", e, "falling back to F16")+ data_qtype = gguf.GGMLQuantizationType.F16+ data = gguf.quants.quantize(data, data_qtype)else:- token: str = reverse_vocab[i]- if token in added_vocab:- # The tokenizer in llama.cpp assumes the CONTROL and USER_DEFINED tokens are pre-normalized.- # To avoid unexpected issues - we make sure to normalize non-normalized tokens- if not added_tokens_decoder[i].normalized:- previous_token = token- token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))- if previous_token != token:- logger.info(- f"{repr(previous_token)} is encoded and decoded back to"- f" {repr(token)} using AutoTokenizer"- )-- if added_tokens_decoder[i].special or self.does_token_look_special(token):- toktypes.append(gguf.TokenType.CONTROL)- else:- # NOTE: this was added for Gemma.- # Encoding and decoding the tokens above isn't sufficient for this case.- token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")- toktypes.append(gguf.TokenType.USER_DEFINED)+ # for deepseek v2+ if name.endswith("kv_b_proj.weight") and cls.model_arch.name == "DEEPSEEK2":+ layer_name = name[: -len(".weight")]+ module = get_module(cls.model, layer_name)+ n_head_kv = cls.hparams["num_key_value_heads"]+ v_head_dim = cls.hparams["v_head_dim"]+ qk_nope_head_dim = cls.hparams["qk_nope_head_dim"]++ attr_list = {"scale": None, "zp": None, "d_scale": None, "wmin": None, "d_wmin": None}+ for attr in attr_list:+ if hasattr(module, attr) or hasattr(module, "w_" + attr):+ if attr in ["scale", "zp"]:+ attr_tensor = getattr(module, attr)+ else:+ attr_tensor = getattr(module, "w_" + attr)+ if attr_tensor is None:+ continue+ kv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)+ k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)+ k_b = k_b.transpose(1, 2)++ name_kb = modify_name.replace("kv_b_proj", "k_b_proj")+ if new_name == cls.map_tensor_name(name_kb):+ attr_list[attr] = k_b+ else:+ attr_list[attr] = v_b+ attr_list["imatrix"] = module.imatrix if hasattr(module, "imatrix") else None+ data = _quant_data_with_args(data_torch, data_qtype, **attr_list)++ # for MOE model+ elif len(data_torch.shape) == 3 and len(re.findall("\d+", name)) == 2:+ new_data = []+ for idx, arr in enumerate(data_torch):+ arr_name = name.split(".")+ for i in range(len(arr_name) - 1, -1, -1):+ if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):+ arr_name[i] = str(idx)+ arr_name = ".".join(arr_name)+ arr, data_qtype = _quant_data(cls, arr, data_qtype, arr_name, modify_name, bid)+ new_data.append(arr)+ data = np.array(new_data)+ del new_dataelse:- toktypes.append(gguf.TokenType.NORMAL)- tokens.append(token)-- return tokens, toktypes, tokpre-- # NOTE: this function is generated by convert_hf_to_gguf_update.py- # do not modify it manually!- # ref: https://github.com/ggml-org/llama.cpp/pull/6920- # Marker: Start get_vocab_base_pre- def get_vocab_base_pre(self, tokenizer) -> str:- # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that- # is specific for the BPE pre-tokenizer used by the model- # we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can- # use in llama.cpp to implement the same pre-tokenizer-- chktxt = "\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български ''''''```````\"\"\"\"......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL" # pylint: disable=C0301-- chktok = tokenizer.encode(chktxt)- chkhsh = sha256(str(chktok).encode()).hexdigest()-- logger.debug(f"chktok: {chktok}")- logger.debug(f"chkhsh: {chkhsh}")-- res = None-- # NOTE: if you get an error here, you need to update the convert_hf_to_gguf_update.py script- # or pull the latest version of the model from Huggingface- # don't edit the hashes manually!- if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b":- # ref: https://huggingface.co/THUDM/glm-4-9b-chat- res = "chatglm-bpe"- if chkhsh == "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516":- # ref: https://huggingface.co/THUDM/glm-4-9b-chat- res = "chatglm-bpe"- if chkhsh == "a1336059768a55c99a734006ffb02203cd450fed003e9a71886c88acf24fdbc2":- # ref: https://huggingface.co/THUDM/glm-4-9b-hf- res = "glm4"- if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":- # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0- res = "minerva-7b"- if chkhsh == "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664":- # ref: https://huggingface.co/tencent/Hunyuan-A13B-Instruct- res = "hunyuan"- if chkhsh == "a6b57017d60e6edb4d88ecc2845188e0eb333a70357e45dcc9b53964a73bbae6":- # ref: https://huggingface.co/tiiuae/Falcon-H1-0.5B-Base- res = "falcon-h1"- if chkhsh == "60476e1243776c4fb1b993dbd7a5f15ac22f83c80afdf425fa5ae01c8d44ef86":- # ref: https://huggingface.co/tiiuae/Falcon-H1-1B-Base- res = "falcon-h1"- if chkhsh == "3eda48b4c4dc7de733d1a8b3e3b4a85243dbbf704da2ee9d42c6beced8897896":- # ref: https://huggingface.co/tiiuae/Falcon-H1-7B-Base- res = "falcon-h1"- if chkhsh == "48f8e02c0359c0bbdd82f26909171fac1c18a457bb47573ed1fe3bbb2c1cfd4b":- # ref: https://huggingface.co/tiiuae/Falcon-H1-34B-Base- res = "falcon-h1"- if chkhsh == "81212dc7cdb7e0c1074ca62c5aeab0d43c9f52b8a737be7b12a777c953027890":- # ref: https://huggingface.co/moonshotai/Kimi-K2-Base- res = "kimi-k2"- if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":- # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B- res = "llama-bpe"- if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754":- # ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base- res = "deepseek-llm"- if chkhsh == "347715f544604f9118bb75ed199f68779f423cabb20db6de6f31b908d04d7821":- # ref: https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base- res = "deepseek-coder"- if chkhsh == "8aeee3860c56296a157a1fe2fad249ec40aa59b1bb5709f4ade11c4e6fe652ed":- # ref: https://huggingface.co/tiiuae/falcon-7b- res = "falcon"- if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f":- # ref: https://huggingface.co/BAAI/bge-small-en-v1.5- res = "bert-bge"- if chkhsh == "9d032fcbd5501f4a38150912590928bfb36091efb5df11b8e2124b0390e3fb1e":- # ref: https://huggingface.co/tiiuae/Falcon3-7B-Base- res = "falcon3"- if chkhsh == "8e62295832751ca1e8f92f2226f403dea30dc5165e448b5bfa05af5340c64ec7":- # ref: https://huggingface.co/BAAI/bge-large-zh-v1.5- res = "bert-bge-large"- if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166":- # ref: https://huggingface.co/mosaicml/mpt-7b- res = "mpt"- if chkhsh == "35d91631860c815f952d711435f48d356ebac988362536bed955d43bfa436e34":- # ref: https://huggingface.co/bigcode/starcoder2-3b- res = "starcoder"- if chkhsh == "3ce83efda5659b07b1ad37ca97ca5797ea4285d9b9ab0dc679e4a720c9da7454":- # ref: https://huggingface.co/openai-community/gpt2- res = "gpt-2"- if chkhsh == "32d85c31273f8019248f2559fed492d929ea28b17e51d81d3bb36fff23ca72b3":- # ref: https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b- res = "stablelm2"- if chkhsh == "6221ad2852e85ce96f791f476e0b390cf9b474c9e3d1362f53a24a06dc8220ff":- # ref: https://huggingface.co/smallcloudai/Refact-1_6-base- res = "refact"- if chkhsh == "9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8":- # ref: https://huggingface.co/CohereForAI/c4ai-command-r-v01- res = "command-r"- if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea":- # ref: https://huggingface.co/Qwen/Qwen1.5-7B- res = "qwen2"- if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166":- # ref: https://huggingface.co/allenai/OLMo-1.7-7B-hf- res = "olmo"- if chkhsh == "a8594e3edff7c29c003940395316294b2c623e09894deebbc65f33f1515df79e":- # ref: https://huggingface.co/databricks/dbrx-base- res = "dbrx"- if chkhsh == "c7699093ba4255a91e702aa38a596aa81669f3525dae06c2953267dde580f448":- # ref: https://huggingface.co/jinaai/jina-reranker-v1-tiny-en- res = "jina-v1-en"- if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f":- # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-en- res = "jina-v2-en"- if chkhsh == "171aeeedd6fb548d418a7461d053f11b6f1f1fc9b387bd66640d28a4b9f5c643":- # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-es- res = "jina-v2-es"- if chkhsh == "27949a2493fc4a9f53f5b9b029c82689cfbe5d3a1929bb25e043089e28466de6":- # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-de- res = "jina-v2-de"- if chkhsh == "c136ed14d01c2745d4f60a9596ae66800e2b61fa45643e72436041855ad4089d":- # ref: https://huggingface.co/abacusai/Smaug-Llama-3-70B-Instruct- res = "smaug-bpe"- if chkhsh == "c7ea5862a53e4272c035c8238367063e2b270d51faa48c0f09e9d5b54746c360":- # ref: https://huggingface.co/LumiOpen/Poro-34B-chat- res = "poro-chat"- if chkhsh == "7967bfa498ade6b757b064f31e964dddbb80f8f9a4d68d4ba7998fcf281c531a":- # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-code- res = "jina-v2-code"- if chkhsh == "7fc505bd3104ca1083b150b17d088b59534ede9bde81f0dd2090967d7fe52cee":- # ref: https://huggingface.co/LumiOpen/Viking-7B- res = "viking"- if chkhsh == "b53802fb28e26d645c3a310b34bfe07da813026ec7c7716883404d5e0f8b1901":- # ref: https://huggingface.co/core42/jais-13b- res = "jais"- if chkhsh == "7b3e7548e4308f52a76e8229e4e6cc831195d0d1df43aed21ac6c93da05fec5f":- # ref: https://huggingface.co/WisdomShell/CodeShell-7B- res = "codeshell"- if chkhsh == "63b97e4253352e6f357cc59ea5b583e3a680eaeaf2632188c2b952de2588485e":- # ref: https://huggingface.co/mistralai/Mistral-Nemo-Base-2407- res = "tekken"- if chkhsh == "855059429035d75a914d1eda9f10a876752e281a054a7a3d421ef0533e5b6249":- # ref: https://huggingface.co/HuggingFaceTB/SmolLM-135M- res = "smollm"- if chkhsh == "3c30d3ad1d6b64202cd222813e7736c2db6e1bd6d67197090fc1211fbc612ae7":- # ref: https://huggingface.co/bigscience/bloom- res = "bloom"- if chkhsh == "bc01ce58980e1db43859146dc51b1758b3b88729b217a74792e9f8d43e479d21":- # ref: https://huggingface.co/TurkuNLP/gpt3-finnish-small- res = "gpt3-finnish"- if chkhsh == "4e2b24cc4770243d65a2c9ec19770a72f08cffc161adbb73fcbb6b7dd45a0aae":- # ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct- res = "exaone"- if chkhsh == "fcace8b9cac38ce847670c970cd5892031a753a1ef381abd1d9af00f713da085":- # ref: https://huggingface.co/microsoft/phi-2- res = "phi-2"- if chkhsh == "60824e3c0d9401f89943cbb2fff727f0e2d4c545ba4df2d6e4f09a6db0f5b450":- # ref: https://huggingface.co/facebook/chameleon-7b- res = "chameleon"- if chkhsh == "8b5a93ed704057481f240da0be7e7dca721d7f8f4755263b6807227a2cbeae65":- # ref: https://huggingface.co/sentence-transformers/stsb-roberta-base- res = "roberta-bpe"- if chkhsh == "ad851be1dba641f2e3711822f816db2c265f788b37c63b4e1aeacb9ee92de8eb":- # ref: https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct- res = "gigachat"- if chkhsh == "d4c8f286ea6b520b3d495c4455483cfa2302c0cfcd4be05d781b6a8a0a7cdaf1":- # ref: https://huggingface.co/Infinigence/Megrez-3B-Instruct- res = "megrez"- if chkhsh == "877081d19cf6996e2c4ff0e1236341e9b7bde288f5311a56a937f0afbbb3aeb5":- # ref: https://huggingface.co/deepseek-ai/DeepSeek-V3- res = "deepseek-v3"- if chkhsh == "b3f499bb4255f8ca19fccd664443283318f2fd2414d5e0b040fbdd0cc195d6c5":- # ref: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B- res = "deepseek-r1-qwen"- if chkhsh == "ccc2ef013c104be7bae2965776d611e1d7a8a2a9c547dd93a682c9a9fc80352e":- # ref: https://huggingface.co/Xenova/gpt-4o- res = "gpt-4o"- if chkhsh == "7dec86086fcc38b66b7bc1575a160ae21cf705be7718b9d5598190d7c12db76f":- # ref: https://huggingface.co/UW/OLMo2-8B-SuperBPE-t180k- res = "superbpe"- if chkhsh == "1994ffd01900cfb37395608534236ecd63f2bd5995d6cb1004dda1af50240f15":- # ref: https://huggingface.co/trillionlabs/Trillion-7B-preview- res = "trillion"- if chkhsh == "96a5f08be6259352137b512d4157e333e21df7edd3fcd152990608735a65b224":- # ref: https://huggingface.co/inclusionAI/Ling-lite- res = "bailingmoe"- if chkhsh == "d353350c764d8c3b39c763113960e4fb4919bea5fbf208a0e3b22e8469dc7406":- # ref: https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct- res = "llama4"- if chkhsh == "0e9433cbbb161f89e264eb32e8e64bfe69e834973ffca5d41d3948a604a3e2a3":- # ref: https://huggingface.co/mistral-community/pixtral-12b- res = "pixtral"- if chkhsh == "d5f1dd6f980fec569fb218a81a7658ac45fc56b38c5a0adeb1c232fbe04ef5ec":- # ref: https://huggingface.co/ByteDance-Seed/Seed-Coder-8B-Base- res = "seed-coder"- if chkhsh == "b0a6b1c0bd5998ebd9df08611efde34a4ff03faed45ae09c43e6b31ebd4b94cf":- # ref: https://huggingface.co/skt/A.X-4.0- res = "a.x-4.0"- if chkhsh == "f6791d196f87ce6b56a7d234be618e0d58f8cda3549416635b2bebcd22cd95c4":- # ref: https://huggingface.co/K-intelligence/Midm-2.0-Base-Instruct- res = "midm-2.0"- if chkhsh == "169bf0296a13c4d9b7672313f749eb36501d931022de052aad6e36f2bf34dd51":- # ref: https://huggingface.co/LiquidAI/LFM2-Tokenizer- res = "lfm2"- if chkhsh == "2085e1638f6c377a0aa4ead21b27bb4cb941bf800df86ed391011769c1758dfb":- # ref: https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B- res = "exaone4"-- if res is None:- logger.warning("\n")- logger.warning("**************************************************************************************")- logger.warning("** WARNING: The BPE pre-tokenizer was not recognized!")- logger.warning("** There are 2 possible reasons for this:")- logger.warning("** - the model has not been added to convert_hf_to_gguf_update.py yet")- logger.warning("** - the pre-tokenization config has changed upstream")- logger.warning(- "** Check your model files and convert_hf_to_gguf_update.py and update them accordingly."- )- logger.warning("** ref: https://github.com/ggml-org/llama.cpp/pull/6920")- logger.warning("**")- logger.warning(f"** chkhsh: {chkhsh}")- logger.warning("**************************************************************************************")- logger.warning("\n")- raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")-- logger.debug(f"tokenizer.ggml.pre: {repr(res)}")- logger.debug(f"chkhsh: {chkhsh}")-- return res- # Marker: End get_vocab_base_pre-- def _set_vocab_none(self) -> None:- self.gguf_writer.add_tokenizer_model("none")-- def _set_vocab_gpt2(self) -> None:- tokens, toktypes, tokpre = self.get_vocab_base()- self.gguf_writer.add_tokenizer_model("gpt2")- self.gguf_writer.add_tokenizer_pre(tokpre)- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)- special_vocab.add_to_gguf(self.gguf_writer)-- def _set_vocab_qwen(self):- dir_model = self.dir_model- hparams = self.hparams- tokens: list[str] = []- toktypes: list[int] = []+ data, data_qtype = _quant_data(cls, data_torch, data_qtype, name, modify_name, bid)- from transformers import AutoTokenizer+ shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape- tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)- vocab_size = hparams["vocab_size"]- assert max(tokenizer.get_vocab().values()) < vocab_size+ # reverse shape to make it similar to the internal ggml dimension order+ shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"- tokpre = self.get_vocab_base_pre(tokenizer)-- merges = []- vocab = {}- mergeable_ranks = tokenizer.mergeable_ranks- for token, rank in mergeable_ranks.items():- vocab[QwenModel.token_bytes_to_string(token)] = rank- if len(token) == 1:- continue- merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)- assert len(merged) == 2- merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))-- # for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined- added_vocab = tokenizer.special_tokens- reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in {**vocab, **added_vocab}.items()}-- for i in range(vocab_size):- if i not in reverse_vocab:- tokens.append(f"[PAD{i}]")- toktypes.append(gguf.TokenType.UNUSED)- elif reverse_vocab[i] in added_vocab:- tokens.append(reverse_vocab[i])- toktypes.append(gguf.TokenType.CONTROL)- else:- tokens.append(reverse_vocab[i])- toktypes.append(gguf.TokenType.NORMAL)-- self.gguf_writer.add_tokenizer_model("gpt2")- self.gguf_writer.add_tokenizer_pre(tokpre)- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(dir_model, load_merges=False)- special_vocab.merges = merges- # only add special tokens when they were not already loaded from config.json- if len(special_vocab.special_token_ids) == 0:- special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"])- special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"])- # this one is usually not in config.json anyway- special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"])- special_vocab.add_to_gguf(self.gguf_writer)-- def _set_vocab_sentencepiece(self, add_to_gguf=True):- tokens, scores, toktypes = self._create_vocab_sentencepiece()-- self.gguf_writer.add_tokenizer_model("llama")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)-- def _create_vocab_sentencepiece(self):- from sentencepiece import SentencePieceProcessor-- tokenizer_path = self.dir_model / "tokenizer.model"-- if not tokenizer_path.is_file():- raise FileNotFoundError(f"File not found: {tokenizer_path}")-- tokenizer = SentencePieceProcessor()- tokenizer.LoadFromFile(str(tokenizer_path))-- vocab_size = (- self.find_hparam(- [- "vocab_size_per_layer_input", # gemma3n- "vocab_size",- ],- optional=True,+ # n_dims is implicit in the shape+ logger.info(+ f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}" f" --> {data_qtype.name}, shape = {shape_str}")- or tokenizer.vocab_size()- )-- tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]- scores: list[float] = [-10000.0] * vocab_size- toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size-- for token_id in range(tokenizer.vocab_size()):- if token_id >= vocab_size:- logger.warning(f"ignore tokens from {token_id}: id is out of range, max={vocab_size - 1}")- break-- piece = tokenizer.IdToPiece(token_id)- text = piece.encode("utf-8")- score = tokenizer.GetScore(token_id)-- toktype = SentencePieceTokenTypes.NORMAL- if tokenizer.IsUnknown(token_id):- toktype = SentencePieceTokenTypes.UNKNOWN- elif tokenizer.IsControl(token_id):- toktype = SentencePieceTokenTypes.CONTROL- elif tokenizer.IsUnused(token_id):- toktype = SentencePieceTokenTypes.UNUSED- elif tokenizer.IsByte(token_id):- toktype = SentencePieceTokenTypes.BYTE-- tokens[token_id] = text- scores[token_id] = score- toktypes[token_id] = toktype-- added_tokens_file = self.dir_model / "added_tokens.json"- if added_tokens_file.is_file():- with open(added_tokens_file, "r", encoding="utf-8") as f:- added_tokens_json = json.load(f)- for key in added_tokens_json:- token_id = added_tokens_json[key]- if token_id >= vocab_size:- logger.warning(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")- continue-- tokens[token_id] = key.encode("utf-8")- scores[token_id] = -1000.0- toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED-- tokenizer_config_file = self.dir_model / "tokenizer_config.json"- if tokenizer_config_file.is_file():- with open(tokenizer_config_file, "r", encoding="utf-8") as f:- tokenizer_config_json = json.load(f)- added_tokens_decoder = tokenizer_config_json.get("added_tokens_decoder", {})- for token_id, token_data in added_tokens_decoder.items():- token_id = int(token_id)- token: str = token_data["content"]- if token_id >= vocab_size:- logger.warning(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")- continue- if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:- if tokens[token_id] != token.encode("utf-8"):- logger.warning(- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token!r}'- )- if token_data.get("special") or self.does_token_look_special(token):- toktypes[token_id] = SentencePieceTokenTypes.CONTROL- else:- token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spaces- toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED-- scores[token_id] = -1000.0- tokens[token_id] = token.encode("utf-8")-- if vocab_size > len(tokens):- pad_count = vocab_size - len(tokens)- logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")- for i in range(1, pad_count + 1):- tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))- scores.append(-1000.0)- toktypes.append(SentencePieceTokenTypes.UNUSED)-- return tokens, scores, toktypes-- def _set_vocab_llama_hf(self):- vocab = gguf.LlamaHfVocab(self.dir_model)- tokens = []- scores = []- toktypes = []-- for text, score, toktype in vocab.all_tokens():- tokens.append(text)- scores.append(score)- toktypes.append(toktype)-- assert len(tokens) == vocab.vocab_size-- self.gguf_writer.add_tokenizer_model("llama")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)- def _set_vocab_rwkv_world(self):- assert (self.dir_model / "rwkv_vocab_v20230424.txt").is_file()- vocab_size = self.hparams.get("vocab_size", 65536)-- tokens: list[bytes] = ["<s>".encode("utf-8")]- toktypes: list[int] = [gguf.TokenType.CONTROL]-- with open(self.dir_model / "rwkv_vocab_v20230424.txt", "r", encoding="utf-8") as f:- lines = f.readlines()- for line in lines:- parts = line.split(" ")- assert len(parts) >= 3- token, token_len = ast.literal_eval(" ".join(parts[1:-1])), int(parts[-1])- token = token.encode("utf-8") if isinstance(token, str) else token- assert isinstance(token, bytes)- assert len(token) == token_len- token_text: str = repr(token)[2:-1] # "b'\xff'" -> "\xff"- tokens.append(token_text.encode("utf-8"))- toktypes.append(gguf.TokenType.NORMAL)- remainder = vocab_size - len(tokens)- assert remainder >= 0- for i in range(len(tokens), vocab_size):- tokens.append(f"[PAD{i}]".encode("utf-8"))- toktypes.append(gguf.TokenType.UNUSED)-- self.gguf_writer.add_tokenizer_model("rwkv")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)- if special_vocab.chat_template is None:- template_path = Path(__file__).parent / "models" / "templates" / "llama-cpp-rwkv-world.jinja"- if template_path.is_file():- with open(template_path, "r", encoding="utf-8") as f:- template = f.read()- else:- template = "rwkv-world"- special_vocab.chat_template = template- # hack: Add '\n\n' as the EOT token to make it chat normally- special_vocab._set_special_token("eot", 261)- # hack: Override these as they have already been set (incorrectly)- special_vocab.special_token_ids["bos"] = 0- special_vocab.special_token_ids["eos"] = 0-- special_vocab.add_to_gguf(self.gguf_writer)-- def _set_vocab_builtin(self, model_name: Literal["gpt-neox", "llama-spm"], vocab_size: int):- tokenizer_path = Path(sys.path[0]) / "models" / f"ggml-vocab-{model_name}.gguf"- logger.warning(f"Using tokenizer from '{os.path.relpath(tokenizer_path, os.getcwd())}'")- vocab_reader = gguf.GGUFReader(tokenizer_path, "r")-- default_pre = "mpt" if model_name == "gpt-neox" else "default"-- field = vocab_reader.get_field(gguf.Keys.Tokenizer.MODEL)- assert field # tokenizer model- self.gguf_writer.add_tokenizer_model(bytes(field.parts[-1]).decode("utf-8"))-- field = vocab_reader.get_field(gguf.Keys.Tokenizer.PRE)- self.gguf_writer.add_tokenizer_pre(bytes(field.parts[-1]).decode("utf-8") if field else default_pre)-- field = vocab_reader.get_field(gguf.Keys.Tokenizer.LIST)- assert field # token list- self.gguf_writer.add_token_list([bytes(field.parts[i]) for i in field.data][:vocab_size])-- if model_name == "llama-spm":- field = vocab_reader.get_field(gguf.Keys.Tokenizer.SCORES)- assert field # token scores- self.gguf_writer.add_token_scores([field.parts[i].tolist()[0] for i in field.data][:vocab_size])-- field = vocab_reader.get_field(gguf.Keys.Tokenizer.TOKEN_TYPE)- assert field # token types- self.gguf_writer.add_token_types([field.parts[i].tolist()[0] for i in field.data][:vocab_size])-- if model_name != "llama-spm":- field = vocab_reader.get_field(gguf.Keys.Tokenizer.MERGES)- assert field # token merges- self.gguf_writer.add_token_merges([bytes(field.parts[i]) for i in field.data])-- if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.BOS_ID)) is not None:- self.gguf_writer.add_bos_token_id(field.parts[-1].tolist()[0])- if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.EOS_ID)) is not None:- self.gguf_writer.add_eos_token_id(field.parts[-1].tolist()[0])- if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.UNK_ID)) is not None:- self.gguf_writer.add_unk_token_id(field.parts[-1].tolist()[0])- if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.PAD_ID)) is not None:- self.gguf_writer.add_pad_token_id(field.parts[-1].tolist()[0])- if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.ADD_BOS)) is not None:- self.gguf_writer.add_add_bos_token(field.parts[-1].tolist()[0])- if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.ADD_EOS)) is not None:- self.gguf_writer.add_add_eos_token(field.parts[-1].tolist()[0])-- def _try_set_pooling_type(self) -> None:- # get pooling path- pooling_path = None- module_path = self.dir_model / "modules.json"- if module_path.is_file():- with open(module_path, encoding="utf-8") as f:- modules = json.load(f)- for mod in modules:- if mod["type"] == "sentence_transformers.models.Pooling":- pooling_path = mod["path"]- break-- # get pooling type- if pooling_path is not None:- with open(self.dir_model / pooling_path / "config.json", encoding="utf-8") as f:- pooling = json.load(f)- if pooling["pooling_mode_mean_tokens"]:- pooling_type = gguf.PoolingType.MEAN- elif pooling["pooling_mode_cls_token"]:- pooling_type = gguf.PoolingType.CLS- elif pooling["pooling_mode_lasttoken"]:- pooling_type = gguf.PoolingType.LAST- else:- raise NotImplementedError("Only MEAN, CLS, and LAST pooling types supported")- self.gguf_writer.add_pooling_type(pooling_type)---class MmprojModel(ModelBase):- model_type = ModelType.MMPROJ- model_arch = gguf.MODEL_ARCH.MMPROJ- preprocessor_config: dict[str, Any]- global_config: dict[str, Any]-- n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]-- has_vision_encoder: bool = True # by default- has_audio_encoder: bool = False-- # for models having multiple encoders, we need to separate their hparams- hparams_vision: dict[str, Any] | None = None- hparams_audio: dict[str, Any] | None = None-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)-- if self.model_arch != gguf.MODEL_ARCH.MMPROJ:- raise TypeError("MmprojModel must be subclassed with model_arch = gguf.MODEL_ARCH.MMPROJ")-- # get n_embd of the text model- if "text_config" not in self.hparams:- self.hparams["text_config"] = {}- if "audio_config" not in self.hparams:- self.hparams["audio_config"] = {}- text_config = {**self.hparams, **self.hparams["text_config"]}- self.n_embd_text = text_config.get("hidden_size", text_config.get("n_embd", 0))- assert self.n_embd_text > 0, "n_embd not found in hparams"-- # move vision config to the top level, while preserving the original hparams in global_config- import copy-- self.global_config = copy.deepcopy(self.hparams)- self.hparams_vision = self.get_vision_config()- self.hparams_audio = self.get_audio_config()-- if self.hparams_vision is None and self.hparams_audio is None:- raise ValueError("vision_config / audio_config not found in hparams")-- # for compat with vision-only models- self.hparams = self.hparams_vision or self.hparams_audio or self.hparams-- # TODO @ngxson : this is a hack to support both vision and audio encoders- have_multiple_encoders = self.has_audio_encoder and self.has_vision_encoder- self.block_count = 128 if have_multiple_encoders else self.find_hparam(self.n_block_keys, True)- self.tensor_map = gguf.get_tensor_name_map(gguf.MODEL_ARCH.MMPROJ, self.block_count)-- # load preprocessor config- with open(self.dir_model / "preprocessor_config.json", "r", encoding="utf-8") as f:- self.preprocessor_config = json.load(f)-- def get_vision_config(self) -> dict[str, Any] | None:- return self.global_config.get("vision_config")-- def get_audio_config(self) -> dict[str, Any] | None:- return self.global_config.get("audio_config")-- def set_type(self):- self.gguf_writer.add_type(gguf.GGUFType.MMPROJ)-- def set_gguf_parameters(self):- self.gguf_writer.add_file_type(self.ftype)-- if self.has_vision_encoder:- self.gguf_writer.add_clip_has_vision_encoder(True)- self.gguf_writer.add_vision_projection_dim(self.n_embd_text)-- # vision config- self.gguf_writer.add_vision_image_size(self.find_vparam(["image_size"]))- self.gguf_writer.add_vision_patch_size(self.find_vparam(["patch_size"]))- self.gguf_writer.add_vision_embedding_length(self.find_vparam(["hidden_size"]))- self.gguf_writer.add_vision_feed_forward_length(self.find_vparam(["intermediate_size"]))- self.gguf_writer.add_vision_block_count(self.find_vparam(self.n_block_keys))- self.gguf_writer.add_vision_head_count(self.find_vparam(["num_attention_heads"]))-- # preprocessor config- self.gguf_writer.add_vision_image_mean(self.preprocessor_config["image_mean"])- self.gguf_writer.add_vision_image_std(self.preprocessor_config["image_std"])-- if self.has_audio_encoder:- self.gguf_writer.add_clip_has_audio_encoder(True)- self.gguf_writer.add_audio_projection_dim(self.n_embd_text)-- # audio config- self.gguf_writer.add_audio_embedding_length(self.find_aparam(["hidden_size"]))- self.gguf_writer.add_audio_feed_forward_length(self.find_aparam(["intermediate_size"]))- self.gguf_writer.add_audio_block_count(self.find_aparam(self.n_block_keys))- self.gguf_writer.add_audio_head_count(self.find_aparam(["num_attention_heads"]))-- if not self.has_vision_encoder and not self.has_audio_encoder:- raise ValueError("MmprojModel must have either vision or audio encoder")-- def write_vocab(self):- raise ValueError("MmprojModel does not support vocab writing")-- def find_vparam(self, keys: Iterable[str], optional: bool = False) -> Any:- assert self.hparams_vision is not None- return self._find_param(self.hparams_vision, keys, optional)-- def find_aparam(self, keys: Iterable[str], optional: bool = False) -> Any:- assert self.hparams_audio is not None- return self._find_param(self.hparams_audio, keys, optional)-- def _find_param(self, obj: dict[str, Any], keys: Iterable[str], optional: bool = False) -> Any:- key = next((k for k in keys if k in obj), None)- if key is not None:- return obj[key]- if optional:- return None- raise KeyError(f"could not find any of: {keys}")--[email protected]("GPTNeoXForCausalLM")-class GPTNeoXModel(TextModel):- model_arch = gguf.MODEL_ARCH.GPTNEOX-- def set_gguf_parameters(self):- block_count = self.hparams["num_hidden_layers"]-- self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])- self.gguf_writer.add_rope_dimension_count(- int(self.hparams["rotary_pct"] * (self.hparams["hidden_size"] // self.hparams["num_attention_heads"])),- )- self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])- self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True))- self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))- n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))-- tensors: list[tuple[str, Tensor]] = []-- if re.match(r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.weight", name):- qkv_weights = data_torch.reshape((n_head, 3, n_embed // n_head, n_embed))- data_torch = torch.cat(- (- qkv_weights[:, 0, :, :].reshape((-1, n_embed)),- qkv_weights[:, 1, :, :].reshape((-1, n_embed)),- qkv_weights[:, 2, :, :].reshape((-1, n_embed)),- ),- dim=0,- )- logger.info("re-format attention.linear_qkv.weight")- elif re.match(r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.bias", name):- qkv_bias = data_torch.reshape((n_head, 3, n_embed // n_head))- data_torch = torch.cat(- (- qkv_bias[:, 0, :].reshape((n_embed,)),- qkv_bias[:, 1, :].reshape((n_embed,)),- qkv_bias[:, 2, :].reshape((n_embed,)),- ),- dim=0,- )- logger.info("re-format attention.linear_qkv.bias")-- tensors.append((self.map_tensor_name(name), data_torch))-- return tensors--[email protected]("BloomForCausalLM", "BloomModel")-class BloomModel(TextModel):- model_arch = gguf.MODEL_ARCH.BLOOM-- def set_gguf_parameters(self):- n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))- n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))- self.gguf_writer.add_context_length(self.hparams.get("seq_length", n_embed))- self.gguf_writer.add_embedding_length(n_embed)- self.gguf_writer.add_feed_forward_length(4 * n_embed)- self.gguf_writer.add_block_count(self.hparams["n_layer"])- self.gguf_writer.add_head_count(n_head)- self.gguf_writer.add_head_count_kv(n_head)- self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_file_type(self.ftype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))- n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))-- name = re.sub(r"transformer\.", "", name)-- tensors: list[tuple[str, Tensor]] = []-- if re.match(r"h\.\d+\.self_attention\.query_key_value\.weight", name):- qkv_weights = data_torch.reshape((n_head, 3, n_embed // n_head, n_embed))- data_torch = torch.cat(- (- qkv_weights[:, 0, :, :].reshape((-1, n_embed)),- qkv_weights[:, 1, :, :].reshape((-1, n_embed)),- qkv_weights[:, 2, :, :].reshape((-1, n_embed)),- ),- dim=0,- )- logger.info("re-format attention.linear_qkv.weight")- elif re.match(r"h\.\d+\.self_attention\.query_key_value\.bias", name):- qkv_bias = data_torch.reshape((n_head, 3, n_embed // n_head))- data_torch = torch.cat(- (- qkv_bias[:, 0, :].reshape((n_embed,)),- qkv_bias[:, 1, :].reshape((n_embed,)),- qkv_bias[:, 2, :].reshape((n_embed,)),- ),- dim=0,- )- logger.info("re-format attention.linear_qkv.bias")-- tensors.append((self.map_tensor_name(name), data_torch))-- return tensors--[email protected]("MPTForCausalLM")-class MPTModel(TextModel):- model_arch = gguf.MODEL_ARCH.MPT-- def set_vocab(self):- try:- self._set_vocab_gpt2()- except Exception:- # Fallback for SEA-LION model- self._set_vocab_sentencepiece()- self.gguf_writer.add_add_bos_token(False)- self.gguf_writer.add_pad_token_id(3)- self.gguf_writer.add_eos_token_id(1)- self.gguf_writer.add_unk_token_id(0)-- def set_gguf_parameters(self):- block_count = self.hparams["n_layers"]- self.gguf_writer.add_context_length(self.hparams["max_seq_len"])- self.gguf_writer.add_embedding_length(self.hparams["d_model"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(4 * self.hparams["d_model"])- self.gguf_writer.add_head_count(self.hparams["n_heads"])- if kv_n_heads := self.hparams["attn_config"].get("kv_n_heads"):- self.gguf_writer.add_head_count_kv(kv_n_heads)- self.gguf_writer.add_layer_norm_eps(1e-5)- if self.hparams["attn_config"]["clip_qkv"] is not None:- self.gguf_writer.add_clamp_kqv(self.hparams["attn_config"]["clip_qkv"])- if self.hparams["attn_config"]["alibi"]:- self.gguf_writer.add_max_alibi_bias(self.hparams["attn_config"]["alibi_bias_max"])- else:- self.gguf_writer.add_max_alibi_bias(0.0)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- if "scales" in name:- new_name = self.map_tensor_name(name, try_suffixes=(".weight", ".bias", ".scales"))- new_name = new_name.replace("scales", "act.scales")- else:- new_name = self.map_tensor_name(name, try_suffixes=(".weight", ".bias"))-- return [(new_name, data_torch)]--[email protected]("OrionForCausalLM")-class OrionModel(TextModel):- model_arch = gguf.MODEL_ARCH.ORION-- def set_vocab(self):- self._set_vocab_sentencepiece()-- def set_gguf_parameters(self):- block_count = self.hparams["num_hidden_layers"]- head_count = self.hparams["num_attention_heads"]- head_count_kv = self.hparams.get("num_key_value_heads", head_count)-- ctx_length = 0- if "max_sequence_length" in self.hparams:- ctx_length = self.hparams["max_sequence_length"]- elif "max_position_embeddings" in self.hparams:- ctx_length = self.hparams["max_position_embeddings"]- elif "model_max_length" in self.hparams:- ctx_length = self.hparams["model_max_length"]- else:- raise ValueError("gguf: can not find ctx length parameter.")-- self.gguf_writer.add_file_type(self.ftype)- self.gguf_writer.add_tensor_data_layout("Meta AI original pth")- self.gguf_writer.add_context_length(ctx_length)- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])- self.gguf_writer.add_head_count(head_count)- self.gguf_writer.add_head_count_kv(head_count_kv)- self.gguf_writer.add_layer_norm_eps(self.hparams["rms_norm_eps"])--[email protected]("BaichuanForCausalLM", "BaiChuanForCausalLM")-class BaichuanModel(TextModel):- model_arch = gguf.MODEL_ARCH.BAICHUAN-- def set_vocab(self):- self._set_vocab_sentencepiece()-- def set_gguf_parameters(self):- block_count = self.hparams["num_hidden_layers"]- head_count = self.hparams["num_attention_heads"]- head_count_kv = self.hparams.get("num_key_value_heads", head_count)-- ctx_length = 0- if "max_sequence_length" in self.hparams:- ctx_length = self.hparams["max_sequence_length"]- elif "max_position_embeddings" in self.hparams:- ctx_length = self.hparams["max_position_embeddings"]- elif "model_max_length" in self.hparams:- ctx_length = self.hparams["model_max_length"]- else:- raise ValueError("gguf: can not find ctx length parameter.")-- self.gguf_writer.add_tensor_data_layout("Meta AI original pth")- self.gguf_writer.add_context_length(ctx_length)- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])- self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- self.gguf_writer.add_head_count(head_count)- self.gguf_writer.add_head_count_kv(head_count_kv)- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])- self.gguf_writer.add_file_type(self.ftype)-- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- head_count = self.hparams["num_attention_heads"]- head_count_kv = self.hparams.get("num_key_value_heads", head_count)-- tensors: list[tuple[str, Tensor]] = []-- if bid is not None and name == f"model.layers.{bid}.self_attn.W_pack.weight":- logger.info(f"Unpacking and permuting layer {bid}")- tensors = [- (- self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid),- self._reverse_hf_permute_part(data_torch, 0, head_count, head_count),- ),- (- self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid),- self._reverse_hf_permute_part(data_torch, 1, head_count, head_count_kv),- ),- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), self._reverse_hf_part(data_torch, 2)),- ]- else:- tensors = [(self.map_tensor_name(name), data_torch)]-- return tensors-- def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor:- if n_kv_head is not None and n_head != n_kv_head:- n_head //= n_kv_head-- return (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])- .swapaxes(1, 2)- .reshape(weights.shape)- )-- def _reverse_hf_permute_part(- self,- weights: Tensor,- n_part: int,- n_head: int,- n_head_kv: int | None = None,- ) -> Tensor:- r = weights.shape[0] // 3- return self._reverse_hf_permute(weights[r * n_part : r * n_part + r, ...], n_head, n_head_kv)-- def _reverse_hf_part(self, weights: Tensor, n_part: int) -> Tensor:- r = weights.shape[0] // 3- return weights[r * n_part : r * n_part + r, ...]--[email protected]("XverseForCausalLM")-class XverseModel(TextModel):- model_arch = gguf.MODEL_ARCH.XVERSE-- def set_vocab(self):- assert (self.dir_model / "tokenizer.json").is_file()- dir_model = self.dir_model- hparams = self.hparams-- tokens: list[bytes] = []- toktypes: list[int] = []-- from transformers import AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(dir_model)- vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))- # Since we are checking the maximum index, we need to ensure it's strictly less than vocab_size,- # because vocab_size is the count of items, and indexes start at 0.- max_vocab_index = max(tokenizer.get_vocab().values())- if max_vocab_index >= vocab_size:- raise ValueError("Vocabulary size exceeds expected maximum size.")-- reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}- added_vocab = tokenizer.get_added_vocab()-- for token_id in range(vocab_size):- token_text = reverse_vocab[token_id].encode("utf-8")- # replace "\x00" to string with length > 0- if token_text == b"\x00":- toktype = gguf.TokenType.BYTE # special- token_text = f"<{token_text}>".encode("utf-8")- elif re.fullmatch(rb"<0x[0-9A-Fa-f]{2}>", token_text):- toktype = gguf.TokenType.BYTE # special- elif reverse_vocab[token_id] in added_vocab:- if tokenizer.added_tokens_decoder[token_id].special:- toktype = gguf.TokenType.CONTROL- else:- toktype = gguf.TokenType.USER_DEFINED- else:- toktype = gguf.TokenType.NORMAL-- tokens.append(token_text)- toktypes.append(toktype)-- self.gguf_writer.add_tokenizer_model("llama")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- block_count = self.hparams["num_hidden_layers"]- head_count = self.hparams["num_attention_heads"]- head_count_kv = self.hparams.get("num_key_value_heads", head_count)-- ctx_length = 0- if "max_sequence_length" in self.hparams:- ctx_length = self.hparams["max_sequence_length"]- elif "max_position_embeddings" in self.hparams:- ctx_length = self.hparams["max_position_embeddings"]- elif "model_max_length" in self.hparams:- ctx_length = self.hparams["model_max_length"]- else:- raise ValueError("gguf: can not find ctx length parameter.")-- self.gguf_writer.add_tensor_data_layout("Meta AI original pth")- self.gguf_writer.add_context_length(ctx_length)- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])- self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- self.gguf_writer.add_head_count(head_count)- self.gguf_writer.add_head_count_kv(head_count_kv)- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])- self.gguf_writer.add_file_type(self.ftype)-- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- head_count = self.hparams["num_attention_heads"]- head_count_kv = self.hparams.get("num_key_value_heads", head_count)-- # HF models permute some of the tensors, so we need to undo that- if name.endswith("q_proj.weight"):- data_torch = self._reverse_hf_permute(data_torch, head_count, head_count)- if name.endswith("k_proj.weight"):- data_torch = self._reverse_hf_permute(data_torch, head_count, head_count_kv)-- return [(self.map_tensor_name(name), data_torch)]-- def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor:- if n_kv_head is not None and n_head != n_kv_head:- n_head //= n_kv_head-- return (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])- .swapaxes(1, 2)- .reshape(weights.shape)- )--[email protected]("FalconForCausalLM", "RWForCausalLM")-class FalconModel(TextModel):- model_arch = gguf.MODEL_ARCH.FALCON-- def set_gguf_parameters(self):- block_count = self.hparams.get("num_hidden_layers")- if block_count is None:- block_count = self.hparams["n_layer"] # old name-- n_head = self.hparams.get("num_attention_heads")- if n_head is None:- n_head = self.hparams["n_head"] # old name-- n_head_kv = self.hparams.get("num_kv_heads")- if n_head_kv is None:- n_head_kv = self.hparams.get("n_head_kv", 1) # old name-- self.gguf_writer.add_context_length(2048) # not in config.json- self.gguf_writer.add_tensor_data_layout("jploski") # qkv tensor transform- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_feed_forward_length(4 * self.hparams["hidden_size"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_head_count(n_head)- self.gguf_writer.add_head_count_kv(n_head_kv)- self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_file_type(self.ftype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- # QKV tensor transform- # The original query_key_value tensor contains n_head_kv "kv groups",- # each consisting of n_head/n_head_kv query weights followed by one key- # and one value weight (shared by all query heads in the kv group).- # This layout makes it a big pain to work with in GGML.- # So we rearrange them here,, so that we have n_head query weights- # followed by n_head_kv key weights followed by n_head_kv value weights,- # in contiguous fashion.- # ref: https://github.com/jploski/ggml/blob/falcon40b/examples/falcon/convert-hf-to-ggml.py-- if "query_key_value" in name:- n_head = self.find_hparam(["num_attention_heads", "n_head"])- n_head_kv = self.find_hparam(["num_kv_heads", "n_head_kv"], optional=True) or 1- head_dim = self.hparams["hidden_size"] // n_head-- qkv = data_torch.view(n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)- q = qkv[:, :-2].reshape(n_head * head_dim, head_dim * n_head)- k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)- v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)- data_torch = torch.cat((q, k, v)).reshape_as(data_torch)-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("GPTBigCodeForCausalLM")-class StarCoderModel(TextModel):- model_arch = gguf.MODEL_ARCH.STARCODER-- def set_gguf_parameters(self):- block_count = self.hparams["n_layer"]-- self.gguf_writer.add_context_length(self.hparams["n_positions"])- self.gguf_writer.add_embedding_length(self.hparams["n_embd"])- self.gguf_writer.add_feed_forward_length(4 * self.hparams["n_embd"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_head_count(self.hparams["n_head"])- self.gguf_writer.add_head_count_kv(1)- self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_file_type(self.ftype)--[email protected]("GPTRefactForCausalLM")-class RefactModel(TextModel):- model_arch = gguf.MODEL_ARCH.REFACT-- def set_vocab(self):- super().set_vocab()-- # TODO: how to determine special FIM tokens automatically?- special_vocab = gguf.SpecialVocab(- self.dir_model, load_merges=False, special_token_types=["prefix", "suffix", "middle", "eot"]- )- special_vocab._set_special_token("prefix", 1)- special_vocab._set_special_token("suffix", 3)- special_vocab._set_special_token("middle", 2)- special_vocab.chat_template = None # do not add it twice- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- hidden_dim = self.hparams["n_embd"]- inner_dim = 4 * hidden_dim- hidden_dim = int(2 * inner_dim / 3)- multiple_of = 256- ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)-- block_count = self.hparams["n_layer"]-- # refact uses Alibi. So this is from config.json which might be used by training.- self.gguf_writer.add_context_length(self.hparams["n_positions"])- self.gguf_writer.add_embedding_length(self.hparams["n_embd"])-- self.gguf_writer.add_feed_forward_length(ff_dim)- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_head_count(self.hparams["n_head"])- self.gguf_writer.add_head_count_kv(1)- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_file_type(self.ftype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- hidden_dim = self.hparams["n_embd"]- inner_dim = 4 * hidden_dim- hidden_dim = int(2 * inner_dim / 3)- multiple_of = 256- ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)- n_head = self.hparams["n_head"]- n_head_kv = 1- head_dim = self.hparams["n_embd"] // n_head-- tensors: list[tuple[str, Tensor]] = []-- if bid is not None:- if name == f"transformer.h.{bid}.attn.kv.weight":- tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), data_torch[: n_head_kv * head_dim])- )- tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), data_torch[n_head_kv * head_dim :])- )- elif name == f"transformer.h.{bid}.attn.q.weight":- tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), data_torch))- elif name == f"transformer.h.{bid}.mlp.gate_up_proj.weight":- tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), data_torch[:ff_dim]))- tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), data_torch[ff_dim:]))-- if len(tensors) == 0:- tensors.append((self.map_tensor_name(name), data_torch))-- return tensors--[email protected]("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM")-class StableLMModel(TextModel):- model_arch = gguf.MODEL_ARCH.STABLELM-- def set_vocab(self):- if (self.dir_model / "tokenizer.json").is_file():- self._set_vocab_gpt2()- else:- # StableLM 2 1.6B used to have a vocab in a similar format to Qwen's vocab- self._set_vocab_qwen()-- def set_gguf_parameters(self):- hparams = self.hparams- block_count = hparams["num_hidden_layers"]-- self.gguf_writer.add_context_length(hparams["max_position_embeddings"])- self.gguf_writer.add_embedding_length(hparams["hidden_size"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])- rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"])- self.gguf_writer.add_rope_dimension_count(- int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))- )- self.gguf_writer.add_head_count(hparams["num_attention_heads"])- self.gguf_writer.add_head_count_kv(hparams["num_key_value_heads"])- self.gguf_writer.add_parallel_residual(- hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True- )- self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_eps", "norm_eps"]))- self.gguf_writer.add_file_type(self.ftype)-- _q_norms: list[dict[str, Tensor]] | None = None- _k_norms: list[dict[str, Tensor]] | None = None-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- n_head = self.hparams["num_attention_heads"]- n_kv_head = self.hparams["num_key_value_heads"]-- if name.find("q_layernorm.norms") != -1:- assert bid is not None-- if self._q_norms is None:- self._q_norms = [{} for _ in range(self.block_count)]-- self._q_norms[bid][name] = data_torch-- if len(self._q_norms[bid]) >= n_head:- return self._stack_qk_norm(bid, n_head, self._q_norms[bid], "q_layernorm")- else:- return []-- if name.find("k_layernorm.norms") != -1:- assert bid is not None-- if self._k_norms is None:- self._k_norms = [{} for _ in range(self.block_count)]-- self._k_norms[bid][name] = data_torch-- if len(self._k_norms[bid]) >= n_kv_head:- return self._stack_qk_norm(bid, n_kv_head, self._k_norms[bid], "k_layernorm")- else:- return []-- return [(self.map_tensor_name(name), data_torch)]-- def _stack_qk_norm(self, bid: int, n_head: int, norms: dict[str, Tensor], layer_name: str = "q_layernorm"):- data: list[Tensor] = []- # extract the norms in order- for xid in range(n_head):- ename = f"model.layers.{bid}.self_attn.{layer_name}.norms.{xid}.weight"- data.append(norms[ename])- del norms[ename]- data_torch = torch.stack(data, dim=0)-- merged_name = f"model.layers.{bid}.self_attn.{layer_name}.weight"- new_name = self.map_tensor_name(merged_name)-- return [(new_name, data_torch)]-- def prepare_tensors(self):- super().prepare_tensors()-- if self._q_norms is not None or self._k_norms is not None:- # flatten two `list[dict[str, Tensor]]` into a single `list[str]`- norms = ([k for d in self._q_norms for k in d.keys()] if self._q_norms is not None else []) + (- [k for d in self._k_norms for k in d.keys()] if self._k_norms is not None else []- )- if len(norms) > 0:- raise ValueError(f"Unprocessed norms: {norms}")--- "LLaMAForCausalLM",- "LlamaForCausalLM",- "MistralForCausalLM",- "MixtralForCausalLM",- "VLlama3ForCausalLM",- "LlavaForConditionalGeneration",- "LlamaModel",-)-class LlamaModel(TextModel):- model_arch = gguf.MODEL_ARCH.LLAMA- undo_permute = True-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- # fix for SmolVLM2, missing `num_attention_heads` in config.json- if self.hf_arch == "VLlama3ForCausalLM":- self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 32)-- def set_vocab(self):- try:- self._set_vocab_sentencepiece()- except FileNotFoundError:- try:- self._set_vocab_llama_hf()- except (FileNotFoundError, TypeError):- # Llama 3- self._set_vocab_gpt2()-- # Apply to CodeLlama only (and ignore for Llama 3 with a vocab size of 128256)- if self.hparams.get("vocab_size", 32000) == 32016:- special_vocab = gguf.SpecialVocab(- self.dir_model, load_merges=False, special_token_types=["prefix", "suffix", "middle", "eot"]- )- special_vocab._set_special_token("prefix", 32007)- special_vocab._set_special_token("suffix", 32008)- special_vocab._set_special_token("middle", 32009)- special_vocab._set_special_token("eot", 32010)- special_vocab.add_to_gguf(self.gguf_writer)-- tokenizer_config_file = self.dir_model / "tokenizer_config.json"- if tokenizer_config_file.is_file():- with open(tokenizer_config_file, "r", encoding="utf-8") as f:- tokenizer_config_json = json.load(f)- if "add_prefix_space" in tokenizer_config_json:- self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])-- # Apply to granite small models only- if self.hparams.get("vocab_size", 32000) == 49152:- self.gguf_writer.add_add_bos_token(False)-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- self.gguf_writer.add_vocab_size(hparams["vocab_size"])-- if (rope_dim := hparams.get("head_dim")) is None:- rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]- self.gguf_writer.add_rope_dimension_count(rope_dim)-- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])-- @staticmethod- def permute(weights: Tensor, n_head: int, n_head_kv: int | None):- if n_head_kv is not None and n_head != n_head_kv:- n_head = n_head_kv- return (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])- .swapaxes(1, 2)- .reshape(weights.shape)- )-- _experts: list[dict[str, Tensor]] | None = None-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- n_head = self.hparams["num_attention_heads"]- n_kv_head = self.hparams.get("num_key_value_heads")- is_vision_tensor = (- "vision_tower" in name- or "vision_model" in name- or "model.connector" in name- or "multi_modal_projector" in name- )-- if is_vision_tensor:- return [] # skip vision tensors- elif self.hf_arch == "LlamaModel":- name = "model." + name- elif name.startswith("model.text_model"):- name = name.replace("text_model.", "") # for SmolVLM- elif name.startswith("language_model."):- name = name.replace("language_model.", "") # for the rest-- if self.undo_permute:- if name.endswith(("q_proj.weight", "q_proj.bias")):- data_torch = LlamaModel.permute(data_torch, n_head, n_head)- if name.endswith(("k_proj.weight", "k_proj.bias")):- data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)-- # process the experts separately- if name.find("block_sparse_moe.experts") != -1:- n_experts = self.hparams["num_local_experts"]-- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- tensors: list[tuple[str, Tensor]] = []-- # merge the experts into a single 3d tensor- for wid in ["w1", "w2", "w3"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{wid}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)-- merged_name = f"layers.{bid}.feed_forward.experts.{wid}.weight"-- new_name = self.map_tensor_name(merged_name)-- tensors.append((new_name, data_torch))- return tensors- else:- return []-- return [(self.map_tensor_name(name), data_torch)]-- def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):- if rope_scaling.get("rope_type", "").lower() == "llama3":- base = self.hparams.get("rope_theta", 10000.0)- if (dim := self.hparams.get("head_dim")) is None:- dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]- freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))-- factor = rope_scaling.get("factor", 8.0)- low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)- high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)- old_context_len = self.hparams.get("original_max_position_embeddings", 8192)-- low_freq_wavelen = old_context_len / low_freq_factor- high_freq_wavelen = old_context_len / high_freq_factor- # assert low_freq_wavelen != high_freq_wavelen # Errors for Llama4-- rope_factors = []- for freq in freqs:- wavelen = 2 * math.pi / freq- if wavelen < high_freq_wavelen:- rope_factors.append(1)- elif wavelen > low_freq_wavelen:- rope_factors.append(factor)- else:- smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)- rope_factors.append(1 / ((1 - smooth) / factor + smooth))-- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32),- )-- def prepare_tensors(self):- super().prepare_tensors()-- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--[email protected]("ArceeForCausalLM")-class ArceeModel(LlamaModel):- model_arch = gguf.MODEL_ARCH.ARCEE-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self._try_set_pooling_type()- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])--- "LlavaForConditionalGeneration", # pixtral- "Mistral3ForConditionalGeneration", # mistral small 3.1-)-class LlavaVisionModel(MmprojModel):- img_break_tok_id = -1-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- if self.hparams["model_type"] == "pixtral":- # layer_norm_eps is not in config.json, it is hard-coded in modeling_pixtral.py- self.hparams["layer_norm_eps"] = self.hparams.get("layer_norm_eps", 1e-5)- self.img_break_tok_id = self.get_token_id("[IMG_BREAK]")- logger.info(f"Image break token id: {self.img_break_tok_id}")- else:- raise ValueError(f"Unsupported model type: {self.hparams['model_type']}")-- def get_token_id(self, token: str) -> int:- tokenizer_config_file = self.dir_model / "tokenizer_config.json"- with open(tokenizer_config_file, "r", encoding="utf-8") as f:- added_tokens_decoder = json.load(f)["added_tokens_decoder"]- for id_, token_data in added_tokens_decoder.items():- if token_data["content"] == token:- return int(id_)- raise ValueError(f"Token '{token}' not found in tokenizer config.")-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- if hparams["model_type"] == "pixtral":- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.PIXTRAL)- self.gguf_writer.add_vision_attention_layernorm_eps(hparams["layer_norm_eps"])-- # hidden_act- if hparams["hidden_act"] == "silu":- self.gguf_writer.add_vision_use_silu(True)- elif hparams["hidden_act"] == "gelu":- self.gguf_writer.add_vision_use_gelu(True)- else:- raise ValueError(f"Unsupported hidden_act: {hparams['hidden_act']}")-- # spatial_merge_size- if "spatial_merge_size" in self.global_config:- self.gguf_writer.add_vision_spatial_merge_size(self.global_config["spatial_merge_size"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused- n_head = self.hparams["num_attention_heads"]- n_kv_head = n_head-- if name.startswith("multi_modal_projector.") or name.startswith("vision_tower."):- # process vision tensors- if name.endswith(("q_proj.weight", "q_proj.bias")):- data_torch = LlamaModel.permute(data_torch, n_head, n_head)- if name.endswith(("k_proj.weight", "k_proj.bias")):- data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)- return [(self.map_tensor_name(name), data_torch)]-- if self.img_break_tok_id > 0 and "embed_tokens.weight" in name:- logger.info(f"Extracting [IMG_BREAK] token embedding from {name}")- # for pixtral model, we need to extract the [IMG_BREAK] token embedding- img_break_embd = data_torch[self.img_break_tok_id]- name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK]- return [(self.map_tensor_name(name), img_break_embd)]-- return [] # skip other tensors--[email protected]("Idefics3ForConditionalGeneration", "SmolVLMForConditionalGeneration")-class SmolVLMModel(MmprojModel):-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- if self.hparams["model_type"] == "smolvlm_vision":- # fix for SmolVLM2, missing some keys in config.json- # default values are taken from transformers code- self.hparams["hidden_size"] = self.hparams.get("hidden_size", 1152)- self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 16)- self.hparams["intermediate_size"] = self.hparams.get("intermediate_size", 3072)-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.IDEFICS3)- self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))- self.gguf_writer.add_vision_projector_scale_factor(self.global_config.get("scale_factor", 2))- self.gguf_writer.add_vision_use_gelu(True)-- def tensor_force_quant(self, name, new_name, bid, n_dims):- del bid, new_name, n_dims # unused- if ".embeddings." in name:- return gguf.GGMLQuantizationType.F32- return False-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused- is_vision_tensor = "vision_tower" in name or "vision_model" in name or "model.connector" in name-- if is_vision_tensor:- return [(self.map_tensor_name(name), data_torch)]-- return [] # skip other tensors--[email protected]("Llama4ForConditionalGeneration")-class Llama4Model(LlamaModel):- model_arch = gguf.MODEL_ARCH.LLAMA4- undo_permute = False-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- # IMPORTANT: the normal "intermediate_size" is renamed to "intermediate_size_mlp", we need to undo this- self.hparams["intermediate_size_moe"] = self.hparams["intermediate_size"]- self.hparams["intermediate_size"] = self.hparams["intermediate_size_mlp"]-- def set_vocab(self):- self._set_vocab_gpt2()-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_interleave_moe_layer_step(self.hparams["interleave_moe_layer_step"])- self.gguf_writer.add_expert_feed_forward_length(self.hparams["intermediate_size_moe"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):- if name.startswith("language_model."):- name = name.replace("language_model.", "")-- # split the gate_up into gate and up- if "gate_up_proj" in name:- name_up = name.replace("gate_up_proj", "up_proj.weight")- name_gate = name.replace("gate_up_proj", "gate_proj.weight")- dim_half = data_torch.shape[-1] // 2- gate_proj_weight, up_proj_weight = data_torch.transpose(-1, -2).split(dim_half, dim=-2)- return [- (self.map_tensor_name(name_gate), gate_proj_weight),- (self.map_tensor_name(name_up), up_proj_weight),- ]-- if name.endswith("down_proj"):- name += ".weight"- data_torch = data_torch.transpose(-1, -2)-- if "multi_modal_projector" in name or "vision_model" in name:- return []- return super().modify_tensors(data_torch, name, bid)--[email protected]("Llama4ForConditionalGeneration")-class Llama4VisionModel(MmprojModel):-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.LLAMA4)- self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams["norm_eps"])- self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / self.hparams["pixel_shuffle_ratio"]))- assert self.hparams["hidden_act"] == "gelu"- self.gguf_writer.add_vision_use_gelu(True)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused- if "multi_modal_projector" in name or "vision_model" in name:- # process vision tensors- if "positional_embedding_vlm" in name and ".weight" not in name:- name += ".weight"- if "multi_modal_projector.linear_1" in name:- # despite the name with number postfix, this is a single fully connected layer- return [(gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_MMPROJ_FC] + ".weight", data_torch)]- return [(self.map_tensor_name(name), data_torch)]- return []--[email protected]("Mistral3ForConditionalGeneration")-class Mistral3Model(LlamaModel):- model_arch = gguf.MODEL_ARCH.LLAMA-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):- name = name.replace("language_model.", "")- if "multi_modal_projector" in name or "vision_tower" in name:- return []- return super().modify_tensors(data_torch, name, bid)--[email protected]("DeciLMForCausalLM")-class DeciModel(TextModel):- model_arch = gguf.MODEL_ARCH.DECI-- @staticmethod- def _ffn_mult_to_intermediate_size(ffn_mult: float, n_embd: int) -> int:- # DeciLM-specific code- intermediate_size = int(2 * ffn_mult * n_embd / 3)- return DeciModel._find_multiple(intermediate_size, 256)-- @staticmethod- def _find_multiple(n: int, k: int) -> int:- # DeciLM-specific code- if n % k == 0:- return n- return n + k - (n % k)-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)-- if "block_configs" in self.hparams: # Llama-3_1-Nemotron-51B- _block_configs: list[dict[str, Any]] = self.hparams["block_configs"]- assert self.block_count == len(_block_configs)- self._num_kv_heads = list()- self._num_heads = list()- _ffn_multipliers = list()- # ***linear attention layer***- # if n_heads_in_group is None and replace_with_linear is True- # then _num_kv_heads[il] is 0 and _num_heads[il] is num_attention_heads- # ***attention-free layer***- # if n_heads_in_group is None and replace_with_linear is False- # then _num_kv_heads[il] is 0 and _num_heads[il] is 0- # ***normal attention-layer***- # if n_heads_in_group is not None, then- # _num_kv_heads[il] is num_attention_head // n_heads_in_group and- # _num_heads[il] is num_attention_head- # ***dummy layer*** for nemotron 253B- # if n_heads_in_group is None and ffn_mult is None- # then _num_kv_heads[il] is 0 and _num_heads[il] is 0 and _ffn_dims is 0- for il in range(len(_block_configs)):- if _block_configs[il]["attention"]["n_heads_in_group"] is None:- if _block_configs[il]["attention"]["replace_with_linear"] is True:- self._num_kv_heads.append(0)- self._num_heads.append(self.hparams["num_attention_heads"])- else:- self._num_kv_heads.append(0)- self._num_heads.append(0)- else:- self._num_kv_heads.append(- self.hparams["num_attention_heads"] // _block_configs[il]["attention"]["n_heads_in_group"]- )- self._num_heads.append(self.hparams["num_attention_heads"])- if _block_configs[il]["ffn"]["ffn_mult"] is None: # dummy layer- _ffn_multipliers.append(0.0)- else:- _ffn_multipliers.append(_block_configs[il]["ffn"]["ffn_mult"])- assert self.block_count == len(self._num_kv_heads)- assert self.block_count == len(self._num_heads)- assert self.block_count == len(_ffn_multipliers)- assert isinstance(self._num_kv_heads, list) and isinstance(self._num_kv_heads[0], int)- assert isinstance(self._num_heads, list) and isinstance(self._num_heads[0], int)- assert isinstance(_ffn_multipliers, list) and isinstance(_ffn_multipliers[0], float)- self._ffn_dims: list[int] = [- DeciModel._ffn_mult_to_intermediate_size(multiplier, self.hparams["hidden_size"])- for multiplier in _ffn_multipliers- ]-- def set_vocab(self):- # Please change tokenizer_config.json of Llama-3_1-Nemotron-51B's- # eos_token from '|eot_id|' to '|end_of_text|'- if self.hparams.get("vocab_size", 128256) == 128256:- tokens, toktypes, tokpre = self.get_vocab_base()- self.gguf_writer.add_tokenizer_model("gpt2")- self.gguf_writer.add_tokenizer_pre(tokpre)- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)- special_vocab.add_to_gguf(self.gguf_writer)- else:- # DeciLM-7B- self._set_vocab_llama_hf()-- def set_gguf_parameters(self):- if "block_configs" in self.hparams: # Llama-3_1-Nemotron-51B- assert self.block_count == len(self._num_kv_heads)- assert self.block_count == len(self._num_heads)- assert self.block_count == len(self._ffn_dims)- if (rope_theta := self.hparams.get("rope_theta")) is not None:- self.gguf_writer.add_rope_freq_base(rope_theta)- self.gguf_writer.add_head_count_kv(self._num_kv_heads)- self.gguf_writer.add_head_count(self._num_heads)- self.gguf_writer.add_feed_forward_length(self._ffn_dims)- self.gguf_writer.add_block_count(self.block_count)- self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])- self.gguf_writer.add_key_length(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- self.gguf_writer.add_value_length(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- self.gguf_writer.add_file_type(self.ftype)- else: # DeciLM-7B- super().set_gguf_parameters()- if "num_key_value_heads_per_layer" in self.hparams: # DeciLM-7B- self._num_kv_heads: list[int] = self.hparams["num_key_value_heads_per_layer"]- assert self.block_count == len(self._num_kv_heads)- self.gguf_writer.add_head_count_kv(self._num_kv_heads)- hparams = self.hparams- self.gguf_writer.add_vocab_size(hparams["vocab_size"])-- if (rope_dim := hparams.get("head_dim")) is None:- rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]- self.gguf_writer.add_rope_dimension_count(rope_dim)-- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])-- @staticmethod- def permute(weights: Tensor, n_head: int, n_head_kv: int | None):- if n_head_kv is not None and n_head != n_head_kv:- n_head = n_head_kv- return (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])- .swapaxes(1, 2)- .reshape(weights.shape)- )-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- n_head = self.hparams["num_attention_heads"]- if bid is not None:- if "num_key_value_heads_per_layer" in self.hparams:- n_kv_head = self.hparams["num_key_value_heads_per_layer"][bid]- elif "block_configs" in self.hparams:- n_kv_head = self._num_kv_heads[bid]- n_head = self._num_heads[bid]- else:- n_kv_head = self.hparams.get("num_key_value_heads")- else:- n_kv_head = self.hparams.get("num_key_value_heads")-- if name.endswith(("q_proj.weight", "q_proj.bias")):- data_torch = DeciModel.permute(data_torch, n_head, n_head)- if name.endswith(("k_proj.weight", "k_proj.bias")):- data_torch = DeciModel.permute(data_torch, n_head, n_kv_head)- return [(self.map_tensor_name(name), data_torch)]-- def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):- if rope_scaling.get("rope_type", "").lower() == "llama3":- base = self.hparams.get("rope_theta", 10000.0)- if (dim := self.hparams.get("head_dim")) is None:- dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]- freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))-- factor = rope_scaling.get("factor", 8.0)- low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)- high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)- old_context_len = self.hparams.get("original_max_position_embeddings", 8192)-- low_freq_wavelen = old_context_len / low_freq_factor- high_freq_wavelen = old_context_len / high_freq_factor- assert low_freq_wavelen != high_freq_wavelen-- rope_factors = []- for freq in freqs:- wavelen = 2 * math.pi / freq- if wavelen < high_freq_wavelen:- rope_factors.append(1)- elif wavelen > low_freq_wavelen:- rope_factors.append(factor)- else:- smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)- rope_factors.append(1 / ((1 - smooth) / factor + smooth))-- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32),- )-- def prepare_tensors(self):- super().prepare_tensors()--[email protected]("BitnetForCausalLM")-class BitnetModel(TextModel):- model_arch = gguf.MODEL_ARCH.BITNET-- def set_vocab(self):- self._set_vocab_sentencepiece()-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(1.0)-- def weight_quant(self, weight: Tensor) -> Tensor:- dtype = weight.dtype- weight = weight.float()- scale = weight.abs().mean().clamp(min=1e-5)- iscale = 1 / scale- result = (weight * iscale).round().clamp(-1, 1) / iscale- return result.type(dtype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- new_name = self.map_tensor_name(name)-- if any(- self.match_model_tensor_name(new_name, key, bid)- for key in [- gguf.MODEL_TENSOR.ATTN_Q,- gguf.MODEL_TENSOR.ATTN_K,- gguf.MODEL_TENSOR.ATTN_V,- gguf.MODEL_TENSOR.ATTN_OUT,- gguf.MODEL_TENSOR.FFN_UP,- gguf.MODEL_TENSOR.FFN_DOWN,- gguf.MODEL_TENSOR.FFN_GATE,- ]- ):- # transform weight into 1/0/-1 (in fp32)- data_torch = self.weight_quant(data_torch)-- yield (new_name, data_torch)--[email protected]("GrokForCausalLM")-class GrokModel(TextModel):- model_arch = gguf.MODEL_ARCH.GROK-- def set_vocab(self):- self._set_vocab_sentencepiece()-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)-- def set_gguf_parameters(self):- super().set_gguf_parameters()-- _experts: list[dict[str, Tensor]] | None = None-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # process the experts separately- if name.find(".moe.") != -1:- n_experts = self.hparams["num_local_experts"]-- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- tensors: list[tuple[str, Tensor]] = []-- # merge the experts into a single 3d tensor- for wid in ["linear", "linear_1", "linear_v"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"transformer.decoder_layer.{bid}.moe.{xid}.{wid}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)-- merged_name = f"transformer.decoder_layer.{bid}.moe.{wid}.weight"-- new_name = self.map_tensor_name(merged_name)-- tensors.append((new_name, data_torch))- return tensors- else:- return []-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("DbrxForCausalLM")-class DbrxModel(TextModel):- model_arch = gguf.MODEL_ARCH.DBRX-- def set_gguf_parameters(self):- ffn_config = self.hparams["ffn_config"]- attn_config = self.hparams["attn_config"]- self.gguf_writer.add_block_count(self.hparams["n_layers"])-- self.gguf_writer.add_context_length(self.hparams["max_seq_len"])- self.gguf_writer.add_embedding_length(self.hparams["d_model"])- self.gguf_writer.add_feed_forward_length(ffn_config["ffn_hidden_size"])-- self.gguf_writer.add_head_count(self.hparams["n_heads"])- self.gguf_writer.add_head_count_kv(attn_config["kv_n_heads"])-- self.gguf_writer.add_rope_freq_base(attn_config["rope_theta"])-- self.gguf_writer.add_clamp_kqv(attn_config["clip_qkv"])-- self.gguf_writer.add_expert_count(ffn_config["moe_num_experts"])- self.gguf_writer.add_expert_used_count(ffn_config["moe_top_k"])-- self.gguf_writer.add_layer_norm_eps(1e-5)-- self.gguf_writer.add_file_type(self.ftype)- logger.info(f"gguf: file type = {self.ftype}")-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- n_expert = self.hparams["ffn_config"]["moe_num_experts"]- n_ff = self.hparams["ffn_config"]["ffn_hidden_size"]- n_embd = self.hparams["d_model"]-- # Specific behavior for experts tensors: suffix .weight, view as 3D and transpose- # original implementation expects (n_expert, n_ff, n_embd) for all experts weights- # But llama.cpp moe graph works differently- # AND the dimensions in ggml are typically in the reverse order of the pytorch dimensions- # so (n_expert, n_ff, n_embd) in pytorch is {n_embd, n_ff, n_expert} in ggml_tensor- exp_tensor_names = {- "ffn.experts.mlp.w1": None, # LLM_TENSOR_FFN_GATE_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}- "ffn.experts.mlp.w2": (0, 2, 1), # LLM_TENSOR_FFN_DOWN_EXPS ggml_tensor->ne{n_ff, n_embd, n_expert}- "ffn.experts.mlp.v1": None,- } # LLM_TENSOR_FFN_UP_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}- experts = False-- for exp_tensor_name in exp_tensor_names.keys():- if name.find(exp_tensor_name) != -1 and name.find(".weight") == -1:- experts = True- data_torch = data_torch.view(n_expert, n_ff, n_embd)- if (permute_tensor := exp_tensor_names[exp_tensor_name]) is not None:- data_torch = data_torch.permute(*permute_tensor)- break-- # map tensor names- # In MoE models the ffn tensors are typically most of the model weights,- # and need to be quantizable. Quantize expects tensor names to be suffixed by .weight.- # Every other model has the weight names ending in .weight,- # let's assume that is the convention which is not the case for dbrx:- # https://huggingface.co/databricks/dbrx-instruct/blob/main/model.safetensors.index.json#L15- new_name = self.map_tensor_name(name if not experts else name + ".weight", try_suffixes=(".weight",))-- return [(new_name, data_torch)]-- def tensor_force_quant(- self, name: str, new_name: str, bid: int | None, n_dims: int- ) -> gguf.GGMLQuantizationType | bool:- del name, new_name, bid # unused-- return n_dims > 1--[email protected]("MiniCPMForCausalLM")-class MiniCPMModel(TextModel):- model_arch = gguf.MODEL_ARCH.MINICPM-- def set_gguf_parameters(self):- super().set_gguf_parameters()- embedding_scale = float(self.hparams["scale_emb"])- self.gguf_writer.add_embedding_scale(embedding_scale)- logger.info(f"gguf: (minicpm) embedding_scale = {embedding_scale}")- residual_scale = self.hparams["scale_depth"] / self.hparams["num_hidden_layers"] ** 0.5- self.gguf_writer.add_residual_scale(residual_scale)- logger.info(f"gguf: (minicpm) residual_scale = {residual_scale}")- logit_scale = self.hparams["hidden_size"] / self.hparams["dim_model_base"]- self.gguf_writer.add_logit_scale(logit_scale)- logger.info(f"gguf: (minicpm) logit_scale = {logit_scale}")- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "longrope":- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LONGROPE)- logger.info(f"gguf: (minicpm) rope_scaling_type = {gguf.RopeScalingType.LONGROPE}")-- def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- rope_dims = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]-- rope_scaling = self.find_hparam(["rope_scaling"], True)- if rope_scaling is not None:- long_factors = rope_scaling.get("long_factor", None)- short_factors = rope_scaling.get("short_factor", None)-- if long_factors is None or short_factors is None:- raise KeyError("Missing the required key rope_scaling.long_factor or rope_scaling_short_factor")-- if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:- raise ValueError(f"The length of rope long and short factors must be {rope_dims / 2}")-- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),- torch.tensor(long_factors, dtype=torch.float32),- )- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),- torch.tensor(short_factors, dtype=torch.float32),- )-- def set_vocab(self):- self._set_vocab_sentencepiece()-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- n_head = self.hparams["num_attention_heads"]- n_kv_head = self.hparams.get("num_key_value_heads")-- # HF models permute some of the tensors, so we need to undo that- if name.endswith(("q_proj.weight")):- data_torch = LlamaModel.permute(data_torch, n_head, n_head)- if name.endswith(("k_proj.weight")):- data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("MiniCPM3ForCausalLM")-class MiniCPM3Model(TextModel):- model_arch = gguf.MODEL_ARCH.MINICPM3-- def set_gguf_parameters(self):- hparams = self.hparams-- self.gguf_writer.add_file_type(self.ftype)- self.gguf_writer.add_context_length(hparams["max_position_embeddings"])- self.gguf_writer.add_embedding_length(hparams["hidden_size"])- self.gguf_writer.add_block_count(self.block_count)- self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])- self.gguf_writer.add_head_count(hparams["num_attention_heads"])- self.gguf_writer.add_head_count_kv(hparams["num_key_value_heads"])- self.gguf_writer.add_layer_norm_rms_eps(hparams["rms_norm_eps"])- self.gguf_writer.add_vocab_size(hparams["vocab_size"])- if "q_lora_rank" in hparams and hparams["q_lora_rank"] is not None:- self.gguf_writer.add_q_lora_rank(hparams["q_lora_rank"])- self.gguf_writer.add_kv_lora_rank(hparams["kv_lora_rank"])- self.gguf_writer.add_key_length(hparams["qk_nope_head_dim"] + hparams["qk_rope_head_dim"])- self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])-- def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- rope_scaling = self.find_hparam(["rope_scaling"], True)- if rope_scaling is not None:- rope_dims = self.hparams["qk_rope_head_dim"]-- long_factors = rope_scaling.get("long_factor", None)- short_factors = rope_scaling.get("short_factor", None)-- if long_factors is None or short_factors is None:- raise KeyError("Missing the required key rope_scaling.long_factor or rope_scaling_short_factor")-- if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:- raise ValueError(f"The length of rope long and short factors must be {rope_dims / 2}")-- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),- torch.tensor(long_factors, dtype=torch.float32),- )- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),- torch.tensor(short_factors, dtype=torch.float32),- )-- def set_vocab(self):- self._set_vocab_sentencepiece()-- def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor:- if n_kv_head is not None and n_head != n_kv_head:- n_head //= n_kv_head-- return (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])- .swapaxes(1, 2)- .reshape(weights.shape)- )--[email protected]("QWenLMHeadModel")-class QwenModel(TextModel):- model_arch = gguf.MODEL_ARCH.QWEN-- @staticmethod- def token_bytes_to_string(b):- from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode-- byte_encoder = bytes_to_unicode()- return "".join([byte_encoder[ord(char)] for char in b.decode("latin-1")])-- @staticmethod- def bpe(mergeable_ranks: dict[bytes, int], token: bytes, max_rank: int | None = None) -> list[bytes]:- parts = [bytes([b]) for b in token]- while True:- min_idx = None- min_rank = None- for i, pair in enumerate(zip(parts[:-1], parts[1:])):- rank = mergeable_ranks.get(pair[0] + pair[1])- if rank is not None and (min_rank is None or rank < min_rank):- min_idx = i- min_rank = rank- if min_rank is None or (max_rank is not None and min_rank >= max_rank):- break- assert min_idx is not None- parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2 :]- return parts-- def set_vocab(self):- self._set_vocab_qwen()-- def set_gguf_parameters(self):- self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])- self.gguf_writer.add_block_count(self.hparams["num_hidden_layers"])- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])- self.gguf_writer.add_rope_freq_base(self.hparams["rotary_emb_base"])- self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])- self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_file_type(self.ftype)--[email protected]("Qwen2Model", "Qwen2ForCausalLM", "Qwen2AudioForConditionalGeneration")-class Qwen2Model(TextModel):- model_arch = gguf.MODEL_ARCH.QWEN2-- def set_vocab(self):- try:- self._set_vocab_sentencepiece()- except FileNotFoundError:- self._set_vocab_gpt2()-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self._try_set_pooling_type()- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- if self.hf_arch == "Qwen2Model":- name = f"model.{name}" # map to Qwen2ForCausalLM tensors- if "language_model." in name:- name = name.replace("language_model.", "") # for InternVL- if (- name.startswith("mlp")- or name.startswith("multi_modal_projector")- or name.startswith("vision_model")- or name.startswith("audio_tower")- ):- # skip vision and audio tensors- return []- yield from super().modify_tensors(data_torch, name, bid)--[email protected]("DreamModel")-class DreamModel(TextModel):- model_arch = gguf.MODEL_ARCH.DREAM-- def get_vocab_base(self) -> tuple[list[str], list[int], str]:- tokens: list[str] = []- toktypes: list[int] = []-- from transformers import AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)-- vocab_dict = tokenizer.get_vocab()- vocab_size = self.hparams.get("vocab_size", len(vocab_dict))- assert max(vocab_dict.values()) < vocab_size-- tokpre = self.get_vocab_base_pre(tokenizer)-- reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()}- added_vocab = tokenizer.get_added_vocab()-- for i in range(vocab_size):- if i not in reverse_vocab:- tokens.append(f"[PAD{i}]")- toktypes.append(gguf.TokenType.UNUSED)- elif reverse_vocab[i] in added_vocab:- tokens.append(reverse_vocab[i])- # Check if it's a special token - treat special tokens as CONTROL tokens- if hasattr(tokenizer, "added_tokens_decoder") and i in tokenizer.added_tokens_decoder:- if tokenizer.added_tokens_decoder[i].special:- toktypes.append(gguf.TokenType.CONTROL)- else:- toktypes.append(gguf.TokenType.USER_DEFINED)- else:- # Fallback: treat all added vocab as control tokens for special tokens like <|im_start|>- toktypes.append(gguf.TokenType.CONTROL)- else:- tokens.append(reverse_vocab[i])- toktypes.append(gguf.TokenType.NORMAL)-- return tokens, toktypes, tokpre-- def set_vocab(self):- try:- self._set_vocab_sentencepiece()- except FileNotFoundError:- self._set_vocab_gpt2()-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self._try_set_pooling_type()-- # Dream models use non-causal attention for diffusion- self.gguf_writer.add_causal_attention(False)- # Handle RoPE scaling similar to Qwen2- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])-- # Add Dream-specific parameters- mask_token_id = self.hparams.get("mask_token_id")- if mask_token_id is not None:- self.gguf_writer.add_mask_token_id(mask_token_id)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # Dream model tensors should be mapped directly since it's the base model- yield from super().modify_tensors(data_torch, name, bid)--[email protected]("Ernie4_5_ForCausalLM")-class Ernie4_5Model(TextModel):- model_arch = gguf.MODEL_ARCH.ERNIE4_5-- def set_vocab(self):- self._set_vocab_sentencepiece()-- def set_gguf_parameters(self):- super().set_gguf_parameters()-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- num_heads = self.hparams["num_attention_heads"]- num_kv_heads = self.hparams["num_key_value_heads"]- if (head_dim := self.hparams.get("head_dim")) is None:- head_dim = self.hparams["hidden_size"] // num_heads-- if "ernie." in name:- name = name.replace("ernie.", "model.")- # split the qkv weights- # qkv_proj shape: [(num_heads + 2 * num_kv_heads) * head_dim, hidden_size]- if "qkv_proj" in name:- name_q = name.replace("qkv_proj.weight", "q_proj.weight")- name_k = name.replace("qkv_proj.weight", "k_proj.weight")- name_v = name.replace("qkv_proj.weight", "v_proj.weight")- total_q_dim = num_heads * head_dim- total_k_dim = num_kv_heads * head_dim- total_v_dim = num_kv_heads * head_dim- q_proj_weight, k_proj_weight, v_proj_weight = data_torch.split(- [total_q_dim, total_k_dim, total_v_dim], dim=0- )- return [- (self.map_tensor_name(name_q), q_proj_weight),- (self.map_tensor_name(name_k), k_proj_weight),- (self.map_tensor_name(name_v), v_proj_weight),- ]- # split the up_gate_proj into gate and up- # up_gate_proj shape: [2 * intermediate_size, hidden_size]- if "up_gate_proj" in name:- name_up = name.replace("up_gate_proj.weight", "up_proj.weight")- name_gate = name.replace("up_gate_proj.weight", "gate_proj.weight")- dim_half = data_torch.shape[0] // 2- gate_proj_weight, up_proj_weight = data_torch.split(dim_half, dim=0)- return [- (self.map_tensor_name(name_gate), gate_proj_weight),- (self.map_tensor_name(name_up), up_proj_weight),- ]- return [(self.map_tensor_name(name), data_torch)]--[email protected]("Ernie4_5_MoeForCausalLM")-class Ernie4_5MoeModel(Ernie4_5Model):- model_arch = gguf.MODEL_ARCH.ERNIE4_5_MOE- _experts: list[dict[str, Tensor]] | None = None-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self._experts = [{} for _ in range(self.block_count)]-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_expert_count(self.hparams["moe_num_experts"])- self.gguf_writer.add_expert_used_count(self.hparams["moe_k"])- self.gguf_writer.add_interleave_moe_layer_step(self.hparams["moe_layer_interval"])- self.gguf_writer.add_leading_dense_block_count(self.hparams["moe_layer_start_index"])- if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:- self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)- if (shared_expert_count := self.hparams.get("moe_num_shared_experts")) is not None:- self.gguf_writer.add_expert_shared_count(shared_expert_count)- if (- shared_expert_count > 0- and (shared_expert_intermediate_size := self.hparams.get("intermediate_size")) is not None- and (num_key_value_heads := self.hparams.get("num_key_value_heads")) is not None- ):- self.gguf_writer.add_expert_shared_feed_forward_length(- shared_expert_intermediate_size // num_key_value_heads- )-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # Modify correction bias name as in DeepseekV2- if name.endswith("e_score_correction_bias"):- name = name.replace("e_score_correction_bias", "e_score_correction.bias")-- # skip Multi-Token Prediction (MTP) layers (again, same as DeepseekV2)- match = re.match(r"model.mtp_block.(\d+)", name)- if match:- return []-- # skip all other MTP tensors for now- match = re.match(r"model.mtp_emb_norm.(\d+)", name)- if match:- return []-- match = re.match(r"model.mtp_hidden_norm.(\d+)", name)- if match:- return []-- match = re.match(r"model.mtp_linear_proj.(\d+)", name)- if match:- return []-- # process the experts separately- if name.find("mlp.experts") != -1:- n_experts = self.hparams["moe_num_experts"]- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- tensors: list[tuple[str, Tensor]] = []-- # merge the experts into a single 3d tensor- for w_name in ["gate_proj", "up_proj", "down_proj"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename_to_retrieve = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"- data.append(self._experts[bid][ename_to_retrieve])- del self._experts[bid][ename_to_retrieve]-- data_torch = torch.stack(data, dim=0)- merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"- new_name = self.map_tensor_name(merged_name)- tensors.append((new_name, data_torch))-- return tensors- else:- return []- return [(self.map_tensor_name(name), data_torch)]-- def prepare_tensors(self):- super().prepare_tensors()-- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--- "Qwen2VLModel",- "Qwen2VLForConditionalGeneration",- "Qwen2_5_VLForConditionalGeneration",- "Qwen2_5OmniModel",-)-class Qwen2VLModel(TextModel):- model_arch = gguf.MODEL_ARCH.QWEN2VL-- def set_gguf_parameters(self):- super().set_gguf_parameters()- mrope_section = self.hparams["rope_scaling"]["mrope_section"]- mrope_section += [0] * max(0, 4 - len(mrope_section))- self.gguf_writer.add_rope_dimension_sections(mrope_section)-- def set_vocab(self):- try:- self._set_vocab_sentencepiece()- except FileNotFoundError:- self._set_vocab_gpt2()-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused- if name.startswith("thinker."):- name = name.replace("thinker.", "")- if (- name.startswith("visual")- or name.startswith("audio")- or name.startswith("talker")- or name.startswith("token2wav")- ):- # skip multimodal tensors- return []- return [(self.map_tensor_name(name), data_torch)]--[email protected]("Qwen2VLModel", "Qwen2VLForConditionalGeneration", "Qwen2_5_VLForConditionalGeneration")-class Qwen2VLVisionModel(MmprojModel):-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- assert self.hparams_vision is not None- self.hparams_vision["image_size"] = self.hparams_vision.get("image_size", 560)- # rename config.json values- self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_heads")- self.hparams_vision["num_hidden_layers"] = self.hparams_vision.get("depth")- if "embed_dim" in self.hparams_vision: # qwen2vl- self.hparams_vision["intermediate_size"] = self.hparams_vision.get("hidden_size")- self.hparams_vision["hidden_size"] = self.hparams_vision.get("embed_dim")-- def set_gguf_parameters(self):- super().set_gguf_parameters()- assert self.hparams_vision is not None- hparams = self.hparams_vision- model_type = self.global_config["model_type"]- if model_type == "qwen2_vl":- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN2VL)- elif model_type == "qwen2_5_vl" or model_type == "qwen2_5_omni":- if model_type == "qwen2_5_omni":- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN25O)- else:- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN25VL)- self.gguf_writer.add_vision_use_silu(True)- # find n_wa_pattern (window attention pattern)- fullatt_block_indexes = hparams.get("fullatt_block_indexes")- assert fullatt_block_indexes is not None, "fullatt_block_indexes is required for qwen2_5_vl"- n_wa_pattern = fullatt_block_indexes[0] + 1- # validate n_wa_pattern- for i in range(1, len(fullatt_block_indexes)):- if fullatt_block_indexes[i] - fullatt_block_indexes[i - 1] != n_wa_pattern:- raise ValueError(f"Invalid fullatt_block_indexes: {fullatt_block_indexes}")- self.gguf_writer.add_vision_n_wa_pattern(n_wa_pattern)- else:- raise ValueError(f"Unknown QwenVL model type: {self.global_config['model_type']}")- # default values below are taken from HF transformers code- self.gguf_writer.add_vision_attention_layernorm_eps(self.global_config.get("rms_norm_eps", 1e-6))-- def tensor_force_quant(self, name, new_name, bid, n_dims):- del bid, name, n_dims # unused- if ".patch_embd." in new_name:- return gguf.GGMLQuantizationType.F16- if ".position_embd." in new_name:- return gguf.GGMLQuantizationType.F32- return False-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused- if name.startswith("visual."):- # process visual tensors- # split QKV tensors if needed- if ".qkv." in name:- if data_torch.ndim == 2: # weight- c3, _ = data_torch.shape- else: # bias- c3 = data_torch.shape[0]- assert c3 % 3 == 0- c = c3 // 3- wq = data_torch[:c]- wk = data_torch[c : c * 2]- wv = data_torch[c * 2 :]- return [- (self.map_tensor_name(name.replace("qkv", "q")), wq),- (self.map_tensor_name(name.replace("qkv", "k")), wk),- (self.map_tensor_name(name.replace("qkv", "v")), wv),- ]- elif "patch_embed.proj.weight" in name:- # split Conv3D into Conv2Ds- c1, c2, kt, kh, kw = data_torch.shape- del c1, c2, kh, kw # unused- assert kt == 2, "Current implementation only support temporal_patch_size of 2"- return [- (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight", data_torch[:, :, 0, ...]),- (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch[:, :, 1, ...]),- ]- else:- return [(self.map_tensor_name(name), data_torch)]- return [] # skip other tensors--[email protected]("Qwen2_5OmniModel")-class Qwen25OmniModel(Qwen2VLVisionModel):- has_vision_encoder = True- has_audio_encoder = True-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- assert self.hparams_audio is not None- self.hparams_audio["hidden_size"] = self.hparams_audio["d_model"]- self.hparams_audio["intermediate_size"] = self.hparams_audio["encoder_ffn_dim"]- self.hparams_audio["num_attention_heads"] = self.hparams_audio["encoder_attention_heads"]-- def set_gguf_parameters(self):- super().set_gguf_parameters()- assert self.hparams_audio is not None- self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["num_mel_bins"])- self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio.get("layer_norm_eps", 1e-5))-- def get_vision_config(self) -> dict[str, Any] | None:- return self.global_config["thinker_config"].get("vision_config")-- def get_audio_config(self) -> dict[str, Any] | None:- return self.global_config["thinker_config"].get("audio_config")-- def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- # SinusoidsPositionEmbedding- assert self.hparams_audio is not None- max_timescale = 10000- length = 1500- channels = self.hparams_audio["hidden_size"]- log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)- inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2).float())- scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]- pos_embd = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1).to(dtype=torch.float32)- yield ("audio_tower.embed_positions.weight", pos_embd)-- def tensor_force_quant(self, name, new_name, bid, n_dims):- del bid, new_name, n_dims # unused- if ".conv" in name and ".weight" in name:- return gguf.GGMLQuantizationType.F16- return False-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- if name.startswith("thinker."):- name = name.replace("thinker.", "")-- if name.startswith("audio_tower"):- # process audio tensors- if "conv1.bias" in name or "conv2.bias" in name:- # transpose conv1 and conv2 bias- data_torch = data_torch.unsqueeze(-1)- if "audio_bos_eos_token" in name:- return []- return [(self.map_tensor_name(name), data_torch)]-- return super().modify_tensors(data_torch, name, bid)--[email protected]("InternVisionModel")-class InternVisionModel(MmprojModel):-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.INTERNVL)- self.gguf_writer.add_vision_attention_layernorm_eps(hparams["layer_norm_eps"])- # hidden_act- if hparams["hidden_act"] == "silu":- self.gguf_writer.add_vision_use_silu(True)- elif hparams["hidden_act"] == "gelu":- self.gguf_writer.add_vision_use_gelu(True)- else:- raise ValueError(f"Unsupported hidden_act: {hparams['hidden_act']}")- # downsample_ratio- downsample_ratio = self.global_config.get("downsample_ratio")- assert downsample_ratio is not None- self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / downsample_ratio))-- def tensor_force_quant(self, name, new_name, bid, n_dims):- del bid, name, n_dims # unused- if ".patch_embd." in new_name:- return gguf.GGMLQuantizationType.F16- if ".position_embd." in new_name:- return gguf.GGMLQuantizationType.F32- return False-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused- if name.startswith("vision_model") or name.startswith("mlp"):- # process visual tensors- # correct name- if name.startswith("vision_model"):- name = "vision_tower." + name- if (".ls" in name or "position_embedding" in name) and not name.endswith(".weight"):- name += ".weight"- # split QKV tensors if needed- if ".qkv." in name:- if data_torch.ndim == 2: # weight- c3, _ = data_torch.shape- else: # bias- c3 = data_torch.shape[0]- assert c3 % 3 == 0- c = c3 // 3- wq = data_torch[:c]- wk = data_torch[c : c * 2]- wv = data_torch[c * 2 :]- return [- (self.map_tensor_name(name.replace("attn.qkv", "self_attn.q_proj")), wq),- (self.map_tensor_name(name.replace("attn.qkv", "self_attn.k_proj")), wk),- (self.map_tensor_name(name.replace("attn.qkv", "self_attn.v_proj")), wv),- ]- return [(self.map_tensor_name(name), data_torch)]- return [] # skip other tensors--[email protected]("WavTokenizerDec")-class WavTokenizerDecModel(TextModel):- model_arch = gguf.MODEL_ARCH.WAVTOKENIZER_DEC-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- if (- name.endswith("codebook.cluster_size")- or name.endswith("codebook.embed_avg")- or name.endswith("codebook.inited")- ):- logger.debug(f"Skipping {name!r}")- return []-- logger.info(f"{self.map_tensor_name(name)} -> {data_torch.shape}")-- return [(self.map_tensor_name(name), data_torch)]-- def set_vocab(self):- self._set_vocab_none()-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])- self.gguf_writer.add_features_length(self.hparams["n_embd_features"])- self.gguf_writer.add_feed_forward_length(self.hparams["n_ff"])- self.gguf_writer.add_group_norm_eps(self.hparams["group_norm_epsilon"])- self.gguf_writer.add_group_norm_groups(self.hparams["group_norm_groups"])-- self.gguf_writer.add_posnet_embedding_length(self.hparams["posnet"]["n_embd"])- self.gguf_writer.add_posnet_block_count(self.hparams["posnet"]["n_layer"])-- self.gguf_writer.add_convnext_embedding_length(self.hparams["convnext"]["n_embd"])- self.gguf_writer.add_convnext_block_count(self.hparams["convnext"]["n_layer"])-- self.gguf_writer.add_causal_attention(False)--[email protected]("Qwen2MoeForCausalLM")-class Qwen2MoeModel(TextModel):- model_arch = gguf.MODEL_ARCH.QWEN2MOE-- def set_gguf_parameters(self):- super().set_gguf_parameters()- if (n_experts := self.hparams.get("num_experts")) is not None:- self.gguf_writer.add_expert_count(n_experts)- if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:- self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)- logger.info(f"gguf: expert feed forward length = {moe_intermediate_size}")- if (shared_expert_intermediate_size := self.hparams.get("shared_expert_intermediate_size")) is not None:- self.gguf_writer.add_expert_shared_feed_forward_length(shared_expert_intermediate_size)- logger.info(f"gguf: expert shared feed forward length = {shared_expert_intermediate_size}")- # YaRN is not enabled by default- # To enable it, please refer to this guide: https://huggingface.co/Qwen/Qwen3-30B-A3B#processing-long-texts- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])-- _experts: list[dict[str, Tensor]] | None = None-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # process the experts separately- if name.find("experts") != -1:- n_experts = self.hparams["num_experts"]- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- tensors: list[tuple[str, Tensor]] = []-- # merge the experts into a single 3d tensor- for w_name in ["down_proj", "gate_proj", "up_proj"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)-- merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"-- new_name = self.map_tensor_name(merged_name)-- tensors.append((new_name, data_torch))- return tensors- else:- return []-- return [(self.map_tensor_name(name), data_torch)]-- def prepare_tensors(self):- super().prepare_tensors()-- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--[email protected]("Qwen3ForCausalLM")-class Qwen3Model(Qwen2Model):- model_arch = gguf.MODEL_ARCH.QWEN3--[email protected]("Qwen3MoeForCausalLM")-class Qwen3MoeModel(Qwen2MoeModel):- model_arch = gguf.MODEL_ARCH.QWEN3MOE--[email protected]("GPT2LMHeadModel")-class GPT2Model(TextModel):- model_arch = gguf.MODEL_ARCH.GPT2-- def set_gguf_parameters(self):- self.gguf_writer.add_block_count(self.hparams["n_layer"])- self.gguf_writer.add_context_length(self.hparams["n_ctx"])- self.gguf_writer.add_embedding_length(self.hparams["n_embd"])- self.gguf_writer.add_feed_forward_length(4 * self.hparams["n_embd"])- self.gguf_writer.add_head_count(self.hparams["n_head"])- self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_file_type(self.ftype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- tensors: list[tuple[str, Tensor]] = []-- # we don't need these- if name.endswith((".attn.bias", ".attn.masked_bias")):- return tensors-- if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight", ".c_proj.weight")):- data_torch = data_torch.transpose(1, 0)-- new_name = self.map_tensor_name(name)-- tensors.append((new_name, data_torch))-- return tensors--[email protected]("PhiForCausalLM")-class Phi2Model(TextModel):- model_arch = gguf.MODEL_ARCH.PHI2-- def set_gguf_parameters(self):- block_count = self.find_hparam(["num_hidden_layers", "n_layer"])-- rot_pct = self.find_hparam(["partial_rotary_factor"])- n_embd = self.find_hparam(["hidden_size", "n_embd"])- n_head = self.find_hparam(["num_attention_heads", "n_head"])-- self.gguf_writer.add_context_length(self.find_hparam(["n_positions", "max_position_embeddings"]))-- self.gguf_writer.add_embedding_length(n_embd)- self.gguf_writer.add_feed_forward_length(4 * n_embd)- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_head_count(n_head)- self.gguf_writer.add_head_count_kv(n_head)- self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_epsilon", "layer_norm_eps"]))- self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head)- self.gguf_writer.add_file_type(self.ftype)- self.gguf_writer.add_add_bos_token(False)--[email protected]("Phi3ForCausalLM")-class Phi3MiniModel(TextModel):- model_arch = gguf.MODEL_ARCH.PHI3-- def set_vocab(self):- # Phi-4 model uses GPT2Tokenizer- tokenizer_config_file = self.dir_model / "tokenizer_config.json"- if tokenizer_config_file.is_file():- with open(tokenizer_config_file, "r", encoding="utf-8") as f:- tokenizer_config_json = json.load(f)- tokenizer_class = tokenizer_config_json["tokenizer_class"]- if tokenizer_class == "GPT2Tokenizer":- return self._set_vocab_gpt2()-- from sentencepiece import SentencePieceProcessor-- tokenizer_path = self.dir_model / "tokenizer.model"-- if not tokenizer_path.is_file():- raise ValueError(f"Error: Missing {tokenizer_path}")-- tokenizer = SentencePieceProcessor()- tokenizer.LoadFromFile(str(tokenizer_path))-- vocab_size = self.hparams.get("vocab_size", tokenizer.vocab_size())-- tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]- scores: list[float] = [-10000.0] * vocab_size- toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size-- for token_id in range(tokenizer.vocab_size()):-- piece = tokenizer.IdToPiece(token_id)- text = piece.encode("utf-8")- score = tokenizer.GetScore(token_id)-- toktype = SentencePieceTokenTypes.NORMAL- if tokenizer.IsUnknown(token_id):- toktype = SentencePieceTokenTypes.UNKNOWN- elif tokenizer.IsControl(token_id):- toktype = SentencePieceTokenTypes.CONTROL- elif tokenizer.IsUnused(token_id):- toktype = SentencePieceTokenTypes.UNUSED- elif tokenizer.IsByte(token_id):- toktype = SentencePieceTokenTypes.BYTE-- tokens[token_id] = text- scores[token_id] = score- toktypes[token_id] = toktype-- added_tokens_file = self.dir_model / "added_tokens.json"- if added_tokens_file.is_file():- with open(added_tokens_file, "r", encoding="utf-8") as f:- added_tokens_json = json.load(f)-- for key in added_tokens_json:- token_id = added_tokens_json[key]- if token_id >= vocab_size:- logger.debug(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")- continue-- tokens[token_id] = key.encode("utf-8")- scores[token_id] = -1000.0- toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED-- tokenizer_config_file = self.dir_model / "tokenizer_config.json"- if tokenizer_config_file.is_file():- with open(tokenizer_config_file, "r", encoding="utf-8") as f:- tokenizer_config_json = json.load(f)- added_tokens_decoder = tokenizer_config_json.get("added_tokens_decoder", {})- for token_id, foken_data in added_tokens_decoder.items():- token_id = int(token_id)- token = foken_data["content"].encode("utf-8")- if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:- if tokens[token_id] != token:- logger.warning(- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '- f'-> {token.decode("utf-8")!r}'- )- tokens[token_id] = token- scores[token_id] = -1000.0- toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED- if foken_data.get("special"):- toktypes[token_id] = SentencePieceTokenTypes.CONTROL-- tokenizer_file = self.dir_model / "tokenizer.json"- if tokenizer_file.is_file():- with open(tokenizer_file, "r", encoding="utf-8") as f:- tokenizer_json = json.load(f)- added_tokens = tokenizer_json.get("added_tokens", [])- for foken_data in added_tokens:- token_id = int(foken_data["id"])- token = foken_data["content"].encode("utf-8")- if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:- if tokens[token_id] != token:- logger.warning(- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '- f'-> {token.decode("utf-8")!r}'- )- tokens[token_id] = token- scores[token_id] = -1000.0- toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED- if foken_data.get("special"):- toktypes[token_id] = SentencePieceTokenTypes.CONTROL-- self.gguf_writer.add_tokenizer_model("llama")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- block_count = self.find_hparam(["num_hidden_layers", "n_layer"])-- n_embd = self.find_hparam(["hidden_size", "n_embd"])- n_head = self.find_hparam(["num_attention_heads", "n_head"])- n_head_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"])- rms_eps = self.find_hparam(["rms_norm_eps"])- max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"])- orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"])- rot_pct = self.hparams.get("partial_rotary_factor", 1.0)- rope_dims = int(rot_pct * n_embd) // n_head-- self.gguf_writer.add_context_length(max_pos_embds)- self.gguf_writer.add_rope_scaling_orig_ctx_len(orig_max_pos_embds)- self.gguf_writer.add_embedding_length(n_embd)- self.gguf_writer.add_feed_forward_length(self.find_hparam(["intermediate_size"]))- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_head_count(n_head)- self.gguf_writer.add_head_count_kv(n_head_kv)- self.gguf_writer.add_layer_norm_rms_eps(rms_eps)- self.gguf_writer.add_rope_dimension_count(rope_dims)- self.gguf_writer.add_rope_freq_base(self.find_hparam(["rope_theta"]))- self.gguf_writer.add_file_type(self.ftype)- sliding_window = self.hparams.get("sliding_window")- # use zero value of sliding_window to distinguish Phi-4 from other PHI3 models- if sliding_window is None:- sliding_window = 0- self.gguf_writer.add_sliding_window(sliding_window)-- def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- n_embd = self.find_hparam(["hidden_size", "n_embd"])- n_head = self.find_hparam(["num_attention_heads", "n_head"])- max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"])- orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"])- rot_pct = self.hparams.get("partial_rotary_factor", 1.0)- rope_dims = int(rot_pct * n_embd) // n_head-- # write rope scaling for long context (128k) model- rope_scaling = self.find_hparam(["rope_scaling"], True)- if rope_scaling is None:- return-- scale = max_pos_embds / orig_max_pos_embds-- rope_scaling_type = rope_scaling.get("rope_type", rope_scaling.get("type", "")).lower()- if len(rope_scaling_type) == 0:- raise KeyError("Missing the required key rope_scaling.type")-- if rope_scaling_type == "su" or rope_scaling_type == "longrope":- attn_factor = math.sqrt(1 + math.log(scale) / math.log(orig_max_pos_embds)) if scale > 1.0 else 1.0- elif rope_scaling_type == "yarn":- attn_factor = 0.1 * math.log(scale) + 1.0 if scale > 1.0 else 1.0- else:- raise NotImplementedError(f"The rope scaling type {rope_scaling_type} is not supported yet")-- self.gguf_writer.add_rope_scaling_attn_factors(attn_factor)-- long_factors = rope_scaling.get("long_factor", None)- short_factors = rope_scaling.get("short_factor", None)-- if long_factors is None or short_factors is None:- raise KeyError("Missing the required key rope_scaling.long_factor or rope_scaling_short_factor")-- if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:- raise ValueError(- f"The length of rope long and short factors must be {rope_dims / 2}."- f" long_factors = {len(long_factors)}, short_factors = {len(short_factors)}."- )-- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),- torch.tensor(long_factors, dtype=torch.float32),- )- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),- torch.tensor(short_factors, dtype=torch.float32),- )--[email protected]("PhiMoEForCausalLM")-class PhiMoeModel(Phi3MiniModel):- model_arch = gguf.MODEL_ARCH.PHIMOE-- _experts: list[dict[str, Tensor]] | None = None-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])- self.gguf_writer.add_expert_count(self.hparams["num_local_experts"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # process the experts separately- if name.find("block_sparse_moe.experts") != -1:- n_experts = self.hparams["num_local_experts"]- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- tensors: list[tuple[str, Tensor]] = []-- # merge the experts into a single 3d tensor- for w_name in ["w1", "w2", "w3"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{w_name}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)-- merged_name = f"model.layers.{bid}.block_sparse_moe.experts.{w_name}.weight"-- new_name = self.map_tensor_name(merged_name)-- tensors.append((new_name, data_torch))- return tensors- else:- return []-- return [(self.map_tensor_name(name), data_torch)]-- def prepare_tensors(self):- super().prepare_tensors()-- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--[email protected]("PlamoForCausalLM")-class PlamoModel(TextModel):- model_arch = gguf.MODEL_ARCH.PLAMO-- def set_vocab(self):- self._set_vocab_sentencepiece()-- def set_gguf_parameters(self):- hparams = self.hparams- block_count = hparams["num_hidden_layers"]-- self.gguf_writer.add_context_length(4096) # not in config.json- self.gguf_writer.add_embedding_length(hparams["hidden_size"])- self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_head_count(hparams["num_attention_heads"])- self.gguf_writer.add_head_count_kv(5) # hparams["num_key_value_heads"]) is wrong- self.gguf_writer.add_layer_norm_rms_eps(hparams["rms_norm_eps"])- self.gguf_writer.add_file_type(self.ftype)-- def shuffle_attn_q_weight(self, data_torch):- assert data_torch.size() == (5120, 5120)- data_torch = data_torch.reshape(8, 5, 128, 5120)- data_torch = torch.permute(data_torch, (1, 0, 2, 3))- data_torch = torch.reshape(data_torch, (5120, 5120))- return data_torch-- def shuffle_attn_output_weight(self, data_torch):- assert data_torch.size() == (5120, 5120)- data_torch = data_torch.reshape(5120, 8, 5, 128)- data_torch = torch.permute(data_torch, (0, 2, 1, 3))- data_torch = torch.reshape(data_torch, (5120, 5120))- return data_torch-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- new_name = self.map_tensor_name(name)-- # shuffle for broadcasting of gqa in ggml_mul_mat- if new_name.endswith("attn_q.weight"):- data_torch = self.shuffle_attn_q_weight(data_torch)- elif new_name.endswith("attn_output.weight"):- data_torch = self.shuffle_attn_output_weight(data_torch)-- return [(new_name, data_torch)]--[email protected]("Plamo2ForCausalLM", "PLaMo2ForCausalLM")-class Plamo2Model(TextModel):- model_arch = gguf.MODEL_ARCH.PLAMO2-- def set_vocab(self):- # PLaMo 2 uses a custom tokenizer with a .jsonl file- # We need to handle this specially- tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl"- tokenizer_config_path = self.dir_model / "tokenizer_config.json"-- if not tokenizer_jsonl_path.is_file():- raise FileNotFoundError(f"PLaMo 2 tokenizer file not found: {tokenizer_jsonl_path}")-- # Load tokenizer config- with open(tokenizer_config_path, "r", encoding="utf-8") as f:- tokenizer_config = json.load(f)-- # Load tokens from JSONL file (actually a list format)- tokens = []- scores = []- toktypes = []-- with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f:- for line_num, line in enumerate(f):- if line.strip():- token_data = json.loads(line)- # Format: [token, score, type, ?, ?, ?, ?]- token = token_data[0].encode("utf-8")- score = float(token_data[1])- token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL"-- tokens.append(token)- scores.append(score)-- # Map token type strings to GGUF token types- if token_type_str == "UNKNOWN":- toktypes.append(gguf.TokenType.UNKNOWN)- elif token_type_str == "CONTROL":- toktypes.append(gguf.TokenType.CONTROL)- elif token_type_str == "BYTE":- toktypes.append(gguf.TokenType.BYTE)- else:- # Check for PLaMo-2 special tokens- token_str = token_data[0]- if token_str.startswith("<|plamo:") and token_str.endswith("|>"):- toktypes.append(gguf.TokenType.CONTROL)- else:- toktypes.append(gguf.TokenType.NORMAL)-- vocab_size = self.hparams["vocab_size"]- if vocab_size > len(tokens):- pad_count = vocab_size - len(tokens)- logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")- for i in range(1, pad_count + 1):- tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))- scores.append(-1000.0)- toktypes.append(gguf.TokenType.UNUSED)-- # Use "plamo2" tokenizer type for PLaMo-2's custom Aho-Corasick tokenizer- self.gguf_writer.add_tokenizer_model("plamo2")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)-- # Add special tokens from config- if "bos_token" in tokenizer_config and tokenizer_config["bos_token"] is not None:- token_id = tokens.index(tokenizer_config["bos_token"].encode("utf-8"))- self.gguf_writer.add_bos_token_id(token_id)- if "eos_token" in tokenizer_config and tokenizer_config["eos_token"] is not None:- token_id = tokens.index(tokenizer_config["eos_token"].encode("utf-8"))- self.gguf_writer.add_eos_token_id(token_id)- if "pad_token" in tokenizer_config and tokenizer_config["pad_token"] is not None:- token_id = tokens.index(tokenizer_config["pad_token"].encode("utf-8"))- self.gguf_writer.add_pad_token_id(token_id)- if "sep_token" in tokenizer_config and tokenizer_config["sep_token"] is not None:- token_id = tokens.index(tokenizer_config["sep_token"].encode("utf-8"))- self.gguf_writer.add_sep_token_id(token_id)- if "unk_token" in tokenizer_config and tokenizer_config["unk_token"] is not None:- token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8"))- self.gguf_writer.add_unk_token_id(token_id)-- # Add <|plamo:op|> as EOT to ensure appropriate end of generation- self.gguf_writer.add_eot_token_id(4)-- self.gguf_writer.add_add_space_prefix(False)-- def set_gguf_parameters(self):- hparams = self.hparams- block_count = hparams["num_hidden_layers"]- self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])-- # Which layers are Mamba layers- # PLaMo 2 uses mamba_step to indicate the pattern (e.g., 2 means every other layer)- # This logic matches modeling_plamo.py's is_mamba function- mamba_step = hparams.get("mamba_step", 2)- mamba_enabled = hparams.get("mamba_enabled", True)- mamba_layers = []-- if mamba_enabled:- for i in range(block_count):- if block_count <= (mamba_step // 2):- # use attention in last layer- is_mamba = i != block_count - 1- else:- is_mamba = (i % mamba_step) != (mamba_step // 2)- if is_mamba:- mamba_layers.append(0)- else:- mamba_layers.append(hparams.get("num_key_value_heads", 4))-- if mamba_layers:- self.gguf_writer.add_head_count_kv(mamba_layers)-- self.gguf_writer.add_context_length(hparams.get("max_position_embeddings", 2048))- self.gguf_writer.add_embedding_length(hparams.get("hidden_size", 4096))- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_head_count(hparams.get("num_attention_heads", 32))- self.gguf_writer.add_layer_norm_rms_eps(hparams.get("rms_norm_eps", 1e-06))- self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 1000000.0))-- # Mamba parameters- self.gguf_writer.add_ssm_state_size(hparams.get("mamba_d_state", 64))- self.gguf_writer.add_ssm_conv_kernel(hparams.get("mamba_d_conv", 4))- self.gguf_writer.add_ssm_time_step_rank(hparams.get("mamba_num_heads", 64))- intermediate_size = hparams.get("mamba_num_heads", 64) * hparams.get("hidden_size_per_head", 128)- self.gguf_writer.add_ssm_inner_size(intermediate_size)- self.gguf_writer.add_ssm_group_count(0)-- # MLP feed forward parameters (for attention layers)- self.gguf_writer.add_feed_forward_length(hparams.get("intermediate_size", 16384))- self.gguf_writer.add_file_type(self.ftype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- if name.endswith(".A_log"):- data_torch = -torch.exp(data_torch)- elif name.endswith(".dt_bias"):- name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"- elif name.endswith(".dt_norm_weight"):- name = name.rpartition(".dt_norm_weight")[0] + ".dt_norm.weight"- elif name.endswith(".B_norm_weight"):- name = name.rpartition(".B_norm_weight")[0] + ".B_norm.weight"- elif name.endswith(".C_norm_weight"):- name = name.rpartition(".C_norm_weight")[0] + ".C_norm.weight"- elif name.endswith(".k_weight"):- name = name.rpartition(".k_weight")[0] + ".k.weight"- elif name.endswith(".q_weight"):- name = name.rpartition(".q_weight")[0] + ".q.weight"- elif name.endswith(".conv1d.weight"):- data_torch = torch.squeeze(data_torch) # remove (, 1, )- assert data_torch.ndim == 2- elif name.endswith(".pre_mixer_norm.weight"):- data_torch += 1.0- elif name.endswith(".post_mixer_norm.weight"):- data_torch += 1.0 / 5- elif name.endswith(".pre_mlp_norm.weight"):- data_torch += 1.0- elif name.endswith(".post_mlp_norm.weight"):- data_torch += 1.0 / (5**1.5)- elif name.endswith(".norm.weight"):- data_torch += 1.0-- new_name = self.map_tensor_name(name)-- return [(new_name, data_torch)]--[email protected]("CodeShellForCausalLM")-class CodeShellModel(TextModel):- model_arch = gguf.MODEL_ARCH.CODESHELL-- def set_gguf_parameters(self):- block_count = self.hparams["n_layer"]-- self.gguf_writer.add_context_length(self.hparams["n_positions"])- self.gguf_writer.add_embedding_length(self.hparams["n_embd"])- self.gguf_writer.add_feed_forward_length(4 * self.hparams["n_embd"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_head_count(self.hparams["n_head"])- self.gguf_writer.add_head_count_kv(self.hparams["num_query_groups"])- self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_file_type(self.ftype)- self.gguf_writer.add_rope_freq_base(10000.0)- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(1.0)-- _has_tok_embd = False-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- output_name = self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT)- tok_embd_name = self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD)-- new_name = self.map_tensor_name(name)-- # assuming token_embd.weight is seen before output.weight- if not self._has_tok_embd and new_name == self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT):- # even though the tensor file(s) does not contain the word embeddings they are still in the weight map- if self.tensor_names and "transformer.wte.weight" in self.tensor_names:- logger.debug(f"{tok_embd_name} not found before {output_name}, assuming they are tied")- self.tensor_names.remove("transformer.wte.weight")- elif new_name == tok_embd_name:- self._has_tok_embd = True-- return [(new_name, data_torch)]--[email protected]("InternLM2ForCausalLM")-class InternLM2Model(TextModel):- model_arch = gguf.MODEL_ARCH.INTERNLM2-- def set_vocab(self):- # (TODO): Is there a better way?- # Copy from _set_vocab_sentencepiece, The only difference is that we will treat the character- # \x00 specially and convert it into an emoji character to prevent it from being mistakenly- # recognized as an empty string in C++.- from sentencepiece import SentencePieceProcessor- from sentencepiece import sentencepiece_model_pb2 as model-- tokenizer_path = self.dir_model / "tokenizer.model"-- tokens: list[bytes] = []- scores: list[float] = []- toktypes: list[int] = []-- if not tokenizer_path.is_file():- logger.error(f"Error: Missing {tokenizer_path}")- sys.exit(1)-- sentencepiece_model = model.ModelProto() # pylint: disable=E1101- sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())- add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix-- tokenizer = SentencePieceProcessor()- tokenizer.LoadFromFile(str(tokenizer_path))-- vocab_size = self.hparams.get("vocab_size", tokenizer.vocab_size())-- for token_id in range(vocab_size):- piece = tokenizer.IdToPiece(token_id)- text = piece.encode("utf-8")- score = tokenizer.GetScore(token_id)- if text == b"\x00":- # (TODO): fixme- # Hack here and replace the \x00 characters.- logger.warning(f"InternLM2 convert token '{text}' to '🐉'!")- text = "🐉".encode("utf-8")-- toktype = SentencePieceTokenTypes.NORMAL- if tokenizer.IsUnknown(token_id):- toktype = SentencePieceTokenTypes.UNKNOWN- elif tokenizer.IsControl(token_id):- toktype = SentencePieceTokenTypes.CONTROL- elif tokenizer.IsUnused(token_id):- toktype = SentencePieceTokenTypes.UNUSED- elif tokenizer.IsByte(token_id):- toktype = SentencePieceTokenTypes.BYTE- # take care of unused raw token- if piece.startswith("[UNUSED"):- toktype = SentencePieceTokenTypes.UNUSED-- tokens.append(text)- scores.append(score)- toktypes.append(toktype)-- added_tokens_file = self.dir_model / "added_tokens.json"- if added_tokens_file.is_file():- with open(added_tokens_file, "r", encoding="utf-8") as f:- added_tokens_json = json.load(f)-- for key in added_tokens_json:- tokens.append(key.encode("utf-8"))- scores.append(-1000.0)- toktypes.append(SentencePieceTokenTypes.USER_DEFINED)-- chat_eos_token = "<|im_end|>"- chat_eos_token_id = None-- tokenizer_config_file = self.dir_model / "tokenizer_config.json"- if tokenizer_config_file.is_file():- with open(tokenizer_config_file, "r", encoding="utf-8") as f:- tokenizer_config_json = json.load(f)- added_tokens_decoder = tokenizer_config_json.get("added_tokens_decoder", {})- for token_id, foken_data in added_tokens_decoder.items():- token_id = int(token_id)- token = foken_data["content"]- if token == chat_eos_token:- chat_eos_token_id = token_id- token = token.encode("utf-8")- if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:- if tokens[token_id] != token:- logger.warning(- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '- f'-> {token.decode("utf-8")!r}'- )- tokens[token_id] = token- scores[token_id] = -1000.0- toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED- if foken_data.get("special"):- toktypes[token_id] = SentencePieceTokenTypes.CONTROL-- tokenizer_file = self.dir_model / "tokenizer.json"- if tokenizer_file.is_file():- with open(tokenizer_file, "r", encoding="utf-8") as f:- tokenizer_json = json.load(f)- added_tokens = tokenizer_json.get("added_tokens", [])- for foken_data in added_tokens:- token_id = int(foken_data["id"])- token = foken_data["content"]- if token == chat_eos_token:- chat_eos_token_id = token_id- token = token.encode("utf-8")- if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:- if tokens[token_id] != token:- logger.warning(- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} '- f'-> {token.decode("utf-8")!r}'- )- tokens[token_id] = token- scores[token_id] = -1000.0- toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED- if foken_data.get("special"):- toktypes[token_id] = SentencePieceTokenTypes.CONTROL-- self.gguf_writer.add_tokenizer_model("llama")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)- self.gguf_writer.add_add_space_prefix(add_prefix)-- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- old_eos = special_vocab.special_token_ids["eos"]- if chat_eos_token_id is not None:- # For the chat model, we replace the eos with '<|im_end|>'.- # TODO: this is a hack, should be fixed- # https://github.com/ggml-org/llama.cpp/pull/6745#issuecomment-2067687048- special_vocab.special_token_ids["eos"] = chat_eos_token_id- logger.warning(- f"Replace eos:{old_eos} with a special token:{chat_eos_token_id}"- " in chat mode so that the conversation can end normally."- )-- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])- self.gguf_writer.add_block_count(self.hparams["num_hidden_layers"])- self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])- self.gguf_writer.add_rope_freq_base(self.hparams["rope_theta"])- self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])- self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"])- self.gguf_writer.add_file_type(self.ftype)- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- num_heads = self.hparams["num_attention_heads"]- num_kv_heads = self.hparams["num_key_value_heads"]- n_embd = self.hparams["hidden_size"]- q_per_kv = num_heads // num_kv_heads- head_dim = n_embd // num_heads- num_groups = num_heads // q_per_kv-- name = name.replace("language_model.", "") # InternVL- if name.startswith("mlp") or name.startswith("vision_model"):- # skip visual tensors- return []-- if bid is not None and f"model.layers.{bid}.attention.wqkv" in name:- qkv = data_torch-- qkv = qkv.reshape((num_groups, q_per_kv + 2, head_dim, n_embd))- q, k, v = qkv[:, :q_per_kv], qkv[:, -2], qkv[:, -1]-- # The model weights of q and k require additional reshape.- q = LlamaModel.permute(q.reshape((-1, q.shape[-1])), num_heads, num_heads)- k = LlamaModel.permute(k.reshape((-1, k.shape[-1])), num_heads, num_kv_heads)- v = v.reshape((-1, v.shape[-1]))-- return [- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), q),- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), k),- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), v),- ]- else:- return [(self.map_tensor_name(name), data_torch)]--[email protected]("InternLM3ForCausalLM")-class InternLM3Model(TextModel):- model_arch = gguf.MODEL_ARCH.LLAMA-- def set_vocab(self):- tokens, scores, toktypes = self._create_vocab_sentencepiece()-- self.gguf_writer.add_tokenizer_model("llama")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))-- tokenizer_config_file = self.dir_model / "tokenizer_config.json"- if tokenizer_config_file.is_file():- with open(tokenizer_config_file, "r", encoding="utf-8") as f:- tokenizer_config_json = json.load(f)- if "add_prefix_space" in tokenizer_config_json:- self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])-- if "added_tokens_decoder" in tokenizer_config_json:- for token_id, token_data in tokenizer_config_json["added_tokens_decoder"].items():- if token_data.get("special"):- token_id = int(token_id)- token = token_data["content"]- special_vocab._set_special_token(token, token_id)- # update eos token- if token == "<|im_end|>" and "eos" in special_vocab.special_token_ids:- special_vocab.special_token_ids["eos"] = token_id-- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- self.gguf_writer.add_vocab_size(hparams["vocab_size"])-- if (rope_dim := hparams.get("head_dim")) is None:- rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]- self.gguf_writer.add_rope_dimension_count(rope_dim)-- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- n_head = self.hparams["num_attention_heads"]- n_kv_head = self.hparams.get("num_key_value_heads")- name = name.replace("language_model.", "") # InternVL- if name.startswith("mlp") or name.startswith("vision_model"):- # skip visual tensors- return []- if name.endswith(("q_proj.weight", "q_proj.bias")):- data_torch = LlamaModel.permute(data_torch, n_head, n_head)- if name.endswith(("k_proj.weight", "k_proj.bias")):- data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)- return [(self.map_tensor_name(name), data_torch)]--[email protected]("BertModel", "BertForMaskedLM", "CamembertModel", "BertForSequenceClassification")-class BertModel(TextModel):- model_arch = gguf.MODEL_ARCH.BERT-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.vocab_size = None-- if cls_out_labels := self.hparams.get("id2label"):- if len(cls_out_labels) == 2 and cls_out_labels[0] == "LABEL_0":- # Remove dummy labels added by AutoConfig- cls_out_labels = None- self.cls_out_labels = cls_out_labels-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_causal_attention(False)- self._try_set_pooling_type()-- if self.cls_out_labels:- self.gguf_writer.add_classifier_output_labels([v for k, v in sorted(self.cls_out_labels.items())])-- def set_vocab(self):- tokens, toktypes, tokpre = self.get_vocab_base()- self.vocab_size = len(tokens)-- # we need this to validate the size of the token_type embeddings- # though currently we are passing all zeros to the token_type embeddings- # "Sequence A" or "Sequence B"- self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))-- # convert to phantom space vocab- def phantom(tok):- if tok.startswith("[") and tok.endswith("]"):- return tok- if tok.startswith("##"):- return tok[2:]- return "\u2581" + tok-- tokens = list(map(phantom, tokens))-- # add vocab to gguf- self.gguf_writer.add_tokenizer_model("bert")- self.gguf_writer.add_tokenizer_pre(tokpre)- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)-- # handle special tokens- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- name = name.removeprefix("bert.")-- if name.endswith(".gamma"):- name = name[:-6] + ".weight"-- if name.endswith(".beta"):- name = name[:-5] + ".bias"-- # we are only using BERT for embeddings so we don't need the pooling layer- if name in ("embeddings.position_ids", "pooler.dense.weight", "pooler.dense.bias"):- return [] # we don't need these-- if name.startswith("cls.predictions"):- return []-- if name.startswith("cls.seq_relationship"):- return []-- if self.cls_out_labels:- # For BertForSequenceClassification (direct projection layer)- if name == "classifier.weight":- name = "classifier.out_proj.weight"-- if name == "classifier.bias":- name = "classifier.out_proj.bias"-- return [(self.map_tensor_name(name), data_torch)]-- def _xlmroberta_tokenizer_init(self) -> None:- # we need the pad_token_id to know how to chop down position_embd matrix- if (pad_token_id := self.hparams.get("pad_token_id")) is not None:- self._position_offset = 1 + pad_token_id- if "max_position_embeddings" in self.hparams:- self.hparams["max_position_embeddings"] -= self._position_offset- else:- self._position_offset = None-- def _xlmroberta_set_vocab(self) -> None:- # to avoid TypeError: Descriptors cannot be created directly- # exception when importing sentencepiece_model_pb2- os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"- from sentencepiece import SentencePieceProcessor- from sentencepiece import sentencepiece_model_pb2 as model-- tokenizer_path = self.dir_model / "sentencepiece.bpe.model"-- tokenizer_json = {}- tokenizer_config_json = {}- if not tokenizer_path.is_file():- tokenizer_path = self.dir_model / "tokenizer.json"- tokenizer_config_path = self.dir_model / "tokenizer_config.json"-- if not tokenizer_path.is_file():- raise FileNotFoundError(f"File not found: {tokenizer_path}")-- from base64 import b64decode-- from transformers import AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(self.dir_model)-- with open(tokenizer_path, "r", encoding="utf-8") as fp:- tokenizer_json = json.load(fp)-- if tokenizer_config_path.is_file():- with open(tokenizer_config_path, "r", encoding="utf-8") as fp:- tokenizer_config_json = json.load(fp)-- add_prefix = tokenizer.add_prefix_space- remove_whitespaces = tokenizer.clean_up_tokenization_spaces- precompiled_charsmap = b64decode(tokenizer_json["normalizer"]["precompiled_charsmap"])-- vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size)- else:- sentencepiece_model = model.ModelProto() # pylint: disable=E1101- sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())- assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM-- add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix- remove_whitespaces = sentencepiece_model.normalizer_spec.remove_extra_whitespaces- precompiled_charsmap = sentencepiece_model.normalizer_spec.precompiled_charsmap-- tokenizer = SentencePieceProcessor()- tokenizer.LoadFromFile(str(tokenizer_path))-- vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size())-- tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]- scores: list[float] = [-10000.0] * vocab_size- toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size-- if isinstance(tokenizer, SentencePieceProcessor):- for token_id in range(tokenizer.vocab_size()):- piece = tokenizer.IdToPiece(token_id)- text = piece.encode("utf-8")- score = tokenizer.GetScore(token_id)-- toktype = SentencePieceTokenTypes.NORMAL- if tokenizer.IsUnknown(token_id):- toktype = SentencePieceTokenTypes.UNKNOWN- elif tokenizer.IsControl(token_id):- toktype = SentencePieceTokenTypes.CONTROL- elif tokenizer.IsUnused(token_id):- toktype = SentencePieceTokenTypes.UNUSED- elif tokenizer.IsByte(token_id):- toktype = SentencePieceTokenTypes.BYTE-- tokens[token_id] = text- scores[token_id] = score- toktypes[token_id] = toktype- else:- added_vocab = tokenizer.get_added_vocab()- unk_token = tokenizer_config_json.get("unk_token")- unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3))-- for token_id in range(tokenizer.vocab_size):- piece = tokenizer._convert_id_to_token(token_id)- if (piece := tokenizer._convert_id_to_token(token_id)) is not None:- text = piece.encode("utf-8")- score = tokenizer_json["model"]["vocab"][token_id][1]-- toktype = SentencePieceTokenTypes.NORMAL- if token_id == unk_token_id:- toktype = SentencePieceTokenTypes.UNKNOWN- elif token_id in tokenizer.all_special_ids:- toktype = SentencePieceTokenTypes.CONTROL- elif token_id in added_vocab.values():- toktype = SentencePieceTokenTypes.USER_DEFINED- # No reliable way to detect this, but jina doesn't have any- # elif tokenizer.IsByte(token_id):- # toktype = SentencePieceTokenTypes.BYTE-- tokens[token_id] = text- scores[token_id] = score- toktypes[token_id] = toktype-- if isinstance(tokenizer, SentencePieceProcessor):- # realign tokens (see HF tokenizer code)- tokens = [b"<s>", b"<pad>", b"</s>", b"<unk>"] + tokens[3:-1]- scores = [0.0, 0.0, 0.0, 0.0] + scores[3:-1]- toktypes = [- SentencePieceTokenTypes.CONTROL,- SentencePieceTokenTypes.CONTROL,- SentencePieceTokenTypes.CONTROL,- SentencePieceTokenTypes.UNKNOWN,- ] + toktypes[3:-1]-- if self.model_arch == gguf.MODEL_ARCH.NOMIC_BERT_MOE:- # Add mask token missing from sentencepiece.bpe.model- tokens[250001] = b"<mask>"- scores[250001] = 0.0- toktypes[250001] = SentencePieceTokenTypes.CONTROL-- self.gguf_writer.add_tokenizer_model("t5")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)- self.gguf_writer.add_add_space_prefix(add_prefix)- self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))- self.gguf_writer.add_remove_extra_whitespaces(remove_whitespaces)- if precompiled_charsmap:- self.gguf_writer.add_precompiled_charsmap(precompiled_charsmap)-- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)--[email protected]("DistilBertModel", "DistilBertForMaskedLM", "DistilBertForSequenceClassification")-class DistilBertModel(BertModel):- model_arch = gguf.MODEL_ARCH.BERT-- def set_gguf_parameters(self):- self.gguf_writer.add_layer_norm_eps(1e-12)- logger.info("gguf: layer norm epsilon = 1e-12")- super().set_gguf_parameters()-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- name = name.removeprefix("distilbert.")-- # These layers act as MLM head, so we don't need them- if name.startswith("vocab_"):- return []-- return super().modify_tensors(data_torch, name, bid)--[email protected]("RobertaModel", "RobertaForSequenceClassification")-class RobertaModel(BertModel):- model_arch = gguf.MODEL_ARCH.BERT-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)-- # we need the pad_token_id to know how to chop down position_embd matrix- if (pad_token_id := self.hparams.get("pad_token_id")) is not None:- self._position_offset = 1 + pad_token_id- if "max_position_embeddings" in self.hparams:- self.hparams["max_position_embeddings"] -= self._position_offset- else:- self._position_offset = None-- def set_vocab(self):- """Support BPE tokenizers for roberta models"""- bpe_tok_path = self.dir_model / "tokenizer.json"- if bpe_tok_path.exists():- self._set_vocab_gpt2()-- # we need this to validate the size of the token_type embeddings- # though currently we are passing all zeros to the token_type embeddings- # "Sequence A" or "Sequence B"- self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))-- else:- return super().set_vocab()-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # if name starts with "roberta.", remove the prefix- # e.g. https://huggingface.co/BAAI/bge-reranker-v2-m3/tree/main- name = name.removeprefix("roberta.")-- # position embeddings start at pad_token_id + 1, so just chop down the weight tensor- if name == "embeddings.position_embeddings.weight":- if self._position_offset is not None:- data_torch = data_torch[self._position_offset :, :]-- return super().modify_tensors(data_torch, name, bid)--[email protected]("NomicBertModel")-class NomicBertModel(BertModel):- model_arch = gguf.MODEL_ARCH.BERT-- def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, **kwargs: Any):- hparams = kwargs.pop("hparams", None)- if hparams is None:- hparams = ModelBase.load_hparams(dir_model)-- self.is_moe = bool(hparams.get("moe_every_n_layers"))- self.model_arch = gguf.MODEL_ARCH.NOMIC_BERT_MOE if self.is_moe else gguf.MODEL_ARCH.NOMIC_BERT-- super().__init__(dir_model, ftype, fname_out, hparams=hparams, **kwargs)-- self._tokenizer_is_xlmroberta = self._is_tokenizer_xlmroberta()- if self._tokenizer_is_xlmroberta:- self._xlmroberta_tokenizer_init()-- npos, mtp = self.hparams["n_positions"], self.hparams.get("max_trained_positions", 2048)- if npos == 8192 and mtp == 2048:- self.hparams["n_positions"] = 2048 # nomic-embed-text v1 and v1.5 are trained for 2048 tokens.- elif npos == 2048 and mtp == 2048:- self.hparams["n_positions"] = 512 # nomic-embed-text-v2-moe is trained for 512 tokens.- else:- raise ValueError(f"unrecognized parameters: n_positions={npos}, max_trained_positions={mtp}")-- assert self.hparams["activation_function"] == "gelu" if self.is_moe else "swiglu"-- # this doesn't do anything in the HF version- assert self.hparams["causal"] is False- # no bias tensors unless MoE- assert self.hparams["qkv_proj_bias"] == self.is_moe- assert self.hparams["mlp_fc1_bias"] == self.is_moe- assert self.hparams["mlp_fc2_bias"] == self.is_moe-- # norm at end of layer- assert self.hparams["prenorm"] is False- # standard RoPE- assert self.hparams["rotary_emb_fraction"] == 1.0- assert self.hparams["rotary_emb_interleaved"] is False- assert self.hparams["rotary_emb_scale_base"] is None-- def set_vocab(self) -> None:- if self._tokenizer_is_xlmroberta:- return self._xlmroberta_set_vocab()- return super().set_vocab()-- def modify_tensors(- self, data_torch: torch.Tensor, name: str, bid: int | None- ) -> Iterable[tuple[str, torch.Tensor]]:- # If the tensor is an experts bias tensor, skip it by returning an empty list.- if "mlp.experts.bias" in name:- return [] # Explicitly return an empty list.-- if "mlp.experts.mlp.w1" in name:- data_torch = data_torch.view(self.hparams["num_experts"], self.hparams["n_inner"], self.hparams["n_embd"])- name += ".weight"-- if "mlp.experts.mlp.w2" in name:- data_torch = data_torch.view(self.hparams["num_experts"], self.hparams["n_inner"], self.hparams["n_embd"])- data_torch = data_torch.transpose(1, 2)- name += ".weight"-- return [(self.map_tensor_name(name), data_torch)]-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_rope_freq_base(self.hparams["rotary_emb_base"])- if self.is_moe:- self.gguf_writer.add_moe_every_n_layers(self.hparams["moe_every_n_layers"])- self.gguf_writer.add_expert_count(self.hparams["num_experts"])- self.gguf_writer.add_expert_used_count(self.hparams["moe_top_k"])-- def _is_tokenizer_xlmroberta(self) -> bool:- with open(self.dir_model / "tokenizer.json") as f:- tokenizer_json = json.load(f)- toktyp = tokenizer_json["model"]["type"]- if toktyp == "Unigram":- return True- if toktyp == "WordPiece":- return False- raise ValueError(f"unknown tokenizer: {toktyp}")--[email protected]("NeoBERT", "NeoBERTLMHead", "NeoBERTForSequenceClassification")-class NeoBert(BertModel):- model_arch = gguf.MODEL_ARCH.NEO_BERT-- def set_gguf_parameters(self):- super().set_gguf_parameters()-- # NeoBERT uses 2/3 of the intermediate size as feed forward length- self.gguf_writer.add_feed_forward_length(int(2 * self.hparams["intermediate_size"] / 3))- self.gguf_writer.add_rope_freq_base(10000.0) # default value for NeoBERT- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)-- f_rms_eps = self.hparams.get("norm_eps", 1e-6) # default value for NeoBERT- self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)- logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")-- self.gguf_writer.add_pooling_type(gguf.PoolingType.CLS) # https://huggingface.co/chandar-lab/NeoBERT#how-to-use-- def modify_tensors(self, data_torch, name, bid):- if name.startswith("decoder."):- return []-- name = name.removeprefix("model.")-- return super().modify_tensors(data_torch, name, bid)--[email protected]("XLMRobertaModel", "XLMRobertaForSequenceClassification")-class XLMRobertaModel(BertModel):- model_arch = gguf.MODEL_ARCH.BERT-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self._xlmroberta_tokenizer_init()-- def set_vocab(self):- self._xlmroberta_set_vocab()-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # if name starts with "roberta.", remove the prefix- # e.g. https://huggingface.co/BAAI/bge-reranker-v2-m3/tree/main- name = name.removeprefix("roberta.")-- # position embeddings start at pad_token_id + 1, so just chop down the weight tensor- if name == "embeddings.position_embeddings.weight":- if self._position_offset is not None:- data_torch = data_torch[self._position_offset :, :]-- return super().modify_tensors(data_torch, name, bid)--[email protected]("GemmaForCausalLM")-class GemmaModel(TextModel):- model_arch = gguf.MODEL_ARCH.GEMMA-- def set_vocab(self):- self._set_vocab_sentencepiece()-- # TODO: these special tokens should be exported only for the CodeGemma family- special_vocab = gguf.SpecialVocab(- self.dir_model, load_merges=False, special_token_types=["prefix", "suffix", "middle", "fsep", "eot"]- )- special_vocab._set_special_token("prefix", 67)- special_vocab._set_special_token("suffix", 69)- special_vocab._set_special_token("middle", 68)- special_vocab._set_special_token("fsep", 70)- special_vocab._set_special_token("eot", 107)- special_vocab.chat_template = None # do not add it twice- special_vocab.add_to_gguf(self.gguf_writer)-- self.gguf_writer.add_add_space_prefix(False)-- def set_gguf_parameters(self):- hparams = self.hparams- block_count = hparams["num_hidden_layers"]-- self.gguf_writer.add_context_length(hparams["max_position_embeddings"])- self.gguf_writer.add_embedding_length(hparams["hidden_size"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])- self.gguf_writer.add_head_count(hparams["num_attention_heads"])- self.gguf_writer.add_head_count_kv(- self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"]- )- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])- self.gguf_writer.add_key_length(hparams["head_dim"])- self.gguf_writer.add_value_length(hparams["head_dim"])- self.gguf_writer.add_file_type(self.ftype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- # lm_head is not used in llama.cpp, while autoawq will include this tensor in model- # To prevent errors, skip loading lm_head.weight.- if name == "lm_head.weight":- logger.debug(f"Skipping get tensor {name!r} in safetensors so that convert can end normally.")- return []-- if name.endswith("norm.weight"):- data_torch = data_torch + 1-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("Gemma2ForCausalLM")-class Gemma2Model(TextModel):- model_arch = gguf.MODEL_ARCH.GEMMA2-- def set_vocab(self):- self._set_vocab_sentencepiece()-- self.gguf_writer.add_add_space_prefix(False)-- def set_gguf_parameters(self):- hparams = self.hparams- block_count = hparams["num_hidden_layers"]-- self.gguf_writer.add_context_length(hparams["max_position_embeddings"])- self.gguf_writer.add_embedding_length(hparams["hidden_size"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])- self.gguf_writer.add_head_count(hparams["num_attention_heads"])- self.gguf_writer.add_head_count_kv(- self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"]- )- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])- self.gguf_writer.add_key_length(hparams["head_dim"])- self.gguf_writer.add_value_length(hparams["head_dim"])- self.gguf_writer.add_file_type(self.ftype)- self.gguf_writer.add_attn_logit_softcapping(self.hparams["attn_logit_softcapping"])- self.gguf_writer.add_final_logit_softcapping(self.hparams["final_logit_softcapping"])- self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- # lm_head is not used in llama.cpp, while autoawq will include this tensor in model- # To prevent errors, skip loading lm_head.weight.- if name == "lm_head.weight":- logger.debug(f"Skipping get tensor {name!r} in safetensors so that convert can end normally.")- return []-- if name.endswith("norm.weight"):- data_torch = data_torch + 1-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("Gemma3ForCausalLM", "Gemma3ForConditionalGeneration")-class Gemma3Model(TextModel):- model_arch = gguf.MODEL_ARCH.GEMMA3- norm_shift = 1.0 # Gemma3RMSNorm adds 1.0 to the norm value-- def set_vocab(self):- self._set_vocab_sentencepiece()-- self.gguf_writer.add_add_space_prefix(False)-- def set_gguf_parameters(self):- hparams = self.hparams- block_count = hparams["num_hidden_layers"]-- # some default values are not specified in the hparams- self.gguf_writer.add_context_length(hparams.get("max_position_embeddings", 131072))- self.gguf_writer.add_embedding_length(hparams["hidden_size"])- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])- self.gguf_writer.add_head_count(hparams.get("num_attention_heads", 8))- self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("rms_norm_eps", 1e-6))- self.gguf_writer.add_key_length(hparams.get("head_dim", 256))- self.gguf_writer.add_value_length(hparams.get("head_dim", 256))- self.gguf_writer.add_file_type(self.ftype)- self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 1_000_000.0)) # for global layers- # attn_logit_softcapping is removed in Gemma3- assert hparams.get("attn_logit_softcapping") is None- self.gguf_writer.add_sliding_window(hparams["sliding_window"])- self.gguf_writer.add_head_count_kv(hparams.get("num_key_value_heads", 4))- if hparams.get("rope_scaling") is not None:- assert hparams["rope_scaling"]["rope_type"] == "linear"- # important: this rope_scaling is only applied for global layers, and not used by 1B model- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(hparams["rope_scaling"]["factor"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- if "language_model." in name:- name = name.replace("language_model.", "")-- elif (- name.startswith("multi_modal_projector.")- or name.startswith("vision_tower.")- or name.startswith("multimodal_projector.")- or name.startswith("vision_model.")- ):- return [] # skip vision tensors-- # remove OOV (out-of-vocabulary) rows in token_embd- if "embed_tokens.weight" in name:- vocab = self._create_vocab_sentencepiece()- tokens = vocab[0]- data_torch = data_torch[: len(tokens)]-- # ref code in Gemma3RMSNorm- # output = output * (1.0 + self.weight.float())- # note: this is not the case on gemma3n- if name.endswith("norm.weight"):- data_torch = data_torch + self.norm_shift-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("Gemma3ForConditionalGeneration")-class Gemma3VisionModel(MmprojModel):-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GEMMA3)- # default values below are taken from HF transformers code- self.gguf_writer.add_vision_attention_layernorm_eps(hparams.get("layer_norm_eps", 1e-6))- self.gguf_writer.add_vision_use_gelu(True)- # calculate proj_scale_factor (used by tinygemma3 test model)- image_seq_length = self.preprocessor_config.get("image_seq_length", 256)- n_per_side = int(image_seq_length**0.5)- image_size = self.hparams["image_size"]- patch_size = self.hparams["patch_size"]- proj_scale_factor = (image_size // patch_size) // n_per_side- if proj_scale_factor > 0 and proj_scale_factor != 4:- # we only need to write this if it's not the default value- # in this case, we are converting a test model- self.gguf_writer.add_vision_projector_scale_factor(proj_scale_factor)-- def tensor_force_quant(self, name, new_name, bid, n_dims):- del bid, new_name, n_dims # unused- # related to https://github.com/ggml-org/llama.cpp/issues/13025- if "input_projection" in name:- return gguf.GGMLQuantizationType.F16- if ".embeddings." in name:- return gguf.GGMLQuantizationType.F32- return False-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- if "vision_model.head." in name:- return [] # skip redundant tensors for tinygemma3-- if (- name.startswith("multi_modal_projector.")- or name.startswith("vision_tower.")- or name.startswith("multimodal_projector.")- or name.startswith("vision_model.")- ):- # process vision tensors- name = name.replace("_weight", ".weight")-- # correct norm value ; only this "soft_emb_norm" need to be corrected as it's part of Gemma projector- # the other norm values are part of SigLIP model, and they are already correct- # ref code: Gemma3RMSNorm- if "soft_emb_norm.weight" in name:- logger.info(f"Correcting norm value for '{name}'")- data_torch = data_torch + 1-- return [(self.map_tensor_name(name), data_torch)]-- return [] # skip other tensors--[email protected]("Gemma3nForConditionalGeneration")-class Gemma3NModel(Gemma3Model):- model_arch = gguf.MODEL_ARCH.GEMMA3N- norm_shift = 0.0 # same value with Gemma3p5RMSNorm scale_shift on python code-- _altup_proj: list[Tensor] = []- _altup_unembd: list[Tensor] = []-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- assert self.hparams["altup_num_inputs"] == 4, "Current conversion only supports 4 altup inputs"- self._altup_proj = [- torch.Tensor(), # to be replaced- torch.Tensor(), # to be replaced- torch.Tensor(), # to be replaced- ]- self._altup_unembd = [- torch.Tensor(), # to be replaced- torch.Tensor(), # to be replaced- torch.Tensor(), # to be replaced- ]-- def set_vocab(self):- super().set_vocab()-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_altup_active_idx(self.hparams["altup_active_idx"])- self.gguf_writer.add_altup_num_inputs(self.hparams["altup_num_inputs"])- self.gguf_writer.add_embedding_length_per_layer_input(self.hparams["hidden_size_per_layer_input"])- self.gguf_writer.add_shared_kv_layers(self.hparams["num_kv_shared_layers"])-- activation_sparsity_scale = []- for s in self.hparams["activation_sparsity_pattern"]:- normal_dist = torch.distributions.normal.Normal(0, 1)- std_multiplier = normal_dist.icdf(torch.tensor(s, dtype=torch.float32))- activation_sparsity_scale.append(std_multiplier.item())- self.gguf_writer.add_activation_sparsity_scale(activation_sparsity_scale)-- sliding_window_pattern = []- for t in self.hparams["layer_types"]:- sliding_window_pattern.append(t == "sliding_attention")- self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)-- def _stack_matrices(self, matrices: list[Tensor]) -> Tensor | None:- has_all = all(m.numel() > 0 for m in matrices)- if not has_all:- return None- else:- return torch.stack(matrices, dim=0)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- if name.endswith("_scale"):- name = name + ".weight"-- # TODO: implement self.prediction_coefs.weight.clamp_(...)-- if "language_model." not in name:- return [] # skip non-language model tensors-- if "altup_unembed_projections" in name:- data_torch = data_torch.to(device="cpu")- if ".0." in name:- self._altup_unembd[0] = data_torch- elif ".1." in name:- self._altup_unembd[1] = data_torch- elif ".2." in name:- self._altup_unembd[2] = data_torch- else:- raise ValueError(f"Unknown name: {name}")- out = self._stack_matrices(self._altup_unembd)- if out is not None:- return [(self.map_tensor_name("model.altup_unembed_projections.weight"), out)]- else:- return []-- if "altup_projections" in name:- data_torch = data_torch.to(device="cpu")- if ".0." in name:- self._altup_proj[0] = data_torch- elif ".1." in name:- self._altup_proj[1] = data_torch- elif ".2." in name:- self._altup_proj[2] = data_torch- else:- raise ValueError(f"Unknown name: {name}")- out = self._stack_matrices(self._altup_proj)- if out is not None:- return [(self.map_tensor_name("model.altup_projections.weight"), out)]- else:- return []-- return super().modify_tensors(data_torch, name, bid)--[email protected]("Starcoder2ForCausalLM")-class StarCoder2Model(TextModel):- model_arch = gguf.MODEL_ARCH.STARCODER2--[email protected]("Rwkv6ForCausalLM")-class Rwkv6Model(TextModel):- model_arch = gguf.MODEL_ARCH.RWKV6-- def set_vocab(self):- self._set_vocab_rwkv_world()-- def set_gguf_parameters(self):- block_count = self.hparams["num_hidden_layers"]- head_size = self.hparams["head_size"]- hidden_size = self.hparams["hidden_size"]- layer_norm_eps = self.hparams["layer_norm_epsilon"]- rescale_every_n_layers = self.hparams["rescale_every"]- intermediate_size = (- self.hparams["intermediate_size"]- if self.hparams["intermediate_size"] is not None- else int((hidden_size * 3.5) // 32 * 32)- )- time_mix_extra_dim = 64 if hidden_size == 4096 else 32- time_decay_extra_dim = 128 if hidden_size == 4096 else 64-- # RWKV isn't context limited- self.gguf_writer.add_context_length(1048576)- self.gguf_writer.add_embedding_length(hidden_size)- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_layer_norm_eps(layer_norm_eps)- self.gguf_writer.add_rescale_every_n_layers(rescale_every_n_layers)- self.gguf_writer.add_wkv_head_size(head_size)- self.gguf_writer.add_time_mix_extra_dim(time_mix_extra_dim)- self.gguf_writer.add_time_decay_extra_dim(time_decay_extra_dim)- self.gguf_writer.add_feed_forward_length(intermediate_size)- self.gguf_writer.add_file_type(self.ftype)-- # required by llama.cpp, unused- self.gguf_writer.add_head_count(0)-- lerp_weights: dict[int, dict[str, Tensor]] = {}-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- new_name = self.map_tensor_name(name)-- if not (new_name.endswith(".weight") or new_name.endswith(".bias")):- new_name += ".weight"-- if (- new_name.endswith("time_mix_w1.weight")- or new_name.endswith("time_mix_decay_w1.weight")- or new_name.endswith("time_mix_decay_w2.weight")- ):- data_torch = data_torch.transpose(0, 1)-- if new_name.endswith("time_mix_w2.weight"):- data_torch = data_torch.permute(0, 2, 1)-- if new_name.endswith("time_mix_decay.weight") or "lerp" in new_name:- data_torch = data_torch.squeeze()-- try:- rescale_every_n_layers = self.hparams["rescale_every"]- if rescale_every_n_layers > 0:- if new_name.endswith("time_mix_output.weight") or new_name.endswith("channel_mix_value.weight"):- data_torch = data_torch.div_(2 ** int(bid // rescale_every_n_layers))- except KeyError:- pass-- # concat time_mix_lerp weights to reduce some cpu overhead- # also reduces the number of tensors in the model- if bid is not None and "time_mix_lerp" in new_name and "time_mix_lerp_x" not in new_name:- try:- self.lerp_weights[bid][new_name] = data_torch- except KeyError:- self.lerp_weights[bid] = {new_name: data_torch}- if all(- f"blk.{bid}.time_mix_lerp_{i}.weight" in self.lerp_weights[bid].keys()- for i in ["w", "k", "v", "r", "g"]- ):- new_name = f"blk.{bid}.time_mix_lerp_fused.weight"- data = torch.stack(- [- self.lerp_weights[bid][f"blk.{bid}.time_mix_lerp_{i}.weight"].unsqueeze(0)- for i in ["w", "k", "v", "r", "g"]- ],- dim=0,- ).unsqueeze(1)- yield (new_name, data)- return-- yield (new_name, data_torch)--[email protected]("RWKV6Qwen2ForCausalLM")-class RWKV6Qwen2Model(Rwkv6Model):- model_arch = gguf.MODEL_ARCH.RWKV6QWEN2-- def set_vocab(self):- try:- self._set_vocab_sentencepiece()- except FileNotFoundError:- self._set_vocab_gpt2()-- def set_gguf_parameters(self):- block_count = self.hparams["num_hidden_layers"]- num_attention_heads = self.hparams["num_attention_heads"]- num_key_value_heads = self.hparams["num_key_value_heads"]- hidden_size = self.hparams["hidden_size"]- head_size = hidden_size // num_attention_heads- rms_norm_eps = self.hparams["rms_norm_eps"]- intermediate_size = self.hparams["intermediate_size"]- time_mix_extra_dim = self.hparams.get("lora_rank_tokenshift", 64 if hidden_size >= 4096 else 32)- time_decay_extra_dim = self.hparams.get("lora_rank_decay", 128 if hidden_size >= 4096 else 64)-- # RWKV isn't context limited- self.gguf_writer.add_context_length(1048576)- self.gguf_writer.add_embedding_length(hidden_size)- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_wkv_head_size(head_size)- self.gguf_writer.add_time_mix_extra_dim(time_mix_extra_dim)- self.gguf_writer.add_time_decay_extra_dim(time_decay_extra_dim)- self.gguf_writer.add_feed_forward_length(intermediate_size)- self.gguf_writer.add_file_type(self.ftype)-- # special parameters for time_mixing in RWKV6QWEN2- self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)- self.gguf_writer.add_token_shift_count(1)- # RWKV6QWEN2 use grouped key/value like GQA- self.gguf_writer.add_head_count_kv(num_key_value_heads)-- # required by llama.cpp, unused- self.gguf_writer.add_head_count(0)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- for new_name, data in super().modify_tensors(data_torch, name, bid):- if "time_mix_w1" in new_name or "time_mix_w2" in new_name:- data = data.view(5, -1, data.shape[-1])- # rwkv6qwen2 has a different order of rkvwg instead of the original wkvrg- # permute them here to avoid code changes- data = torch.stack([data[3], data[1], data[2], data[0], data[4]], dim=0).view(-1, data.shape[-1])- if "w2" in new_name:- data = data.view(5, -1, data.shape[-1])- yield (new_name, data)- continue- yield (new_name, data)--[email protected]("Rwkv7ForCausalLM", "RWKV7ForCausalLM")-class Rwkv7Model(TextModel):- model_arch = gguf.MODEL_ARCH.RWKV7-- def set_vocab(self):- self._set_vocab_rwkv_world()-- def calc_lora_rank(self, hidden_size, exponent, multiplier):- return max(1, round(hidden_size**exponent * multiplier / 32)) * 32-- def set_gguf_parameters(self):- block_count = self.hparams["num_hidden_layers"]- try:- head_size = self.hparams["head_size"]- layer_norm_eps = self.hparams["layer_norm_epsilon"]- except KeyError:- head_size = self.hparams["head_dim"]- layer_norm_eps = self.hparams["norm_eps"]- hidden_size = self.hparams["hidden_size"]- intermediate_size = (- self.hparams["intermediate_size"] if self.hparams["intermediate_size"] is not None else (hidden_size * 4)- )-- # ICLR: In-Context-Learning-Rate- try:- lora_rank_decay = (- self.hparams["lora_rank_decay"]- if self.hparams["lora_rank_decay"] is not None- else self.calc_lora_rank(hidden_size, 0.5, 1.8)- )- lora_rank_iclr = (- self.hparams["lora_rank_iclr"]- if self.hparams["lora_rank_iclr"] is not None- else self.calc_lora_rank(hidden_size, 0.5, 1.8)- )- lora_rank_value_residual_mix = (- self.hparams["lora_rank_value_residual_mix"]- if self.hparams["lora_rank_value_residual_mix"] is not None- else self.calc_lora_rank(hidden_size, 0.5, 1.3)- )- lora_rank_gate = (- self.hparams["lora_rank_gate"]- if self.hparams["lora_rank_gate"] is not None- else self.calc_lora_rank(hidden_size, 0.8, 0.6)- )- except KeyError:- lora_rank_decay = (- self.hparams["decay_low_rank_dim"]- if self.hparams["decay_low_rank_dim"] is not None- else self.calc_lora_rank(hidden_size, 0.5, 1.8)- )- lora_rank_iclr = (- self.hparams["a_low_rank_dim"]- if self.hparams["a_low_rank_dim"] is not None- else self.calc_lora_rank(hidden_size, 0.5, 1.8)- )- lora_rank_value_residual_mix = (- self.hparams["v_low_rank_dim"]- if self.hparams["v_low_rank_dim"] is not None- else self.calc_lora_rank(hidden_size, 0.5, 1.3)- )- lora_rank_gate = (- self.hparams["gate_low_rank_dim"]- if self.hparams["gate_low_rank_dim"] is not None- else self.calc_lora_rank(hidden_size, 0.8, 0.6)- )-- # RWKV isn't context limited- self.gguf_writer.add_context_length(1048576)- self.gguf_writer.add_embedding_length(hidden_size)- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_layer_norm_eps(layer_norm_eps)- self.gguf_writer.add_wkv_head_size(head_size)- self.gguf_writer.add_decay_lora_rank(lora_rank_decay)- self.gguf_writer.add_iclr_lora_rank(lora_rank_iclr)- self.gguf_writer.add_value_residual_mix_lora_rank(lora_rank_value_residual_mix)- self.gguf_writer.add_gate_lora_rank(lora_rank_gate)- self.gguf_writer.add_feed_forward_length(intermediate_size)- self.gguf_writer.add_file_type(self.ftype)-- # required by llama.cpp, unused- self.gguf_writer.add_head_count(0)-- lerp_weights: dict[int, dict[str, Tensor]] = {}- lora_needs_transpose: bool = True-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # unify tensor names here to make life easier- name = name.replace("blocks", "layers").replace("ffn", "feed_forward")- name = name.replace("self_attn", "attention").replace("attn", "attention")- name = name.replace("time_mixer.", "")- # lora layer names in fla-hub's impl- if "_lora.lora" in name:- self.lora_needs_transpose = False- name = name.replace("_lora.lora.0.weight", "1.weight")- name = name.replace("_lora.lora.2.weight", "2.weight")- name = name.replace("_lora.lora.2.bias", "0.weight")-- name = name.replace("feed_forward_norm", "ln2")- name = name.replace("g_norm", "ln_x")-- if "attention.v" in name and "value" not in self.map_tensor_name(name) and bid == 0:- # some models have dummy v0/v1/v2 on first layer while others don't- # ignore them all since they are not used- return-- wkv_has_gate = self.hparams.get("wkv_has_gate", True)- lerp_list = ["r", "w", "k", "v", "a", "g"] if wkv_has_gate else ["r", "w", "k", "v", "a"]-- if bid is not None and "attention.x_" in name:- if "attention.x_x" in name:- # already concatenated- new_name = f"blk.{bid}.time_mix_lerp_fused.weight"- data = data_torch.reshape(len(lerp_list), 1, 1, -1)- yield (new_name, data)- else:- try:- self.lerp_weights[bid][name] = data_torch- except KeyError:- self.lerp_weights[bid] = {name: data_torch}- if all(f"model.layers.{bid}.attention.x_{i}" in self.lerp_weights[bid].keys() for i in lerp_list):- new_name = f"blk.{bid}.time_mix_lerp_fused.weight"- data = torch.stack(- [self.lerp_weights[bid][f"model.layers.{bid}.attention.x_{i}"] for i in lerp_list], dim=0- )- yield (new_name, data)- return- else:- data_torch = data_torch.squeeze()- new_name = self.map_tensor_name(name)-- if not (new_name.endswith(".weight") or new_name.endswith(".bias")):- new_name += ".weight"-- if self.lora_needs_transpose and any(- new_name.endswith(t)- for t in [- "time_mix_w1.weight",- "time_mix_w2.weight",- "time_mix_a1.weight",- "time_mix_a2.weight",- "time_mix_v1.weight",- "time_mix_v2.weight",- "time_mix_g1.weight",- "time_mix_g2.weight",- ]- ):- data_torch = data_torch.transpose(0, 1)-- if "r_k" in new_name:- data_torch = data_torch.flatten()-- if bid == 0 and "time_mix_a" in new_name:- # dummy v0/v1/v2 on first layer- # easiest way to make llama happy- yield (new_name.replace("time_mix_a", "time_mix_v"), data_torch)-- yield (new_name, data_torch)--[email protected]("RwkvHybridForCausalLM")-class ARwkv7Model(Rwkv7Model):- model_arch = gguf.MODEL_ARCH.ARWKV7-- def set_vocab(self):- try:- self._set_vocab_sentencepiece()- except FileNotFoundError:- self._set_vocab_gpt2()-- def set_gguf_parameters(self):- block_count = self.hparams["num_hidden_layers"]- hidden_size = self.hparams["hidden_size"]- head_size = self.hparams["head_size"]- rms_norm_eps = self.hparams["rms_norm_eps"]- intermediate_size = self.hparams["intermediate_size"]- wkv_has_gate = self.hparams["wkv_has_gate"]- assert self.hparams["wkv_version"] == 7-- # ICLR: In-Context-Learning-Rate- lora_rank_decay = 64- lora_rank_iclr = 64- lora_rank_value_residual_mix = 32- lora_rank_gate = 128 if wkv_has_gate else 0-- # RWKV isn't context limited- self.gguf_writer.add_context_length(1048576)- self.gguf_writer.add_embedding_length(hidden_size)- self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)- self.gguf_writer.add_wkv_head_size(head_size)- self.gguf_writer.add_decay_lora_rank(lora_rank_decay)- self.gguf_writer.add_iclr_lora_rank(lora_rank_iclr)- self.gguf_writer.add_value_residual_mix_lora_rank(lora_rank_value_residual_mix)- self.gguf_writer.add_gate_lora_rank(lora_rank_gate)- self.gguf_writer.add_feed_forward_length(intermediate_size)- self.gguf_writer.add_file_type(self.ftype)- self.gguf_writer.add_token_shift_count(1)-- # required by llama.cpp, unused- self.gguf_writer.add_head_count(0)--[email protected]("MambaForCausalLM", "MambaLMHeadModel", "FalconMambaForCausalLM")-class MambaModel(TextModel):- model_arch = gguf.MODEL_ARCH.MAMBA-- def __init__(self, dir_model: Path, *args, **kwargs):- # Avoid using AutoConfig for hparams- hparams = kwargs.pop("hparams", None)- if hparams is None:- with open(dir_model / "config.json", "r", encoding="utf-8") as f:- hparams = json.load(f)- super().__init__(dir_model, *args, hparams=hparams, **kwargs)-- def set_vocab(self):- vocab_size = self.hparams["vocab_size"]- # Round vocab size to next multiple of 8- pad_vocab = self.hparams.get("pad_vocab_size_multiple", 8)- # pad using ceiling division- # ref: https://stackoverflow.com/a/17511341/22827863- vocab_size = -(vocab_size // -pad_vocab) * pad_vocab- self.hparams["vocab_size"] = vocab_size-- if (self.dir_model / "tokenizer.json").is_file():- self._set_vocab_gpt2()- elif (self.dir_model / "tokenizer.model").is_file():- self._set_vocab_sentencepiece()- else:- # Use the GPT-NeoX tokenizer when no tokenizer files are present- self._set_vocab_builtin("gpt-neox", vocab_size)-- def set_gguf_parameters(self):- d_model = self.find_hparam(["hidden_size", "d_model"])- d_conv = self.find_hparam(["conv_kernel", "d_conv"], optional=True) or 4- d_inner = self.find_hparam(["intermediate_size", "d_inner"], optional=True) or 2 * d_model- d_state = self.find_hparam(["state_size", "d_state"], optional=True) or 16- dt_rank = self.find_hparam(["time_step_rank", "dt_rank"], optional=True) or -(d_model // -16)- rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-5- use_dt_b_c_norm = False- # For falconmamba we do apply RMS norm on B / DT and C layers- if self.find_hparam(["model_type"], optional=True) in ("falcon_mamba",):- use_dt_b_c_norm = True- # Fail early for models which don't have a block expansion factor of 2- assert d_inner == 2 * d_model-- self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default- self.gguf_writer.add_embedding_length(d_model)- self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loading- self.gguf_writer.add_head_count(0) # unused, but seemingly required when loading- self.gguf_writer.add_block_count(self.block_count)- self.gguf_writer.add_ssm_conv_kernel(d_conv)- self.gguf_writer.add_ssm_inner_size(d_inner)- self.gguf_writer.add_ssm_state_size(d_state)- self.gguf_writer.add_ssm_time_step_rank(dt_rank)- self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)- self.gguf_writer.add_ssm_dt_b_c_rms(- use_dt_b_c_norm- ) # For classic Mamba we don't apply rms norm on B / DT layers- self.gguf_writer.add_file_type(self.ftype)-- _tok_embd = None-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- output_name = self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT)- tok_embd_name = self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD)-- new_name = self.map_tensor_name(name)-- if name.endswith(".A_log"):- logger.debug("A_log --> A ==> " + new_name)- data_torch = -torch.exp(data_torch)-- # [4 1 8192 1] -> [4 8192 1 1]- if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_CONV1D, bid):- data_torch = data_torch.squeeze()-- # assuming token_embd.weight is seen before output.weight- if self._tok_embd is not None and new_name == output_name:- if torch.equal(self._tok_embd, data_torch):- logger.debug(f"{output_name} is equivalent to {tok_embd_name}, omitting")- return []- elif new_name == tok_embd_name:- self._tok_embd = data_torch-- return [(new_name, data_torch)]--[email protected]("Mamba2ForCausalLM")-class Mamba2Model(TextModel):- model_arch = gguf.MODEL_ARCH.MAMBA2-- def __init__(self, dir_model: Path, *args, **kwargs):- # Avoid using AutoConfig for hparams- # It wrongly assumes all Mamba2 models are Mamba-Codestral-7B-v0.1- hparams = kwargs.pop("hparams", None)- if hparams is None:- with open(dir_model / "config.json", "r", encoding="utf-8") as f:- hparams = json.load(f)- super().__init__(dir_model, *args, hparams=hparams, **kwargs)- self.d_model = self.find_hparam(["hidden_size", "d_model", "dim"])- self.d_inner = (- self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"], optional=True) or 2 * self.d_model- )- self.n_group = self.find_hparam(["n_groups"], optional=True) or 1-- def set_vocab(self):- vocab_size = self.hparams["vocab_size"]- # Round vocab size to next multiple of 16- pad_vocab = self.hparams.get("pad_vocab_size_multiple", 16)- # pad using ceiling division- # ref: https://stackoverflow.com/a/17511341/22827863- vocab_size = -(vocab_size // -pad_vocab) * pad_vocab- self.hparams["vocab_size"] = vocab_size-- if (self.dir_model / "tokenizer.model").is_file():- self._set_vocab_sentencepiece()- elif (self.dir_model / "tokenizer.model.v3").is_file():- # mamba-codestral- raise NotImplementedError(- f"Please rename {self.dir_model / 'tokenizer.model.v3'} to {self.dir_model / 'tokenizer.model'}"- )- elif (self.dir_model / "tokenizer.json").is_file():- self._set_vocab_gpt2()- else:- # Use the GPT-NeoX tokenizer when no tokenizer files are present- self._set_vocab_builtin("gpt-neox", vocab_size)-- def set_gguf_parameters(self):- d_conv = self.find_hparam(["conv_kernel", "d_conv"], optional=True) or 4- d_state = self.find_hparam(["state_size", "d_state"], optional=True) or 128- head_dim = self.find_hparam(["mamba_d_head", "head_dim"], optional=True) or 64-- rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-5-- # Fail early for models which don't have a block expansion factor of 2- # TODO: does this really matter?- # skip the assertion for FalconH1 Model- if self.model_arch != gguf.MODEL_ARCH.FALCON_H1:- assert self.d_inner == 2 * self.d_model- assert self.d_inner % head_dim == 0-- self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default- self.gguf_writer.add_embedding_length(self.d_model)- self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loading- self.gguf_writer.add_head_count(0) # unused, but seemingly required when loading- self.gguf_writer.add_block_count(self.block_count)- self.gguf_writer.add_ssm_conv_kernel(d_conv)- self.gguf_writer.add_ssm_inner_size(self.d_inner)- self.gguf_writer.add_ssm_state_size(d_state)- self.gguf_writer.add_ssm_time_step_rank(self.d_inner // head_dim)- self.gguf_writer.add_ssm_group_count(self.n_group)- self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)- self.gguf_writer.add_file_type(self.ftype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:-- if name.startswith("model.backbone") or name.startswith("model.lm_head"):- # map Mamba-Codestral-7B-v0.1 tensor names to the names used by Mamba-2- name = name.removeprefix("model.")-- if name.endswith(".dt_bias"):- name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"-- new_name = self.map_tensor_name(name)-- if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_CONV1D, bid):- data_torch = data_torch.squeeze()- elif any(- self.match_model_tensor_name(new_name, t, bid, suffix="")- for t in [- gguf.MODEL_TENSOR.SSM_A,- gguf.MODEL_TENSOR.SSM_D,- ]- ):- # unsqueeze A to use similar shape semantics as Mamba-1- # (D is also unsqueezed, but for more straightforward broadcast internally)- data_torch = data_torch.reshape((*data_torch.shape, 1))- elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_NORM, bid):- data_torch = data_torch.reshape((self.n_group, self.d_inner // self.n_group))-- if name.endswith(".A_log"):- logger.debug("A_log --> A ==> " + new_name)- data_torch = -torch.exp(data_torch)-- yield (new_name, data_torch)--[email protected]("JambaForCausalLM")-class JambaModel(TextModel):- model_arch = gguf.MODEL_ARCH.JAMBA-- def get_vocab_base_pre(self, tokenizer) -> str:- del tokenizer # unused-- return "gpt-2"-- def set_vocab(self):- if (self.dir_model / "tokenizer.model").is_file():- # Using Jamba's tokenizer.json causes errors on model load- # (something about "byte not found in vocab"),- # but there's a working tokenizer.model- self._set_vocab_sentencepiece()- else:- # Some Jamba models only have a tokenizer.json, which works.- self._set_vocab_gpt2()-- def set_gguf_parameters(self):- d_model = self.find_hparam(["hidden_size", "mamba_d_model"])- d_conv = self.find_hparam(["mamba_d_conv"], optional=True) or 4- d_inner = self.hparams["mamba_expand"] * d_model- d_state = self.find_hparam(["mamba_d_state"], optional=True) or 16- dt_rank = self.find_hparam(["mamba_dt_rank"], optional=True) or -(d_model // -16)- rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-6- n_kv_head = self.hparams["num_key_value_heads"]- attn_offset = self.hparams["attn_layer_offset"]- attn_period = self.hparams["attn_layer_period"]- n_kv_vec = [0 for _ in range(attn_offset)] + [- n_kv_head if (i - attn_offset) % attn_period == 0 else 0 for i in range(attn_offset, self.block_count)- ]-- self.gguf_writer.add_block_count(self.block_count)- self.gguf_writer.add_context_length(self.find_hparam(["max_position_embeddings", "n_ctx"]))- self.gguf_writer.add_embedding_length(d_model)- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])- self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])- self.gguf_writer.add_head_count_kv(n_kv_vec)- self.gguf_writer.add_ssm_conv_kernel(d_conv)- self.gguf_writer.add_ssm_inner_size(d_inner)- self.gguf_writer.add_ssm_state_size(d_state)- self.gguf_writer.add_ssm_time_step_rank(dt_rank)- self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)- self.gguf_writer.add_expert_count(self.hparams["num_experts"])- self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])- self.gguf_writer.add_file_type(self.ftype)-- _experts: list[dict[str, Tensor]] | None = None-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:-- # Mini-Jamba- name = name.replace(".moe.", ".feed_forward.")- if bid is not None:- moe_offset = self.hparams["expert_layer_offset"]- moe_period = self.hparams["expert_layer_period"]-- if not (bid >= moe_offset and (bid - moe_offset) % moe_period == 0):- name = name.replace(".experts.0.", ".")-- # process the experts separately- if ".feed_forward.experts." in name:- n_experts = self.hparams["num_experts"]-- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:-- # merge the experts into a single 3d tensor- for wid in ["down_proj", "gate_proj", "up_proj"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"model.layers.{bid}.feed_forward.experts.{xid}.{wid}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)-- # using the same merged name as qwen2moe- merged_name = f"model.layers.{bid}.mlp.experts.{wid}.weight"-- new_name = self.map_tensor_name(merged_name)-- yield new_name, data_torch- return-- new_name = self.map_tensor_name(name)-- if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_CONV1D, bid):- data_torch = data_torch.squeeze()-- if name.endswith(".A_log"):- logger.debug("A_log --> A ==> " + new_name)- data_torch = -torch.exp(data_torch)-- yield (new_name, data_torch)-- def prepare_tensors(self):- super().prepare_tensors()-- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--[email protected]("CohereForCausalLM")-class CommandR2Model(TextModel):- model_arch = gguf.MODEL_ARCH.COMMAND_R-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)-- # max_position_embeddings = 8192 in config.json but model was actually- # trained on 128k context length- # aya-23 models don't have model_max_length specified- self.hparams["max_position_embeddings"] = self.find_hparam(["model_max_length", "max_position_embeddings"])-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)--[email protected]("Cohere2ForCausalLM")-class Cohere2Model(TextModel):- model_arch = gguf.MODEL_ARCH.COHERE2-- def set_gguf_parameters(self):- super().set_gguf_parameters()-- self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])- self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])- self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])-- rotary_pct = self.hparams["rotary_pct"]- hidden_size = self.hparams["hidden_size"]- num_attention_heads = self.hparams["num_attention_heads"]- self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size // num_attention_heads)))- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)--[email protected]("OlmoForCausalLM")[email protected]("OLMoForCausalLM")-class OlmoModel(TextModel):- model_arch = gguf.MODEL_ARCH.OLMO-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_layer_norm_eps(1e-5)- clip_qkv = self.hparams.get("clip_qkv")- if clip_qkv is not None:- self.gguf_writer.add_clamp_kqv(clip_qkv)-- # Same as super class, but permuting q_proj, k_proj- # Copied from: LlamaModel- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- n_head = self.hparams["num_attention_heads"]- n_kv_head = self.hparams.get("num_key_value_heads")-- if name.endswith("q_proj.weight"):- data_torch = LlamaModel.permute(data_torch, n_head, n_head)- if name.endswith("k_proj.weight"):- data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("Olmo2ForCausalLM")-class Olmo2Model(TextModel):- model_arch = gguf.MODEL_ARCH.OLMO2--[email protected]("OlmoeForCausalLM")-class OlmoeModel(TextModel):- model_arch = gguf.MODEL_ARCH.OLMOE-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_layer_norm_rms_eps(1e-5)- if (n_experts := self.hparams.get("num_experts")) is not None:- self.gguf_writer.add_expert_count(n_experts)-- _experts: list[dict[str, Tensor]] | None = None-- # Copied from: Qwen2MoeModel- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # process the experts separately- if name.find("experts") != -1:- n_experts = self.hparams["num_experts"]- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- tensors: list[tuple[str, Tensor]] = []-- # merge the experts into a single 3d tensor- for w_name in ["down_proj", "gate_proj", "up_proj"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)-- merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"-- new_name = self.map_tensor_name(merged_name)-- tensors.append((new_name, data_torch))- return tensors- else:- return []-- return [(self.map_tensor_name(name), data_torch)]-- # Copied from: Qwen2MoeModel- def prepare_tensors(self):- super().prepare_tensors()-- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--[email protected]("JinaBertModel", "JinaBertForMaskedLM")-class JinaBertV2Model(BertModel):- model_arch = gguf.MODEL_ARCH.JINA_BERT_V2-- def set_vocab(self):- tokenizer_class = "BertTokenizer"- with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f:- tokenizer_class = json.load(f)["tokenizer_class"]-- if tokenizer_class == "BertTokenizer":- super().set_vocab()- elif tokenizer_class == "RobertaTokenizer":- self._set_vocab_gpt2()- self.gguf_writer.add_token_type_count(2)- else:- raise NotImplementedError(f"Tokenizer {tokenizer_class} is not supported for JinaBertModel")--[email protected]("OpenELMForCausalLM")-class OpenELMModel(TextModel):- model_arch = gguf.MODEL_ARCH.OPENELM-- @staticmethod- def _make_divisible(v: float | int, divisor: int) -> int:- new_v = max(divisor, int(v + divisor / 2) // divisor * divisor)- # Make sure that round down does not go down by more than 10%.- if new_v < 0.9 * v:- new_v += divisor- return new_v-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)-- ffn_multipliers: list[float] = self.hparams["ffn_multipliers"]- ffn_dim_divisor: int = self.hparams["ffn_dim_divisor"]- self._n_embd: int = self.hparams["model_dim"]- self._num_kv_heads: list[int] = self.hparams["num_kv_heads"]- self._num_query_heads: list[int] = self.hparams["num_query_heads"]- self._ffn_dims: list[int] = [- OpenELMModel._make_divisible(multiplier * self._n_embd, ffn_dim_divisor) for multiplier in ffn_multipliers- ]- assert isinstance(self._num_kv_heads, list) and isinstance(self._num_kv_heads[0], int)- assert isinstance(self._num_query_heads, list) and isinstance(self._num_query_heads[0], int)-- # Uses the tokenizer from meta-llama/Llama-2-7b-hf- def set_vocab(self):- try:- self._set_vocab_sentencepiece()- except FileNotFoundError:- self._set_vocab_builtin("llama-spm", self.hparams["vocab_size"])-- def set_gguf_parameters(self):- n_embd = self._n_embd- head_dim = self.hparams["head_dim"]- rot_pct = 1.0- assert self.block_count == len(self._num_kv_heads)- assert self.block_count == len(self._num_query_heads)- assert self.block_count == len(self._ffn_dims)-- self.gguf_writer.add_block_count(self.block_count)- self.gguf_writer.add_context_length(self.hparams["max_context_length"])- self.gguf_writer.add_embedding_length(n_embd)- self.gguf_writer.add_feed_forward_length(self._ffn_dims)- self.gguf_writer.add_head_count(self._num_query_heads)- self.gguf_writer.add_head_count_kv(self._num_kv_heads)- self.gguf_writer.add_rope_freq_base(self.hparams["rope_freq_constant"])- # https://huggingface.co/apple/OpenELM-270M-Instruct/blob/c401df2/modeling_openelm.py#L30- self.gguf_writer.add_layer_norm_rms_eps(1e-6)- self.gguf_writer.add_rope_dimension_count(int(rot_pct * head_dim))- self.gguf_writer.add_key_length(head_dim)- self.gguf_writer.add_value_length(head_dim)- self.gguf_writer.add_file_type(self.ftype)-- def find_hparam(self, keys: Iterable[str], optional: bool = False) -> Any:- if "n_layers" in keys:- return self.hparams["num_transformer_layers"]-- return super().find_hparam(keys, optional)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:-- # split ff- if bid is not None and name == f"transformer.layers.{bid}.ffn.proj_1.weight":- ff_dim = self._ffn_dims[bid]- yield (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), data_torch[:ff_dim])- yield (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), data_torch[ff_dim:])- return-- yield (self.map_tensor_name(name), data_torch)--[email protected]("ArcticForCausalLM")-class ArcticModel(TextModel):- model_arch = gguf.MODEL_ARCH.ARCTIC-- def set_vocab(self):- # The reason for using a custom implementation here is that the- # snowflake-arctic-instruct model redefined tokens 31998 and 31999 from- # tokenizer.model and used them as BOS and EOS instead of adding new tokens.- from sentencepiece import SentencePieceProcessor-- tokenizer_path = self.dir_model / "tokenizer.model"-- if not tokenizer_path.is_file():- logger.error(f"Error: Missing {tokenizer_path}")- sys.exit(1)-- # Read the whole vocabulary from the tokenizer.model file- tokenizer = SentencePieceProcessor()- tokenizer.LoadFromFile(str(tokenizer_path))-- vocab_size = self.hparams.get("vocab_size", tokenizer.vocab_size())-- tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]- scores: list[float] = [-10000.0] * vocab_size- toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size-- for token_id in range(tokenizer.vocab_size()):-- piece = tokenizer.IdToPiece(token_id)- text = piece.encode("utf-8")- score = tokenizer.GetScore(token_id)-- toktype = SentencePieceTokenTypes.NORMAL- if tokenizer.IsUnknown(token_id):- toktype = SentencePieceTokenTypes.UNKNOWN- elif tokenizer.IsControl(token_id):- toktype = SentencePieceTokenTypes.CONTROL- elif tokenizer.IsUnused(token_id):- toktype = SentencePieceTokenTypes.UNUSED- elif tokenizer.IsByte(token_id):- toktype = SentencePieceTokenTypes.BYTE-- tokens[token_id] = text- scores[token_id] = score- toktypes[token_id] = toktype-- # Use the added_tokens_decoder field from tokeniser_config.json as the source- # of information about added/redefined tokens and modify them accordingly.- tokenizer_config_file = self.dir_model / "tokenizer_config.json"- if tokenizer_config_file.is_file():- with open(tokenizer_config_file, "r", encoding="utf-8") as f:- tokenizer_config_json = json.load(f)-- if "added_tokens_decoder" in tokenizer_config_json:- added_tokens_decoder = tokenizer_config_json["added_tokens_decoder"]- for token_id, token_json in added_tokens_decoder.items():- token_id = int(token_id)- if token_id >= vocab_size:- logger.debug(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")- continue-- token_content = token_json["content"]- token_type = SentencePieceTokenTypes.USER_DEFINED- token_score = -10000.0-- # Map unk_token to UNKNOWN, other special tokens to CONTROL- # Set the score to 0.0 as in the original tokenizer.model- if ("special" in token_json) and token_json["special"]:- if token_content == tokenizer_config_json["unk_token"]:- token_type = SentencePieceTokenTypes.UNKNOWN- else:- token_type = SentencePieceTokenTypes.CONTROL- token_score = 0.0-- logger.info(- f"Setting added token {token_id} to '{token_content}'"- f" (type: {token_type}, score: {token_score:.2f})"- )- tokens[token_id] = token_content.encode("utf-8")- toktypes[token_id] = token_type- scores[token_id] = token_score-- self.gguf_writer.add_tokenizer_model("llama")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- self.gguf_writer.add_vocab_size(hparams["vocab_size"])- self.gguf_writer.add_rope_dimension_count(hparams["hidden_size"] // hparams["num_attention_heads"])-- _experts: list[dict[str, Tensor]] | None = None-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- n_head = self.hparams["num_attention_heads"]- n_kv_head = self.hparams.get("num_key_value_heads")-- if name.endswith("q_proj.weight"):- data_torch = LlamaModel.permute(data_torch, n_head, n_head)- if name.endswith("k_proj.weight"):- data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)-- # process the experts separately- if name.find("block_sparse_moe.experts") != -1:- n_experts = self.hparams["num_local_experts"]-- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- tensors: list[tuple[str, Tensor]] = []-- # merge the experts into a single 3d tensor- for wid in ["w1", "w2", "w3"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{wid}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)-- merged_name = f"layers.{bid}.feed_forward.experts.{wid}.weight"-- new_name = self.map_tensor_name(merged_name)-- tensors.append((new_name, data_torch))- return tensors- else:- return []-- return [(self.map_tensor_name(name), data_torch)]-- def prepare_tensors(self):- super().prepare_tensors()-- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--[email protected]("DeepseekForCausalLM")-class DeepseekModel(TextModel):- model_arch = gguf.MODEL_ARCH.DEEPSEEK-- def set_vocab(self):- try:- self._set_vocab_sentencepiece()- except FileNotFoundError:- self._set_vocab_gpt2()-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- if (rope_dim := hparams.get("head_dim")) is None:- rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]-- self.gguf_writer.add_rope_dimension_count(rope_dim)- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)- self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])- self.gguf_writer.add_vocab_size(hparams["vocab_size"])- self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])- self.gguf_writer.add_expert_weights_scale(1.0)- self.gguf_writer.add_expert_count(hparams["n_routed_experts"])- self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"])-- _experts: list[dict[str, Tensor]] | None = None-- @staticmethod- def permute(weights: Tensor, n_head: int, n_head_kv: int | None):- if n_head_kv is not None and n_head != n_head_kv:- n_head = n_head_kv- return (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])- .swapaxes(1, 2)- .reshape(weights.shape)- )-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- n_head = self.hparams["num_attention_heads"]- n_kv_head = self.hparams.get("num_key_value_heads")-- if name.endswith(("q_proj.weight", "q_proj.bias")):- data_torch = DeepseekModel.permute(data_torch, n_head, n_head)- if name.endswith(("k_proj.weight", "k_proj.bias")):- data_torch = DeepseekModel.permute(data_torch, n_head, n_kv_head)-- # process the experts separately- if name.find("mlp.experts") != -1:- n_experts = self.hparams["n_routed_experts"]- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- tensors: list[tuple[str, Tensor]] = []-- # merge the experts into a single 3d tensor- for w_name in ["down_proj", "gate_proj", "up_proj"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)-- merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"-- new_name = self.map_tensor_name(merged_name)-- tensors.append((new_name, data_torch))- return tensors- else:- return []-- return [(self.map_tensor_name(name), data_torch)]-- def prepare_tensors(self):- super().prepare_tensors()-- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--[email protected]("DeepseekV2ForCausalLM")[email protected]("DeepseekV3ForCausalLM")-class DeepseekV2Model(TextModel):- model_arch = gguf.MODEL_ARCH.DEEPSEEK2-- def set_vocab(self):- try:- self._set_vocab_gpt2()- return- except Exception:- pass-- from transformers import AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)- tokpre = self.get_vocab_base_pre(tokenizer)-- if tokpre == "kimi-k2":- # Build merges list using the approach similar to HunYuanMoE- merges = []- vocab = {}- mergeable_ranks = tokenizer.model._mergeable_ranks- for token, rank in mergeable_ranks.items():- vocab[QwenModel.token_bytes_to_string(token)] = rank- if len(token) == 1:- continue- merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)- if len(merged) == 2:- merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))-- # Build token list- vocab_size = self.hparams["vocab_size"]- special_tokens = tokenizer.special_tokens- reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}- tokens: list[str] = []- toktypes: list[int] = []-- for i in range(vocab_size):- if i not in reverse_vocab:- tokens.append(f"[PAD{i}]")- toktypes.append(gguf.TokenType.UNUSED)- else:- token = reverse_vocab[i]- tokens.append(token)- if i in special_tokens.values():- toktypes.append(gguf.TokenType.CONTROL)- else:- toktypes.append(gguf.TokenType.NORMAL)-- self.gguf_writer.add_tokenizer_model("gpt2")- self.gguf_writer.add_tokenizer_pre(tokpre)- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)- self.gguf_writer.add_token_merges(merges)-- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)- special_vocab.add_to_gguf(self.gguf_writer)- else:- raise NotImplementedError(f"Deepseek pre-tokenizer {tokpre!r} is not supported yet!")-- def set_gguf_parameters(self):-- # note: deepseek2 using MLA converts into MQA (ie: GQA with 1 group)- self.hparams["num_key_value_heads"] = 1-- super().set_gguf_parameters()- hparams = self.hparams-- self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])- self.gguf_writer.add_vocab_size(hparams["vocab_size"])- if "q_lora_rank" in hparams and hparams["q_lora_rank"] is not None:- self.gguf_writer.add_q_lora_rank(hparams["q_lora_rank"])- self.gguf_writer.add_kv_lora_rank(hparams["kv_lora_rank"])-- # note: deepseek2 using MLA converts into MQA with larger heads, then decompresses to MHA- self.gguf_writer.add_key_length(hparams["kv_lora_rank"] + hparams["qk_rope_head_dim"])- self.gguf_writer.add_value_length(hparams["kv_lora_rank"])- self.gguf_writer.add_key_length_mla(hparams["qk_nope_head_dim"] + hparams["qk_rope_head_dim"])- self.gguf_writer.add_value_length_mla(hparams["v_head_dim"])-- self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])- self.gguf_writer.add_expert_count(hparams["n_routed_experts"])- self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"])- self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])- self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])-- if hparams["scoring_func"] == "sigmoid":- self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)- elif hparams["scoring_func"] == "softmax":- self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)- else:- raise ValueError(f"Unsupported scoring_func value: {hparams['scoring_func']}")-- self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])-- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])- self.gguf_writer.add_rope_scaling_yarn_log_mul(0.1 * rope_scaling["mscale_all_dim"])-- _experts: list[dict[str, Tensor]] | None = None-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # rename e_score_correction_bias tensors- if name.endswith("e_score_correction_bias"):- name = name.replace("e_score_correction_bias", "e_score_correction.bias")-- # skip Multi-Token Prediction (MTP) layers- block_count = self.hparams["num_hidden_layers"]- match = re.match(r"model.layers.(\d+)", name)- if match and int(match.group(1)) >= block_count:- return []-- # process the experts separately- if name.find("mlp.experts") != -1:- n_experts = self.hparams["n_routed_experts"]- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- tensors: list[tuple[str, Tensor]] = []-- # merge the experts into a single 3d tensor- for w_name in ["down_proj", "gate_proj", "up_proj"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)-- merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"-- new_name = self.map_tensor_name(merged_name)-- tensors.append((new_name, data_torch))- return tensors- else:- return []-- # note: MLA with the absorption optimization, needs these two split and k_b_proj transposed- if name.endswith("kv_b_proj.weight"):- name_kb = name.replace("kv_b_proj", "k_b_proj")- name_vb = name.replace("kv_b_proj", "v_b_proj")-- n_head_kv = self.hparams["num_key_value_heads"]- v_head_dim = self.hparams["v_head_dim"]- qk_nope_head_dim = self.hparams["qk_nope_head_dim"]-- assert data_torch.shape[0] == n_head_kv * (v_head_dim + qk_nope_head_dim)-- kv_b = data_torch.view(n_head_kv, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])- k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)- k_b = k_b.transpose(1, 2)-- return [(self.map_tensor_name(name_kb), k_b), (self.map_tensor_name(name_vb), v_b)]-- return [(self.map_tensor_name(name), data_torch)]-- def prepare_tensors(self):- super().prepare_tensors()-- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--[email protected]("Dots1ForCausalLM")-class Dots1Model(Qwen2MoeModel):- model_arch = gguf.MODEL_ARCH.DOTS1-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.hparams["num_experts"] = self.hparams["n_routed_experts"]-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"])- self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"])- self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])- self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])-- if self.hparams["scoring_func"] == "noaux_tc":- self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)- else:- raise ValueError(f"Unsupported scoring_func value: {self.hparams['scoring_func']}")-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):- if name.endswith("e_score_correction_bias"):- name = name.replace("e_score_correction_bias", "e_score_correction.bias")- if "shared_experts" in name:- return [(self.map_tensor_name(name), data_torch)]- return super().modify_tensors(data_torch, name, bid)--[email protected]("PLMForCausalLM")-class PLMModel(TextModel):- model_arch = gguf.MODEL_ARCH.PLM-- def set_vocab(self):- self._set_vocab_gpt2()-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- self.gguf_writer.add_vocab_size(hparams["vocab_size"])- self.gguf_writer.add_kv_lora_rank(hparams["kv_lora_rank"])- self.gguf_writer.add_key_length(hparams["qk_nope_head_dim"] + hparams["qk_rope_head_dim"])- self.gguf_writer.add_value_length(hparams["v_head_dim"])- self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- return [(self.map_tensor_name(name), data_torch)]-- def prepare_tensors(self):- super().prepare_tensors()--[email protected]("T5WithLMHeadModel")[email protected]("T5ForConditionalGeneration")[email protected]("MT5ForConditionalGeneration")[email protected]("UMT5ForConditionalGeneration")-class T5Model(TextModel):- model_arch = gguf.MODEL_ARCH.T5-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.shared_token_embeddings_found = False-- def set_vocab(self):- # to avoid TypeError: Descriptors cannot be created directly- # exception when importing sentencepiece_model_pb2- os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"- from sentencepiece import SentencePieceProcessor- from sentencepiece import sentencepiece_model_pb2 as model-- tokenizer_path = self.dir_model / "tokenizer.model"-- # many older models use spiece.model tokenizer model filename- if not tokenizer_path.is_file():- tokenizer_path = self.dir_model / "spiece.model"-- if not tokenizer_path.is_file():- raise FileNotFoundError(f"File not found: {tokenizer_path}")-- sentencepiece_model = model.ModelProto() # pylint: disable=E1101- sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())-- # some models like Pile-T5 family use BPE tokenizer instead of Unigram- if sentencepiece_model.trainer_spec.model_type == 2: # BPE- # assure the tokenizer model file name is correct- assert tokenizer_path.name == "tokenizer.model"- return self._set_vocab_sentencepiece()- else:- assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM-- add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix- remove_whitespaces = sentencepiece_model.normalizer_spec.remove_extra_whitespaces- precompiled_charsmap = sentencepiece_model.normalizer_spec.precompiled_charsmap-- tokenizer = SentencePieceProcessor()- tokenizer.LoadFromFile(str(tokenizer_path))-- vocab_size = self.hparams.get("vocab_size", tokenizer.vocab_size())-- tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]- scores: list[float] = [-10000.0] * vocab_size- toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size-- for token_id in range(tokenizer.vocab_size()):- piece = tokenizer.IdToPiece(token_id)- text = piece.encode("utf-8")- score = tokenizer.GetScore(token_id)-- toktype = SentencePieceTokenTypes.NORMAL- if tokenizer.IsUnknown(token_id):- toktype = SentencePieceTokenTypes.UNKNOWN- elif tokenizer.IsControl(token_id):- toktype = SentencePieceTokenTypes.CONTROL- elif tokenizer.IsUnused(token_id):- toktype = SentencePieceTokenTypes.UNUSED- elif tokenizer.IsByte(token_id):- toktype = SentencePieceTokenTypes.BYTE-- tokens[token_id] = text- scores[token_id] = score- toktypes[token_id] = toktype-- added_tokens_file = self.dir_model / "added_tokens.json"- if added_tokens_file.is_file():- with open(added_tokens_file, "r", encoding="utf-8") as f:- added_tokens_json = json.load(f)- for key in added_tokens_json:- token_id = added_tokens_json[key]- if token_id >= vocab_size:- logger.warning(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")- continue-- tokens[token_id] = key.encode("utf-8")- scores[token_id] = -1000.0- toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED-- if vocab_size > len(tokens):- pad_count = vocab_size - len(tokens)- logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")- for i in range(1, pad_count + 1):- tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))- scores.append(-1000.0)- toktypes.append(SentencePieceTokenTypes.UNUSED)-- self.gguf_writer.add_tokenizer_model("t5")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)- self.gguf_writer.add_add_space_prefix(add_prefix)- self.gguf_writer.add_remove_extra_whitespaces(remove_whitespaces)- if precompiled_charsmap:- self.gguf_writer.add_precompiled_charsmap(precompiled_charsmap)-- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- if (n_ctx := self.find_hparam(["n_positions"], optional=True)) is None:- logger.warning("Couldn't find context length in config.json, assuming default value of 512")- n_ctx = 512- self.gguf_writer.add_context_length(n_ctx)- self.gguf_writer.add_embedding_length(self.hparams["d_model"])- self.gguf_writer.add_feed_forward_length(self.hparams["d_ff"])- self.gguf_writer.add_block_count(self.hparams["num_layers"])- self.gguf_writer.add_head_count(self.hparams["num_heads"])- self.gguf_writer.add_key_length(self.hparams["d_kv"])- self.gguf_writer.add_value_length(self.hparams["d_kv"])- self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_relative_attn_buckets_count(self.hparams["relative_attention_num_buckets"])- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_decoder_start_token_id(self.hparams["decoder_start_token_id"])- self.gguf_writer.add_file_type(self.ftype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- if name in ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight", "shared.weight"]:- if not self.shared_token_embeddings_found:- name = "shared.weight"- self.shared_token_embeddings_found = True- else:- logger.debug(f"Skipping shared tensor {name!r} in safetensors so that convert can end normally.")- return []-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("T5EncoderModel")-class T5EncoderModel(TextModel):- model_arch = gguf.MODEL_ARCH.T5ENCODER-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.shared_token_embeddings_found = False-- def set_vocab(self):- # to avoid TypeError: Descriptors cannot be created directly- # exception when importing sentencepiece_model_pb2- os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"- from sentencepiece import SentencePieceProcessor- from sentencepiece import sentencepiece_model_pb2 as model-- tokenizer_path = self.dir_model / "tokenizer.model"-- # many older models use spiece.model tokenizer model filename- if not tokenizer_path.is_file():- tokenizer_path = self.dir_model / "spiece.model"-- if not tokenizer_path.is_file():- raise FileNotFoundError(f"File not found: {tokenizer_path}")-- sentencepiece_model = model.ModelProto() # pylint: disable=E1101- sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())-- # some models like Pile-T5 family use BPE tokenizer instead of Unigram- if sentencepiece_model.trainer_spec.model_type == 2: # BPE- # assure the tokenizer model file name is correct- assert tokenizer_path.name == "tokenizer.model"- return self._set_vocab_sentencepiece()- else:- assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM-- add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix- remove_whitespaces = sentencepiece_model.normalizer_spec.remove_extra_whitespaces- precompiled_charsmap = sentencepiece_model.normalizer_spec.precompiled_charsmap-- tokenizer = SentencePieceProcessor()- tokenizer.LoadFromFile(str(tokenizer_path))-- vocab_size = self.hparams.get("vocab_size", tokenizer.vocab_size())-- tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]- scores: list[float] = [-10000.0] * vocab_size- toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size-- for token_id in range(tokenizer.vocab_size()):- piece = tokenizer.IdToPiece(token_id)- text = piece.encode("utf-8")- score = tokenizer.GetScore(token_id)-- toktype = SentencePieceTokenTypes.NORMAL- if tokenizer.IsUnknown(token_id):- toktype = SentencePieceTokenTypes.UNKNOWN- elif tokenizer.IsControl(token_id):- toktype = SentencePieceTokenTypes.CONTROL- elif tokenizer.IsUnused(token_id):- toktype = SentencePieceTokenTypes.UNUSED- elif tokenizer.IsByte(token_id):- toktype = SentencePieceTokenTypes.BYTE-- tokens[token_id] = text- scores[token_id] = score- toktypes[token_id] = toktype-- added_tokens_file = self.dir_model / "added_tokens.json"- if added_tokens_file.is_file():- with open(added_tokens_file, "r", encoding="utf-8") as f:- added_tokens_json = json.load(f)- for key in added_tokens_json:- token_id = added_tokens_json[key]- if token_id >= vocab_size:- logger.warning(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")- continue-- tokens[token_id] = key.encode("utf-8")- scores[token_id] = -1000.0- toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED-- if vocab_size > len(tokens):- pad_count = vocab_size - len(tokens)- logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")- for i in range(1, pad_count + 1):- tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))- scores.append(-1000.0)- toktypes.append(SentencePieceTokenTypes.UNUSED)-- self.gguf_writer.add_tokenizer_model("t5")- self.gguf_writer.add_tokenizer_pre("default")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)- self.gguf_writer.add_add_space_prefix(add_prefix)- self.gguf_writer.add_remove_extra_whitespaces(remove_whitespaces)- if precompiled_charsmap:- self.gguf_writer.add_precompiled_charsmap(precompiled_charsmap)-- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- if (n_ctx := self.find_hparam(["n_positions"], optional=True)) is None:- logger.warning("Couldn't find context length in config.json, assuming default value of 512")- n_ctx = 512- self.gguf_writer.add_context_length(n_ctx)- self.gguf_writer.add_embedding_length(self.hparams["d_model"])- self.gguf_writer.add_feed_forward_length(self.hparams["d_ff"])- self.gguf_writer.add_block_count(self.hparams["num_layers"])- self.gguf_writer.add_head_count(self.hparams["num_heads"])- self.gguf_writer.add_key_length(self.hparams["d_kv"])- self.gguf_writer.add_value_length(self.hparams["d_kv"])- self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_relative_attn_buckets_count(self.hparams["relative_attention_num_buckets"])- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_file_type(self.ftype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- if name in ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight", "shared.weight"]:- if not self.shared_token_embeddings_found:- name = "shared.weight"- self.shared_token_embeddings_found = True- else:- logger.debug(f"Skipping shared tensor {name!r} in safetensors so that convert can end normally.")- return []-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("JAISLMHeadModel")-class JaisModel(TextModel):- model_arch = gguf.MODEL_ARCH.JAIS-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)-- # SwigLU activation- assert self.hparams["activation_function"] == "swiglu"- # ALiBi position embedding- assert self.hparams["position_embedding_type"] == "alibi"-- # Embeddings scale- self.embeddings_scale = 1.0- if "mup_embeddings_scale" in self.hparams:- self.embeddings_scale = self.hparams["mup_embeddings_scale"]- elif "embeddings_scale" in self.hparams:- self.embeddings_scale = self.hparams["embeddings_scale"]- else:- assert False-- self.width_scale = 1.0- if "mup_output_alpha" in self.hparams:- assert "mup_width_scale" in self.hparams- self.width_scale = self.hparams["mup_output_alpha"] * self.hparams["mup_width_scale"]- elif "width_scale" in self.hparams:- self.width_scale = self.hparams["width_scale"]- else:- assert False-- self.max_alibi_bias = 8.0-- def set_vocab(self):- self._set_vocab_gpt2()-- def set_gguf_parameters(self):- self.gguf_writer.add_block_count(self.hparams["n_layer"])- self.gguf_writer.add_context_length(self.hparams["n_positions"])- self.gguf_writer.add_embedding_length(self.hparams["n_embd"])- self.gguf_writer.add_feed_forward_length(self.hparams["n_inner"])- self.gguf_writer.add_head_count(self.hparams["n_head"])- self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_file_type(self.ftype)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- tensors: list[tuple[str, Tensor]] = []-- # we don't need these- if name.endswith((".attn.bias")):- return tensors-- if name.endswith(("relative_pe.slopes")):- # Calculate max ALiBi bias (this is the inverse of the ALiBi calculation)- # Some other models has max_alibi_bias spelled out explicitly in the hyperparams,- # but Jais's PyTorch model simply precalculates the slope values and places them- # in relative_pes.slopes- n_head_closest_log2 = 2 ** math.floor(math.log2(self.hparams["n_head"]))- first_val = float(data_torch[0].item())- self.max_alibi_bias = -round(math.log2(first_val) * n_head_closest_log2)-- return tensors-- if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight", ".c_fc2.weight")):- data_torch = data_torch.transpose(1, 0)-- new_name = self.map_tensor_name(name)-- if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD):- tensors.append((new_name, data_torch * self.embeddings_scale))- elif new_name == self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT):- tensors.append((new_name, data_torch * self.width_scale))- else:- tensors.append((new_name, data_torch))-- return tensors-- def prepare_tensors(self):- super().prepare_tensors()- self.gguf_writer.add_max_alibi_bias(self.max_alibi_bias)--[email protected]("Glm4ForCausalLM")-class Glm4Model(TextModel):- model_arch = gguf.MODEL_ARCH.GLM4-- def set_vocab(self):- from transformers import AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)- tokens, toktypes, tokpre = self.get_vocab_base()- self.gguf_writer.add_tokenizer_model("gpt2")- self.gguf_writer.add_tokenizer_pre(tokpre)- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)- special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])- special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])- special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- super().set_gguf_parameters()- rope_dim = self.hparams["head_dim"]- self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)))- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])--[email protected]("GlmForCausalLM", "ChatGLMModel", "ChatGLMForConditionalGeneration")-class ChatGLMModel(TextModel):- model_arch = gguf.MODEL_ARCH.CHATGLM-- def set_vocab_chatglm3(self):- dir_model = self.dir_model- hparams = self.hparams- tokens: list[bytes] = []- toktypes: list[int] = []- scores: list[float] = []-- from transformers import AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)- vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab()))- assert max(tokenizer.get_vocab().values()) < vocab_size- role_special_tokens = ["<|system|>", "<|user|>", "<|assistant|>", "<|observation|>"]- special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"] + role_special_tokens- for token_id in range(vocab_size):- piece = tokenizer._convert_id_to_token(token_id)- if token_id == 0:- piece = "<unk>"- elif token_id == 1:- piece = "<bos>"- elif token_id == 2:- piece = "<eos>"-- text = piece.encode("utf-8")- score = 0.0- if len(piece) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size():- score = tokenizer.tokenizer.sp_model.get_score(token_id)-- if token_id >= tokenizer.tokenizer.sp_model.vocab_size():- if piece in special_tokens:- toktype = SentencePieceTokenTypes.CONTROL- elif len(piece) == 0:- text = f"[PAD{token_id}]".encode("utf-8")- toktype = SentencePieceTokenTypes.UNUSED- else:- toktype = SentencePieceTokenTypes.USER_DEFINED- tokens.append(text)- scores.append(score)- toktypes.append(toktype)- continue-- toktype = SentencePieceTokenTypes.NORMAL- if tokenizer.tokenizer.sp_model.is_unknown(token_id):- toktype = SentencePieceTokenTypes.UNKNOWN- elif tokenizer.tokenizer.sp_model.is_control(token_id):- toktype = SentencePieceTokenTypes.CONTROL- elif tokenizer.tokenizer.sp_model.is_unused(token_id):- toktype = SentencePieceTokenTypes.UNUSED- elif tokenizer.tokenizer.sp_model.is_byte(token_id):- toktype = SentencePieceTokenTypes.BYTE-- tokens.append(text)- scores.append(score)- toktypes.append(toktype)-- self.gguf_writer.add_tokenizer_model("llama")- # glm3 needs prefix and suffix formatted as:- # prompt = "[gMASK]sop<|user|>\n" + prompt + "<|assistant|>"- self.gguf_writer.add_tokenizer_pre("chatglm-spm")- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_scores(scores)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- special_vocab.add_to_gguf(self.gguf_writer)-- @staticmethod- def token_bytes_to_string(b):- from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode-- byte_encoder = bytes_to_unicode()- return "".join([byte_encoder[ord(char)] for char in b.decode("latin-1")])-- @staticmethod- def bpe(mergeable_ranks: dict[bytes, int], token: bytes, max_rank: int | None = None) -> list[bytes]:- parts = [bytes([b]) for b in token]- while True:- min_idx = None- min_rank = None- for i, pair in enumerate(zip(parts[:-1], parts[1:])):- rank = mergeable_ranks.get(pair[0] + pair[1])- if rank is not None and (min_rank is None or rank < min_rank):- min_idx = i- min_rank = rank- if min_rank is None or (max_rank is not None and min_rank >= max_rank):- break- assert min_idx is not None- parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2 :]- return parts-- def set_vocab(self):- if "THUDM/chatglm3-6b" in self.hparams.get("_name_or_path", ""):- self.set_vocab_chatglm3()- return-- dir_model = self.dir_model- hparams = self.hparams- tokens: list[str] = []- toktypes: list[int] = []-- from transformers import AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)- vocab_size = hparams.get("padded_vocab_size", hparams["vocab_size"])- assert max(tokenizer.get_vocab().values()) < vocab_size-- tokens, toktypes, tokpre = self.get_vocab_base()- self.gguf_writer.add_tokenizer_model("gpt2")- self.gguf_writer.add_tokenizer_pre(tokpre)- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)- # only add special tokens when they were not already loaded from config.json- special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])- # this one is usually not in config.json anyway- special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))- n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))- n_head_kv = self.hparams.get("multi_query_group_num", self.hparams.get("num_key_value_heads", n_head))- self.gguf_writer.add_context_length(self.hparams.get("seq_length", n_embed))- self.gguf_writer.add_embedding_length(n_embed)- self.gguf_writer.add_feed_forward_length(- self.hparams.get("ffn_hidden_size", self.hparams.get("intermediate_size", 4 * n_embed))- )- self.gguf_writer.add_block_count(self.hparams.get("num_layers", self.hparams["num_hidden_layers"]))- self.gguf_writer.add_head_count(n_head)- self.gguf_writer.add_head_count_kv(n_head_kv)- self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("layernorm_epsilon", 1e-5))- self.gguf_writer.add_file_type(self.ftype)- if "attention_dim" in self.hparams:- rope_dim = self.hparams["attention_dim"]- else:- rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]- self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)))- self.gguf_writer.add_add_bos_token(False)- rope_freq = 10000- if "rope_ratio" in self.hparams:- rope_freq = rope_freq * self.hparams["rope_ratio"]- self.gguf_writer.add_rope_freq_base(rope_freq)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision."):- return []-- name = name.removeprefix("transformer.")- return [(self.map_tensor_name(name), data_torch)]--[email protected]("NemotronForCausalLM")-class NemotronModel(TextModel):- model_arch = gguf.MODEL_ARCH.NEMOTRON-- def set_vocab(self):- self._set_vocab_sentencepiece()- self.gguf_writer.add_pad_token_id(0)- self.gguf_writer.add_unk_token_id(1)-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- self.gguf_writer.add_vocab_size(hparams["vocab_size"])-- f_norm_eps = self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon", "norm_eps"])- self.gguf_writer.add_layer_norm_eps(f_norm_eps)-- # * Partial RoPE- rot_pct = self.find_hparam(["partial_rotary_factor", "rope_pct", "rope_percent"])- n_embd = self.find_hparam(["hidden_size", "n_embd"])- n_head = self.find_hparam(["num_attention_heads", "n_head"])- self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head)-- # * RopeScaling for Nemotron- if "rope_scaling" not in self.hparams or self.hparams["rope_scaling"] is None:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)- else:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(self.hparams["factor"])-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- if name.endswith("norm.weight"):- data_torch = data_torch + 1-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("ExaoneForCausalLM")-class ExaoneModel(TextModel):- model_arch = gguf.MODEL_ARCH.EXAONE-- def set_gguf_parameters(self):- hparams = self.hparams-- assert hparams["activation_function"] == "silu"-- max_position_embeddings = hparams["max_position_embeddings"]- embed_dim = hparams["hidden_size"]- num_heads = hparams["num_attention_heads"]- num_kv_heads = hparams.get("num_key_value_heads", num_heads)- layer_norm_eps = hparams["layer_norm_epsilon"]- intermediate_size = hparams["intermediate_size"] if "intermediate_size" in hparams else 4 * embed_dim- num_layers = hparams["num_layers"]- # ignore for now as EXAONE-3.0-7.8B-Instruct attentino_dropout is 0.0- # attention_dropout_rate = hparams["attention_dropout"]- # ignore for now as EXAONE-3.0-7.8B-Instruct embed_dropout is 0.0- # embed_dropout_rate = hparams["embed_dropout"]- self.gguf_writer.add_embedding_length(embed_dim)- self.gguf_writer.add_head_count(num_heads)- self.gguf_writer.add_head_count_kv(num_kv_heads)- self.gguf_writer.add_context_length(max_position_embeddings)- self.gguf_writer.add_layer_norm_rms_eps(layer_norm_eps)- self.gguf_writer.add_feed_forward_length(intermediate_size)- self.gguf_writer.add_block_count(num_layers)- self.gguf_writer.add_file_type(self.ftype)-- if (rope_theta := self.hparams.get("rope_theta")) is not None:- self.gguf_writer.add_rope_freq_base(rope_theta)- rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"], optional=True)- rotary_factor = rotary_factor if rotary_factor is not None else 1.0- self.gguf_writer.add_rope_dimension_count(- int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))- )- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])-- def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):- if rope_scaling.get("rope_type", "").lower() == "llama3":- base = self.hparams.get("rope_theta", 10000.0)- if (dim := self.hparams.get("head_dim")) is None:- dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]- freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))-- factor = rope_scaling.get("factor", 8.0)- low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)- high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)- old_context_len = self.hparams.get("original_max_position_embeddings", 8192)-- low_freq_wavelen = old_context_len / low_freq_factor- high_freq_wavelen = old_context_len / high_freq_factor- assert low_freq_wavelen != high_freq_wavelen-- rope_factors = []- for freq in freqs:- wavelen = 2 * math.pi / freq- if wavelen < high_freq_wavelen:- rope_factors.append(1)- elif wavelen > low_freq_wavelen:- rope_factors.append(factor)- else:- smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)- rope_factors.append(1 / ((1 - smooth) / factor + smooth))-- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32),- )--[email protected]("Exaone4ForCausalLM")-class Exaone4Model(TextModel):- model_arch = gguf.MODEL_ARCH.EXAONE4-- def set_vocab(self):- tokens, toktypes, tokpre = self.get_vocab_base()- self.gguf_writer.add_tokenizer_model("gpt2")- self.gguf_writer.add_tokenizer_pre(tokpre)- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)- special_vocab.add_to_gguf(self.gguf_writer)-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- self.gguf_writer.add_vocab_size(hparams["vocab_size"])-- if hparams.get("sliding_window") is not None:- self.gguf_writer.add_sliding_window(hparams["sliding_window"])- if "layer_types" in hparams:- self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]])- elif "sliding_window_pattern" in hparams:- sliding_window_pattern = []- if isinstance(hparams["sliding_window_pattern"], str): # e.g. LLLG- for i in range(hparams["num_hidden_layers"]):- sliding_window_pattern.append(- hparams["sliding_window_pattern"][i % len(hparams["sliding_window_pattern"])] == "L"- )- if isinstance(hparams["sliding_window_pattern"], int): # e.g. 4- for i in range(hparams["num_hidden_layers"]):- sliding_window_pattern.append((i + 1) % hparams["sliding_window_pattern"] != 0)- if len(sliding_window_pattern) == hparams["num_hidden_layers"]:- self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)-- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])-- def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):- if rope_scaling.get("rope_type", "").lower() == "llama3":- base = self.hparams.get("rope_theta", 10_000.0)- if (dim := self.hparams.get("head_dim")) is None:- dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]- freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))-- factor = rope_scaling.get("factor", 16.0)- low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)- high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)- old_context_len = self.hparams.get("original_max_position_embeddings", 8192)-- low_freq_wavelen = old_context_len / low_freq_factor- high_freq_wavelen = old_context_len / high_freq_factor-- rope_factors = []- for freq in freqs:- wavelen = 2 * math.pi / freq- if wavelen < high_freq_wavelen:- rope_factors.append(1)- elif wavelen > low_freq_wavelen:- rope_factors.append(factor)- else:- smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)- rope_factors.append(1 / ((1 - smooth) / factor + smooth))-- yield (- self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32),- )--[email protected]("GraniteForCausalLM")-class GraniteModel(LlamaModel):- """Conversion for IBM's GraniteForCausalLM"""-- model_arch = gguf.MODEL_ARCH.GRANITE-- def set_gguf_parameters(self):- """Granite uses standard llama parameters with the following differences:-- - No head_dim support- - New multiplier params:- - attention_scale- - embedding_scale- - residual_scale- - logits_scaling- """- if head_dim := self.hparams.pop("head_dim", None):- logger.warning("Ignoring head_dim (%s) from config for Granite", head_dim)- super().set_gguf_parameters()- # NOTE: Convert _multiplier params to _scale params for naming- # consistency- if attention_scale := self.hparams.get("attention_multiplier"):- self.gguf_writer.add_attention_scale(attention_scale)- logger.info("gguf: (granite) attention_scale = %s", attention_scale)- if embedding_scale := self.hparams.get("embedding_multiplier"):- self.gguf_writer.add_embedding_scale(embedding_scale)- logger.info("gguf: (granite) embedding_scale = %s", embedding_scale)- if residual_scale := self.hparams.get("residual_multiplier"):- self.gguf_writer.add_residual_scale(residual_scale)- logger.info("gguf: (granite) residual_scale = %s", residual_scale)- if logits_scale := self.hparams.get("logits_scaling"):- self.gguf_writer.add_logit_scale(logits_scale)- logger.info("gguf: (granite) logits_scale = %s", logits_scale)--[email protected]("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM")-class GraniteMoeModel(GraniteModel):- """Conversion for IBM's GraniteMoeForCausalLM"""-- model_arch = gguf.MODEL_ARCH.GRANITE_MOE-- def set_gguf_parameters(self):- """GraniteMoeShared uses GraniteMoe parameters plus the following:- - shared_intermediate_size- """- super().set_gguf_parameters()- if shared_feed_forward_length := self.hparams.get("shared_intermediate_size"):- self.gguf_writer.add_expert_shared_feed_forward_length(shared_feed_forward_length)- logger.info("gguf: (granitemoeshared) shared_feed_forward_length = %s", shared_feed_forward_length)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- """In modeling_granitemoe, the JetMoe implementation of parallel experts- is used. This essentially merges w1 and w3 into a single tensor with 2x- the hidden size that is then split during forward. To keep compatibility- with existing mixtral support, we pull them apart here.- """-- if name.endswith("block_sparse_moe.input_linear.weight"):- ffn_dim = self.hparams["intermediate_size"]- assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size"- gate, up = data_torch.split(ffn_dim, dim=-2)- return [- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), gate),- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), up),- ]-- has_experts = bool(self.hparams.get("num_local_experts"))-- if name.endswith("shared_mlp.input_linear.weight"):- ffn_dim = self.hparams["shared_intermediate_size"]- assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size"- gate, up = data_torch.split(ffn_dim, dim=-2)- if has_experts:- return [- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_SHEXP, bid), gate),- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), up),- ]- return [- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), gate),- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), up),- ]-- if not has_experts and name.endswith("shared_mlp.output_linear.weight"):- return [(self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN, bid), data_torch)]-- return super().modify_tensors(data_torch, name, bid)--[email protected]("GraniteMoeHybridForCausalLM", "BambaForCausalLM")-class GraniteHybridModel(Mamba2Model, GraniteMoeModel):- """GraniteHybrid is a hybrid SSM + Attention model that uses Mamba2 SSM- layers and optionally uses MoE w/ a shared expert"""-- model_arch = gguf.MODEL_ARCH.GRANITE_HYBRID- undo_permute = True-- def __init__(self, *args, **kwargs):-- # Hybrid mamba models use a prefix for the mamba-specific params.- # TODO: Extend this if the prefix(es) need to be configurable- self.hparam_prefixes = ["mamba"]-- super().__init__(*args, **kwargs)-- # Lists of which layers use ssm vs attention- self._attn_layers = self.get_attn_layers()- self._ssm_layers = [i for i in range(self.block_count) if i not in self._attn_layers]-- # n_group and d_inner are used during reshape_tensors for mamba2- self.d_model = self.find_hparam(["hidden_size", "d_model"])- self.n_group = self.find_hparam(["n_groups"])- self.d_inner = self.find_hparam(["expand"]) * self.d_model-- def get_attn_layers(self):- # Explicit list of layer type names- if layer_types := self.hparams.get("layer_types"):- return [i for i, typ in enumerate(layer_types) if typ == "attention"]-- # Layer types indicated by index or period- attn_layers = self.hparams.get("attn_layer_indices", [])- if not attn_layers:- attn_period = self.hparams.get("attn_layer_period")- assert attn_period, "Didn't find attn_layer_indices or attn_layer_period"- attn_offset = self.hparams.get("attn_layer_offset")- assert attn_offset is not None, "No attention layer offset set with attn_layer_period"- attn_layers = [i for i in range(self.block_count) if i % attn_period == attn_offset]- return attn_layers-- def find_hparam(self, keys: Iterable[str], *args, **kwargs) -> Any:- prefixed = []- for pfx in self.hparam_prefixes:- prefixed.extend("_".join([pfx, k]) for k in keys)- keys = list(keys) + prefixed- return Mamba2Model.find_hparam(self, keys, *args, **kwargs)-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- if name.endswith("block_sparse_moe.input_linear.weight") or "shared_mlp" in name:- return GraniteMoeModel.modify_tensors(self, data_torch, name, bid)-- # Determine whether this is a mamba layer or an attention layer- if bid in self._ssm_layers:- return Mamba2Model.modify_tensors(self, data_torch, name, bid)- elif bid in self._attn_layers:- return GraniteMoeModel.modify_tensors(self, data_torch, name, bid)- return [(self.map_tensor_name(name), data_torch)]-- def set_gguf_parameters(self):- """This method merges params from both parents and some that are- specific to this model. The result is some duplication of how the params- get set. The following warnings are expected during conversion:-- WARNING:Duplicated key name 'granitehybrid.attention.head_count_kv'- WARNING:Duplicated key name 'granitehybrid.context_length'- """- GraniteMoeModel.set_gguf_parameters(self)-- ## Mamba mixer params ##- self.gguf_writer.add_ssm_conv_kernel(self.find_hparam(["conv_kernel", "d_conv"]))- self.gguf_writer.add_ssm_state_size(self.find_hparam(["state_size", "d_state"]))- self.gguf_writer.add_ssm_group_count(self.n_group)- self.gguf_writer.add_ssm_inner_size(self.d_inner)- # NOTE: The mamba_dt_rank is _not_ the right field for how this is used- # in llama.cpp- self.gguf_writer.add_ssm_time_step_rank(self.find_hparam(["n_heads"]))-- ## Attention params ##- head_count_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"])- head_count_kv_vec = [head_count_kv if i in self._attn_layers else 0 for i in range(self.block_count)]- if rope_dim := self.hparams.get("attn_rotary_emb"):- self.gguf_writer.add_rope_dimension_count(rope_dim)- self.gguf_writer.add_head_count_kv(head_count_kv_vec)-- ## If Bamba, use rope, otherwise don't- use_rope = "BambaForCausalLM" in self.hparams["architectures"]- self.gguf_writer.add_rope_scaling_finetuned(use_rope)- if not use_rope:- self.gguf_writer.add_context_length(2**20)-- ## Validation ##- d_head = self.find_hparam(["d_head"], optional=True) or 64- assert self.hparams.get("hidden_act") in [None, "silu"], "Only SILU activation supported"- assert self.d_inner % d_head == 0, f"SSM inner size {self.d_inner} not a multiple of head dim {d_head}"-- def set_vocab(self):- self.hparams["pad_vocab_size_multiple"] = 8- Mamba2Model.set_vocab(self)--[email protected]("BailingMoeForCausalLM")-class BailingMoeModel(TextModel):- model_arch = gguf.MODEL_ARCH.BAILINGMOE-- def set_vocab(self):- self._set_vocab_gpt2()-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams- if (rope_dim := hparams.get("head_dim")) is None:- rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]-- self.gguf_writer.add_rope_dimension_count(rope_dim)- rope_scaling = self.hparams.get("rope_scaling") or {}- if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])- self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])- else:- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)- self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])- self.gguf_writer.add_vocab_size(hparams["vocab_size"])- self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])- self.gguf_writer.add_expert_weights_scale(1.0)- self.gguf_writer.add_expert_count(hparams["num_experts"])- self.gguf_writer.add_expert_shared_count(hparams["num_shared_experts"])- self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])-- _experts: list[dict[str, Tensor]] | None = None-- @staticmethod- def permute(weights: Tensor, n_head: int, n_head_kv: int | None):- if n_head_kv is not None and n_head != n_head_kv:- n_head = n_head_kv- return (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])- .swapaxes(1, 2)- .reshape(weights.shape)- )-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- n_head = self.hparams["num_attention_heads"]- n_kv_head = self.hparams.get("num_key_value_heads")- n_embd = self.hparams["hidden_size"]- if (head_dim := self.hparams.get("head_dim")) is None:- head_dim = n_embd // n_head-- output_name = self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT)-- if name.endswith("attention.dense.weight"):- return [(self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_OUT, bid), data_torch)]- elif name.endswith("query_key_value.weight"):- q, k, v = data_torch.split([n_head * head_dim, n_kv_head * head_dim, n_kv_head * head_dim], dim=-2)-- return [- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), BailingMoeModel.permute(q, n_head, n_head)),- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), BailingMoeModel.permute(k, n_head, n_kv_head)),- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), v),- ]- elif name.find("mlp.experts") != -1:- n_experts = self.hparams["num_experts"]- assert bid is not None-- tensors: list[tuple[str, Tensor]] = []-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- # merge the experts into a single 3d tensor- for w_name in ["down_proj", "gate_proj", "up_proj"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)-- merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"-- new_name = self.map_tensor_name(merged_name)-- tensors.append((new_name, data_torch))-- return tensors-- new_name = self.map_tensor_name(name)-- if new_name == output_name and self.hparams.get("norm_head"):- data_torch = data_torch.float()- data_torch /= torch.norm(data_torch, p=2, dim=0, keepdim=True) + 1e-7-- return [(new_name, data_torch)]-- def prepare_tensors(self):- super().prepare_tensors()-- if self._experts is not None:- # flatten `list[dict[str, Tensor]]` into `list[str]`- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--[email protected]("ChameleonForConditionalGeneration")[email protected]("ChameleonForCausalLM") # obsolete-class ChameleonModel(TextModel):- model_arch = gguf.MODEL_ARCH.CHAMELEON-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_swin_norm(self.hparams.get("swin_norm", False))-- def set_vocab(self):- self._set_vocab_gpt2()-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # ignore image tokenizer for now- # TODO: remove this once image support is implemented for Chameleon- if name.startswith("model.vqmodel"):- return []-- n_head = self.hparams["num_attention_heads"]- n_kv_head = self.hparams.get("num_key_value_heads")- hidden_dim = self.hparams.get("hidden_size")-- if name.endswith(("q_proj.weight", "q_proj.bias")):- data_torch = LlamaModel.permute(data_torch, n_head, n_head)- if name.endswith(("k_proj.weight", "k_proj.bias")):- data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)- if name.endswith(("q_norm.weight", "q_norm.bias")):- data_torch = ChameleonModel._reverse_hf_permute(data_torch, n_head, hidden_dim)- if name.endswith(("k_norm.weight", "k_norm.bias")):- data_torch = ChameleonModel._reverse_hf_permute(data_torch, n_kv_head, hidden_dim)-- return [(self.map_tensor_name(name), data_torch)]-- @staticmethod- def _reverse_hf_permute(data_torch, n_heads, hidden_dim):- head_dim = hidden_dim // n_heads- data_torch = data_torch[0].view(2, head_dim // 2).t().reshape(1, -1)- data_torch = data_torch.repeat_interleave(n_heads, 0)- return data_torch--[email protected]("UltravoxModel")-class UltravoxModel(TextModel):- model_arch = gguf.MODEL_ARCH.LLAMA # dummy-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- raise NotImplementedError(- "Ultravox does not have text decoder. Instead, it uses Llama or other models for text."- " If you want to get the audio encoder, please use --mmproj argument"- )--[email protected]("Qwen2AudioForConditionalGeneration")-class WhisperEncoderModel(MmprojModel):- has_vision_encoder = False # no vision encoder- has_audio_encoder = True-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.hparams["hidden_size"] = self.hparams["d_model"]- self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"]- self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"]-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN2A)- self.gguf_writer.add_audio_num_mel_bins(self.hparams["num_mel_bins"])- self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))-- def tensor_force_quant(self, name, new_name, bid, n_dims):- del bid, new_name, n_dims # unused- if ".conv" in name and ".weight" in name:- return gguf.GGMLQuantizationType.F16- return False-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- del bid # unused-- if name.startswith("language_model."):- # skip language model tensors- return []-- # prevent clash naming with vision tensors- if name.startswith("multi_modal_projector"):- name = "audio." + name-- if "conv1.bias" in name or "conv2.bias" in name:- # transpose conv1 and conv2 bias- data_torch = data_torch.unsqueeze(-1)-- return [(self.map_tensor_name(name), data_torch)]--[email protected]("UltravoxModel")-class UltravoxWhisperEncoderModel(WhisperEncoderModel):- has_vision_encoder = False # no vision encoder- has_audio_encoder = True-- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_audio_stack_factor(self.global_config["stack_factor"])--[email protected]("FalconH1ForCausalLM")-class FalconH1Model(Mamba2Model):- model_arch = gguf.MODEL_ARCH.FALCON_H1-- def __init__(self, *args, **kwargs):- # Set the hparam prefixes for Falcon Mamba2- self.hparam_prefixes = ["mamba"]-- # Initialize the base Mamba2Model- super().__init__(*args, **kwargs)-- # Use Llama conversion for attention- self._transformer_model_class = LlamaModel-- # n_group and d_inner are used during reshape_tensors for mamba2- self.n_group = self.find_hparam(["n_groups"])- self.d_inner = self.find_hparam(["mamba_d_ssm"])- self.d_head = self.find_hparam(["d_head"])-- # Initialize any Falcon Mamba2 specific attributes- self.has_attention = True # Falcon Mamba2 has attention components-- # Load Falcon-H1 multipliers from hyperparameters- self.attention_in_multiplier = self.find_hparam(["attention_in_multiplier"], optional=True)- self.attention_out_multiplier = self.find_hparam(["attention_out_multiplier"], optional=True)- self.ssm_in_multiplier = self.find_hparam(["ssm_in_multiplier"], optional=True)- self.ssm_out_multiplier = self.find_hparam(["ssm_out_multiplier"], optional=True)- self.mlp_multipliers = self.find_hparam(["mlp_multipliers"], optional=True)- self.ssm_multipliers = self.find_hparam(["ssm_multipliers"], optional=True)- self.intermediate_size = self.find_hparam(["intermediate_size"])- self.key_multiplier = self.find_hparam(["key_multiplier"], optional=True)-- def find_hparam(self, keys: Iterable[str], *args, **kwargs) -> Any:- prefixed = []- for pfx in self.hparam_prefixes:- prefixed.extend("_".join([pfx, k]) for k in keys)- keys = list(keys) + prefixed- return super().find_hparam(keys, *args, **kwargs)-- def set_vocab(self):- self._set_vocab_gpt2()-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- tensors = list(super().modify_tensors(data_torch, name, bid))- tensor = tensors[0][1]-- if "down_proj" in name:- tensor = tensor * self.mlp_multipliers[1]- elif "gate_proj" in name:- tensor = tensor * self.mlp_multipliers[0]- elif "k_proj" in name:- tensor = tensor * self.key_multiplier * self.attention_in_multiplier- elif "q_proj" in name:- tensor = tensor * self.attention_in_multiplier- elif "v_proj" in name:- tensor = tensor * self.attention_in_multiplier- elif "o_proj" in name:- tensor = tensor * self.attention_out_multiplier- elif "out_proj" in name:- tensor = tensor * self.ssm_out_multiplier- elif "in_proj" in name:- tensor = tensor * self.ssm_in_multiplier- zxbcdt_multipliers = self.hparams["ssm_multipliers"]- intermediate_size = self.hparams["mamba_d_ssm"]- groups_time_state_size = self.hparams["mamba_n_groups"] * self.hparams["mamba_d_state"]- tensor[:intermediate_size, :] *= zxbcdt_multipliers[0]- tensor[intermediate_size : 2 * intermediate_size, :] *= zxbcdt_multipliers[1]- tensor[2 * intermediate_size : 2 * intermediate_size + groups_time_state_size, :] *= zxbcdt_multipliers[2]- tensor[- 2 * intermediate_size + groups_time_state_size : 2 * intermediate_size + 2 * groups_time_state_size, :- ] *= zxbcdt_multipliers[3]- tensor[2 * intermediate_size + 2 * groups_time_state_size :, :] *= zxbcdt_multipliers[4]- elif "lm_head" in name:- tensor = tensor * self.hparams["lm_head_multiplier"]- elif "embed_tokens" in name:- tensor = tensor * self.hparams["embedding_multiplier"]- elif "mamba.norm" in name:- tensor = tensor.reshape(self.n_group, self.d_inner // self.n_group)-- tensors = [(tensors[0][0], tensor)]- return tensors-- def set_gguf_parameters(self):- super().set_gguf_parameters()-- ## General Params ##- self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])- # Override some Mamba2 defaults- self.gguf_writer.add_block_count(self.block_count)- self.gguf_writer.add_context_length(self.hparams.get("max_position_embeddings", 0))- self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])-- ## Attention params ##- self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) # Override value 0 from Mamba2- self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"])- self.gguf_writer.add_key_length(self.hparams["head_dim"])- self.gguf_writer.add_value_length(self.hparams["head_dim"])-- ## Validation ##- assert self.hparams.get("hidden_act") in [None, "silu"], "Only SILU activation supported"- assert (- self.d_inner % self.d_head == 0- ), f"SSM inner size {self.d_inner} not a multiple of head dim {self.d_head}"-- # Add any other Falcon Mamba2 specific configuration- self.gguf_writer.add_rope_freq_base(self.find_hparam(["rope_theta"]))--[email protected]("HunYuanMoEV1ForCausalLM")-class HunYuanMoEModel(TextModel):- model_arch = gguf.MODEL_ARCH.HUNYUAN_MOE-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- # For handling tied embeddings- self._tok_embd = None-- def set_vocab(self):- from transformers import AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)-- # 1. Get the pre-tokenizer identifier hash- tokpre = self.get_vocab_base_pre(tokenizer)-- # 2. Reverse-engineer the merges list from mergeable_ranks- merges = []- vocab = {}- mergeable_ranks = tokenizer.mergeable_ranks- for token, rank in mergeable_ranks.items():- vocab[QwenModel.token_bytes_to_string(token)] = rank- if len(token) == 1:- continue- merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)- if len(merged) == 2: # todo this is an assert in Qwen, why?- merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))-- # 3. Generate the tokens and toktypes lists- vocab_size = self.hparams["vocab_size"]- assert tokenizer.vocab_size == vocab_size- special_tokens = tokenizer.special_tokens- reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}- tokens: list[str] = []- toktypes: list[int] = []- for i in range(vocab_size):- if i not in reverse_vocab:- tokens.append(f"[PAD{i}]")- toktypes.append(gguf.TokenType.UNUSED)- else:- token = reverse_vocab[i]- tokens.append(token)- if i in special_tokens.values():- toktypes.append(gguf.TokenType.CONTROL)- else:- toktypes.append(gguf.TokenType.NORMAL)-- # 4. Write all vocab-related fields to the GGUF writer- self.gguf_writer.add_tokenizer_model("gpt2")- self.gguf_writer.add_tokenizer_pre(tokpre)- self.gguf_writer.add_token_list(tokens)- self.gguf_writer.add_token_types(toktypes)- self.gguf_writer.add_token_merges(merges)-- # 5. Add special tokens and chat templates- special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)- special_vocab.add_to_gguf(self.gguf_writer)- # FIX for BOS token: Overwrite incorrect id read from config.json- self.gguf_writer.add_bos_token_id(127959) # <|bos|>-- def set_gguf_parameters(self):- super().set_gguf_parameters()- hparams = self.hparams-- self.gguf_writer.add_expert_count(hparams["num_experts"])- self.gguf_writer.add_expert_shared_feed_forward_length(hparams["intermediate_size"])-- moe_intermediate_size = hparams["moe_intermediate_size"]- assert all(n == moe_intermediate_size[0] for n in moe_intermediate_size)- self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size[0])-- moe_topk = hparams["moe_topk"]- assert all(topk == moe_topk[0] for topk in moe_topk)- self.gguf_writer.add_expert_used_count(moe_topk[0])-- moe_shared_expert = hparams["num_shared_expert"]- assert all(n == moe_shared_expert[0] for n in moe_shared_expert)- self.gguf_writer.add_expert_shared_count(moe_shared_expert[0])-- # Rope- rope_scaling = hparams.get("rope_scaling", {})- if rope_scaling.get("type") == "dynamic":- alpha = rope_scaling.get("alpha", 1000)- base = hparams.get("rope_theta", 10000.0)- dim = hparams["hidden_size"] // hparams["num_attention_heads"] # 128- scaled_base = base * (alpha ** (dim / (dim - 2))) # 10000 * (1000 ** (128 / 126)) = 11158839.9251- self.gguf_writer.add_rope_freq_base(scaled_base)- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)- self.gguf_writer.add_rope_scaling_factor(1)- # There is no consistent way to calculate ctx from alpha, and the config is incorrectly set to 32k- self.gguf_writer.add_rope_scaling_orig_ctx_len(256 * 1024) # 256k context length- self.gguf_writer.add_context_length(256 * 1024) # 256k context length-- assert (- alpha == 1000- and base == 10000.0- and dim == 128- and self.hparams["max_position_embeddings"] in [32 * 1024, 256 * 1024]- ), "HunYuan dynamic RoPE scaling assumptions changed, please update the logic or context length manually"-- _experts: list[dict[str, Tensor]] | None = None-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- if name == "model.embed_tokens.weight":- self._tok_embd = data_torch.clone()-- if name == "lm_head.weight":- if self.hparams.get("tie_word_embeddings", False):- logger.info("Skipping tied output layer 'lm_head.weight'")- return []-- if name.find("mlp.experts") != -1:- n_experts = self.hparams["num_experts"]- assert bid is not None-- if self._experts is None:- self._experts = [{} for _ in range(self.block_count)]-- self._experts[bid][name] = data_torch-- if len(self._experts[bid]) >= n_experts * 3:- # merge the experts into a single 3d tensor- tensors: list[tuple[str, Tensor]] = []- for w_name in ["down_proj", "gate_proj", "up_proj"]:- data: list[Tensor] = []-- for xid in range(n_experts):- ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"- data.append(self._experts[bid][ename])- del self._experts[bid][ename]-- data_torch = torch.stack(data, dim=0)- merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"- new_name = self.map_tensor_name(merged_name)- tensors.append((new_name, data_torch))-- return tensors- else:- return []-- return [(self.map_tensor_name(name), data_torch)]-- def prepare_tensors(self):- super().prepare_tensors()- if self._experts is not None:- experts = [k for d in self._experts for k in d.keys()]- if len(experts) > 0:- raise ValueError(f"Unprocessed experts: {experts}")--[email protected]("SmolLM3ForCausalLM")-class SmolLM3Model(LlamaModel):- model_arch = gguf.MODEL_ARCH.SMOLLM3-- def set_vocab(self):- super().set_vocab()- # remove unsupported array slicing in chat template- # ref: https://huggingface.co/ggml-org/SmolLM3-3B-GGUF/discussions/1- from transformers import AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(self.dir_model)- if tokenizer.chat_template is not None:- chat_template = tokenizer.chat_template.replace("[:]", "")- self.gguf_writer.add_chat_template(chat_template)--[email protected]("Lfm2ForCausalLM")[email protected]("LFM2ForCausalLM")-class LFM2Model(TextModel):- model_arch = gguf.MODEL_ARCH.LFM2-- def _add_feed_forward_length(self):- ff_dim = self.hparams["block_ff_dim"]-- auto_adjust_ff_dim = self.hparams["block_auto_adjust_ff_dim"]- ff_dim = self.hparams["block_ff_dim"]- ffn_dim_multiplier = self.hparams["block_ffn_dim_multiplier"]- multiple_of = self.hparams["block_multiple_of"]-- if auto_adjust_ff_dim:- ff_dim = int(2 * ff_dim / 3)- # custom dim factor multiplier- if ffn_dim_multiplier is not None:- ff_dim = int(ffn_dim_multiplier * ff_dim)- ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)-- self.gguf_writer.add_feed_forward_length(ff_dim)-- def set_gguf_parameters(self):- # set num_key_value_heads only for attention layers- self.hparams["num_key_value_heads"] = [- self.hparams["num_key_value_heads"] if layer_type == "full_attention" else 0- for layer_type in self.hparams["layer_types"]- ]-- super().set_gguf_parameters()- self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])- self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])- self.gguf_writer.add_layer_norm_rms_eps(self.hparams["norm_eps"])- self._add_feed_forward_length()-- def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- # conv op requires 2d tensor- if "conv.conv" in name:- data_torch = data_torch.squeeze(1)-- return [(self.map_tensor_name(name), data_torch)]---###### CONVERSION LOGIC ######---# tree of lazy tensors-class LazyTorchTensor(gguf.LazyBase):- _tensor_type = torch.Tensor- # to keep the type-checker happy- dtype: torch.dtype- shape: torch.Size-- # only used when converting a torch.Tensor to a np.ndarray- _dtype_map: dict[torch.dtype, type] = {- torch.float16: np.float16,- torch.float32: np.float32,- }-- _dtype_str_map: dict[str, torch.dtype] = {- "F64": torch.float64,- "F32": torch.float32,- "BF16": torch.bfloat16,- "F16": torch.float16,- # "U64": torch.uint64,- "I64": torch.int64,- # "U32": torch.uint32,- "I32": torch.int32,- # "U16": torch.uint16,- "I16": torch.int16,- "U8": torch.uint8,- "I8": torch.int8,- "BOOL": torch.bool,- "F8_E4M3": torch.float8_e4m3fn,- "F8_E5M2": torch.float8_e5m2,- }-- def numpy(self) -> gguf.LazyNumpyTensor:- dtype = self._dtype_map[self.dtype]- return gguf.LazyNumpyTensor(- meta=gguf.LazyNumpyTensor.meta_with_dtype_and_shape(dtype, self.shape),- args=(self,),- func=(lambda s: s.numpy()),- )-- @classmethod- def meta_with_dtype_and_shape(cls, dtype: torch.dtype, shape: tuple[int, ...]) -> Tensor:- return torch.empty(size=shape, dtype=dtype, device="meta")-- @classmethod- def from_safetensors_slice(cls, st_slice: Any) -> Tensor:- dtype = cls._dtype_str_map[st_slice.get_dtype()]- shape: tuple[int, ...] = tuple(st_slice.get_shape())- lazy = cls(meta=cls.meta_with_dtype_and_shape(dtype, shape), args=(st_slice,), func=lambda s: s[:])- return cast(torch.Tensor, lazy)-- @classmethod- def from_remote_tensor(cls, remote_tensor: gguf.utility.RemoteTensor):- dtype = cls._dtype_str_map[remote_tensor.dtype]- shape = remote_tensor.shape- meta = cls.meta_with_dtype_and_shape(dtype, shape)- lazy = cls(- meta=meta, args=(remote_tensor,), func=lambda r: torch.frombuffer(r.data(), dtype=dtype).reshape(shape)- )- return cast(torch.Tensor, lazy)-- @classmethod- def __torch_function__(cls, func, types, args=(), kwargs=None):- del types # unused-- if kwargs is None:- kwargs = {}-- if func is torch.Tensor.numpy:- return args[0].numpy()-- return cls._wrap_fn(func)(*args, **kwargs)---def split_str_to_n_bytes(split_str: str) -> int:- if split_str.endswith("K"):- n = int(split_str[:-1]) * 1000- elif split_str.endswith("M"):- n = int(split_str[:-1]) * 1000 * 1000- elif split_str.endswith("G"):- n = int(split_str[:-1]) * 1000 * 1000 * 1000- elif split_str.isnumeric():- n = int(split_str)- else:- raise ValueError(f"Invalid split size: {split_str}, must be a number, optionally followed by K, M, or G")-- if n < 0:- raise ValueError(f"Invalid split size: {split_str}, must be positive")-- return n---def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> str:- # TODO @ngxson : this won't work correctly if the model has both audio & vision encoders- # maybe we should fallback to text model's arch in that case, since not many models have both- text_config = hparams.get("text_config", {})- vision_config = hparams.get("vision_config", {})- arch = None- if (arches := hparams.get("architectures")) is not None and len(arches) > 0:- arch = arches[0]- elif "ssm_cfg" in hparams:- # For non-hf Mamba and Mamba2 models- arch = hparams["ssm_cfg"].get("layer", "Mamba") + "ForCausalLM"-- # if "architectures" is found in the sub-config, use that instead- if model_type == ModelType.TEXT and text_config.get("architectures") is not None:- arch = text_config["architectures"][0]- elif model_type == ModelType.MMPROJ and vision_config.get("architectures") is not None:- arch = vision_config["architectures"][0]- if arch is None:- raise ValueError("Failed to detect model architecture")- return arch+ cls.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)++ # # save cpu memory, but slow+ if cls.low_cpu_mem_usage:+ for weight_name in clean_weight_list:+ module = get_module(cls.model, ".".join(weight_name.split(".")[:-1]))+ for key in ["scale", "zp", "d_scale", "wmin", "d_wmin", "imatrix"]:+ if hasattr(module, key):+ setattr(module, key, None)+ if hasattr(module, "w_" + key):+ setattr(module, "w_" + key, None)+ if cls.model_arch == gguf.MODEL_ARCH.LLAMA and "embed_tokens.weight" in weight_name:+ continue+ clean_module_parameter(module, weight_name.split(".")[-1])+++def load_hparams(dir_model: Path):+ try:+ # for security reason, we don't allow loading remote code by default+ # if a model need remote code, we will fallback to config.json+ config = AutoConfig.from_pretrained(dir_model, trust_remote_code=False).to_dict()+ except Exception as e:+ logger.warning(f"Failed to load model config from {dir_model}: {e}")+ logger.warning("Trying to load config.json instead")+ with open(dir_model / "config.json", "r", encoding="utf-8") as f:+ config = json.load(f)+ if "llm_config" in config:+ # rename for InternVL+ config["text_config"] = config["llm_config"]+ if "thinker_config" in config:+ # rename for Qwen2.5-Omni+ config["text_config"] = config["thinker_config"]["text_config"]+ return configdiff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 9074deb0a..51f2e285c 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -18,9 +18,10 @@ import sysimport timefrom pathlib import Path+import requestsimport torch-from auto_round.export.export_to_gguf.convert import ModelBase, ModelType, get_model_architecture+# from auto_round.export.export_to_gguf.convert_hf_to_gguf import ModelBase, ModelType, get_model_architecturefrom auto_round.utils import (LazyImport,check_to_quantized,@@ -31,6 +32,10 @@ from auto_round.utils import (logger,)+convert_hf_to_gguf = LazyImport("auto_round.export.export_to_gguf.convert_hf_to_gguf")+from auto_round.export.export_to_gguf.convert import wrapper_model_instance+from auto_round.export.export_to_gguf.special_handle import handle_special_model+TMP_DIR_NAME = "tmp_dir"gguf = LazyImport("gguf")@@ -59,12 +64,17 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {def create_model_class(- output_dir, model, layer_config, backend="gguf:q4_0", low_cpu_mem_usage=False, model_type=ModelType.TEXT+ output_dir,+ model,+ layer_config,+ backend="gguf:q4_0",+ low_cpu_mem_usage=False,+ model_type=convert_hf_to_gguf.ModelType.TEXT,):tmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))with torch.inference_mode():- hparams = ModelBase.load_hparams(tmp_work_dir)- model_architecture = get_model_architecture(hparams=hparams, model_type=model_type)+ hparams = convert_hf_to_gguf.ModelBase.load_hparams(tmp_work_dir)+ model_architecture = convert_hf_to_gguf.get_model_architecture(hparams=hparams, model_type=model_type)# if "architectures" in hparams:# model_architecture = hparams["architectures"][0]# else:@@ -75,11 +85,13 @@ def create_model_class(# elif model_architecture.replace("ConditionalGeneration", "CausalLM") in ModelBase._model_classes:# model_architecture = model_architecture.replace("ConditionalGeneration", "CausalLM")try:- model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)+ model_class = convert_hf_to_gguf.ModelBase.from_model_architecture(+ model_architecture, model_type=model_type+ )except NotImplementedError:- logger.error(f"Model {model_architecture} is not supported")+ logger.error(f"Model {model_architecture} is not supported to export gguf format.")sys.exit(1)- model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)+ model_class = convert_hf_to_gguf.ModelBase.from_model_architecture(model_architecture, model_type=model_type)model_name = model.name_or_path.split("/")if len(model_name[-1]) == 0:model_name = model_name[-2]@@ -92,12 +104,9 @@ def create_model_class(output_type = FTYPE_MAP.get(output_type.lower())model_instance = model_class(- model,- layer_config,dir_model=tmp_work_dir,ftype=output_type,fname_out=Path(output_dir),- low_cpu_mem_usage=low_cpu_mem_usage, # pylint: disable=E0401is_big_endian=False,model_name=model_name,split_max_tensors=False,@@ -105,6 +114,10 @@ def create_model_class(dry_run=False,small_first_shard=False,)+ model_instance = wrapper_model_instance(+ model_instance, model=model, layer_config=layer_config, low_cpu_mem_usage=low_cpu_mem_usage+ )+ model_instance = handle_special_model(model_instance, model_architecture)return model_instance@@ -118,7 +131,7 @@ def pack_gguf_layer(tokenizer,processor=None,image_processor=None,- model_type=ModelType.TEXT,+ model_type=convert_hf_to_gguf.ModelType.TEXT,):"""Export the model to gguf format."""global gguf_model_instance_global@@ -138,13 +151,23 @@ def pack_gguf_layer(gguf_model_instance_global = [create_model_class(- output_dir, model, layer_config, backend, low_cpu_mem_usage=True, model_type=ModelType.TEXT+ output_dir,+ model,+ layer_config,+ backend,+ low_cpu_mem_usage=True,+ model_type=convert_hf_to_gguf.ModelType.TEXT,)]- if model_type == ModelType.MMPROJ:+ if model_type == convert_hf_to_gguf.ModelType.MMPROJ:gguf_model_instance_global.append(create_model_class(- output_dir, model, layer_config, backend, low_cpu_mem_usage=True, model_type=ModelType.MMPROJ+ output_dir,+ model,+ layer_config,+ backend,+ low_cpu_mem_usage=True,+ model_type=convert_hf_to_gguf.ModelType.MMPROJ,))if not hasattr(model, "last_layer_name_to_block_name"):@@ -209,11 +232,13 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, vimage_processor.save_pretrained(tmp_work_dir)gguf_model_instance_global = [- create_model_class(output_dir, model, layer_config, backend, model_type=ModelType.TEXT)+ create_model_class(output_dir, model, layer_config, backend, model_type=convert_hf_to_gguf.ModelType.TEXT)]if vlm:gguf_model_instance_global.append(- create_model_class(output_dir, model, layer_config, backend, model_type=ModelType.MMPROJ)+ create_model_class(+ output_dir, model, layer_config, backend, model_type=convert_hf_to_gguf.ModelType.MMPROJ+ ))for gguf_model in gguf_model_instance_global:diff --git a/auto_round/export/export_to_gguf/special_handle.py b/auto_round/export/export_to_gguf/special_handle.pynew file mode 100644index 000000000..490788d6d--- /dev/null+++ b/auto_round/export/export_to_gguf/special_handle.py@@ -0,0 +1,135 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import json+import os+from functools import partial+from typing import Iterable++import torch+from safetensors import safe_open+from torch import Tensor++from auto_round.utils import download_hf_model+++def handle_special_model(cls, model_architecture):+ if model_architecture == "GptOssForCausalLM":+ cls.generate_extra_tensors = partial(gptoss_generate_extra_tensors, cls)+ cls.modify_tensors = partial(gptoss_modify_tensors, cls)+ return cls+++def get_tensor_from_file(dir_path, tensor_name):+ if not os.path.isdir(dir_path):+ dir_path = download_hf_model(dir_path)+ INDEX_FILE = "model.safetensors.index.json"+ # get filename+ if INDEX_FILE in os.listdir(dir_path):+ tensor_index = json.load(open(os.path.join(dir_path, INDEX_FILE)))+ filename = tensor_index["weight_map"][tensor_name]+ else:+ filename = "model.safetensors"++ # get tensor+ f = safe_open(os.path.join(dir_path, filename), framework="pt")+ return f.get_tensor(tensor_name)+++GPTOSS_RELOAD = True+++def gptoss_generate_extra_tensors(cls) -> Iterable[tuple[str, Tensor]]:+ blocks0: Tensor = torch.zeros(1)+ blocks1: Tensor = torch.zeros(1)+ found_mxfp4_tensors = False++ # we assume that tensors are loaded in the correct order+ def repack(name, data_torch, blocks0, blocks1):+ if "mlp.experts.down_proj_blocks" in name:+ blocks0 = data_torch+ elif "mlp.experts.down_proj_scales" in name:+ new_name = cls.map_tensor_name(name.replace("_scales", ".weight"))+ for t in cls.gguf_writer.tensors:+ if new_name in t.keys():+ return blocks0, blocks1+ cls.repack_mxfp4(new_name, blocks0, data_torch)+ found_mxfp4_tensors = True+ elif "mlp.experts.gate_up_proj_blocks" in name:+ blocks0, blocks1 = data_torch[:, ::2, :, :], data_torch[:, 1::2, :, :]+ elif "mlp.experts.gate_up_proj_scales" in name:+ scales0, scales1 = data_torch[:, ::2, :], data_torch[:, 1::2, :]+ new_name_gate = cls.map_tensor_name(name.replace("gate_up_proj_scales", "gate_proj.weight"))+ new_name_up = cls.map_tensor_name(name.replace("gate_up_proj_scales", "up_proj.weight"))+ for t in cls.gguf_writer.tensors:+ if new_name_gate in t.keys() or new_name_up in t.keys():+ return blocks0, blocks1+ cls.repack_mxfp4(new_name_gate, blocks0, scales0)+ cls.repack_mxfp4(new_name_up, blocks1, scales1)+ found_mxfp4_tensors = True+ return blocks0, blocks1++ for name, data_torch in cls.get_tensors():+ if GPTOSS_RELOAD and (name.endswith("mlp.experts.down_proj") or name.endswith("mlp.experts.gate_up_proj")):+ block_name = name + "_blocks"+ block_data_torch = get_tensor_from_file(cls.model.name_or_path, block_name)+ blocks0, blocks1 = repack(block_name, block_data_torch, blocks0, blocks1)+ scale_name = name + "_scales"+ scale_data_torch = get_tensor_from_file(cls.model.name_or_path, scale_name)+ blocks0, blocks1 = repack(scale_name, scale_data_torch, blocks0, blocks1)+ else:+ blocks0, blocks1 = repack(name, data_torch, blocks0, blocks1)++ # convert to bf16 model, foud_mxfp4_tensors is False+ # if not found_mxfp4_tensors:+ # raise ValueError("No MXFP4 tensors found in the model. Please make sure you are using MXFP4 model.")+ return []+++def gptoss_modify_tensors(cls, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused++ if "sinks" in name:+ name += ".weight"++ # correct naming for down_proj+ if "down_proj" in name:+ if name.endswith("_bias"):+ name = name.replace("down_proj_bias", "down_proj.bias")+ else:+ if GPTOSS_RELOAD:+ return []+ if data_torch.dtype != torch.uint8:+ return [(cls.map_tensor_name(name + ".weight"), data_torch)]+ return []++ # split the gate_up into gate and up+ if "gate_up_proj" in name:+ if name.endswith("_bias"):+ name_up = name.replace("gate_up_proj_bias", "up_proj.bias")+ name_gate = name.replace("gate_up_proj_bias", "gate_proj.bias")+ gate_proj_bias, up_proj_bias = data_torch[..., ::2], data_torch[..., 1::2]+ return [(cls.map_tensor_name(name_gate), gate_proj_bias), (cls.map_tensor_name(name_up), up_proj_bias)]+ else:+ if GPTOSS_RELOAD:+ return []+ if data_torch.dtype != torch.uint8:+ new_name_gate = cls.map_tensor_name(name.replace("gate_up_proj", "gate_proj.weight"))+ new_name_up = cls.map_tensor_name(name.replace("gate_up_proj", "up_proj.weight"))+ gate_data, up_data = data_torch[:, ::2, :], data_torch[:, 1::2, :]+ # gate_data = gate_data.reshape(-1, 2880, 2880)+ # up_data = up_data.reshape(-1, 2880, 2880)+ return [(new_name_gate, gate_data), (new_name_up, up_data)]+ return []++ return [(cls.map_tensor_name(name), data_torch)]diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 68dad2468..68cf14e97 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1178,6 +1178,7 @@ def get_layer_features(layer):def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):import argparse+ from auto_round.export.export_to_gguf.convert import download_convert_filefrom auto_round.utils import loggerif format_str is None:@@ -1187,9 +1188,71 @@ def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):format_str = format_str.replace("q*_", f"q{args_or_ar.bits}_")formats = format_str.lower().replace(" ", "").split(",")formats = sorted(formats, key=lambda x: len(x))+ export_gguf = Falsefor f in formats:+ if f.startswith("gguf"):+ export_gguf = True+if f.startswith("gguf") and f not in GGUF_CONFIG:logger.error(f"{f} is not supported, please check.")++ redownload = False+ if export_gguf:+ try:+ from auto_round.export.export_to_gguf.convert_hf_to_gguf import ( # pylint: disable=E0401+ ModelBase,+ ModelType,+ get_model_architecture,+ )++ if isinstance(args_or_ar.model, str):+ model_path = args_or_ar.model+ else:+ model_path = args_or_ar.model.name_or_path+ if not os.path.isdir(model_path):+ model_path = download_hf_model(model_path)+ hparams = ModelBase.load_hparams(model_path)+ model_architecture = get_model_architecture(hparams=hparams, model_type=ModelType.TEXT)+ if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:+ logger.warning(+ f"Current version of gguf export does not support for {model_architecture},"+ " will re-download dependency file."+ )+ redownload = True+ except ModuleNotFoundError as e:+ if "convert_hf_to_gguf" in str(e):+ logger.warning("GGUF export dependency file is not found, download from github.")+ redownload = True+ else:+ raise ImportError(+ "Please use the latest gguf-py, you can use the following command to install it:\n"+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install ."+ )+ download_convert_file(redownload)++ try:+ from auto_round.export.export_to_gguf.convert_hf_to_gguf import ( # pylint: disable=E0401+ ModelBase,+ ModelType,+ get_model_architecture,+ )+ except ImportError as e:+ raise ImportError(+ "Please use the latest gguf-py, you can use the following command to install it:\n"+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install ."+ )+ if isinstance(args_or_ar.model, str):+ model_path = args_or_ar.model+ else:+ model_path = args_or_ar.model.name_or_path+ if not os.path.isdir(model_path):+ model_path = download_hf_model(model_path)+ hparams = ModelBase.load_hparams(model_path)+ model_architecture = get_model_architecture(hparams=hparams, model_type=ModelType.TEXT)+ if model_architecture not in ModelBase._model_classes[ModelType.TEXT]:+ logger.error(f"Model {model_architecture} is not supported to export gguf format.")+ sys.exit(1)+pattern = re.compile("q\d_k")pre_dq_format = ""unsupport_list, reset_list = [], []@@ -1197,13 +1260,7 @@ def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):if format in formats:if format == "q6_k_s":logger.warning("Please note that q6_k_s is q6_k.")- try:- from auto_round.export.export_to_gguf.convert import ModelBase- except:- raise ImportError(- f"Please use the latest gguf-py for {format}, you can use the following command to install it:\n"- "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install ."- )+if re.search(pattern, format):if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")@@ -1211,29 +1268,6 @@ def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):else:pre_dq_format = format- if isinstance(args_or_ar.model, str) and os.path.isdir(args_or_ar.model):- from pathlib import Path-- from auto_round.export.export_to_gguf.convert import ModelBase-- hparams = ModelBase.load_hparams(Path(args_or_ar.model))- model_architecture = hparams["architectures"][0]- try:- model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)- except NotImplementedError:- logger.error(f"Model {model_architecture} is not supported to export GGUF format")- sys.exit(1)-- if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % QK_K != 0):- model_name = args_or_ar.model.split("/")- model_name = model_name[-1] if model_name[-1] else model_name[-2]- hidden_size = hparams["hidden_size"]- logger.error(- f"Currently only support pure mode for format: {format}. "- f"{model_name} is not supported, cause hidden_size({hidden_size}) % 256 !=0"- )- sys.exit(-1)-unsupport_list, reset_list = [], []gguf_config = GGUF_CONFIG[format]for k, v in gguf_config.items():@@ -1724,11 +1758,14 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_typeimport gguf # pylint: disable=E0401- from auto_round.export.export_to_gguf.convert import ModelBase, get_model_architecture+ # from auto_round.export.export_to_gguf.convert import ModelBase, get_model_architecture+ convert_hf_to_gguf = LazyImport("auto_round.export.export_to_gguf.convert_hf_to_gguf")- model_architecture = get_model_architecture(hparams=model.config.to_dict(), model_type=model_type)+ model_architecture = convert_hf_to_gguf.get_model_architecture(+ hparams=model.config.to_dict(), model_type=model_type+ )try:- model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)+ model_class = convert_hf_to_gguf.ModelBase.from_model_architecture(model_architecture, model_type=model_type)except NotImplementedError:return layer_config, {}@@ -2156,3 +2193,35 @@ def out_of_vram(error_msg):if "HIP out of memory. Tried to allocate" in error_msg:return Truereturn False+++def download_hf_model(repo_id, cache_dir=None, repo_type=None, revision=None):+ """Download hugging face model from hf hub."""+ from huggingface_hub.constants import DEFAULT_REVISION, HUGGINGFACE_HUB_CACHE+ from huggingface_hub.file_download import REGEX_COMMIT_HASH, repo_folder_name+ from huggingface_hub.utils import EntryNotFoundError++ if cache_dir is None:+ cache_dir = HUGGINGFACE_HUB_CACHE+ if revision is None:+ revision = DEFAULT_REVISION+ if repo_type is None:+ repo_type = "model"+ storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))+ commit_hash = None+ if REGEX_COMMIT_HASH.match(revision):+ commit_hash = revision+ else:+ ref_path = os.path.join(storage_folder, "refs", revision)+ if os.path.exists(ref_path):+ with open(ref_path) as f:+ commit_hash = f.read()+ if storage_folder and commit_hash:+ pointer_path = os.path.join(storage_folder, "snapshots", commit_hash)+ if os.path.isdir(pointer_path):+ return pointer_path+ else: # pragma: no cover+ from huggingface_hub import snapshot_download++ model_path = snapshot_download(repo_id)+ return model_pathdiff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex cacaca71e..f1a7d9ccd 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -204,6 +204,7 @@ class TestGGUF(unittest.TestCase):print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))shutil.rmtree("./saved", ignore_errors=True)+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)autoround = AutoRound(model,self.tokenizer,@@ -266,6 +267,7 @@ class TestGGUF(unittest.TestCase):self.assertEqual(autoround.layer_config["model.layers.10.mlp.gate_proj"]["mostly"], "gguf:q8_0")shutil.rmtree("./saved", ignore_errors=True)+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)autoround = AutoRound(model, tokenizer, iters=0, nsamples=1, seqlen=128, disable_opt_rtn=False)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake")