PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support to export gemma3 gguf format (#470)
Heng Guo43629a729708
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 9d3c4b852..89db51361 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -99,23 +99,23 @@ class OriModel:# subclasses should define this!model_arch: gguf.MODEL_ARCH- def __init__(self,- dir_model: Path,- ftype: gguf.LlamaFileType,- fname_out: Path,- is_big_endian: bool = False,- use_temp_file: bool = False,- eager: bool = False,- metadata_override: Path | None = None,- model_name: str | None = None,- split_max_tensors: int = 0,- split_max_size: int = 0,- dry_run: bool = False,- small_first_shard: bool = False,- hparams: dict[str, Any] | None = None):+ def __init__(+ self,+ dir_model: Path,+ ftype: gguf.LlamaFileType,+ fname_out: Path,+ is_big_endian: bool = False,+ use_temp_file: bool = False,+ eager: bool = False,+ metadata_override: Path | None = None,+ model_name: str | None = None,+ split_max_tensors: int = 0,+ split_max_size: int = 0,+ dry_run: bool = False,+ small_first_shard: bool = False,+ hparams: dict[str, Any] | None = None):if type(self) is Model:- raise TypeError(- f"{type(self).__name__!r} should not be directly instantiated")+ raise TypeError(f"{type(self).__name__!r} should not be directly instantiated")self.dir_model = dir_modelself.ftype = ftype@@ -124,18 +124,13 @@ class OriModel:self.endianess = gguf.GGUFEndian.BIG if is_big_endian else gguf.GGUFEndian.LITTLEself.use_temp_file = use_temp_fileself.lazy = not eager- self.part_names = Model.get_model_part_names(self.dir_model, "model",- ".safetensors")+ self.part_names = Model.get_model_part_names(self.dir_model, "model", ".safetensors")self.is_safetensors = len(self.part_names) > 0if not self.is_safetensors:- self.part_names = Model.get_model_part_names(- self.dir_model, "pytorch_model", ".bin")- self.hparams = Model.load_hparams(- self.dir_model) if hparams is None else hparams- self.block_count = self.find_hparam(- ["n_layers", "num_hidden_layers", "n_layer", "num_layers"])- self.tensor_map = gguf.get_tensor_name_map(self.model_arch,- self.block_count)+ self.part_names = Model.get_model_part_names(self.dir_model, "pytorch_model", ".bin")+ self.hparams = Model.load_hparams(self.dir_model) if hparams is None else hparams+ self.block_count = self.find_hparam(["n_layers", "num_hidden_layers", "n_layer", "num_layers"])+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)self.tensor_names = Noneself.metadata_override = metadata_overrideself.model_name = model_name@@ -146,14 +141,10 @@ class OriModel:# NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie._, first_tensor = next(self.get_tensors())if first_tensor.dtype == torch.float16:- logger.info(- f"choosing --outtype f16 from first tensor type ({first_tensor.dtype})"- )+ logger.info(f"choosing --outtype f16 from first tensor type ({first_tensor.dtype})")self.ftype = gguf.LlamaFileType.MOSTLY_F16else:- logger.info(- f"choosing --outtype bf16 from first tensor type ({first_tensor.dtype})"- )+ logger.info(f"choosing --outtype bf16 from first tensor type ({first_tensor.dtype})")self.ftype = gguf.LlamaFileType.MOSTLY_BF16# Configure GGUF Writer@@ -192,8 +183,7 @@ class OriModel:index: dict[str, Any] = json.load(f)weight_map = index.get("weight_map")if weight_map is None or not isinstance(weight_map, dict):- raise ValueError(- f"Can't load 'weight_map' from {index_name!r}")+ raise ValueError(f"Can't load 'weight_map' from {index_name!r}")self.tensor_names.update(weight_map.keys())else:self.tensor_names = tensor_names_from_parts@@ -204,17 +194,10 @@ class OriModel:ctx: ContextManager[Any]if self.is_safetensors:from safetensors import safe_open- ctx = cast(- ContextManager[Any],- safe_open(self.dir_model / part_name,- framework="pt",- device="cpu"))+ ctx = cast(ContextManager[Any], safe_open(self.dir_model / part_name, framework="pt", device="cpu"))else:ctx = contextlib.nullcontext(- torch.load(str(self.dir_model / part_name),- map_location="cpu",- mmap=True,- weights_only=True))+ torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True))with ctx as model_part:tensor_names_from_parts.update(model_part.keys())@@ -233,41 +216,29 @@ class OriModel:yield name, data# verify tensor name presence and identify potentially missing files- if len(tensor_names_from_parts.symmetric_difference(- self.tensor_names)) > 0:- missing = sorted(- self.tensor_names.difference(tensor_names_from_parts))- extra = sorted(- tensor_names_from_parts.difference(self.tensor_names))- missing_files = sorted(- set(weight_map[n] for n in missing if n in weight_map))+ if len(tensor_names_from_parts.symmetric_difference(self.tensor_names)) > 0:+ missing = sorted(self.tensor_names.difference(tensor_names_from_parts))+ extra = sorted(tensor_names_from_parts.difference(self.tensor_names))+ missing_files = sorted(set(weight_map[n] for n in missing if n in weight_map))if len(extra) == 0 and len(missing_files) > 0:- raise ValueError(- f"Missing or incomplete model files: {missing_files}")+ raise ValueError(f"Missing or incomplete model files: {missing_files}")else:raise ValueError("Mismatch between weight map and model parts for tensor names:\n"f"Missing tensors: {missing}\n"f"Extra tensors: {extra}")- def format_tensor_name(self,- key: gguf.MODEL_TENSOR,- bid: int | None = None,- suffix: str = ".weight") -> str:+ def format_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str:if key not in gguf.MODEL_TENSORS[self.model_arch]:- raise ValueError(- f"Missing {key!r} for MODEL_TENSORS of {self.model_arch!r}")+ raise ValueError(f"Missing {key!r} for MODEL_TENSORS of {self.model_arch!r}")name: str = gguf.TENSOR_NAMES[key]if "{bid}" in name:assert bid is not Nonename = name.format(bid=bid)return name + suffix- def match_model_tensor_name(self,- name: str,- key: gguf.MODEL_TENSOR,- bid: int | None,- suffix: str = ".weight") -> bool:+ def match_model_tensor_name(+ self, name: str, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> bool:if key not in gguf.MODEL_TENSORS[self.model_arch]:return Falsekey_name: str = gguf.TENSOR_NAMES[key]@@ -280,11 +251,8 @@ class OriModel:return Falsereturn name == (key_name + suffix)- def map_tensor_name(- self, name: str,- try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:- new_name = self.tensor_map.get_name(key=name,- try_suffixes=try_suffixes)+ def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:+ new_name = self.tensor_map.get_name(key=name, try_suffixes=try_suffixes)if new_name is None:raise ValueError(f"Can not map tensor {name!r}")return new_name@@ -292,23 +260,19 @@ class OriModel:def set_gguf_parameters(self):self.gguf_writer.add_block_count(self.block_count)- if (n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx"],- optional=True)) is not None:+ if (n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx"], optional=True)) is not None:self.gguf_writer.add_context_length(n_ctx)logger.info(f"gguf: context length = {n_ctx}")- if (n_embd := self.find_hparam(["hidden_size", "n_embd"],- optional=True)) is not None:+ if (n_embd := self.find_hparam(["hidden_size", "n_embd"], optional=True)) is not None:self.gguf_writer.add_embedding_length(n_embd)logger.info(f"gguf: embedding length = {n_embd}")- if (n_ff := self.find_hparam(["intermediate_size", "n_inner"],- optional=True)) is not None:+ if (n_ff := self.find_hparam(["intermediate_size", "n_inner"], optional=True)) is not None:self.gguf_writer.add_feed_forward_length(n_ff)logger.info(f"gguf: feed forward length = {n_ff}")- if (n_head := self.find_hparam(["num_attention_heads", "n_head"],- optional=True)) is not None:+ if (n_head := self.find_hparam(["num_attention_heads", "n_head"], optional=True)) is not None:self.gguf_writer.add_head_count(n_head)logger.info(f"gguf: head count = {n_head}")@@ -322,16 +286,14 @@ class OriModel:if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None:self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")- if (f_norm_eps := self.find_hparam(- ["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"],- optional=True)) is not None:+ if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"],+ optional=True)) is not None:self.gguf_writer.add_layer_norm_eps(f_norm_eps)logger.info(f"gguf: layer norm epsilon = {f_norm_eps}")if (n_experts := self.hparams.get("num_local_experts")) is not None:self.gguf_writer.add_expert_count(n_experts)logger.info(f"gguf: expert count = {n_experts}")- if (n_experts_used :=- self.hparams.get("num_experts_per_tok")) is not None:+ if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:self.gguf_writer.add_expert_used_count(n_experts_used)logger.info(f"gguf: experts used count = {n_experts_used}")@@ -342,14 +304,13 @@ class OriModel:self.gguf_writer.add_file_type(self.ftype)logger.info(f"gguf: file type = {self.ftype}")- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedreturn [(self.map_tensor_name(name), data_torch)]- def tensor_force_quant(self, name: str, new_name: str, bid: int | None,- n_dims: int) -> gguf.GGMLQuantizationType | bool:+ def tensor_force_quant(+ self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:del name, new_name, bid, n_dims # unusedreturn False@@ -359,15 +320,11 @@ class OriModel:return ()def prepare_tensors(self):- max_name_len = max(- len(s)- for _, s in self.tensor_map.mapping.values()) + len(".weight,")+ max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")- for name, data_torch in chain(self.generate_extra_tensors(),- self.get_tensors()):+ for name, data_torch in chain(self.generate_extra_tensors(), self.get_tensors()):# we don't need these- if name.endswith((".attention.masked_bias", ".attention.bias",- ".rotary_emb.inv_freq")):+ if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):continueold_dtype = data_torch.dtype@@ -383,8 +340,7 @@ class OriModel:bid = int(part)break- for new_name, data_torch in (self.modify_tensors(- data_torch, name, bid)):+ for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):# TODO: why do we squeeze here?# data = data_torch.squeeze().numpy()data = data_torch.numpy()@@ -394,8 +350,7 @@ class OriModel:data = data_torch.numpy()n_dims = len(data.shape)- data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(- name, new_name, bid, n_dims)+ data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)# Most of the codebase that takes in 1D tensors or norms only handles F32 tensorsif n_dims <= 1 or new_name.endswith("_norm.weight"):@@ -403,30 +358,26 @@ class OriModel:# Conditions should closely match those in llama_model_quantize_internal in llama.cpp# Some tensor types are always in float32- if data_qtype is False and (any(- self.match_model_tensor_name(new_name, key, bid)- for key in (- gguf.MODEL_TENSOR.FFN_GATE_INP,- gguf.MODEL_TENSOR.POS_EMBD,- gguf.MODEL_TENSOR.TOKEN_TYPES,- gguf.MODEL_TENSOR.SSM_CONV1D,- gguf.MODEL_TENSOR.TIME_MIX_FIRST,- gguf.MODEL_TENSOR.TIME_MIX_W1,- gguf.MODEL_TENSOR.TIME_MIX_W2,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,- gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,- gguf.MODEL_TENSOR.POSNET_NORM1,- gguf.MODEL_TENSOR.POSNET_NORM2,- )) or not new_name.endswith(".weight")):+ if data_qtype is False and (any(self.match_model_tensor_name(new_name, key, bid) for key in (+ gguf.MODEL_TENSOR.FFN_GATE_INP,+ gguf.MODEL_TENSOR.POS_EMBD,+ gguf.MODEL_TENSOR.TOKEN_TYPES,+ gguf.MODEL_TENSOR.SSM_CONV1D,+ gguf.MODEL_TENSOR.TIME_MIX_FIRST,+ gguf.MODEL_TENSOR.TIME_MIX_W1,+ gguf.MODEL_TENSOR.TIME_MIX_W2,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,+ gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,+ gguf.MODEL_TENSOR.POSNET_NORM1,+ gguf.MODEL_TENSOR.POSNET_NORM2,+ )) or not new_name.endswith(".weight")):data_qtype = gguf.GGMLQuantizationType.F32- if data_qtype is False and any(- self.match_model_tensor_name(new_name, key, bid)- for key in (- gguf.MODEL_TENSOR.TOKEN_EMBD,- gguf.MODEL_TENSOR.OUTPUT,- )):+ if data_qtype is False and any(self.match_model_tensor_name(new_name, key, bid) for key in (+ gguf.MODEL_TENSOR.TOKEN_EMBD,+ gguf.MODEL_TENSOR.OUTPUT,+ )):if self.ftype in (gguf.LlamaFileType.MOSTLY_TQ1_0,gguf.LlamaFileType.MOSTLY_TQ2_0,@@ -449,8 +400,7 @@ class OriModel:elif self.ftype == gguf.LlamaFileType.MOSTLY_TQ2_0:data_qtype = gguf.GGMLQuantizationType.TQ2_0else:- raise ValueError(- f"Unknown file type: {self.ftype.name}")+ raise ValueError(f"Unknown file type: {self.ftype.name}")try:data = gguf.quants.quantize(data, data_qtype)@@ -460,32 +410,25 @@ class OriModel:data = gguf.quants.quantize(data, data_qtype)shape = gguf.quant_shape_from_byte_shape(- data.shape,- data_qtype) if data.dtype == np.uint8 else data.shape+ data.shape, data_qtype) if data.dtype == np.uint8 else data.shape# reverse shape to make it similar to the internal ggml dimension ordershape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"# n_dims is implicit in the shapelogger.info(- f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"- )+ f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}")- self.gguf_writer.add_tensor(new_name,- data,- raw_dtype=data_qtype)+ self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)def set_type(self):self.gguf_writer.add_type(gguf.GGUFType.MODEL)def prepare_metadata(self, vocab_only: bool):- total_params, shared_params, expert_params, expert_count = self.gguf_writer.get_total_parameter_count(- )+ total_params, shared_params, expert_params, expert_count = self.gguf_writer.get_total_parameter_count()- self.metadata = gguf.Metadata.load(self.metadata_override,- self.dir_model_card,- self.model_name, total_params)+ self.metadata = gguf.Metadata.load(self.metadata_override, self.dir_model_card, self.model_name, total_params)# Fallback to model directory name if metadata name is still missingif self.metadata.name is None:@@ -493,10 +436,7 @@ class OriModel:# Generate parameter weight class (useful for leader boards) if not yet determinedif self.metadata.size_label is None and total_params > 0:- self.metadata.size_label = gguf.size_label(total_params,- shared_params,- expert_params,- expert_count)+ self.metadata.size_label = gguf.size_label(total_params, shared_params, expert_params, expert_count)# Extract the encoding scheme from the file type name. e.g. 'gguf.LlamaFileType.MOSTLY_Q8_0' --> 'Q8_0'output_type: str = self.ftype.name.partition("_")[2]@@ -531,8 +471,7 @@ class OriModel:# file template strings as it doesn't actually exist as a file# Process templated file name with the output ftype, useful with the "auto" ftype- self.fname_out = self.fname_out.parent / gguf.fill_templated_filename(- self.fname_out.name, output_type)+ self.fname_out = self.fname_out.parent / gguf.fill_templated_filename(self.fname_out.name, output_type)self.set_type()@@ -566,8 +505,7 @@ class OriModel:self.gguf_writer.close()@staticmethod- def get_model_part_names(dir_model: Path, prefix: str,- suffix: str) -> list[str]:+ def get_model_part_names(dir_model: Path, prefix: str, suffix: str) -> list[str]:part_names: list[str] = []for filename in os.listdir(dir_model):if filename.startswith(prefix) and filename.endswith(suffix):@@ -603,8 +541,7 @@ class OriModel:try:return cls._model_classes[arch]except KeyError:- raise NotImplementedError(- f'Architecture {arch!r} not supported!') from None+ raise NotImplementedError(f'Architecture {arch!r} not supported!') from Nonedef does_token_look_special(self, token: str | bytes) -> bool:if isinstance(token, (bytes, bytearray)):@@ -623,16 +560,11 @@ class OriModel:"[@BOS@]", # gemma{,-2})- seems_special = seems_special or (token_text.startswith("<|")- and token_text.endswith("|>"))- seems_special = seems_special or (token_text.startswith("<|")- and token_text.endswith("|>")- ) # deepseek-coder+ seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))+ seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>")) # deepseek-coder# TODO: should these be marked as UNUSED instead? (maybe not)- seems_special = seems_special or (token_text.startswith("<unused")- and token_text.endswith(">")- ) # gemma{,-2}+ seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">")) # gemma{,-2}return seems_special@@ -648,10 +580,7 @@ class OriModel:tokpre = self.get_vocab_base_pre(tokenizer)- reverse_vocab = {- id_: encoded_tok- for encoded_tok, id_ in tokenizer.vocab.items()- }+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}added_vocab = tokenizer.get_added_vocab()for i in range(vocab_size):@@ -665,23 +594,18 @@ class OriModel:# To avoid unexpected issues - we make sure to normalize non-normalized tokensif not tokenizer.added_tokens_decoder[i].normalized:previous_token = token- token = tokenizer.decode(- tokenizer.encode(token, add_special_tokens=False))+ token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))if previous_token != token:logger.info(f"{repr(previous_token)} is encoded and decoded"- f" back to {repr(token)} using AutoTokenizer"- )+ f" back to {repr(token)} using AutoTokenizer")- if tokenizer.added_tokens_decoder[- i].special or self.does_token_look_special(token):+ if tokenizer.added_tokens_decoder[i].special or self.does_token_look_special(token):toktypes.append(gguf.TokenType.CONTROL)else:# NOTE: this was added for Gemma.# Encoding and decoding the tokens above isn't sufficient for this case.- token = token.replace(- b"\xe2\x96\x81".decode("utf-8"),- " ") # pre-normalize user-defined spaces+ token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spacestoktypes.append(gguf.TokenType.USER_DEFINED)else:toktypes.append(gguf.TokenType.NORMAL)@@ -699,7 +623,9 @@ class OriModel:# we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can# use in llama.cpp to implement the same pre-tokenizer- chktxt = ('\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български \'\'\'\'\'\'```````""""......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL') # pylint: disable=C0301+ chktxt = (+ '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български \'\'\'\'\'\'```````""""......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL' # pylint: disable=C0301+ ) # pylint: disable=C0301chktok = tokenizer.encode(chktxt)chkhsh = sha256(str(chktok).encode()).hexdigest()@@ -832,33 +758,19 @@ class OriModel:if res is None:logger.warning("\n")+ logger.warning("**************************************************************************************")+ logger.warning("** WARNING: The BPE pre-tokenizer was not recognized!")+ logger.warning("** There are 2 possible reasons for this:")+ logger.warning("** - the model has not been added to convert_hf_to_gguf_update.py yet")+ logger.warning("** - the pre-tokenization config has changed upstream")logger.warning(- "**************************************************************************************"- )- logger.warning(- "** WARNING: The BPE pre-tokenizer was not recognized!")- logger.warning(- "** There are 2 possible reasons for this:")- logger.warning(- "** - the model has not been added to convert_hf_to_gguf_update.py yet"- )- logger.warning(- "** - the pre-tokenization config has changed upstream"- )- logger.warning(- "** Check your model files and convert_hf_to_gguf_update.py and update them accordingly."- )- logger.warning(- "** ref: https://github.com/ggerganov/llama.cpp/pull/6920")+ "** Check your model files and convert_hf_to_gguf_update.py and update them accordingly.")+ logger.warning("** ref: https://github.com/ggerganov/llama.cpp/pull/6920")logger.warning("**")logger.warning(f"** chkhsh: {chkhsh}")- logger.warning(- "**************************************************************************************"- )+ logger.warning("**************************************************************************************")logger.warning("\n")- raise NotImplementedError(- "BPE pre-tokenizer was not recognized - update get_vocab_base_pre()"- )+ raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")logger.debug(f"tokenizer.ggml.pre: {repr(res)}")logger.debug(f"chkhsh: {chkhsh}")@@ -886,8 +798,7 @@ class OriModel:toktypes: list[int] = []from transformers import AutoTokenizer- tokenizer = AutoTokenizer.from_pretrained(dir_model,- trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)vocab_size = hparams["vocab_size"]assert max(tokenizer.get_vocab().values()) < vocab_size@@ -902,18 +813,11 @@ class OriModel:continuemerged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)assert len(merged) == 2- merges.append(' '.join(map(QwenModel.token_bytes_to_string,- merged)))+ merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))# for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combinedadded_vocab = tokenizer.special_tokens- reverse_vocab = {- id_: encoded_tok- for encoded_tok, id_ in {- **vocab,- **added_vocab- }.items()- }+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in {**vocab, **added_vocab}.items()}for i in range(vocab_size):if i not in reverse_vocab:@@ -935,13 +839,10 @@ class OriModel:special_vocab.merges = merges# only add special tokens when they were not already loaded from config.jsonif len(special_vocab.special_token_ids) == 0:- special_vocab._set_special_token(- "bos", tokenizer.special_tokens["<|endoftext|>"])- special_vocab._set_special_token(- "eos", tokenizer.special_tokens["<|endoftext|>"])+ special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"])+ special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"])# this one is usually not in config.json anyway- special_vocab._set_special_token(- "unk", tokenizer.special_tokens["<|endoftext|>"])+ special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"])special_vocab.add_to_gguf(self.gguf_writer)def _set_vocab_sentencepiece(self, add_to_gguf=True):@@ -969,9 +870,7 @@ class OriModel:vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())- tokens: list[bytes] = [- f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)- ]+ tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_sizetoktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size@@ -1001,9 +900,7 @@ class OriModel:for key in added_tokens_json:token_id = added_tokens_json[key]if token_id >= vocab_size:- logger.warning(- f'ignore token {token_id}: id is out of range, max={vocab_size - 1}'- )+ logger.warning(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')continuetokens[token_id] = key.encode("utf-8")@@ -1014,34 +911,26 @@ class OriModel:if tokenizer_config_file.is_file():with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)- added_tokens_decoder = tokenizer_config_json.get(- "added_tokens_decoder", {})+ added_tokens_decoder = tokenizer_config_json.get("added_tokens_decoder", {})for token_id, token_data in added_tokens_decoder.items():token_id = int(token_id)token: str = token_data["content"]if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:if tokens[token_id] != token.encode("utf-8"):logger.warning(- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token!r}'- )- if token_data.get(- "special") or self.does_token_look_special(token):+ f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token!r}')+ if token_data.get("special") or self.does_token_look_special(token):toktypes[token_id] = SentencePieceTokenTypes.CONTROLelse:- token = token.replace(- b"\xe2\x96\x81".decode("utf-8"),- " ") # pre-normalize user-defined spaces- toktypes[- token_id] = SentencePieceTokenTypes.USER_DEFINED+ token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spaces+ toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINEDscores[token_id] = -1000.0tokens[token_id] = token.encode("utf-8")if vocab_size > len(tokens):pad_count = vocab_size - len(tokens)- logger.debug(- f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]"- )+ logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")for i in range(1, pad_count + 1):tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))scores.append(-1000.0)@@ -1071,65 +960,49 @@ class OriModel:special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))special_vocab.add_to_gguf(self.gguf_writer)- def _set_vocab_builtin(self, model_name: Literal["gpt-neox", "llama-spm"],- vocab_size: int):- tokenizer_path = Path(- sys.path[0]) / "models" / f"ggml-vocab-{model_name}.gguf"- logger.warning(- f"Using tokenizer from '{os.path.relpath(tokenizer_path, os.getcwd())}'"- )+ def _set_vocab_builtin(self, model_name: Literal["gpt-neox", "llama-spm"], vocab_size: int):+ tokenizer_path = Path(sys.path[0]) / "models" / f"ggml-vocab-{model_name}.gguf"+ logger.warning(f"Using tokenizer from '{os.path.relpath(tokenizer_path, os.getcwd())}'")vocab_reader = gguf.GGUFReader(tokenizer_path, "r")default_pre = "mpt" if model_name == "gpt-neox" else "default"field = vocab_reader.get_field(gguf.Keys.Tokenizer.MODEL)assert field # tokenizer model- self.gguf_writer.add_tokenizer_model(- bytes(field.parts[-1]).decode("utf-8"))+ self.gguf_writer.add_tokenizer_model(bytes(field.parts[-1]).decode("utf-8"))field = vocab_reader.get_field(gguf.Keys.Tokenizer.PRE)- self.gguf_writer.add_tokenizer_pre(- bytes(field.parts[-1]).decode("utf-8") if field else default_pre)+ self.gguf_writer.add_tokenizer_pre(bytes(field.parts[-1]).decode("utf-8") if field else default_pre)field = vocab_reader.get_field(gguf.Keys.Tokenizer.LIST)assert field # token list- self.gguf_writer.add_token_list(- [bytes(field.parts[i]) for i in field.data][:vocab_size])+ self.gguf_writer.add_token_list([bytes(field.parts[i]) for i in field.data][:vocab_size])if model_name == "llama-spm":field = vocab_reader.get_field(gguf.Keys.Tokenizer.SCORES)assert field # token scores- self.gguf_writer.add_token_scores(- [field.parts[i].tolist()[0] for i in field.data][:vocab_size])+ self.gguf_writer.add_token_scores([field.parts[i].tolist()[0] for i in field.data][:vocab_size])field = vocab_reader.get_field(gguf.Keys.Tokenizer.TOKEN_TYPE)assert field # token types- self.gguf_writer.add_token_types(- [field.parts[i].tolist()[0] for i in field.data][:vocab_size])+ self.gguf_writer.add_token_types([field.parts[i].tolist()[0] for i in field.data][:vocab_size])if model_name != "llama-spm":field = vocab_reader.get_field(gguf.Keys.Tokenizer.MERGES)assert field # token merges- self.gguf_writer.add_token_merges(- [bytes(field.parts[i]) for i in field.data])+ self.gguf_writer.add_token_merges([bytes(field.parts[i]) for i in field.data])- if (field := vocab_reader.get_field(- gguf.Keys.Tokenizer.BOS_ID)) is not None:+ if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.BOS_ID)) is not None:self.gguf_writer.add_bos_token_id(field.parts[-1].tolist()[0])- if (field := vocab_reader.get_field(- gguf.Keys.Tokenizer.EOS_ID)) is not None:+ if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.EOS_ID)) is not None:self.gguf_writer.add_eos_token_id(field.parts[-1].tolist()[0])- if (field := vocab_reader.get_field(- gguf.Keys.Tokenizer.UNK_ID)) is not None:+ if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.UNK_ID)) is not None:self.gguf_writer.add_unk_token_id(field.parts[-1].tolist()[0])- if (field := vocab_reader.get_field(- gguf.Keys.Tokenizer.PAD_ID)) is not None:+ if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.PAD_ID)) is not None:self.gguf_writer.add_pad_token_id(field.parts[-1].tolist()[0])- if (field := vocab_reader.get_field(- gguf.Keys.Tokenizer.ADD_BOS)) is not None:+ if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.ADD_BOS)) is not None:self.gguf_writer.add_add_bos_token(field.parts[-1].tolist()[0])- if (field := vocab_reader.get_field(- gguf.Keys.Tokenizer.ADD_EOS)) is not None:+ if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.ADD_EOS)) is not None:self.gguf_writer.add_add_eos_token(field.parts[-1].tolist()[0])@@ -1172,8 +1045,7 @@ class Model(OriModel):# can't use an abstract property, because overriding it without type errors# would require using decorated functions instead of simply defining the propertyif "model_arch" not in cls.__dict__:- raise TypeError(- f"Missing property 'model_arch' for {cls.__name__!r}")+ raise TypeError(f"Missing property 'model_arch' for {cls.__name__!r}")def get_tensors(self) -> Iterator[tuple[str, Tensor]]:for name, tensor in self.model.named_parameters():@@ -1216,27 +1088,23 @@ class Model(OriModel):# Conditions should closely match those in llama_model_quantize_internal in llama.cpp# Some tensor types are always in float32- if data_qtype is False and (any(- self.match_model_tensor_name(new_name, key, bid)- for key in (- gguf.MODEL_TENSOR.FFN_GATE_INP,- gguf.MODEL_TENSOR.POS_EMBD,- gguf.MODEL_TENSOR.TOKEN_TYPES,- gguf.MODEL_TENSOR.SSM_CONV1D,- gguf.MODEL_TENSOR.TIME_MIX_FIRST,- gguf.MODEL_TENSOR.TIME_MIX_W1,- gguf.MODEL_TENSOR.TIME_MIX_W2,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,- )) or not new_name.endswith(".weight")):+ if data_qtype is False and (any(self.match_model_tensor_name(new_name, key, bid) for key in (+ gguf.MODEL_TENSOR.FFN_GATE_INP,+ gguf.MODEL_TENSOR.POS_EMBD,+ gguf.MODEL_TENSOR.TOKEN_TYPES,+ gguf.MODEL_TENSOR.SSM_CONV1D,+ gguf.MODEL_TENSOR.TIME_MIX_FIRST,+ gguf.MODEL_TENSOR.TIME_MIX_W1,+ gguf.MODEL_TENSOR.TIME_MIX_W2,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,+ )) or not new_name.endswith(".weight")):data_qtype = gguf.GGMLQuantizationType.F32- if data_qtype is False and any(- self.match_model_tensor_name(new_name, key, bid)- for key in (- gguf.MODEL_TENSOR.TOKEN_EMBD,- gguf.MODEL_TENSOR.OUTPUT,- )):+ if data_qtype is False and any(self.match_model_tensor_name(new_name, key, bid) for key in (+ gguf.MODEL_TENSOR.TOKEN_EMBD,+ gguf.MODEL_TENSOR.OUTPUT,+ )):if self.ftype in (gguf.LlamaFileType.MOSTLY_TQ1_0,gguf.LlamaFileType.MOSTLY_TQ2_0,@@ -1264,8 +1132,7 @@ class Model(OriModel):elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:data_qtype = gguf.GGMLQuantizationType.Q2_Kelse:- raise ValueError(- f"Unknown file type: {self.ftype.name}")+ raise ValueError(f"Unknown file type: {self.ftype.name}")# try:# data = gguf.quants.quantize(data, data_qtype)@@ -1313,7 +1180,7 @@ class Model(OriModel):new_data = []for idx, arr in enumerate(data):arr_name = name.split('.')- for i in range(len(arr_name)-1, -1, -1):+ for i in range(len(arr_name) - 1, -1, -1):if arr_name[i].isdecimal() and int(arr_name[i]) == (data.shape[0] - 1):arr_name[i] = str(idx)arr_name = ".".join(arr_name)@@ -1325,8 +1192,7 @@ class Model(OriModel):data, data_qtype = _quant_data(data, data_qtype)shape = gguf.quant_shape_from_byte_shape(- data.shape,- data_qtype) if data.dtype == np.uint8 else data.shape+ data.shape, data_qtype) if data.dtype == np.uint8 else data.shape# reverse shape to make it similar to the internal ggml dimension ordershape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"@@ -1336,9 +1202,7 @@ class Model(OriModel):f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}"f" --> {data_qtype.name}, shape = {shape_str}")- self.gguf_writer.add_tensor(new_name,- data,- raw_dtype=data_qtype)+ self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)@Model.register("GPTNeoXForCausalLM")@@ -1348,36 +1212,26 @@ class GPTNeoXModel(Model):def set_gguf_parameters(self):block_count = self.hparams["num_hidden_layers"]- self.gguf_writer.add_context_length(- self.hparams["max_position_embeddings"])+ self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(- self.hparams["intermediate_size"])+ self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])self.gguf_writer.add_rope_dimension_count(- int(self.hparams["rotary_pct"] *- (self.hparams["hidden_size"] //- self.hparams["num_attention_heads"])), )+ int(self.hparams["rotary_pct"] * (self.hparams["hidden_size"] // self.hparams["num_attention_heads"])),)self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])- self.gguf_writer.add_parallel_residual(- self.hparams.get("use_parallel_residual", True))+ self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True))self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"])- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused- n_head = self.hparams.get("n_head",- self.hparams.get("num_attention_heads"))+ n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))tensors: list[tuple[str, Tensor]] = []- if re.match(- r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.weight",- name):- qkv_weights = data_torch.reshape(- (n_head, 3, n_embed // n_head, n_embed))+ if re.match(r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.weight", name):+ qkv_weights = data_torch.reshape((n_head, 3, n_embed // n_head, n_embed))data_torch = torch.cat((qkv_weights[:, 0, :, :].reshape((-1, n_embed)),@@ -1387,15 +1241,13 @@ class GPTNeoXModel(Model):dim=0,)logger.info("re-format attention.linear_qkv.weight")- elif re.match(- r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.bias",- name):+ elif re.match(r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.bias", name):qkv_bias = data_torch.reshape((n_head, 3, n_embed // n_head))data_torch = torch.cat((- qkv_bias[:, 0, :].reshape((n_embed, )),- qkv_bias[:, 1, :].reshape((n_embed, )),- qkv_bias[:, 2, :].reshape((n_embed, )),+ qkv_bias[:, 0, :].reshape((n_embed,)),+ qkv_bias[:, 1, :].reshape((n_embed,)),+ qkv_bias[:, 2, :].reshape((n_embed,)),),dim=0,)@@ -1412,10 +1264,8 @@ class BloomModel(Model):def set_gguf_parameters(self):n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))- n_head = self.hparams.get("n_head",- self.hparams.get("num_attention_heads"))- self.gguf_writer.add_context_length(- self.hparams.get("seq_length", n_embed))+ n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))+ self.gguf_writer.add_context_length(self.hparams.get("seq_length", n_embed))self.gguf_writer.add_embedding_length(n_embed)self.gguf_writer.add_feed_forward_length(4 * n_embed)self.gguf_writer.add_block_count(self.hparams["n_layer"])@@ -1424,12 +1274,10 @@ class BloomModel(Model):self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])self.gguf_writer.add_file_type(self.ftype)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused- n_head = self.hparams.get("n_head",- self.hparams.get("num_attention_heads"))+ n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))name = re.sub(r'transformer\.', '', name)@@ -1437,8 +1285,7 @@ class BloomModel(Model):tensors: list[tuple[str, Tensor]] = []if re.match(r"h\.\d+\.self_attention\.query_key_value\.weight", name):- qkv_weights = data_torch.reshape(- (n_head, 3, n_embed // n_head, n_embed))+ qkv_weights = data_torch.reshape((n_head, 3, n_embed // n_head, n_embed))data_torch = torch.cat((qkv_weights[:, 0, :, :].reshape((-1, n_embed)),@@ -1452,9 +1299,9 @@ class BloomModel(Model):qkv_bias = data_torch.reshape((n_head, 3, n_embed // n_head))data_torch = torch.cat((- qkv_bias[:, 0, :].reshape((n_embed, )),- qkv_bias[:, 1, :].reshape((n_embed, )),- qkv_bias[:, 2, :].reshape((n_embed, )),+ qkv_bias[:, 0, :].reshape((n_embed,)),+ qkv_bias[:, 1, :].reshape((n_embed,)),+ qkv_bias[:, 2, :].reshape((n_embed,)),),dim=0,)@@ -1466,11 +1313,8 @@ class BloomModel(Model):assert self.tensor_names is not None# TODO: tie them at runtime, don't duplicate in the model file- if all(s not in self.tensor_names- for s in ("lm_head.weight", "output.weight")):- tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT),- data_torch))+ if all(s not in self.tensor_names for s in ("lm_head.weight", "output.weight")):+ tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch))return tensors@@ -1501,26 +1345,20 @@ class MPTModel(Model):self.gguf_writer.add_head_count_kv(kv_n_heads)self.gguf_writer.add_layer_norm_eps(1e-5)if self.hparams["attn_config"]["clip_qkv"] is not None:- self.gguf_writer.add_clamp_kqv(- self.hparams["attn_config"]["clip_qkv"])+ self.gguf_writer.add_clamp_kqv(self.hparams["attn_config"]["clip_qkv"])if self.hparams["attn_config"]["alibi"]:- self.gguf_writer.add_max_alibi_bias(- self.hparams["attn_config"]["alibi_bias_max"])+ self.gguf_writer.add_max_alibi_bias(self.hparams["attn_config"]["alibi_bias_max"])else:self.gguf_writer.add_max_alibi_bias(0.0)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedif "scales" in name:- new_name = self.map_tensor_name(name,- try_suffixes=(".weight", ".bias",- ".scales"))+ new_name = self.map_tensor_name(name, try_suffixes=(".weight", ".bias", ".scales"))new_name = new_name.replace("scales", "act.scales")else:- new_name = self.map_tensor_name(name,- try_suffixes=(".weight", ".bias"))+ new_name = self.map_tensor_name(name, try_suffixes=(".weight", ".bias"))return [(new_name, data_torch)]@@ -1552,11 +1390,9 @@ class OrionModel(Model):self.gguf_writer.add_context_length(ctx_length)self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(- self.hparams["intermediate_size"])+ self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])self.gguf_writer.add_head_count(head_count)self.gguf_writer.add_head_count_kv(head_count_kv)- # note: config provides rms norm but it is actually layer normself.gguf_writer.add_layer_norm_eps(self.hparams["rms_norm_eps"])@@ -1586,26 +1422,19 @@ class BaichuanModel(Model):self.gguf_writer.add_context_length(ctx_length)self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(- self.hparams["intermediate_size"])- self.gguf_writer.add_rope_dimension_count(- self.hparams["hidden_size"] // self.hparams["num_attention_heads"])+ self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])+ self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])self.gguf_writer.add_head_count(head_count)self.gguf_writer.add_head_count_kv(head_count_kv)self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_file_type(self.ftype)- if self.hparams.get(- "rope_scaling"- ) is not None and "factor" in self.hparams["rope_scaling"]:+ if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:if self.hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(- gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(- self.hparams["rope_scaling"]["factor"])+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:head_count = self.hparams["num_attention_heads"]head_count_kv = self.hparams.get("num_key_value_heads", head_count)@@ -1614,30 +1443,26 @@ class BaichuanModel(Model):if bid is not None and name == f"model.layers.{bid}.self_attn.W_pack.weight":logger.info(f"Unpacking and permuting layer {bid}")tensors = [- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid),- self._reverse_hf_permute_part(data_torch, 0, head_count,- head_count)),- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid),- self._reverse_hf_permute_part(data_torch, 1, head_count,- head_count_kv)),- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid),- self._reverse_hf_part(data_torch, 2)),+ (+ self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q,+ bid), self._reverse_hf_permute_part(data_torch, 0, head_count, head_count)),+ (+ self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid),+ self._reverse_hf_permute_part(data_torch, 1, head_count, head_count_kv)),+ (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), self._reverse_hf_part(data_torch, 2)),]else:tensors = [(self.map_tensor_name(name), data_torch)]return tensors- def _reverse_hf_permute(self,- weights: Tensor,- n_head: int,- n_kv_head: int | None = None) -> Tensor:+ def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor:if n_kv_head is not None and n_head != n_kv_head:n_head //= n_kv_head- return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(- weights.shape))+ return (+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,+ *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))def _reverse_hf_permute_part(self,@@ -1647,12 +1472,11 @@ class BaichuanModel(Model):n_head_kv: int | None = None,) -> Tensor:r = weights.shape[0] // 3- return self._reverse_hf_permute(- weights[r * n_part:r * n_part + r, ...], n_head, n_head_kv)+ return self._reverse_hf_permute(weights[r * n_part:r*n_part + r, ...], n_head, n_head_kv)def _reverse_hf_part(self, weights: Tensor, n_part: int) -> Tensor:r = weights.shape[0] // 3- return weights[r * n_part:r * n_part + r, ...]+ return weights[r * n_part:r*n_part + r, ...]@Model.register("XverseForCausalLM")@@ -1676,10 +1500,7 @@ class XverseModel(Model):if max_vocab_index >= vocab_size:raise ValueError("Vocabulary size exceeds expected maximum size.")- reverse_vocab: dict[int, str] = {- id_: encoded_tok- for encoded_tok, id_ in tokenizer.vocab.items()- }+ reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}added_vocab = tokenizer.get_added_vocab()for token_id in range(vocab_size):@@ -1728,26 +1549,19 @@ class XverseModel(Model):self.gguf_writer.add_context_length(ctx_length)self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])self.gguf_writer.add_block_count(block_count)- self.gguf_writer.add_feed_forward_length(- self.hparams["intermediate_size"])- self.gguf_writer.add_rope_dimension_count(- self.hparams["hidden_size"] // self.hparams["num_attention_heads"])+ self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])+ self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])self.gguf_writer.add_head_count(head_count)self.gguf_writer.add_head_count_kv(head_count_kv)self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_file_type(self.ftype)- if self.hparams.get(- "rope_scaling"- ) is not None and "factor" in self.hparams["rope_scaling"]:+ if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:if self.hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(- gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(- self.hparams["rope_scaling"]["factor"])+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedhead_count = self.hparams["num_attention_heads"]@@ -1755,24 +1569,19 @@ class XverseModel(Model):# HF models permute some of the tensors, so we need to undo thatif name.endswith("q_proj.weight"):- data_torch = self._reverse_hf_permute(data_torch, head_count,- head_count)+ data_torch = self._reverse_hf_permute(data_torch, head_count, head_count)if name.endswith("k_proj.weight"):- data_torch = self._reverse_hf_permute(data_torch, head_count,- head_count_kv)+ data_torch = self._reverse_hf_permute(data_torch, head_count, head_count_kv)return [(self.map_tensor_name(name), data_torch)]- def _reverse_hf_permute(self,- weights: Tensor,- n_head: int,- n_kv_head: int | None = None) -> Tensor:+ def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor:if n_kv_head is not None and n_head != n_kv_head:n_head //= n_kv_head- return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(- weights.shape))+ return (+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,+ *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))@Model.register("FalconForCausalLM", "RWForCausalLM")@@ -1793,19 +1602,16 @@ class FalconModel(Model):n_head_kv = self.hparams.get("n_head_kv", 1) # old nameself.gguf_writer.add_context_length(2048) # not in config.json- self.gguf_writer.add_tensor_data_layout(- "jploski") # qkv tensor transform+ self.gguf_writer.add_tensor_data_layout("jploski") # qkv tensor transformself.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_feed_forward_length(4 *- self.hparams["hidden_size"])+ self.gguf_writer.add_feed_forward_length(4 * self.hparams["hidden_size"])self.gguf_writer.add_block_count(block_count)self.gguf_writer.add_head_count(n_head)self.gguf_writer.add_head_count_kv(n_head_kv)self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])self.gguf_writer.add_file_type(self.ftype)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused# QKV tensor transform@@ -1820,12 +1626,10 @@ class FalconModel(Model):if "query_key_value" in name:n_head = self.find_hparam(["num_attention_heads", "n_head"])- n_head_kv = self.find_hparam(["num_kv_heads", "n_head_kv"],- optional=True) or 1+ n_head_kv = self.find_hparam(["num_kv_heads", "n_head_kv"], optional=True) or 1head_dim = self.hparams["hidden_size"] // n_head- qkv = data_torch.view(n_head_kv, n_head // n_head_kv + 2, head_dim,- head_dim * n_head)+ qkv = data_torch.view(n_head_kv, n_head//n_head_kv + 2, head_dim, head_dim * n_head)q = qkv[:, :-2].reshape(n_head * head_dim, head_dim * n_head)k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)@@ -1860,9 +1664,7 @@ class RefactModel(Model):# TODO: how to determine special FIM tokens automatically?special_vocab = gguf.SpecialVocab(- self.dir_model,- load_merges=False,- special_token_types=['prefix', 'suffix', 'middle', 'eot'])+ self.dir_model, load_merges=False, special_token_types=['prefix', 'suffix', 'middle', 'eot'])special_vocab._set_special_token("prefix", 1)special_vocab._set_special_token("suffix", 3)special_vocab._set_special_token("middle", 2)@@ -1874,7 +1676,7 @@ class RefactModel(Model):inner_dim = 4 * hidden_dimhidden_dim = int(2 * inner_dim / 3)multiple_of = 256- ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)+ ff_dim = multiple_of * ((hidden_dim+multiple_of-1) // multiple_of)block_count = self.hparams["n_layer"]@@ -1886,17 +1688,15 @@ class RefactModel(Model):self.gguf_writer.add_block_count(block_count)self.gguf_writer.add_head_count(self.hparams["n_head"])self.gguf_writer.add_head_count_kv(1)- self.gguf_writer.add_layer_norm_rms_eps(- self.hparams["layer_norm_epsilon"])+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"])self.gguf_writer.add_file_type(self.ftype)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:hidden_dim = self.hparams["n_embd"]inner_dim = 4 * hidden_dimhidden_dim = int(2 * inner_dim / 3)multiple_of = 256- ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)+ ff_dim = multiple_of * ((hidden_dim+multiple_of-1) // multiple_of)n_head = self.hparams["n_head"]n_head_kv = 1head_dim = self.hparams["n_embd"] // n_head@@ -1906,22 +1706,14 @@ class RefactModel(Model):if bid is not None:if name == f"transformer.h.{bid}.attn.kv.weight":tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid),- data_torch[:n_head_kv * head_dim]))+ (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), data_torch[:n_head_kv * head_dim]))tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid),- data_torch[n_head_kv * head_dim:]))+ (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), data_torch[n_head_kv * head_dim:]))elif name == f"transformer.h.{bid}.attn.q.weight":- tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q,- bid), data_torch))+ tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), data_torch))elif name == f"transformer.h.{bid}.mlp.gate_up_proj.weight":- tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE,- bid), data_torch[:ff_dim]))- tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP,- bid), data_torch[ff_dim:]))+ tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), data_torch[:ff_dim]))+ tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), data_torch[ff_dim:]))if len(tensors) == 0:tensors.append((self.map_tensor_name(name), data_torch))@@ -1929,8 +1721,7 @@ class RefactModel(Model):return tensors[email protected]("StableLmForCausalLM", "StableLMEpochForCausalLM",- "LlavaStableLMEpochForCausalLM")[email protected]("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM")class StableLMModel(Model):model_arch = gguf.MODEL_ARCH.STABLELM@@ -1951,22 +1742,18 @@ class StableLMModel(Model):self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"])self.gguf_writer.add_rope_dimension_count(- int(rotary_factor *- (hparams["hidden_size"] // hparams["num_attention_heads"])))+ int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"])))self.gguf_writer.add_head_count(hparams["num_attention_heads"])self.gguf_writer.add_head_count_kv(hparams["num_key_value_heads"])self.gguf_writer.add_parallel_residual(- hparams["use_parallel_residual"] if "use_parallel_residual" in- hparams else True)- self.gguf_writer.add_layer_norm_eps(- self.find_hparam(["layer_norm_eps", "norm_eps"]))+ hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True)+ self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_eps", "norm_eps"]))self.gguf_writer.add_file_type(self.ftype)_q_norms: list[dict[str, Tensor]] | None = None_k_norms: list[dict[str, Tensor]] | None = None- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]n_kv_head = self.hparams["num_key_value_heads"]@@ -1979,8 +1766,7 @@ class StableLMModel(Model):self._q_norms[bid][name] = data_torchif len(self._q_norms[bid]) >= n_head:- return self._stack_qk_norm(bid, n_head, self._q_norms[bid],- "q_layernorm")+ return self._stack_qk_norm(bid, n_head, self._q_norms[bid], "q_layernorm")else:return []@@ -1993,18 +1779,13 @@ class StableLMModel(Model):self._k_norms[bid][name] = data_torchif len(self._k_norms[bid]) >= n_kv_head:- return self._stack_qk_norm(bid, n_kv_head, self._k_norms[bid],- "k_layernorm")+ return self._stack_qk_norm(bid, n_kv_head, self._k_norms[bid], "k_layernorm")else:return []return [(self.map_tensor_name(name), data_torch)]- def _stack_qk_norm(self,- bid: int,- n_head: int,- norms: dict[str, Tensor],- layer_name: str = "q_layernorm"):+ def _stack_qk_norm(self, bid: int, n_head: int, norms: dict[str, Tensor], layer_name: str = "q_layernorm"):data: list[Tensor] = []# extract the norms in orderfor xid in range(n_head):@@ -2023,16 +1804,13 @@ class StableLMModel(Model):if self._q_norms is not None or self._k_norms is not None:# flatten two `list[dict[str, Tensor]]` into a single `list[str]`- norms = ([k for d in self._q_norms- for k in d.keys()] if self._q_norms is not None else- []) + ([k for d in self._k_norms for k in d.keys()]- if self._k_norms is not None else [])+ norms = ([k for d in self._q_norms for k in d.keys()] if self._q_norms is not None else+ []) + ([k for d in self._k_norms for k in d.keys()] if self._k_norms is not None else [])if len(norms) > 0:raise ValueError(f"Unprocessed norms: {norms}")[email protected]("LLaMAForCausalLM", "LlamaForCausalLM", "MistralForCausalLM",- "MixtralForCausalLM")[email protected]("LLaMAForCausalLM", "LlamaForCausalLM", "MistralForCausalLM", "MixtralForCausalLM")class LlamaModel(Model):model_arch = gguf.MODEL_ARCH.LLAMA@@ -2049,9 +1827,7 @@ class LlamaModel(Model):# Apply to CodeLlama only (and ignore for Llama 3 with a vocab size of 128256)if self.hparams.get("vocab_size", 32000) == 32016:special_vocab = gguf.SpecialVocab(- self.dir_model,- load_merges=False,- special_token_types=['prefix', 'suffix', 'middle', 'eot'])+ self.dir_model, load_merges=False, special_token_types=['prefix', 'suffix', 'middle', 'eot'])special_vocab._set_special_token("prefix", 32007)special_vocab._set_special_token("suffix", 32008)special_vocab._set_special_token("middle", 32009)@@ -2063,8 +1839,7 @@ class LlamaModel(Model):with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)if "add_prefix_space" in tokenizer_config_json:- self.gguf_writer.add_add_space_prefix(- tokenizer_config_json["add_prefix_space"])+ self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])# Apply to granite small models onlyif self.hparams.get("vocab_size", 32000) == 49152:@@ -2081,27 +1856,22 @@ class LlamaModel(Model):rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]self.gguf_writer.add_rope_dimension_count(rope_dim)- if self.hparams.get(- "rope_scaling"- ) is not None and "factor" in self.hparams["rope_scaling"]:+ if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:if self.hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(- gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(- self.hparams["rope_scaling"]["factor"])+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])@staticmethoddef permute(weights: Tensor, n_head: int, n_head_kv: int | None):if n_head_kv is not None and n_head != n_head_kv:n_head = n_head_kv- return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(- weights.shape))+ return (+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,+ *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))_experts: list[dict[str, Tensor]] | None = None- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]n_kv_head = self.hparams.get("num_key_value_heads")@@ -2150,17 +1920,13 @@ class LlamaModel(Model):if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):if rope_scaling.get("rope_type", '').lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)- dim = self.hparams.get(- "head_dim", self.hparams["hidden_size"] //- self.hparams["num_attention_heads"])- freqs = 1.0 / (base**(- torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ dim = self.hparams.get("head_dim", self.hparams["hidden_size"] // self.hparams["num_attention_heads"])+ freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)- old_context_len = self.hparams.get(- "original_max_position_embeddings", 8192)+ old_context_len = self.hparams.get("original_max_position_embeddings", 8192)low_freq_wavelen = old_context_len / low_freq_factorhigh_freq_wavelen = old_context_len / high_freq_factor@@ -2174,13 +1940,12 @@ class LlamaModel(Model):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len / wavelen - low_freq_factor- ) / (high_freq_factor - low_freq_factor)- rope_factors.append(1 /- ((1 - smooth) / factor + smooth))+ smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)+ rope_factors.append(1 / ((1-smooth) / factor + smooth))- yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32))+ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),+ torch.tensor(rope_factors, dtype=torch.float32))def prepare_tensors(self):super().prepare_tensors()@@ -2207,14 +1972,13 @@ class DeciModel(Model):# DeciLM-specific codeif n % k == 0:return n- return n + k - (n % k)+ return n + k - (n%k)def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)if "block_configs" in self.hparams: # Llama-3_1-Nemotron-51B- _block_configs: list[dict[str,- Any]] = self.hparams["block_configs"]+ _block_configs: list[dict[str, Any]] = self.hparams["block_configs"]assert self.block_count == len(_block_configs)self._num_kv_heads = list()self._num_heads = list()@@ -2231,32 +1995,25 @@ class DeciModel(Model):# _num_heads[il] is num_attention_headfor il in range(len(_block_configs)):if _block_configs[il]["attention"]["n_heads_in_group"] is None:- if _block_configs[il]["attention"][- "replace_with_linear"] is True:+ if _block_configs[il]["attention"]["replace_with_linear"] is True:self._num_kv_heads.append(0)- self._num_heads.append(- self.hparams["num_attention_heads"])+ self._num_heads.append(self.hparams["num_attention_heads"])else:self._num_kv_heads.append(0)self._num_heads.append(0)else:self._num_kv_heads.append(- self.hparams["num_attention_heads"] //- _block_configs[il]["attention"]["n_heads_in_group"])+ self.hparams["num_attention_heads"] // _block_configs[il]["attention"]["n_heads_in_group"])self._num_heads.append(self.hparams["num_attention_heads"])_ffn_multipliers.append(_block_configs[il]["ffn"]["ffn_mult"])assert self.block_count == len(self._num_kv_heads)assert self.block_count == len(self._num_heads)assert self.block_count == len(_ffn_multipliers)- assert isinstance(self._num_kv_heads, list) and isinstance(- self._num_kv_heads[0], int)- assert isinstance(self._num_heads, list) and isinstance(- self._num_heads[0], int)- assert isinstance(_ffn_multipliers, list) and isinstance(- _ffn_multipliers[0], float)+ assert isinstance(self._num_kv_heads, list) and isinstance(self._num_kv_heads[0], int)+ assert isinstance(self._num_heads, list) and isinstance(self._num_heads[0], int)+ assert isinstance(_ffn_multipliers, list) and isinstance(_ffn_multipliers[0], float)self._ffn_dims: list[int] = [- DeciModel._ffn_mult_to_intermediate_size(- multiplier, self.hparams["hidden_size"])+ DeciModel._ffn_mult_to_intermediate_size(multiplier, self.hparams["hidden_size"])for multiplier in _ffn_multipliers]@@ -2287,23 +2044,16 @@ class DeciModel(Model):self.gguf_writer.add_head_count(self._num_heads)self.gguf_writer.add_feed_forward_length(self._ffn_dims)self.gguf_writer.add_block_count(self.block_count)- self.gguf_writer.add_context_length(- self.hparams["max_position_embeddings"])+ self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_layer_norm_rms_eps(- self.hparams["rms_norm_eps"])- self.gguf_writer.add_key_length(- self.hparams["hidden_size"] //- self.hparams["num_attention_heads"])- self.gguf_writer.add_value_length(- self.hparams["hidden_size"] //- self.hparams["num_attention_heads"])+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])+ self.gguf_writer.add_key_length(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])+ self.gguf_writer.add_value_length(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])self.gguf_writer.add_file_type(self.ftype)else: # DeciLM-7Bsuper().set_gguf_parameters()if "num_key_value_heads_per_layer" in self.hparams: # DeciLM-7B- self._num_kv_heads: list[int] = self.hparams[- "num_key_value_heads_per_layer"]+ self._num_kv_heads: list[int] = self.hparams["num_key_value_heads_per_layer"]assert self.block_count == len(self._num_kv_heads)self.gguf_writer.add_head_count_kv(self._num_kv_heads)hparams = self.hparams@@ -2315,25 +2065,20 @@ class DeciModel(Model):rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]self.gguf_writer.add_rope_dimension_count(rope_dim)- if self.hparams.get(- "rope_scaling"- ) is not None and "factor" in self.hparams["rope_scaling"]:+ if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:if self.hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(- gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(- self.hparams["rope_scaling"]["factor"])+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])@staticmethoddef permute(weights: Tensor, n_head: int, n_head_kv: int | None):if n_head_kv is not None and n_head != n_head_kv:n_head = n_head_kv- return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(- weights.shape))+ return (+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,+ *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]if bid is not None:if "num_key_value_heads_per_layer" in self.hparams:@@ -2356,17 +2101,13 @@ class DeciModel(Model):if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):if rope_scaling.get("rope_type", '').lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)- dim = self.hparams.get(- "head_dim", self.hparams["hidden_size"] //- self.hparams["num_attention_heads"])- freqs = 1.0 / (base**(- torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ dim = self.hparams.get("head_dim", self.hparams["hidden_size"] // self.hparams["num_attention_heads"])+ freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)- old_context_len = self.hparams.get(- "original_max_position_embeddings", 8192)+ old_context_len = self.hparams.get("original_max_position_embeddings", 8192)low_freq_wavelen = old_context_len / low_freq_factorhigh_freq_wavelen = old_context_len / high_freq_factor@@ -2380,13 +2121,12 @@ class DeciModel(Model):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len / wavelen - low_freq_factor- ) / (high_freq_factor - low_freq_factor)- rope_factors.append(1 /- ((1 - smooth) / factor + smooth))+ smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)+ rope_factors.append(1 / ((1-smooth) / factor + smooth))- yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32))+ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),+ torch.tensor(rope_factors, dtype=torch.float32))def prepare_tensors(self):super().prepare_tensors()@@ -2409,25 +2149,21 @@ class BitnetModel(Model):weight = weight.float()scale = weight.abs().mean().clamp(min=1e-5)iscale = 1 / scale- # TODO: multiply by the scale directly instead of inverting it twice- # (this is also unnecessarily doubly inverted upstream)result = (weight * iscale).round().clamp(-1, 1) / iscalereturn result.type(dtype)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:new_name = self.map_tensor_name(name)- if any(- self.match_model_tensor_name(new_name, key, bid) for key in [- gguf.MODEL_TENSOR.ATTN_Q,- gguf.MODEL_TENSOR.ATTN_K,- gguf.MODEL_TENSOR.ATTN_V,- gguf.MODEL_TENSOR.ATTN_OUT,- gguf.MODEL_TENSOR.FFN_UP,- gguf.MODEL_TENSOR.FFN_DOWN,- gguf.MODEL_TENSOR.FFN_GATE,- ]):+ if any(self.match_model_tensor_name(new_name, key, bid) for key in [+ gguf.MODEL_TENSOR.ATTN_Q,+ gguf.MODEL_TENSOR.ATTN_K,+ gguf.MODEL_TENSOR.ATTN_V,+ gguf.MODEL_TENSOR.ATTN_OUT,+ gguf.MODEL_TENSOR.FFN_UP,+ gguf.MODEL_TENSOR.FFN_DOWN,+ gguf.MODEL_TENSOR.FFN_GATE,+ ]):# transform weight into 1/0/-1 (in fp32)data_torch = self.weight_quant(data_torch)@@ -2449,8 +2185,7 @@ class GrokModel(Model):_experts: list[dict[str, Tensor]] | None = None- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# process the experts separatelyif name.find(".moe.") != -1:n_experts = self.hparams["num_local_experts"]@@ -2516,8 +2251,7 @@ class DbrxModel(Model):self.gguf_writer.add_file_type(self.ftype)logger.info(f"gguf: file type = {self.ftype}")- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedn_expert = self.hparams["ffn_config"]["moe_num_experts"]@@ -2530,11 +2264,8 @@ class DbrxModel(Model):# AND the dimensions in ggml are typically in the reverse order of the pytorch dimensions# so (n_expert, n_ff, n_embd) in pytorch is {n_embd, n_ff, n_expert} in ggml_tensorexp_tensor_names = {- "ffn.experts.mlp.w1":- None, # LLM_TENSOR_FFN_GATE_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}- "ffn.experts.mlp.w2": (- 0, 2, 1- ), # LLM_TENSOR_FFN_DOWN_EXPS ggml_tensor->ne{n_ff, n_embd, n_expert}+ "ffn.experts.mlp.w1": None, # LLM_TENSOR_FFN_GATE_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}+ "ffn.experts.mlp.w2": (0, 2, 1), # LLM_TENSOR_FFN_DOWN_EXPS ggml_tensor->ne{n_ff, n_embd, n_expert}"ffn.experts.mlp.v1": None} # LLM_TENSOR_FFN_UP_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}experts = False@@ -2543,8 +2274,7 @@ class DbrxModel(Model):if name.find(exp_tensor_name) != -1 and name.find(".weight") == -1:experts = Truedata_torch = data_torch.view(n_expert, n_ff, n_embd)- if (permute_tensor :=- exp_tensor_names[exp_tensor_name]) is not None:+ if (permute_tensor := exp_tensor_names[exp_tensor_name]) is not None:data_torch = data_torch.permute(*permute_tensor)break@@ -2554,14 +2284,12 @@ class DbrxModel(Model):# Every other model has the weight names ending in .weight,# let's assume that is the convention which is not the case for dbrx:# https://huggingface.co/databricks/dbrx-instruct/blob/main/model.safetensors.index.json#L15- new_name = self.map_tensor_name(name if not experts else name +- ".weight",- try_suffixes=(".weight", ))+ new_name = self.map_tensor_name(name if not experts else name + ".weight", try_suffixes=(".weight",))return [(new_name, data_torch)]- def tensor_force_quant(self, name: str, new_name: str, bid: int | None,- n_dims: int) -> gguf.GGMLQuantizationType | bool:+ def tensor_force_quant(+ self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:del name, new_name, bid # unusedreturn n_dims > 1@@ -2576,25 +2304,19 @@ class MiniCPMModel(Model):embedding_scale = float(self.hparams["scale_emb"])self.gguf_writer.add_embedding_scale(embedding_scale)logger.info(f"gguf: (minicpm) embedding_scale = {embedding_scale}")- residual_scale = self.hparams["scale_depth"] / self.hparams[- "num_hidden_layers"]**0.5+ residual_scale = self.hparams["scale_depth"] / self.hparams["num_hidden_layers"]**0.5self.gguf_writer.add_residual_scale(residual_scale)logger.info(f"gguf: (minicpm) residual_scale = {residual_scale}")- logit_scale = self.hparams["hidden_size"] / self.hparams[- "dim_model_base"]+ logit_scale = self.hparams["hidden_size"] / self.hparams["dim_model_base"]self.gguf_writer.add_logit_scale(logit_scale)logger.info(f"gguf: (minicpm) logit_scale = {logit_scale}")if self.hparams.get("rope_scaling") is not None:if self.hparams["rope_scaling"].get("type") == "longrope":- self.gguf_writer.add_rope_scaling_type(- gguf.RopeScalingType.LONGROPE)- logger.info(- f"gguf: (minicpm) rope_scaling_type = {gguf.RopeScalingType.LONGROPE}"- )+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LONGROPE)+ logger.info(f"gguf: (minicpm) rope_scaling_type = {gguf.RopeScalingType.LONGROPE}")def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- rope_dims = self.hparams["hidden_size"] // self.hparams[- "num_attention_heads"]+ rope_dims = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]rope_scaling = self.find_hparam(['rope_scaling'], True)if rope_scaling is not None:@@ -2602,28 +2324,22 @@ class MiniCPMModel(Model):short_factors = rope_scaling.get('short_factor', None)if long_factors is None or short_factors is None:- raise KeyError(- 'Missing the required key rope_scaling.long_factor or rope_scaling_short_factor'- )+ raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor')- if len(long_factors) != len(short_factors) or len(- long_factors) != rope_dims / 2:- raise ValueError(- f'The length of rope long and short factors must be {rope_dims / 2}'- )+ if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:+ raise ValueError(f'The length of rope long and short factors must be {rope_dims / 2}')- yield (self.format_tensor_name(- gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),+ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),torch.tensor(long_factors, dtype=torch.float32))- yield (self.format_tensor_name(- gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),+ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),torch.tensor(short_factors, dtype=torch.float32))def set_vocab(self):self._set_vocab_sentencepiece()- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedn_head = self.hparams["num_attention_heads"]@@ -2657,8 +2373,7 @@ class MiniCPM3Model(Model):if "q_lora_rank" in hparams and hparams["q_lora_rank"] is not None:self.gguf_writer.add_q_lora_rank(hparams["q_lora_rank"])self.gguf_writer.add_kv_lora_rank(hparams["kv_lora_rank"])- self.gguf_writer.add_key_length(hparams["qk_nope_head_dim"] +- hparams["qk_rope_head_dim"])+ self.gguf_writer.add_key_length(hparams["qk_nope_head_dim"] + hparams["qk_rope_head_dim"])self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:@@ -2670,36 +2385,28 @@ class MiniCPM3Model(Model):short_factors = rope_scaling.get('short_factor', None)if long_factors is None or short_factors is None:- raise KeyError(- 'Missing the required key rope_scaling.long_factor or rope_scaling_short_factor'- )+ raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor')- if len(long_factors) != len(short_factors) or len(- long_factors) != rope_dims / 2:- raise ValueError(- f'The length of rope long and short factors must be {rope_dims / 2}'- )+ if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:+ raise ValueError(f'The length of rope long and short factors must be {rope_dims / 2}')- yield (self.format_tensor_name(- gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),+ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),torch.tensor(long_factors, dtype=torch.float32))- yield (self.format_tensor_name(- gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),+ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),torch.tensor(short_factors, dtype=torch.float32))def set_vocab(self):self._set_vocab_sentencepiece()- def _reverse_hf_permute(self,- weights: Tensor,- n_head: int,- n_kv_head: int | None = None) -> Tensor:+ def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor:if n_kv_head is not None and n_head != n_kv_head:n_head //= n_kv_head- return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(- weights.shape))+ return (+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,+ *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))@Model.register("QWenLMHeadModel")@@ -2710,13 +2417,10 @@ class QwenModel(Model):def token_bytes_to_string(b):from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicodebyte_encoder = bytes_to_unicode()- return ''.join(- [byte_encoder[ord(char)] for char in b.decode('latin-1')])+ return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])@staticmethod- def bpe(mergeable_ranks: dict[bytes, int],- token: bytes,- max_rank: int | None = None) -> list[bytes]:+ def bpe(mergeable_ranks: dict[bytes, int], token: bytes, max_rank: int | None = None) -> list[bytes]:parts = [bytes([b]) for b in token]while True:min_idx = None@@ -2726,30 +2430,24 @@ class QwenModel(Model):if rank is not None and (min_rank is None or rank < min_rank):min_idx = imin_rank = rank- if min_rank is None or (max_rank is not None- and min_rank >= max_rank):+ if min_rank is None or (max_rank is not None and min_rank >= max_rank):breakassert min_idx is not None- parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]- ] + parts[min_idx + 2:]+ parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2:]return partsdef set_vocab(self):self._set_vocab_qwen()def set_gguf_parameters(self):- self.gguf_writer.add_context_length(- self.hparams["max_position_embeddings"])+ self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])self.gguf_writer.add_block_count(self.hparams["num_hidden_layers"])self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_feed_forward_length(- self.hparams["intermediate_size"])+ self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])self.gguf_writer.add_rope_freq_base(self.hparams["rotary_emb_base"])- self.gguf_writer.add_rope_dimension_count(- self.hparams["hidden_size"] // self.hparams["num_attention_heads"])+ self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])- self.gguf_writer.add_layer_norm_rms_eps(- self.hparams["layer_norm_epsilon"])+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"])self.gguf_writer.add_file_type(self.ftype)@@ -2765,17 +2463,12 @@ class Qwen2Model(Model):def set_gguf_parameters(self):super().set_gguf_parameters()- if self.hparams.get(- "rope_scaling"- ) is not None and "factor" in self.hparams["rope_scaling"]:+ if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:if self.hparams["rope_scaling"].get("type") == "yarn":- self.gguf_writer.add_rope_scaling_type(- gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(- self.hparams["rope_scaling"]["factor"])+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)+ self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])self.gguf_writer.add_rope_scaling_orig_ctx_len(- self.hparams["rope_scaling"]- ["original_max_position_embeddings"])+ self.hparams["rope_scaling"]["original_max_position_embeddings"])@Model.register("Qwen2VLForConditionalGeneration")@@ -2805,8 +2498,7 @@ class Qwen2VLModel(Model):class WavTokenizerDecModel(Model):model_arch = gguf.MODEL_ARCH.WAVTOKENIZER_DEC- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedif \@@ -2829,18 +2521,13 @@ class WavTokenizerDecModel(Model):self.gguf_writer.add_features_length(self.hparams["n_embd_features"])self.gguf_writer.add_feed_forward_length(self.hparams["n_ff"])self.gguf_writer.add_group_norm_eps(self.hparams["group_norm_epsilon"])- self.gguf_writer.add_group_norm_groups(- self.hparams["group_norm_groups"])+ self.gguf_writer.add_group_norm_groups(self.hparams["group_norm_groups"])- self.gguf_writer.add_posnet_embedding_length(- self.hparams["posnet"]["n_embd"])- self.gguf_writer.add_posnet_block_count(- self.hparams["posnet"]["n_layer"])+ self.gguf_writer.add_posnet_embedding_length(self.hparams["posnet"]["n_embd"])+ self.gguf_writer.add_posnet_block_count(self.hparams["posnet"]["n_layer"])- self.gguf_writer.add_convnext_embedding_length(- self.hparams["convnext"]["n_embd"])- self.gguf_writer.add_convnext_block_count(- self.hparams["convnext"]["n_layer"])+ self.gguf_writer.add_convnext_embedding_length(self.hparams["convnext"]["n_embd"])+ self.gguf_writer.add_convnext_block_count(self.hparams["convnext"]["n_layer"])self.gguf_writer.add_causal_attention(False)@@ -2853,25 +2540,16 @@ class Qwen2MoeModel(Model):super().set_gguf_parameters()if (n_experts := self.hparams.get("num_experts")) is not None:self.gguf_writer.add_expert_count(n_experts)- if (moe_intermediate_size :=- self.hparams.get("moe_intermediate_size")) is not None:- self.gguf_writer.add_expert_feed_forward_length(- moe_intermediate_size)- logger.info(- f"gguf: expert feed forward length = {moe_intermediate_size}")- if (shared_expert_intermediate_size :=- self.hparams.get('shared_expert_intermediate_size')- ) is not None:- self.gguf_writer.add_expert_shared_feed_forward_length(- shared_expert_intermediate_size)- logger.info(- f"gguf: expert shared feed forward length = {shared_expert_intermediate_size}"- )+ if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:+ self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)+ logger.info(f"gguf: expert feed forward length = {moe_intermediate_size}")+ if (shared_expert_intermediate_size := self.hparams.get('shared_expert_intermediate_size')) is not None:+ self.gguf_writer.add_expert_shared_feed_forward_length(shared_expert_intermediate_size)+ logger.info(f"gguf: expert shared feed forward length = {shared_expert_intermediate_size}")_experts: list[dict[str, Tensor]] | None = None- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# process the experts separatelyif name.find("experts") != -1:n_experts = self.hparams["num_experts"]@@ -2930,8 +2608,7 @@ class GPT2Model(Model):self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])self.gguf_writer.add_file_type(self.ftype)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedtensors: list[tuple[str, Tensor]] = []@@ -2940,8 +2617,7 @@ class GPT2Model(Model):if name.endswith((".attn.bias", ".attn.masked_bias")):return tensors- if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight",- ".c_proj.weight")):+ if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight", ".c_proj.weight")):data_torch = data_torch.transpose(1, 0)new_name = self.map_tensor_name(name)@@ -2950,8 +2626,7 @@ class GPT2Model(Model):# note: GPT2 output is tied to (same as) wte in original modelif new_name == self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD):- tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT),- data_torch))+ tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch))return tensors@@ -2967,18 +2642,15 @@ class Phi2Model(Model):n_embd = self.find_hparam(["hidden_size", "n_embd"])n_head = self.find_hparam(["num_attention_heads", "n_head"])- self.gguf_writer.add_context_length(- self.find_hparam(["n_positions", "max_position_embeddings"]))+ self.gguf_writer.add_context_length(self.find_hparam(["n_positions", "max_position_embeddings"]))self.gguf_writer.add_embedding_length(n_embd)self.gguf_writer.add_feed_forward_length(4 * n_embd)self.gguf_writer.add_block_count(block_count)self.gguf_writer.add_head_count(n_head)self.gguf_writer.add_head_count_kv(n_head)- self.gguf_writer.add_layer_norm_eps(- self.find_hparam(["layer_norm_epsilon", "layer_norm_eps"]))- self.gguf_writer.add_rope_dimension_count(- int(rot_pct * n_embd) // n_head)+ self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_epsilon", "layer_norm_eps"]))+ self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head)self.gguf_writer.add_file_type(self.ftype)self.gguf_writer.add_add_bos_token(False)@@ -3009,9 +2681,7 @@ class Phi3MiniModel(Model):vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())- tokens: list[bytes] = [- f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)- ]+ tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_sizetoktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size@@ -3043,9 +2713,7 @@ class Phi3MiniModel(Model):for key in added_tokens_json:token_id = added_tokens_json[key]if token_id >= vocab_size:- logger.debug(- f'ignore token {token_id}: id is out of range, max={vocab_size - 1}'- )+ logger.debug(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')continuetokens[token_id] = key.encode("utf-8")@@ -3056,16 +2724,15 @@ class Phi3MiniModel(Model):if tokenizer_config_file.is_file():with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)- added_tokens_decoder = tokenizer_config_json.get(- "added_tokens_decoder", {})+ added_tokens_decoder = tokenizer_config_json.get("added_tokens_decoder", {})for token_id, foken_data in added_tokens_decoder.items():token_id = int(token_id)token = foken_data["content"].encode("utf-8")if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:if tokens[token_id] != token:logger.warning(- f"replacing token {token_id}: {tokens[token_id].decode('utf-8')!r} "- f"-> {token.decode('utf-8')!r}"+ f"replacing token {token_id}: {tokens[token_id].decode('utf-8')!r}"+ f" -> {token.decode('utf-8')!r}")tokens[token_id] = tokenscores[token_id] = -1000.0@@ -3084,8 +2751,8 @@ class Phi3MiniModel(Model):if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:if tokens[token_id] != token:logger.warning(- f"replacing token {token_id}: {tokens[token_id].decode('utf-8')!r} "- f"-> {token.decode('utf-8')!r}"+ f"replacing token {token_id}: {tokens[token_id].decode('utf-8')!r}"+ f" -> {token.decode('utf-8')!r}")tokens[token_id] = tokenscores[token_id] = -1000.0@@ -3109,17 +2776,15 @@ class Phi3MiniModel(Model):n_head = self.find_hparam(["num_attention_heads", "n_head"])n_head_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"])rms_eps = self.find_hparam(["rms_norm_eps"])- max_pos_embds = self.find_hparam(- ["n_positions", "max_position_embeddings"])- orig_max_pos_embds = self.find_hparam(- ["original_max_position_embeddings"])- rope_dims = n_embd // n_head+ max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"])+ orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"])+ rot_pct = self.hparams.get("partial_rotary_factor", 1.0)+ rope_dims = int(rot_pct * n_embd) // n_headself.gguf_writer.add_context_length(max_pos_embds)self.gguf_writer.add_rope_scaling_orig_ctx_len(orig_max_pos_embds)self.gguf_writer.add_embedding_length(n_embd)- self.gguf_writer.add_feed_forward_length(- self.find_hparam(["intermediate_size"]))+ self.gguf_writer.add_feed_forward_length(self.find_hparam(["intermediate_size"]))self.gguf_writer.add_block_count(block_count)self.gguf_writer.add_head_count(n_head)self.gguf_writer.add_head_count_kv(n_head_kv)@@ -3136,11 +2801,10 @@ class Phi3MiniModel(Model):def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:n_embd = self.find_hparam(["hidden_size", "n_embd"])n_head = self.find_hparam(["num_attention_heads", "n_head"])- max_pos_embds = self.find_hparam(- ["n_positions", "max_position_embeddings"])- orig_max_pos_embds = self.find_hparam(- ["original_max_position_embeddings"])- rope_dims = n_embd // n_head+ max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"])+ orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"])+ rot_pct = self.hparams.get("partial_rotary_factor", 1.0)+ rope_dims = int(rot_pct * n_embd) // n_head# write rope scaling for long context (128k) modelrope_scaling = self.find_hparam(['rope_scaling'], True)@@ -3154,15 +2818,11 @@ class Phi3MiniModel(Model):raise KeyError('Missing the required key rope_scaling.type')if rope_scaling_type == 'su' or rope_scaling_type == 'longrope':- attn_factor = math.sqrt(- 1 + math.log(scale) /- math.log(orig_max_pos_embds)) if scale > 1.0 else 1.0+ attn_factor = math.sqrt(1 + math.log(scale) / math.log(orig_max_pos_embds)) if scale > 1.0 else 1.0elif rope_scaling_type == 'yarn':attn_factor = 0.1 * math.log(scale) + 1.0 if scale > 1.0 else 1.0else:- raise NotImplementedError(- f'The rope scaling type {rope_scaling_type} is not supported yet'- )+ raise NotImplementedError(f'The rope scaling type {rope_scaling_type} is not supported yet')self.gguf_writer.add_rope_scaling_attn_factors(attn_factor)@@ -3170,20 +2830,20 @@ class Phi3MiniModel(Model):short_factors = rope_scaling.get('short_factor', None)if long_factors is None or short_factors is None:- raise KeyError(- 'Missing the required key rope_scaling.long_factor or rope_scaling_short_factor'- )+ raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor')- if len(long_factors) != len(short_factors) or len(- long_factors) != rope_dims / 2:+ if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:raise ValueError(- f'The length of rope long and short factors must be {rope_dims / 2}'+ f"The length of rope long and short factors must be {rope_dims / 2}."+ f" long_factors = {len(long_factors)}, short_factors = {len(short_factors)}.")- yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),- torch.tensor(long_factors, dtype=torch.float32))- yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),- torch.tensor(short_factors, dtype=torch.float32))+ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),+ torch.tensor(long_factors, dtype=torch.float32))+ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),+ torch.tensor(short_factors, dtype=torch.float32))@Model.register("PhiMoEForCausalLM")@@ -3194,12 +2854,10 @@ class PhiMoeModel(Phi3MiniModel):def set_gguf_parameters(self):super().set_gguf_parameters()- self.gguf_writer.add_expert_used_count(- self.hparams["num_experts_per_tok"])+ self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])self.gguf_writer.add_expert_count(self.hparams["num_local_experts"])- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# process the experts separatelyif name.find("block_sparse_moe.experts") != -1:n_experts = self.hparams["num_local_experts"]@@ -3261,8 +2919,7 @@ class PlamoModel(Model):self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])self.gguf_writer.add_block_count(block_count)self.gguf_writer.add_head_count(hparams["num_attention_heads"])- self.gguf_writer.add_head_count_kv(- 5) # hparams["num_key_value_heads"]) is wrong+ self.gguf_writer.add_head_count_kv(5) # hparams["num_key_value_heads"]) is wrongself.gguf_writer.add_layer_norm_rms_eps(hparams["rms_norm_eps"])self.gguf_writer.add_file_type(self.ftype)@@ -3280,8 +2937,7 @@ class PlamoModel(Model):data_torch = torch.reshape(data_torch, (5120, 5120))return data_torch- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusednew_name = self.map_tensor_name(name)@@ -3314,8 +2970,7 @@ class CodeShellModel(Model):self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)self.gguf_writer.add_rope_scaling_factor(1.0)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusednew_name = self.map_tensor_name(name)@@ -3325,12 +2980,9 @@ class CodeShellModel(Model):if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD):assert self.tensor_names is not None- if all(s not in self.tensor_names- for s in ("lm_head.weight", "output.weight")):+ if all(s not in self.tensor_names for s in ("lm_head.weight", "output.weight")):# copy tok_embd.weight to output.weight- tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT),- data_torch))+ tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch))return tensors@@ -3410,8 +3062,7 @@ class InternLM2Model(Model):if tokenizer_config_file.is_file():with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)- added_tokens_decoder = tokenizer_config_json.get(- "added_tokens_decoder", {})+ added_tokens_decoder = tokenizer_config_json.get("added_tokens_decoder", {})for token_id, foken_data in added_tokens_decoder.items():token_id = int(token_id)token = foken_data["content"]@@ -3444,8 +3095,8 @@ class InternLM2Model(Model):if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:if tokens[token_id] != token:logger.warning(- f"replacing token {token_id}: {tokens[token_id].decode('utf-8')!r}"- f" -> {token.decode('utf-8')!r}"+ f"replacing token {token_id}: {tokens[token_id].decode('utf-8')!r} "+ f"-> {token.decode('utf-8')!r}")tokens[token_id] = tokenscores[token_id] = -1000.0@@ -3465,7 +3116,7 @@ class InternLM2Model(Model):if chat_eos_token_id is not None:# For the chat model, we replace the eos with '<|im_end|>'.# TODO: this is a hack, should be fixed- # https://github.com/ggerganov/llama.cpp/pull/6745#issuecomment-2067687048+ # https://github.com/ggml-org/llama.cpp/pull/6745#issuecomment-2067687048special_vocab.special_token_ids["eos"] = chat_eos_token_idlogger.warning(f"Replace eos:{old_eos} with a special token:{chat_eos_token_id}"@@ -3474,28 +3125,21 @@ class InternLM2Model(Model):special_vocab.add_to_gguf(self.gguf_writer)def set_gguf_parameters(self):- self.gguf_writer.add_context_length(- self.hparams["max_position_embeddings"])+ self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])self.gguf_writer.add_block_count(self.hparams["num_hidden_layers"])self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])- self.gguf_writer.add_feed_forward_length(- self.hparams["intermediate_size"])+ self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])self.gguf_writer.add_rope_freq_base(self.hparams["rope_theta"])self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"])self.gguf_writer.add_file_type(self.ftype)- if self.hparams.get(- "rope_scaling"- ) is not None and "factor" in self.hparams["rope_scaling"]:+ if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:if self.hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(- gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(- self.hparams["rope_scaling"]["factor"])+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:num_heads = self.hparams["num_attention_heads"]num_kv_heads = self.hparams["num_key_value_heads"]n_embd = self.hparams["hidden_size"]@@ -3509,11 +3153,8 @@ class InternLM2Model(Model):qkv = qkv.reshape((num_groups, q_per_kv + 2, head_dim, n_embd))q, k, v = qkv[:, :q_per_kv], qkv[:, -2], qkv[:, -1]- # The model weights of q and k require additional reshape.- q = LlamaModel.permute(q.reshape((-1, q.shape[-1])), num_heads,- num_heads)- k = LlamaModel.permute(k.reshape((-1, k.shape[-1])), num_heads,- num_kv_heads)+ q = LlamaModel.permute(q.reshape((-1, q.shape[-1])), num_heads, num_heads)+ k = LlamaModel.permute(k.reshape((-1, k.shape[-1])), num_heads, num_kv_heads)v = v.reshape((-1, v.shape[-1]))return [@@ -3525,6 +3166,67 @@ class InternLM2Model(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("InternLM3ForCausalLM")+class InternLM3Model(Model):+ model_arch = gguf.MODEL_ARCH.LLAMA++ def set_vocab(self):+ tokens, scores, toktypes = self._create_vocab_sentencepiece()++ self.gguf_writer.add_tokenizer_model("llama")+ self.gguf_writer.add_tokenizer_pre("default")+ self.gguf_writer.add_token_list(tokens)+ self.gguf_writer.add_token_scores(scores)+ self.gguf_writer.add_token_types(toktypes)++ special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))++ tokenizer_config_file = self.dir_model / 'tokenizer_config.json'+ if tokenizer_config_file.is_file():+ with open(tokenizer_config_file, "r", encoding="utf-8") as f:+ tokenizer_config_json = json.load(f)+ if "add_prefix_space" in tokenizer_config_json:+ self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])++ if "added_tokens_decoder" in tokenizer_config_json:+ for token_id, token_data in tokenizer_config_json["added_tokens_decoder"].items():+ if token_data.get("special"):+ token_id = int(token_id)+ token = token_data["content"]+ special_vocab._set_special_token(token, token_id)+ # update eos token+ if token == '<|im_end|>' and "eos" in special_vocab.special_token_ids:+ special_vocab.special_token_ids["eos"] = token_id++ special_vocab.add_to_gguf(self.gguf_writer)++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ hparams = self.hparams+ self.gguf_writer.add_vocab_size(hparams["vocab_size"])++ if "head_dim" in hparams:+ rope_dim = hparams["head_dim"]+ else:+ rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]+ self.gguf_writer.add_rope_dimension_count(rope_dim)++ if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:+ if self.hparams["rope_scaling"].get("type") == "linear" or self.hparams["rope_scaling"].get(+ "rope_type") == "linear":+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ n_head = self.hparams["num_attention_heads"]+ n_kv_head = self.hparams.get("num_key_value_heads")+ if name.endswith(("q_proj.weight", "q_proj.bias")):+ data_torch = LlamaModel.permute(data_torch, n_head, n_head)+ if name.endswith(("k_proj.weight", "k_proj.bias")):+ data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)+ return [(self.map_tensor_name(name), data_torch)]++@Model.register("BertModel", "BertForMaskedLM", "CamembertModel")class BertModel(Model):model_arch = gguf.MODEL_ARCH.BERT@@ -3550,16 +3252,14 @@ class BertModel(Model):# get pooling typeif pooling_path is not None:- with open(self.dir_model / pooling_path / "config.json",- encoding="utf-8") as f:+ with open(self.dir_model / pooling_path / "config.json", encoding="utf-8") as f:pooling = json.load(f)if pooling["pooling_mode_mean_tokens"]:pooling_type = gguf.PoolingType.MEANelif pooling["pooling_mode_cls_token"]:pooling_type = gguf.PoolingType.CLSelse:- raise NotImplementedError(- "Only MEAN and CLS pooling types supported")+ raise NotImplementedError("Only MEAN and CLS pooling types supported")self.gguf_writer.add_pooling_type(pooling_type)def set_vocab(self):@@ -3569,8 +3269,7 @@ class BertModel(Model):# we need this to validate the size of the token_type embeddings# though currently we are passing all zeros to the token_type embeddings# "Sequence A" or "Sequence B"- self.gguf_writer.add_token_type_count(- self.hparams.get("type_vocab_size", 1))+ self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))# convert to phantom space vocabdef phantom(tok):@@ -3592,8 +3291,7 @@ class BertModel(Model):special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))special_vocab.add_to_gguf(self.gguf_writer)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedif name.startswith("bert."):@@ -3606,8 +3304,7 @@ class BertModel(Model):name = name[:-5] + ".bias"# we are only using BERT for embeddings so we don't need the pooling layer- if name in ("embeddings.position_ids", "pooler.dense.weight",- "pooler.dense.bias"):+ if name in ("embeddings.position_ids", "pooler.dense.weight", "pooler.dense.bias"):return [] # we don't need theseif name.startswith("cls.predictions"):@@ -3630,8 +3327,7 @@ class RobertaModel(BertModel):if (pad_token_id := self.hparams.get("pad_token_id")) is not None:self._position_offset = 1 + pad_token_idif "max_position_embeddings" in self.hparams:- self.hparams[- "max_position_embeddings"] -= self._position_offset+ self.hparams["max_position_embeddings"] -= self._position_offsetelse:self._position_offset = None@@ -3646,14 +3342,12 @@ class RobertaModel(BertModel):# we need this to validate the size of the token_type embeddings# though currently we are passing all zeros to the token_type embeddings# "Sequence A" or "Sequence B"- self.gguf_writer.add_token_type_count(- self.hparams.get("type_vocab_size", 1))+ self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))else:return super().set_vocab()- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# if name starts with "roberta.", remove the prefix# e.g. https://huggingface.co/BAAI/bge-reranker-v2-m3/tree/mainif name.startswith("roberta."):@@ -3708,8 +3402,7 @@ class XLMRobertaModel(BertModel):if (pad_token_id := self.hparams.get("pad_token_id")) is not None:self._position_offset = 1 + pad_token_idif "max_position_embeddings" in self.hparams:- self.hparams[- "max_position_embeddings"] -= self._position_offset+ self.hparams["max_position_embeddings"] -= self._position_offsetelse:self._position_offset = None@@ -3737,9 +3430,7 @@ class XLMRobertaModel(BertModel):vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())- tokens: list[bytes] = [- f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)- ]+ tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_sizetoktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size@@ -3764,9 +3455,7 @@ class XLMRobertaModel(BertModel):if vocab_size > len(tokens):pad_count = vocab_size - len(tokens)- logger.debug(- f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]"- )+ logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")for i in range(1, pad_count + 1):tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))scores.append(-1000.0)@@ -3788,8 +3477,7 @@ class XLMRobertaModel(BertModel):self.gguf_writer.add_token_scores(scores)self.gguf_writer.add_token_types(toktypes)self.gguf_writer.add_add_space_prefix(add_prefix)- self.gguf_writer.add_token_type_count(- self.hparams.get("type_vocab_size", 1))+ self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))self.gguf_writer.add_remove_extra_whitespaces(remove_whitespaces)if precompiled_charsmap:self.gguf_writer.add_precompiled_charsmap(precompiled_charsmap)@@ -3800,8 +3488,7 @@ class XLMRobertaModel(BertModel):self.gguf_writer.add_add_bos_token(True)self.gguf_writer.add_add_eos_token(True)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# if name starts with "roberta.", remove the prefix# e.g. https://huggingface.co/BAAI/bge-reranker-v2-m3/tree/mainif name.startswith("roberta."):@@ -3824,9 +3511,7 @@ class GemmaModel(Model):# TODO: these special tokens should be exported only for the CodeGemma familyspecial_vocab = gguf.SpecialVocab(- self.dir_model,- load_merges=False,- special_token_types=['prefix', 'suffix', 'middle', 'fsep', 'eot'])+ self.dir_model, load_merges=False, special_token_types=['prefix', 'suffix', 'middle', 'fsep', 'eot'])special_vocab._set_special_token("prefix", 67)special_vocab._set_special_token("suffix", 69)special_vocab._set_special_token("middle", 68)@@ -3847,23 +3532,19 @@ class GemmaModel(Model):self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])self.gguf_writer.add_head_count(hparams["num_attention_heads"])self.gguf_writer.add_head_count_kv(- self.hparams["num_key_value_heads"] if "num_key_value_heads" in- hparams else hparams["num_attention_heads"])+ self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"])self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_key_length(hparams["head_dim"])self.gguf_writer.add_value_length(hparams["head_dim"])self.gguf_writer.add_file_type(self.ftype)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused# lm_head is not used in llama.cpp, while autoawq will include this tensor in model# To prevent errors, skip loading lm_head.weight.if name == "lm_head.weight":- logger.debug(- f"Skipping get tensor {name!r} in safetensors so that convert can end normally."- )+ logger.debug(f"Skipping get tensor {name!r} in safetensors so that convert can end normally.")return []if name.endswith("norm.weight"):@@ -3891,28 +3572,22 @@ class Gemma2Model(Model):self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])self.gguf_writer.add_head_count(hparams["num_attention_heads"])self.gguf_writer.add_head_count_kv(- self.hparams["num_key_value_heads"] if "num_key_value_heads" in- hparams else hparams["num_attention_heads"])+ self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"])self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_key_length(hparams["head_dim"])self.gguf_writer.add_value_length(hparams["head_dim"])self.gguf_writer.add_file_type(self.ftype)- self.gguf_writer.add_attn_logit_softcapping(- self.hparams["attn_logit_softcapping"])- self.gguf_writer.add_final_logit_softcapping(- self.hparams["final_logit_softcapping"])+ self.gguf_writer.add_attn_logit_softcapping(self.hparams["attn_logit_softcapping"])+ self.gguf_writer.add_final_logit_softcapping(self.hparams["final_logit_softcapping"])self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused# lm_head is not used in llama.cpp, while autoawq will include this tensor in model# To prevent errors, skip loading lm_head.weight.if name == "lm_head.weight":- logger.debug(- f"Skipping get tensor {name!r} in safetensors so that convert can end normally."- )+ logger.debug(f"Skipping get tensor {name!r} in safetensors so that convert can end normally.")return []if name.endswith("norm.weight"):@@ -3921,6 +3596,83 @@ class Gemma2Model(Model):return [(self.map_tensor_name(name), data_torch)][email protected]("Gemma3ForCausalLM", "Gemma3ForConditionalGeneration")+class Gemma3Model(Model):+ model_arch = gguf.MODEL_ARCH.GEMMA3+ has_vision: bool = False++ # we need to merge the text_config into the root level of hparams+ def __init__(self, *args, **kwargs):+ hparams = Model.load_hparams(kwargs["dir_model"])+ if "text_config" in hparams:+ hparams = {**hparams, **hparams["text_config"]}+ kwargs["hparams"] = hparams+ super().__init__(*args, **kwargs)+ if "vision_config" in hparams:+ logger.info("Has vision encoder, but it will be ignored")+ self.has_vision = True++ def write(self):+ super().write()+ if self.has_vision:+ logger.info("NOTE: this script only convert the language model to GGUF")+ logger.info(" for the vision model, please use gemma3_convert_encoder_to_gguf.py")++ def set_vocab(self):+ self._set_vocab_sentencepiece()++ self.gguf_writer.add_add_space_prefix(False)++ def set_gguf_parameters(self):+ hparams = self.hparams+ block_count = hparams["num_hidden_layers"]++ # some default values are not specified in the hparams+ self.gguf_writer.add_context_length(hparams.get("max_position_embeddings", 131072))+ self.gguf_writer.add_embedding_length(hparams["hidden_size"])+ self.gguf_writer.add_block_count(block_count)+ self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])+ self.gguf_writer.add_head_count(hparams.get("num_attention_heads", 8))+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("rms_norm_eps", 1e-6))+ self.gguf_writer.add_key_length(hparams.get("head_dim", 256))+ self.gguf_writer.add_value_length(hparams.get("head_dim", 256))+ self.gguf_writer.add_file_type(self.ftype)+ self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 1_000_000.0)) # for global layers+ # both attn_logit_softcapping and final_logit_softcapping are removed in Gemma3+ assert hparams.get("attn_logit_softcapping") is None+ assert hparams.get("final_logit_softcapping") is None+ self.gguf_writer.add_sliding_window(hparams["sliding_window"])+ self.gguf_writer.add_head_count_kv(hparams.get("num_key_value_heads", 4))+ if hparams.get("rope_scaling") is not None:+ assert hparams["rope_scaling"]["rope_type"] == "linear"+ # important: this rope_scaling is only applied for global layers, and not used by 1B model+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(hparams["rope_scaling"]["factor"])++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused++ if name.startswith("language_model."):+ name = name.replace("language_model.", "")+ elif name.startswith("multi_modal_projector.") or name.startswith("vision_tower.") \+ or name.startswith("multimodal_projector.") or name.startswith("vision_model."):+ # ignore vision tensors+ return []++ # remove OOV (out-of-vocabulary) rows in token_embd+ if "embed_tokens.weight" in name:+ vocab = self._create_vocab_sentencepiece()+ tokens = vocab[0]+ data_torch = data_torch[:len(tokens)]++ # ref code in Gemma3RMSNorm+ # output = output * (1.0 + self.weight.float())+ if name.endswith("norm.weight"):+ data_torch = data_torch + 1++ return [(self.map_tensor_name(name), data_torch)]++@Model.register("Starcoder2ForCausalLM")class StarCoder2Model(Model):model_arch = gguf.MODEL_ARCH.STARCODER2@@ -3937,17 +3689,13 @@ class Rwkv6Model(Model):tokens: list[bytes] = ['<s>'.encode("utf-8")]toktypes: list[int] = [gguf.TokenType.CONTROL]- with open(self.dir_model / "rwkv_vocab_v20230424.txt",- "r",- encoding="utf-8") as f:+ with open(self.dir_model / "rwkv_vocab_v20230424.txt", "r", encoding="utf-8") as f:lines = f.readlines()for line in lines:parts = line.split(' ')assert len(parts) >= 3- token, token_len = ast.literal_eval(' '.join(- parts[1:-1])), int(parts[-1])- token = token.encode("utf-8") if isinstance(token,- str) else token+ token, token_len = ast.literal_eval(' '.join(parts[1:-1])), int(parts[-1])+ token = token.encode("utf-8") if isinstance(token, str) else tokenassert isinstance(token, bytes)assert len(token) == token_lentoken_text: str = repr(token)[2:-1] # "b'\xff'" -> "\xff"@@ -3974,9 +3722,8 @@ class Rwkv6Model(Model):hidden_size = self.hparams["hidden_size"]layer_norm_eps = self.hparams["layer_norm_epsilon"]rescale_every_n_layers = self.hparams["rescale_every"]- intermediate_size = self.hparams["intermediate_size"] if self.hparams[- "intermediate_size"] is not None else int(- (hidden_size * 3.5) // 32 * 32)+ intermediate_size = self.hparams["intermediate_size"] if self.hparams["intermediate_size"] is not None else int(+ (hidden_size*3.5) // 32 * 32)time_mix_extra_dim = 64 if hidden_size == 4096 else 32time_decay_extra_dim = 128 if hidden_size == 4096 else 64@@ -3997,16 +3744,14 @@ class Rwkv6Model(Model):lerp_weights: dict[int, dict[str, Tensor]] = {}- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:new_name = self.map_tensor_name(name)if not (new_name.endswith(".weight") or new_name.endswith(".bias")):new_name += ".weight"if new_name.endswith("time_mix_w1.weight") or new_name.endswith(- "time_mix_decay_w1.weight") or new_name.endswith(- "time_mix_decay_w2.weight"):+ "time_mix_decay_w1.weight") or new_name.endswith("time_mix_decay_w2.weight"):data_torch = data_torch.transpose(0, 1)if new_name.endswith("time_mix_w2.weight"):@@ -4018,11 +3763,8 @@ class Rwkv6Model(Model):try:rescale_every_n_layers = self.hparams["rescale_every"]if rescale_every_n_layers > 0:- if new_name.endswith(- "time_mix_output.weight") or new_name.endswith(- "channel_mix_value.weight"):- data_torch = data_torch.div_(2**int(- bid // rescale_every_n_layers))+ if new_name.endswith("time_mix_output.weight") or new_name.endswith("channel_mix_value.weight"):+ data_torch = data_torch.div_(2**int(bid // rescale_every_n_layers))except KeyError:pass@@ -4033,16 +3775,14 @@ class Rwkv6Model(Model):self.lerp_weights[bid][new_name] = data_torchexcept KeyError:self.lerp_weights[bid] = {new_name: data_torch}- if all(f"blk.{bid}.time_mix_lerp_{i}.weight" in- self.lerp_weights[bid].keys()+ if all(f"blk.{bid}.time_mix_lerp_{i}.weight" in self.lerp_weights[bid].keys()for i in ["w", "k", "v", "r", "g"]):new_name = f"blk.{bid}.time_mix_lerp_fused.weight"data = torch.stack([- self.lerp_weights[bid]- [f"blk.{bid}.time_mix_lerp_{i}.weight"].unsqueeze(0)+ self.lerp_weights[bid][f"blk.{bid}.time_mix_lerp_{i}.weight"].unsqueeze(0)for i in ["w", "k", "v", "r", "g"]],- dim=0).unsqueeze(1)+ dim=0).unsqueeze(1)yield (new_name, data)return@@ -4089,16 +3829,13 @@ class RWKV6Qwen2Model(Rwkv6Model):# required by llama.cpp, unusedself.gguf_writer.add_head_count(0)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:for new_name, data in super().modify_tensors(data_torch, name, bid):if "time_mix_w1" in new_name or "time_mix_w2" in new_name:data = data.view(5, -1, data.shape[-1])# rwkv6qwen2 has a different order of rkvwg instead of the original wkvrg# permute them here to avoid code changes- data = torch.stack(- [data[3], data[1], data[2], data[0], data[4]],- dim=0).view(-1, data.shape[-1])+ data = torch.stack([data[3], data[1], data[2], data[0], data[4]], dim=0).view(-1, data.shape[-1])if "w2" in new_name:data = data.view(5, -1, data.shape[-1])yield (new_name, data)@@ -4106,8 +3843,7 @@ class RWKV6Qwen2Model(Rwkv6Model):yield (new_name, data)[email protected]("MambaForCausalLM", "MambaLMHeadModel",- "FalconMambaForCausalLM")[email protected]("MambaForCausalLM", "MambaLMHeadModel", "FalconMambaForCausalLM")class MambaModel(Model):model_arch = gguf.MODEL_ARCH.MAMBA@@ -4130,32 +3866,24 @@ class MambaModel(Model):def set_gguf_parameters(self):d_model = self.find_hparam(["hidden_size", "d_model"])- d_conv = self.find_hparam(["conv_kernel", "d_conv"],- optional=True) or 4- d_inner = self.find_hparam(["intermediate_size", "d_inner"],- optional=True) or 2 * d_model- d_state = self.find_hparam(["state_size", "d_state"],- optional=True) or 16+ d_conv = self.find_hparam(["conv_kernel", "d_conv"], optional=True) or 4+ d_inner = self.find_hparam(["intermediate_size", "d_inner"], optional=True) or 2 * d_model+ d_state = self.find_hparam(["state_size", "d_state"], optional=True) or 16# ceiling division- dt_rank = self.find_hparam(["time_step_rank", "dt_rank"],- optional=True) or -(d_model // -16)- rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"],- optional=True) or 1e-5+ # ref: https://stackoverflow.com/a/17511341/22827863+ dt_rank = self.find_hparam(["time_step_rank", "dt_rank"], optional=True) or -(d_model // -16)+ rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-5use_dt_b_c_norm = False# For falconmamba we do apply RMS norm on B / DT and C layers- if self.find_hparam(["model_type"],- optional=True) in ("falcon_mamba", ):+ if self.find_hparam(["model_type"], optional=True) in ("falcon_mamba",):use_dt_b_c_norm = True# Fail early for models which don't have a block expansion factor of 2assert d_inner == 2 * d_model- self.gguf_writer.add_context_length(- 2**20) # arbitrary value; for those who use the default+ self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the defaultself.gguf_writer.add_embedding_length(d_model)- self.gguf_writer.add_feed_forward_length(- 0) # unused, but seemingly required when loading- self.gguf_writer.add_head_count(- 0) # unused, but seemingly required when loading+ self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loading+ self.gguf_writer.add_head_count(0) # unused, but seemingly required when loadingself.gguf_writer.add_block_count(self.block_count)self.gguf_writer.add_ssm_conv_kernel(d_conv)self.gguf_writer.add_ssm_inner_size(d_inner)@@ -4163,14 +3891,12 @@ class MambaModel(Model):self.gguf_writer.add_ssm_time_step_rank(dt_rank)self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)self.gguf_writer.add_ssm_dt_b_c_rms(- use_dt_b_c_norm- ) # For classic Mamba we don't apply rms norm on B / DT layers+ use_dt_b_c_norm) # For classic Mamba we don't apply rms norm on B / DT layersself.gguf_writer.add_file_type(self.ftype)_tok_embd = None- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedoutput_name = self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT)@@ -4185,9 +3911,7 @@ class MambaModel(Model):# assuming token_embd.weight is seen before output.weightif self._tok_embd is not None and new_name == output_name:if torch.equal(self._tok_embd, data_torch):- logger.debug(- f"{output_name} is equivalent to {tok_embd_name}, omitting"- )+ logger.debug(f"{output_name} is equivalent to {tok_embd_name}, omitting")return []elif new_name == tok_embd_name:self._tok_embd = data_torch@@ -4205,8 +3929,7 @@ class CommandR2Model(Model):# max_position_embeddings = 8192 in config.json but model was actually# trained on 128k context length# aya-23 models don't have model_max_length specified- self.hparams["max_position_embeddings"] = self.find_hparam(- ["model_max_length", "max_position_embeddings"])+ self.hparams["max_position_embeddings"] = self.find_hparam(["model_max_length", "max_position_embeddings"])def set_gguf_parameters(self):super().set_gguf_parameters()@@ -4228,8 +3951,7 @@ class Cohere2Model(Model):rotary_pct = self.hparams["rotary_pct"]hidden_size = self.hparams["hidden_size"]num_attention_heads = self.hparams["num_attention_heads"]- self.gguf_writer.add_rope_dimension_count(- int(rotary_pct * (hidden_size // num_attention_heads)))+ self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size//num_attention_heads)))self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)@@ -4247,8 +3969,7 @@ class OlmoModel(Model):# Same as super class, but permuting q_proj, k_proj# Copied from: LlamaModel- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedn_head = self.hparams["num_attention_heads"]@@ -4280,8 +4001,7 @@ class OlmoeModel(Model):_experts: list[dict[str, Tensor]] | None = None# Copied from: Qwen2MoeModel- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# process the experts separatelyif name.find("experts") != -1:n_experts = self.hparams["num_experts"]@@ -4353,9 +4073,7 @@ class JinaBertV2Model(BertModel):def set_vocab(self):tokenizer_class = 'BertTokenizer'- with open(self.dir_model / "tokenizer_config.json",- "r",- encoding="utf-8") as f:+ with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f:tokenizer_class = json.load(f)['tokenizer_class']if tokenizer_class == 'BertTokenizer':@@ -4364,14 +4082,11 @@ class JinaBertV2Model(BertModel):self._set_vocab_gpt2()self.gguf_writer.add_token_type_count(2)else:- raise NotImplementedError(- f'Tokenizer {tokenizer_class} is not supported for JinaBertModel'- )+ raise NotImplementedError(f'Tokenizer {tokenizer_class} is not supported for JinaBertModel')self.gguf_writer.add_add_bos_token(True)self.gguf_writer.add_add_eos_token(True)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# if name starts with "bert.", remove the prefix# e.g. https://huggingface.co/jinaai/jina-reranker-v1-tiny-enif name.startswith("bert."):@@ -4386,7 +4101,7 @@ class OpenELMModel(Model):@staticmethoddef _make_divisible(v: float | int, divisor: int) -> int:- new_v = max(divisor, int(v + divisor / 2) // divisor * divisor)+ new_v = max(divisor, int(v + divisor/2) // divisor * divisor)# Make sure that round down does not go down by more than 10%.if new_v < 0.9 * v:new_v += divisor@@ -4401,14 +4116,10 @@ class OpenELMModel(Model):self._num_kv_heads: list[int] = self.hparams["num_kv_heads"]self._num_query_heads: list[int] = self.hparams["num_query_heads"]self._ffn_dims: list[int] = [- OpenELMModel._make_divisible(multiplier * self._n_embd,- ffn_dim_divisor)- for multiplier in ffn_multipliers+ OpenELMModel._make_divisible(multiplier * self._n_embd, ffn_dim_divisor) for multiplier in ffn_multipliers]- assert isinstance(self._num_kv_heads, list) and isinstance(- self._num_kv_heads[0], int)- assert isinstance(self._num_query_heads, list) and isinstance(- self._num_query_heads[0], int)+ assert isinstance(self._num_kv_heads, list) and isinstance(self._num_kv_heads[0], int)+ assert isinstance(self._num_query_heads, list) and isinstance(self._num_query_heads[0], int)# Uses the tokenizer from meta-llama/Llama-2-7b-hfdef set_vocab(self):@@ -4445,16 +4156,13 @@ class OpenELMModel(Model):return super().find_hparam(keys, optional)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# split ffif bid is not None and name == f"transformer.layers.{bid}.ffn.proj_1.weight":ff_dim = self._ffn_dims[bid]- yield (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE,- bid), data_torch[:ff_dim])- yield (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP,- bid), data_torch[ff_dim:])+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), data_torch[:ff_dim])+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), data_torch[ff_dim:])returnyield (self.map_tensor_name(name), data_torch)@@ -4482,9 +4190,7 @@ class ArcticModel(Model):vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())- tokens: list[bytes] = [- f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)- ]+ tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_sizetoktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size@@ -4516,14 +4222,11 @@ class ArcticModel(Model):tokenizer_config_json = json.load(f)if "added_tokens_decoder" in tokenizer_config_json:- added_tokens_decoder = tokenizer_config_json[- "added_tokens_decoder"]+ added_tokens_decoder = tokenizer_config_json["added_tokens_decoder"]for token_id, token_json in added_tokens_decoder.items():token_id = int(token_id)if token_id >= vocab_size:- logger.debug(- f'ignore token {token_id}: id is out of range, max={vocab_size - 1}'- )+ logger.debug(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')continuetoken_content = token_json["content"]@@ -4533,16 +4236,15 @@ class ArcticModel(Model):# Map unk_token to UNKNOWN, other special tokens to CONTROL# Set the score to 0.0 as in the original tokenizer.modelif ("special" in token_json) and token_json["special"]:- if token_content == tokenizer_config_json[- "unk_token"]:+ if token_content == tokenizer_config_json["unk_token"]:token_type = SentencePieceTokenTypes.UNKNOWNelse:token_type = SentencePieceTokenTypes.CONTROLtoken_score = 0.0logger.info(- f"Setting added token {token_id} to '{token_content}' (type: {token_type}, "- f"score: {token_score:.2f})"+ f"Setting added token {token_id} to '{token_content}'"+ f" (type: {token_type}, score: {token_score:.2f})")tokens[token_id] = token_content.encode("utf-8")toktypes[token_id] = token_type@@ -4561,13 +4263,11 @@ class ArcticModel(Model):super().set_gguf_parameters()hparams = self.hparamsself.gguf_writer.add_vocab_size(hparams["vocab_size"])- self.gguf_writer.add_rope_dimension_count(- hparams["hidden_size"] // hparams["num_attention_heads"])+ self.gguf_writer.add_rope_dimension_count(hparams["hidden_size"] // hparams["num_attention_heads"])_experts: list[dict[str, Tensor]] | None = None- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]n_kv_head = self.hparams.get("num_key_value_heads")@@ -4642,11 +4342,9 @@ class DeepseekModel(Model):self.gguf_writer.add_rope_dimension_count(rope_dim)self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)- self.gguf_writer.add_leading_dense_block_count(- hparams["first_k_dense_replace"])+ self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])self.gguf_writer.add_vocab_size(hparams["vocab_size"])- self.gguf_writer.add_expert_feed_forward_length(- hparams["moe_intermediate_size"])+ self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])self.gguf_writer.add_expert_weights_scale(1.0)self.gguf_writer.add_expert_count(hparams["n_routed_experts"])self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"])@@ -4657,12 +4355,11 @@ class DeepseekModel(Model):def permute(weights: Tensor, n_head: int, n_head_kv: int | None):if n_head_kv is not None and n_head != n_head_kv:n_head = n_head_kv- return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(- weights.shape))+ return (+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,+ *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]n_kv_head = self.hparams.get("num_key_value_heads")@@ -4728,57 +4425,42 @@ class DeepseekV2Model(Model):super().set_gguf_parameters()hparams = self.hparams- self.gguf_writer.add_leading_dense_block_count(- hparams["first_k_dense_replace"])+ self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])self.gguf_writer.add_vocab_size(hparams["vocab_size"])if "q_lora_rank" in hparams and hparams["q_lora_rank"] is not None:self.gguf_writer.add_q_lora_rank(hparams["q_lora_rank"])self.gguf_writer.add_kv_lora_rank(hparams["kv_lora_rank"])- self.gguf_writer.add_key_length(hparams["qk_nope_head_dim"] +- hparams["qk_rope_head_dim"])+ self.gguf_writer.add_key_length(hparams["qk_nope_head_dim"] + hparams["qk_rope_head_dim"])self.gguf_writer.add_value_length(hparams["v_head_dim"])- self.gguf_writer.add_expert_feed_forward_length(- hparams["moe_intermediate_size"])+ self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])self.gguf_writer.add_expert_count(hparams["n_routed_experts"])self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"])- self.gguf_writer.add_expert_weights_scale(- hparams["routed_scaling_factor"])+ self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])if hparams["scoring_func"] == "sigmoid":- self.gguf_writer.add_expert_gating_func(- gguf.ExpertGatingFuncType.SIGMOID)+ self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)elif hparams["scoring_func"] == "softmax":- self.gguf_writer.add_expert_gating_func(- gguf.ExpertGatingFuncType.SOFTMAX)+ self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)else:- raise ValueError(- f"Unsupported scoring_func value: {hparams['scoring_func']}")+ raise ValueError(f"Unsupported scoring_func value: {hparams['scoring_func']}")self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])- if self.hparams.get(- "rope_scaling"- ) is not None and "factor" in self.hparams["rope_scaling"]:+ if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:if self.hparams["rope_scaling"].get("type") == "yarn":- self.gguf_writer.add_rope_scaling_type(- gguf.RopeScalingType.YARN)- self.gguf_writer.add_rope_scaling_factor(- self.hparams["rope_scaling"]["factor"])+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)+ self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])self.gguf_writer.add_rope_scaling_orig_ctx_len(- self.hparams["rope_scaling"]- ["original_max_position_embeddings"])- self.gguf_writer.add_rope_scaling_yarn_log_mul(- 0.1 * hparams["rope_scaling"]["mscale_all_dim"])+ self.hparams["rope_scaling"]["original_max_position_embeddings"])+ self.gguf_writer.add_rope_scaling_yarn_log_mul(0.1 * hparams["rope_scaling"]["mscale_all_dim"])_experts: list[dict[str, Tensor]] | None = None- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# rename e_score_correction_bias tensorsif name.endswith("e_score_correction_bias"):- name = name.replace("e_score_correction_bias",- "e_score_correction.bias")+ name = name.replace("e_score_correction_bias", "e_score_correction.bias")# skip Multi-Token Prediction (MTP) layersblock_count = self.hparams["num_hidden_layers"]@@ -4878,9 +4560,7 @@ class T5Model(Model):vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())- tokens: list[bytes] = [- f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)- ]+ tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_sizetoktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size@@ -4910,9 +4590,7 @@ class T5Model(Model):for key in added_tokens_json:token_id = added_tokens_json[key]if token_id >= vocab_size:- logger.warning(- f'ignore token {token_id}: id is out of range, max={vocab_size - 1}'- )+ logger.warning(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')continuetokens[token_id] = key.encode("utf-8")@@ -4921,9 +4599,7 @@ class T5Model(Model):if vocab_size > len(tokens):pad_count = vocab_size - len(tokens)- logger.debug(- f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]"- )+ logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")for i in range(1, pad_count + 1):tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))scores.append(-1000.0)@@ -4947,9 +4623,7 @@ class T5Model(Model):def set_gguf_parameters(self):if (n_ctx := self.find_hparam(["n_positions"], optional=True)) is None:- logger.warning(- "Couldn't find context length in config.json, assuming default value of 512"- )+ logger.warning("Couldn't find context length in config.json, assuming default value of 512")n_ctx = 512self.gguf_writer.add_context_length(n_ctx)self.gguf_writer.add_embedding_length(self.hparams["d_model"])@@ -4959,29 +4633,20 @@ class T5Model(Model):self.gguf_writer.add_key_length(self.hparams["d_kv"])self.gguf_writer.add_value_length(self.hparams["d_kv"])self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_relative_attn_buckets_count(- self.hparams["relative_attention_num_buckets"])- self.gguf_writer.add_layer_norm_rms_eps(- self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_decoder_start_token_id(- self.hparams["decoder_start_token_id"])+ self.gguf_writer.add_relative_attn_buckets_count(self.hparams["relative_attention_num_buckets"])+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"])+ self.gguf_writer.add_decoder_start_token_id(self.hparams["decoder_start_token_id"])self.gguf_writer.add_file_type(self.ftype)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused- if name in [- "decoder.embed_tokens.weight", "encoder.embed_tokens.weight",- "shared.weight"- ]:+ if name in ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight", "shared.weight"]:if not self.shared_token_embeddings_found:name = "shared.weight"self.shared_token_embeddings_found = Trueelse:- logger.debug(- f"Skipping shared tensor {name!r} in safetensors so that convert can end normally."- )+ logger.debug(f"Skipping shared tensor {name!r} in safetensors so that convert can end normally.")return []return [(self.map_tensor_name(name), data_torch)]@@ -5031,9 +4696,7 @@ class T5EncoderModel(Model):vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())- tokens: list[bytes] = [- f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)- ]+ tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_sizetoktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size@@ -5063,9 +4726,7 @@ class T5EncoderModel(Model):for key in added_tokens_json:token_id = added_tokens_json[key]if token_id >= vocab_size:- logger.warning(- f'ignore token {token_id}: id is out of range, max={vocab_size - 1}'- )+ logger.warning(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')continuetokens[token_id] = key.encode("utf-8")@@ -5074,9 +4735,7 @@ class T5EncoderModel(Model):if vocab_size > len(tokens):pad_count = vocab_size - len(tokens)- logger.debug(- f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]"- )+ logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")for i in range(1, pad_count + 1):tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))scores.append(-1000.0)@@ -5100,9 +4759,7 @@ class T5EncoderModel(Model):def set_gguf_parameters(self):if (n_ctx := self.find_hparam(["n_positions"], optional=True)) is None:- logger.warning(- "Couldn't find context length in config.json, assuming default value of 512"- )+ logger.warning("Couldn't find context length in config.json, assuming default value of 512")n_ctx = 512self.gguf_writer.add_context_length(n_ctx)self.gguf_writer.add_embedding_length(self.hparams["d_model"])@@ -5112,27 +4769,19 @@ class T5EncoderModel(Model):self.gguf_writer.add_key_length(self.hparams["d_kv"])self.gguf_writer.add_value_length(self.hparams["d_kv"])self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])- self.gguf_writer.add_relative_attn_buckets_count(- self.hparams["relative_attention_num_buckets"])- self.gguf_writer.add_layer_norm_rms_eps(- self.hparams["layer_norm_epsilon"])+ self.gguf_writer.add_relative_attn_buckets_count(self.hparams["relative_attention_num_buckets"])+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"])self.gguf_writer.add_file_type(self.ftype)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused- if name in [- "decoder.embed_tokens.weight", "encoder.embed_tokens.weight",- "shared.weight"- ]:+ if name in ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight", "shared.weight"]:if not self.shared_token_embeddings_found:name = "shared.weight"self.shared_token_embeddings_found = Trueelse:- logger.debug(- f"Skipping shared tensor {name!r} in safetensors so that convert can end normally."- )+ logger.debug(f"Skipping shared tensor {name!r} in safetensors so that convert can end normally.")return []return [(self.map_tensor_name(name), data_torch)]@@ -5162,8 +4811,7 @@ class JaisModel(Model):self.width_scale = 1.0if 'mup_output_alpha' in self.hparams:assert 'mup_width_scale' in self.hparams- self.width_scale = self.hparams['mup_output_alpha'] * self.hparams[- 'mup_width_scale']+ self.width_scale = self.hparams['mup_output_alpha'] * self.hparams['mup_width_scale']elif 'width_scale' in self.hparams:self.width_scale = self.hparams['width_scale']else:@@ -5183,8 +4831,7 @@ class JaisModel(Model):self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])self.gguf_writer.add_file_type(self.ftype)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unusedtensors: list[tuple[str, Tensor]] = []@@ -5198,16 +4845,13 @@ class JaisModel(Model):# Some other models has max_alibi_bias spelled out explicitly in the hyperparams,# but Jais's PyTorch model simply precalculates the slope values and places them# in relative_pes.slopes- n_head_closest_log2 = 2**math.floor(- math.log2(self.hparams["n_head"]))+ n_head_closest_log2 = 2**math.floor(math.log2(self.hparams["n_head"]))first_val = float(data_torch[0].item())- self.max_alibi_bias = -round(- math.log2(first_val) * n_head_closest_log2)+ self.max_alibi_bias = -round(math.log2(first_val) * n_head_closest_log2)return tensors- if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight",- ".c_fc2.weight")):+ if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight", ".c_fc2.weight")):data_torch = data_torch.transpose(1, 0)new_name = self.map_tensor_name(name)@@ -5226,7 +4870,7 @@ class JaisModel(Model):self.gguf_writer.add_max_alibi_bias(self.max_alibi_bias)[email protected]("ChatGLMModel", "ChatGLMForConditionalGeneration")[email protected]("GlmForCausalLM", "ChatGLMModel", "ChatGLMForConditionalGeneration")class ChatGLMModel(Model):model_arch = gguf.MODEL_ARCH.CHATGLM@@ -5238,16 +4882,11 @@ class ChatGLMModel(Model):scores: list[float] = []from transformers import AutoTokenizer- tokenizer = AutoTokenizer.from_pretrained(dir_model,- trust_remote_code=True)- vocab_size = hparams.get("padded_vocab_size",- len(tokenizer.get_vocab()))+ tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)+ vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab()))assert max(tokenizer.get_vocab().values()) < vocab_size- role_special_tokens = [- "<|system|>", "<|user|>", "<|assistant|>", "<|observation|>"- ]- special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"- ] + role_special_tokens+ role_special_tokens = ["<|system|>", "<|user|>", "<|assistant|>", "<|observation|>"]+ special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"] + role_special_tokensfor token_id in range(vocab_size):piece = tokenizer._convert_id_to_token(token_id)if token_id == 0:@@ -5259,9 +4898,7 @@ class ChatGLMModel(Model):text = piece.encode("utf-8")score = 0.0- if len(- piece- ) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size():+ if len(piece) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size():score = tokenizer.tokenizer.sp_model.get_score(token_id)if token_id >= tokenizer.tokenizer.sp_model.vocab_size():@@ -5306,13 +4943,10 @@ class ChatGLMModel(Model):def token_bytes_to_string(b):from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicodebyte_encoder = bytes_to_unicode()- return ''.join(- [byte_encoder[ord(char)] for char in b.decode('latin-1')])+ return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])@staticmethod- def bpe(mergeable_ranks: dict[bytes, int],- token: bytes,- max_rank: int | None = None) -> list[bytes]:+ def bpe(mergeable_ranks: dict[bytes, int], token: bytes, max_rank: int | None = None) -> list[bytes]:parts = [bytes([b]) for b in token]while True:min_idx = None@@ -5322,12 +4956,10 @@ class ChatGLMModel(Model):if rank is not None and (min_rank is None or rank < min_rank):min_idx = imin_rank = rank- if min_rank is None or (max_rank is not None- and min_rank >= max_rank):+ if min_rank is None or (max_rank is not None and min_rank >= max_rank):breakassert min_idx is not None- parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]- ] + parts[min_idx + 2:]+ parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2:]return partsdef set_vocab(self):@@ -5341,97 +4973,51 @@ class ChatGLMModel(Model):toktypes: list[int] = []from transformers import AutoTokenizer- tokenizer = AutoTokenizer.from_pretrained(dir_model,- trust_remote_code=True)- vocab_size = hparams["padded_vocab_size"]+ tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)+ vocab_size = hparams.get("padded_vocab_size", hparams["vocab_size"])assert max(tokenizer.get_vocab().values()) < vocab_size- tokpre = self.get_vocab_base_pre(tokenizer)-- merges = []- vocab = {}- mergeable_ranks = tokenizer.mergeable_ranks- for token, rank in mergeable_ranks.items():- vocab[ChatGLMModel.token_bytes_to_string(token)] = rank- if len(token) == 1:- continue- merged = ChatGLMModel.bpe(mergeable_ranks, token, max_rank=rank)- assert len(merged) >= 2 and len(merged) <= 7- merges.append(' '.join(- map(ChatGLMModel.token_bytes_to_string, merged)))-- # for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined- added_vocab = tokenizer.get_added_vocab()- reverse_vocab = {- id_: encoded_tok- for encoded_tok, id_ in {- **vocab,- **added_vocab- }.items()- }-- for i in range(vocab_size):- if i not in reverse_vocab:- tokens.append(f"[PAD{i}]")- toktypes.append(gguf.TokenType.UNUSED)- elif reverse_vocab[i] in added_vocab:- tokens.append(reverse_vocab[i])- if tokenizer.added_tokens_decoder[i].special:- toktypes.append(gguf.TokenType.CONTROL)- else:- toktypes.append(gguf.TokenType.USER_DEFINED)- else:- tokens.append(reverse_vocab[i])- toktypes.append(gguf.TokenType.NORMAL)-+ tokens, toktypes, tokpre = self.get_vocab_base()self.gguf_writer.add_tokenizer_model("gpt2")self.gguf_writer.add_tokenizer_pre(tokpre)self.gguf_writer.add_token_list(tokens)self.gguf_writer.add_token_types(toktypes)-- special_vocab = gguf.SpecialVocab(dir_model, load_merges=False)- special_vocab.merges = merges+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)# only add special tokens when they were not already loaded from config.json- special_vocab._set_special_token(- "eos",- tokenizer.get_added_vocab()["<|endoftext|>"])- special_vocab._set_special_token(- "eot",- tokenizer.get_added_vocab()["<|user|>"])+ special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])# this one is usually not in config.json anyway- special_vocab._set_special_token(- "unk",- tokenizer.get_added_vocab()["<|endoftext|>"])+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])special_vocab.add_to_gguf(self.gguf_writer)def set_gguf_parameters(self):n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))- n_head = self.hparams.get("n_head",- self.hparams.get("num_attention_heads"))- n_head_kv = self.hparams.get("multi_query_group_num", n_head)- self.gguf_writer.add_context_length(- self.hparams.get("seq_length", n_embed))+ n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))+ n_head_kv = self.hparams.get("multi_query_group_num", self.hparams.get("num_key_value_heads", n_head))+ self.gguf_writer.add_context_length(self.hparams.get("seq_length", n_embed))self.gguf_writer.add_embedding_length(n_embed)self.gguf_writer.add_feed_forward_length(- self.hparams.get("ffn_hidden_size", 4 * n_embed))- self.gguf_writer.add_block_count(self.hparams["num_layers"])+ self.hparams.get("ffn_hidden_size", self.hparams.get("intermediate_size", 4 * n_embed)))+ self.gguf_writer.add_block_count(self.hparams.get("num_layers", self.hparams["num_hidden_layers"]))self.gguf_writer.add_head_count(n_head)self.gguf_writer.add_head_count_kv(n_head_kv)- self.gguf_writer.add_layer_norm_rms_eps(- self.hparams["layernorm_epsilon"])+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("layernorm_epsilon", 1e-5))self.gguf_writer.add_file_type(self.ftype)- self.gguf_writer.add_rope_dimension_count(64)+ if "attention_dim" in self.hparams:+ rope_dim = self.hparams["attention_dim"]+ else:+ rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]+ self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)))self.gguf_writer.add_add_bos_token(False)rope_freq = 10000if "rope_ratio" in self.hparams:rope_freq = rope_freq * self.hparams["rope_ratio"]self.gguf_writer.add_rope_freq_base(rope_freq)- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused- if name.endswith(".rotary_pos_emb.inv_freq"):+ if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision."):return []name = name.removeprefix("transformer.")@@ -5452,31 +5038,24 @@ class NemotronModel(Model):hparams = self.hparamsself.gguf_writer.add_vocab_size(hparams["vocab_size"])- f_norm_eps = self.find_hparam([- "layer_norm_eps", "layer_norm_epsilon", "norm_epsilon", "norm_eps"- ])+ f_norm_eps = self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon", "norm_eps"])self.gguf_writer.add_layer_norm_eps(f_norm_eps)# * Partial RoPE- rot_pct = self.find_hparam(- ["partial_rotary_factor", "rope_pct", "rope_percent"])+ rot_pct = self.find_hparam(["partial_rotary_factor", "rope_pct", "rope_percent"])n_embd = self.find_hparam(["hidden_size", "n_embd"])n_head = self.find_hparam(["num_attention_heads", "n_head"])- self.gguf_writer.add_rope_dimension_count(- int(rot_pct * n_embd) // n_head)+ self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head)# * RopeScaling for Nemotron- if "rope_scaling" not in self.hparams or self.hparams[- "rope_scaling"] is None:+ if "rope_scaling" not in self.hparams or self.hparams["rope_scaling"] is None:self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)else:self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)self.gguf_writer.add_rope_scaling_factor(self.hparams["factor"])- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:- # * Adding +1 to LayerNorm's weights here to implement layernorm1p w/o- # changing anything on the GGML engine side+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ # * Adding +1 to LayerNorm's weights here to implement layernorm1p w/o changing anything on the GGML engine side# model.layers.{l}.input_layernorm.weight# model.layers.{l}.post_attention_layernorm.weight# model.norm.weight@@ -5500,8 +5079,7 @@ class ExaoneModel(Model):num_heads = hparams["num_attention_heads"]num_kv_heads = hparams.get("num_key_value_heads", num_heads)layer_norm_eps = hparams["layer_norm_epsilon"]- intermediate_size = hparams[- "intermediate_size"] if "intermediate_size" in hparams else 4 * embed_dim+ intermediate_size = hparams["intermediate_size"] if "intermediate_size" in hparams else 4 * embed_dimnum_layers = hparams["num_layers"]# ignore for now as EXAONE-3.0-7.8B-Instruct attentino_dropout is 0.0# attention_dropout_rate = hparams["attention_dropout"]@@ -5518,35 +5096,26 @@ class ExaoneModel(Model):if (rope_theta := self.hparams.get("rope_theta")) is not None:self.gguf_writer.add_rope_freq_base(rope_theta)- rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"],- optional=True)+ rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"], optional=True)rotary_factor = rotary_factor if rotary_factor is not None else 1.0self.gguf_writer.add_rope_dimension_count(- int(rotary_factor *- (hparams["hidden_size"] // hparams["num_attention_heads"])))- if hparams.get("rope_scaling"- ) is not None and "factor" in hparams["rope_scaling"]:+ int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"])))+ if hparams.get("rope_scaling") is not None and "factor" in hparams["rope_scaling"]:if hparams["rope_scaling"].get("type") == "linear":- self.gguf_writer.add_rope_scaling_type(- gguf.RopeScalingType.LINEAR)- self.gguf_writer.add_rope_scaling_factor(- hparams["rope_scaling"]["factor"])+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(hparams["rope_scaling"]["factor"])def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):if rope_scaling.get("rope_type", '').lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)- dim = self.hparams.get(- "head_dim", self.hparams["hidden_size"] //- self.hparams["num_attention_heads"])- freqs = 1.0 / (base**(- torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ dim = self.hparams.get("head_dim", self.hparams["hidden_size"] // self.hparams["num_attention_heads"])+ freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)- old_context_len = self.hparams.get(- "original_max_position_embeddings", 8192)+ old_context_len = self.hparams.get("original_max_position_embeddings", 8192)low_freq_wavelen = old_context_len / low_freq_factorhigh_freq_wavelen = old_context_len / high_freq_factor@@ -5560,13 +5129,12 @@ class ExaoneModel(Model):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len / wavelen - low_freq_factor- ) / (high_freq_factor - low_freq_factor)- rope_factors.append(1 /- ((1 - smooth) / factor + smooth))+ smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)+ rope_factors.append(1 / ((1-smooth) / factor + smooth))- yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32))+ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),+ torch.tensor(rope_factors, dtype=torch.float32))@Model.register("GraniteForCausalLM")@@ -5585,19 +5153,16 @@ class GraniteModel(LlamaModel):- logits_scaling"""if head_dim := self.hparams.pop("head_dim", None):- logger.warning("Ignoring head_dim (%s) from config for Granite",- head_dim)+ logger.warning("Ignoring head_dim (%s) from config for Granite", head_dim)super().set_gguf_parameters()# NOTE: Convert _multiplier params to _scale params for naming# consistencyif attention_scale := self.hparams.get("attention_multiplier"):self.gguf_writer.add_attention_scale(attention_scale)- logger.info("gguf: (granite) attention_scale = %s",- attention_scale)+ logger.info("gguf: (granite) attention_scale = %s", attention_scale)if embedding_scale := self.hparams.get("embedding_multiplier"):self.gguf_writer.add_embedding_scale(embedding_scale)- logger.info("gguf: (granite) embedding_scale = %s",- embedding_scale)+ logger.info("gguf: (granite) embedding_scale = %s", embedding_scale)if residual_scale := self.hparams.get("residual_multiplier"):self.gguf_writer.add_residual_scale(residual_scale)logger.info("gguf: (granite) residual_scale = %s", residual_scale)@@ -5611,8 +5176,7 @@ class GraniteMoeModel(GraniteModel):"""Conversion for IBM's GraniteMoeForCausalLM"""model_arch = gguf.MODEL_ARCH.GRANITE_MOE- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:"""In modeling_granitemoe, the JetMoe implementation of parallel expertsis used. This essentially merges w1 and w3 into a single tensor with 2xthe hidden size that is then split during forward. To keep compatibility@@ -5621,15 +5185,11 @@ class GraniteMoeModel(GraniteModel):if name.endswith("block_sparse_moe.input_linear.weight"):ffn_dim = self.hparams["intermediate_size"]- assert data_torch.shape[- -2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size"- gate, up = data_torch[..., :ffn_dim, :], data_torch[...,- ffn_dim:, :]+ assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size"+ gate, up = data_torch[..., :ffn_dim, :], data_torch[..., ffn_dim:, :]return [- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP,- bid), gate),- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP,- bid), up),+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), gate),+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), up),]return super().modify_tensors(data_torch, name, bid)@@ -5647,8 +5207,7 @@ class ChameleonModel(Model):def set_vocab(self):self._set_vocab_gpt2()- def modify_tensors(self, data_torch: Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, Tensor]]:+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# ignore image tokenizer for now# TODO: remove this once image support is implemented for Chameleonif name.startswith("model.vqmodel"):@@ -5663,11 +5222,9 @@ class ChameleonModel(Model):if name.endswith(("k_proj.weight", "k_proj.bias")):data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head)if name.endswith(("q_norm.weight", "q_norm.bias")):- data_torch = ChameleonModel._reverse_hf_permute(- data_torch, n_head, hidden_dim)+ data_torch = ChameleonModel._reverse_hf_permute(data_torch, n_head, hidden_dim)if name.endswith(("k_norm.weight", "k_norm.bias")):- data_torch = ChameleonModel._reverse_hf_permute(- data_torch, n_kv_head, hidden_dim)+ data_torch = ChameleonModel._reverse_hf_permute(data_torch, n_kv_head, hidden_dim)return [(self.map_tensor_name(name), data_torch)]@@ -5695,6 +5252,8 @@ class LazyTorchTensor(gguf.LazyBase):torch.float32: np.float32,}+ # used for safetensors slices+ # TODO: uncomment U64, U32, and U16, ref: https://github.com/pytorch/pytorch/issues/58734_dtype_str_map: dict[str, torch.dtype] = {"F64": torch.float64,"F32": torch.float32,@@ -5716,23 +5275,19 @@ class LazyTorchTensor(gguf.LazyBase):def numpy(self) -> gguf.LazyNumpyTensor:dtype = self._dtype_map[self.dtype]return gguf.LazyNumpyTensor(- meta=gguf.LazyNumpyTensor.meta_with_dtype_and_shape(- dtype, self.shape),- args=(self, ),+ meta=gguf.LazyNumpyTensor.meta_with_dtype_and_shape(dtype, self.shape),+ args=(self,),func=(lambda s: s.numpy()))@classmethod- def meta_with_dtype_and_shape(cls, dtype: torch.dtype,- shape: tuple[int, ...]) -> Tensor:+ def meta_with_dtype_and_shape(cls, dtype: torch.dtype, shape: tuple[int, ...]) -> Tensor:return torch.empty(size=shape, dtype=dtype, device="meta")@classmethoddef from_safetensors_slice(cls, st_slice: Any) -> Tensor:dtype = cls._dtype_str_map[st_slice.get_dtype()]shape: tuple[int, ...] = tuple(st_slice.get_shape())- lazy = cls(meta=cls.meta_with_dtype_and_shape(dtype, shape),- args=(st_slice, ),- func=lambda s: s[:])+ lazy = cls(meta=cls.meta_with_dtype_and_shape(dtype, shape), args=(st_slice,), func=lambda s: s[:])return cast(torch.Tensor, lazy)@classmethod@@ -5749,8 +5304,7 @@ class LazyTorchTensor(gguf.LazyBase):def parse_args() -> argparse.Namespace:- parser = argparse.ArgumentParser(- description="Convert a huggingface model to a GGML compatible file")+ parser = argparse.ArgumentParser(description="Convert a huggingface model to a GGML compatible file")parser.add_argument("--vocab-only",action="store_true",@@ -5766,9 +5320,11 @@ def parse_args() -> argparse.Namespace:type=str,choices=["f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"],default="f16",- help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0,"- "tq1_0 or tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type depending "- "on the first loaded tensor type"),+ help=+ "output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0,"+ " tq1_0 or tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type "+ "depending on the first loaded tensor type",+ )parser.add_argument("--bigendian",action="store_true",@@ -5822,13 +5378,8 @@ def parse_args() -> argparse.Namespace:"--no-tensor-first-split",action="store_true",help="do not add tensors to the first split (disabled by default)")- parser.add_argument(- "--metadata",- type=Path,- help="Specify the path for an authorship metadata override file")- parser.add_argument("--print-supported-models",- action="store_true",- help="Print the supported models")+ parser.add_argument("--metadata", type=Path, help="Specify the path for an authorship metadata override file")+ parser.add_argument("--print-supported-models", action="store_true", help="Print the supported models")args = parser.parse_args()if not args.print_supported_models and args.model is None:@@ -5846,9 +5397,7 @@ def split_str_to_n_bytes(split_str: str) -> int:elif split_str.isnumeric():n = int(split_str)else:- raise ValueError(- f"Invalid split size: {split_str}, must be a number, optionally followed by K, M, or G"- )+ raise ValueError(f"Invalid split size: {split_str}, must be a number, optionally followed by K, M, or G")if n < 0:raise ValueError(f"Invalid split size: {split_str}, must be positive")@@ -5926,9 +5475,7 @@ def main() -> None:if args.vocab_only:logger.info("Exporting model vocab...")model_instance.write_vocab()- logger.info(- f"Model vocab successfully exported to {model_instance.fname_out}"- )+ logger.info(f"Model vocab successfully exported to {model_instance.fname_out}")else:logger.info("Exporting model...")model_instance.write()diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex b81f86be1..bf6357381 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -278,6 +278,57 @@ def setup_lmeval_parser():args = parser.parse_args()return args+def _gguf_args_check(args):+ from auto_round.utils import logger++ _GGUF_CONFIG = {+ "gguf:q4_0": {+ "bits": 4,+ "act_bits": 16,+ "group_size": 32,+ "asym": False,+ },+ "gguf:q4_1": {+ "bits": 4,+ "act_bits": 16,+ "group_size": 32,+ "asym": True,+ }+ }++ formats = args.format.lower().replace(' ', '').split(",")+ for format in _GGUF_CONFIG:+ if format in formats:+ from pathlib import Path+ from auto_round.export.export_to_gguf.convert import Model+ hparams = Model.load_hparams(Path(args.model))+ model_architecture = hparams["architectures"][0]+ try:+ model_class = Model.from_model_architecture(model_architecture)+ except NotImplementedError:+ logger.error(f"Model {model_architecture} is not supported to export GGUF format")+ sys.exit(1)++ unsupport_list, reset_list = [], []+ gguf_config = _GGUF_CONFIG[format]+ for k, v in gguf_config.items():+ if getattr(args, k) != v:+ unsupport_list.append(f"{k}={getattr(args, k)}")+ reset_list.append(f"{k}={v}")+ setattr(args, k, v)+ if len(unsupport_list) > 0:+ if len(formats) > 1:+ logger.error(+ f"format {format} not support for {', '.join(unsupport_list)},"+ f" please reset to {', '.join(reset_list)}, and retry")+ exit(-1)+ else:+ logger.error(+ f"format {format} not support for {', '.join(unsupport_list)},"+ f" reset to {', '.join(reset_list)}.")+ logger.info(f"export format {format}, sym = {not args.asym}, group_size = {args.group_size}")++ return argsdef tune(args):import transformers@@ -286,7 +337,9 @@ def tune(args):if args.format is None:args.format = "auto_round"- supported_formats = ["auto_round", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_awq", "fake"]+ supported_formats = [+ "auto_round", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_awq", "gguf:q4_0", "gguf:q4_1", "fake"+ ]if not args.quant_nontext_module:supported_formats.extend(["auto_gptq", "auto_gptq:marlin"])@@ -295,6 +348,9 @@ def tune(args):if format not in supported_formats:raise ValueError(f"{format} is not supported, we only support {supported_formats}")+ args = _gguf_args_check(args)++##must set this before import torchset_cuda_visible_devices(args.device)device_str, use_auto_mapping = get_device_and_parallelism(args.device)@@ -603,5 +659,3 @@ def lmms_eval(args):apply_chat_template=False,)return results--