PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
update gguf convert file and fix bug of permute bug (#679)
Heng Guo4e1b81f9ff5c
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex b1e7bd0b3..49d16562f 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -828,10 +828,10 @@ class AutoRound(object):if not hasattr(module, "imatrix"):module.imatrix = squared- module.imatrix_cnt = 1+ module.imatrix_cnt = input.shape[0]else:module.imatrix += squared- module.imatrix_cnt += 1+ module.imatrix_cnt += input.shape[0]hook_handles = []for name, module in model.named_modules():@@ -885,33 +885,36 @@ class AutoRound(object):cnt = 1cnt += 1except RuntimeError as e:- try:- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:- import accelerate- accelerate.hooks.remove_hook_from_submodules(model)- # Fallback: out-of-memory → try CPU blockwise quantization- logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")- model = model.to("cpu")- clear_memory()- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)- except Exception:- # Final fallback: warn and use CPU-only quantization- logger.warning("Fallback to CPU. "- "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`.")- model = model.to("cpu")- clear_memory()- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:- import accelerate- accelerate.hooks.remove_hook_from_submodules(model)-- orig_device = self.device- self.device = "cpu"- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)- self.device = orig_device- finally:- # Always remove hooks- for hook in hooks:- hook.remove()+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ try:+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate+ accelerate.hooks.remove_hook_from_submodules(model)+ # Fallback: out-of-memory → try CPU blockwise quantization+ logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")+ model = model.to("cpu")+ clear_memory()+ self.quantize_via_rtn_blockwise(all_to_quantized_module_names)+ except Exception:+ # Final fallback: warn and use CPU-only quantization+ logger.warning("Fallback to CPU. "+ "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`.")+ model = model.to("cpu")+ clear_memory()+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate+ accelerate.hooks.remove_hook_from_submodules(model)++ orig_device = self.device+ self.device = "cpu"+ self.quantize_via_rtn_blockwise(all_to_quantized_module_names)+ self.device = orig_device+ finally:+ # Always remove hooks+ for hook in hooks:+ hook.remove()+ else:+ raise# Move back to CPU and free memorymodel.to("cpu")@@ -1029,7 +1032,7 @@ class AutoRound(object):# Step 2: Try quantization on GPU first, fall back to CPU if OOM# if only export gguf, using gguf-packing instead of rtn- if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and self.disable_opt_rtn:+ if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:m.scale = Nonem.zp = Noneelse:diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex ae016c914..eba6db810 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -15,6 +15,7 @@import torchfrom auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad, loggerfrom auto_round.data_type.register import register_dtype+from auto_round.utils import get_reciprocal@register_dtype("int_sym_dq")@@ -69,7 +70,7 @@ def quant_tensor_sym_dq(scale = scale.view(-1, 1)zp = torch.full_like(scale, maxq) # pylint: disable=E1130- int_w = torch.where(scale != 0, round_ste(tensor / scale + v), 0)+ int_w = round_ste(tensor * get_reciprocal(scale) + v)q = torch.clamp(int_w + zp, 0, 2 ** bits - 1)qdq_result = (scale * (q - zp)).to(tensor.dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)@@ -140,7 +141,7 @@ def double_quant_tensor(tensor, bits):scale = wmax / maxqscale = scale.view(-1, 1)# inverse_scale = torch.where(scale == 0, 0, 1 / scale)- inverse_scale = torch.where(wmax > 0, maxq / wmax, 0).view(-1, 1)+ inverse_scale = (maxq * get_reciprocal(wmax)).clamp(min=0).view(-1, 1)qdq_tensor = torch.clamp(round_ste(tensor * inverse_scale), max=maxq) * scalereturn qdq_tensor, scale@@ -151,7 +152,7 @@ def double_quant_tensor_sym(tensor, bits):imax = abs(tensor).argmax(axis=-1, keepdims=True)wmax = torch.take_along_dim(tensor, imax, dim=-1)scale = wmax / -maxq- inverse_scale = torch.where(scale == 0, 0, 1 / scale) ##1e-40+ inverse_scale = get_reciprocal(scale)qdq_tensor = torch.clip((round_ste(tensor * inverse_scale)), -maxq, maxq - 1) * scalereturn qdq_tensor, scale@@ -161,7 +162,7 @@ def make_qp_quants(nmax, data, quant_weights):quant_weights = quant_weights.to(torch.float32)group_max = torch.max(data, dim=-1, keepdim=True)[0]scale = group_max / nmax- iscale = torch.where(scale == 0, 0, 1 / scale)+ iscale = get_reciprocal(scale)L = torch.round(iscale * data)diffs = data - scale * L@@ -171,7 +172,7 @@ def make_qp_quants(nmax, data, quant_weights):if _is == 0:continuescale_is = group_max / (0.1 * _is + nmax)- iscale_is = torch.where(scale_is == 0, 0, 1 / scale_is)+ iscale_is = get_reciprocal(scale_is)tmp_L = torch.round(iscale_is * data).clip(max=nmax)diffs = data - scale_is * tmp_L@@ -328,11 +329,11 @@ def quant_tensor_gguf_asym_dq(use_mad=params["use_mad"], weights=quant_weights)scale = scale.to(scale_dtype)- scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)scale = scale.reshape(-1, super_group_size)wmin = wmin.reshape(-1, super_group_size)scale, d_scale = double_quant_tensor(scale, super_bits)- wmin = torch.where(torch.abs(wmin) < 1e-30, 0, wmin)+ wmin = torch.where(torch.abs(wmin) < 1e-30, torch.zeros_like(wmin), wmin)wmin, d_wmin = double_quant_tensor(wmin, super_bits)wmin = wmin.view(-1, 1)scale = scale.view(-1, 1)@@ -386,7 +387,7 @@ def quant_tensor_gguf_asym_dq(use_mad=params["use_mad"], weights=quant_weights)scale = scale.to(scale_dtype)- scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)nmax = 2 ** super_bits - 1scale = scale.reshape(-1, super_group_size)wmin = wmin_0.reshape(-1, super_group_size)@@ -399,7 +400,7 @@ def quant_tensor_gguf_asym_dq(d_wmin = d_wmin.unsqueeze(-1)scale = (d_scale * q_scale).view(-1, 1)wmin = (d_wmin * q_wmin).view(-1, 1)- inverse_scale = torch.where(scale == 0, 0, 1 / scale)+ inverse_scale = get_reciprocal(scale)int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)qdq_result = (scale * int_w - wmin).to(orig_dtype)@@ -436,7 +437,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u# scale = 1 / ((maxq - minq) / (rmax - rmin + 1e-8))scale = (rmax - rmin) / (maxq - minq)- iscale = torch.where(scale == 0, 0, 1 / scale)+ iscale = get_reciprocal(scale)# quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)quant_data = torch.clamp(torch.round(iscale * (data - rmin)), minq, maxq)diff = scale * quant_data + rmin - data@@ -447,7 +448,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, ufactor = rrmin + rdelta * is_ + maxq - minq# iscale_new = factor / (rmax - rmin + 1e-8)scale_new = (rmax - rmin) / factor- iscale_new = torch.where(scale_new == 0, 0, 1 / scale_new)+ iscale_new = get_reciprocal(scale_new)quant_data_new = torch.clamp(torch.round(iscale_new * (data - rmin)), minq, maxq)mul_weights_quant_data = weights * quant_data_new@@ -460,7 +461,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, uthis_min = (sum_l2 * sum_x - sum_l * sum_xl) / Dthis_min[this_min > 0] = 0this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]- reverse_this_scale = torch.where(this_scale == 0, 0, 1 / this_scale)+ reverse_this_scale = get_reciprocal(this_scale)quant_data = torch.clamp(torch.round(reverse_this_scale * (data - this_min)), minq, maxq)diff = this_scale * quant_data + this_min - data@@ -569,13 +570,13 @@ def quant_tensor_gguf_sym_dq(quant_weights[mean_replace_index, :] = tmp_quant_weights[mean_replace_index, :]scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)- scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)# conduct double quantscale, d_scale = double_quant_tensor_sym(scale, super_bits)scale = scale.unsqueeze(-1)zp = torch.full_like(scale, maxq) # pylint: disable=E1130- inverse_scale = torch.where(scale == 0, 0, 1.0 / scale)+ inverse_scale = get_reciprocal(scale)int_w = torch.round(tensor * inverse_scale).clip(-maxq, maxq - 1) + maxqqdq_result = (scale * (int_w - zp)).to(orig_dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 9ddd55585..034b86590 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -356,6 +356,7 @@ class OriModel:gguf.MODEL_TENSOR.POS_EMBD,gguf.MODEL_TENSOR.TOKEN_TYPES,gguf.MODEL_TENSOR.SSM_CONV1D,+ gguf.MODEL_TENSOR.SHORTCONV_CONV,gguf.MODEL_TENSOR.TIME_MIX_FIRST,gguf.MODEL_TENSOR.TIME_MIX_W1,gguf.MODEL_TENSOR.TIME_MIX_W2,@@ -648,8 +649,7 @@ class ModelBase(OriModel):if hasattr(module, attr) and getattr(module, attr) is not None:attr_tensor = getattr(module, attr)ori_shape = attr_tensor.shape- # attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)[0][1]- attr_tensor = self.modify_tensors(attr_tensor, modify_name, bid)[0][1]+ attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)[0][1]attr_tensor = attr_tensor.reshape(ori_shape)setattr(module, attr, attr_tensor)scale = module.scale if hasattr(module, "scale") else None@@ -1136,10 +1136,10 @@ class TextModel(ModelBase):)seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))- seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))+ seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>")) # deepseek-coder# TODO: should these be marked as UNUSED instead? (maybe not)- seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">"))+ seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">")) # gemma{,-2}return seems_special@@ -1149,7 +1149,7 @@ class TextModel(ModelBase):toktypes: list[int] = []from transformers import AutoTokenizer- tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model)vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))assert max(tokenizer.vocab.values()) < vocab_size@@ -1183,7 +1183,7 @@ class TextModel(ModelBase):else:# NOTE: this was added for Gemma.# Encoding and decoding the tokens above isn't sufficient for this case.- token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")+ token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")toktypes.append(gguf.TokenType.USER_DEFINED)else:toktypes.append(gguf.TokenType.NORMAL)@@ -1214,6 +1214,36 @@ class TextModel(ModelBase):# NOTE: if you get an error here, you need to update the convert_hf_to_gguf_update.py script# or pull the latest version of the model from Huggingface# don't edit the hashes manually!+ if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b":+ # ref: https://huggingface.co/THUDM/glm-4-9b-chat+ res = "chatglm-bpe"+ if chkhsh == "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516":+ # ref: https://huggingface.co/THUDM/glm-4-9b-chat+ res = "chatglm-bpe"+ if chkhsh == "a1336059768a55c99a734006ffb02203cd450fed003e9a71886c88acf24fdbc2":+ # ref: https://huggingface.co/THUDM/glm-4-9b-hf+ res = "glm4"+ if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":+ # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0+ res = "minerva-7b"+ if chkhsh == "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664":+ # ref: https://huggingface.co/tencent/Hunyuan-A13B-Instruct+ res = "hunyuan"+ if chkhsh == "a6b57017d60e6edb4d88ecc2845188e0eb333a70357e45dcc9b53964a73bbae6":+ # ref: https://huggingface.co/tiiuae/Falcon-H1-0.5B-Base+ res = "falcon-h1"+ if chkhsh == "60476e1243776c4fb1b993dbd7a5f15ac22f83c80afdf425fa5ae01c8d44ef86":+ # ref: https://huggingface.co/tiiuae/Falcon-H1-1B-Base+ res = "falcon-h1"+ if chkhsh == "3eda48b4c4dc7de733d1a8b3e3b4a85243dbbf704da2ee9d42c6beced8897896":+ # ref: https://huggingface.co/tiiuae/Falcon-H1-7B-Base+ res = "falcon-h1"+ if chkhsh == "48f8e02c0359c0bbdd82f26909171fac1c18a457bb47573ed1fe3bbb2c1cfd4b":+ # ref: https://huggingface.co/tiiuae/Falcon-H1-34B-Base+ res = "falcon-h1"+ if chkhsh == "81212dc7cdb7e0c1074ca62c5aeab0d43c9f52b8a737be7b12a777c953027890":+ # ref: https://huggingface.co/moonshotai/Kimi-K2-Base+ res = "kimi-k2"if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8Bres = "llama-bpe"@@ -1349,18 +1379,18 @@ class TextModel(ModelBase):if chkhsh == "d5f1dd6f980fec569fb218a81a7658ac45fc56b38c5a0adeb1c232fbe04ef5ec":# ref: https://huggingface.co/ByteDance-Seed/Seed-Coder-8B-Baseres = "seed-coder"- if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b":- # ref: https://huggingface.co/THUDM/glm-4-9b-chat- res = "chatglm-bpe"- if chkhsh == "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516":- # ref: https://huggingface.co/THUDM/glm-4-9b-chat- res = "chatglm-bpe"- if chkhsh == "a1336059768a55c99a734006ffb02203cd450fed003e9a71886c88acf24fdbc2":- # ref: https://huggingface.co/THUDM/glm-4-9b-hf- res = "glm4"- if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":- # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0- res = "minerva-7b"+ if chkhsh == "b0a6b1c0bd5998ebd9df08611efde34a4ff03faed45ae09c43e6b31ebd4b94cf":+ # ref: https://huggingface.co/skt/A.X-4.0+ res = "a.x-4.0"+ if chkhsh == "f6791d196f87ce6b56a7d234be618e0d58f8cda3549416635b2bebcd22cd95c4":+ # ref: https://huggingface.co/K-intelligence/Midm-2.0-Base-Instruct+ res = "midm-2.0"+ if chkhsh == "169bf0296a13c4d9b7672313f749eb36501d931022de052aad6e36f2bf34dd51":+ # ref: https://huggingface.co/LiquidAI/LFM2-Tokenizer+ res = "lfm2"+ if chkhsh == "2085e1638f6c377a0aa4ead21b27bb4cb941bf800df86ed391011769c1758dfb":+ # ref: https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B+ res = "exaone4"if res is None:logger.warning("\n")@@ -1607,7 +1637,14 @@ class TextModel(ModelBase):self.gguf_writer.add_token_list(tokens)self.gguf_writer.add_token_types(toktypes)special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)- special_vocab.chat_template = "rwkv-world"+ if special_vocab.chat_template is None:+ template_path = Path(__file__).parent / "models" / "templates" / "llama-cpp-rwkv-world.jinja"+ if template_path.is_file():+ with open(template_path, "r", encoding="utf-8") as f:+ template = f.read()+ else:+ template = "rwkv-world"+ special_vocab.chat_template = template# hack: Add '\n\n' as the EOT token to make it chat normallyspecial_vocab._set_special_token("eot", 261)# hack: Override these as they have already been set (incorrectly)@@ -3290,6 +3327,76 @@ class Qwen2Model(TextModel):yield from super().modify_tensors(data_torch, name, bid)[email protected]("DreamModel")+class DreamModel(TextModel):+ model_arch = gguf.MODEL_ARCH.DREAM++ def get_vocab_base(self) -> tuple[list[str], list[int], str]:+ tokens: list[str] = []+ toktypes: list[int] = []++ from transformers import AutoTokenizer+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)++ vocab_dict = tokenizer.get_vocab()+ vocab_size = self.hparams.get("vocab_size", len(vocab_dict))+ assert max(vocab_dict.values()) < vocab_size++ tokpre = self.get_vocab_base_pre(tokenizer)++ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()}+ added_vocab = tokenizer.get_added_vocab()++ for i in range(vocab_size):+ if i not in reverse_vocab:+ tokens.append(f"[PAD{i}]")+ toktypes.append(gguf.TokenType.UNUSED)+ elif reverse_vocab[i] in added_vocab:+ tokens.append(reverse_vocab[i])+ # Check if it's a special token - treat special tokens as CONTROL tokens+ if hasattr(tokenizer, 'added_tokens_decoder') and i in tokenizer.added_tokens_decoder:+ if tokenizer.added_tokens_decoder[i].special:+ toktypes.append(gguf.TokenType.CONTROL)+ else:+ toktypes.append(gguf.TokenType.USER_DEFINED)+ else:+ # Fallback: treat all added vocab as control tokens for special tokens like <|im_start|>+ toktypes.append(gguf.TokenType.CONTROL)+ else:+ tokens.append(reverse_vocab[i])+ toktypes.append(gguf.TokenType.NORMAL)++ return tokens, toktypes, tokpre++ def set_vocab(self):+ try:+ self._set_vocab_sentencepiece()+ except FileNotFoundError:+ self._set_vocab_gpt2()++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self._try_set_pooling_type()++ # Dream models use non-causal attention for diffusion+ self.gguf_writer.add_causal_attention(False)+ # Handle RoPE scaling similar to Qwen2+ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])+ self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])++ # Add Dream-specific parameters+ mask_token_id = self.hparams.get("mask_token_id")+ if mask_token_id is not None:+ self.gguf_writer.add_mask_token_id(mask_token_id)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ # Dream model tensors should be mapped directly since it's the base model+ yield from super().modify_tensors(data_torch, name, bid)++@ModelBase.register("Ernie4_5_ForCausalLM")class Ernie4_5Model(TextModel):model_arch = gguf.MODEL_ARCH.ERNIE4_5@@ -3303,7 +3410,8 @@ class Ernie4_5Model(TextModel):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:num_heads = self.hparams["num_attention_heads"]num_kv_heads = self.hparams["num_key_value_heads"]- head_dim = self.hparams["head_dim"]+ if (head_dim := self.hparams.get("head_dim")) is None:+ head_dim = self.hparams["hidden_size"] // num_headsif "ernie." in name:name = name.replace("ernie.", "model.")@@ -3332,6 +3440,96 @@ class Ernie4_5Model(TextModel):return [(self.map_tensor_name(name), data_torch)][email protected]("Ernie4_5_MoeForCausalLM")+class Ernie4_5MoeModel(Ernie4_5Model):+ model_arch = gguf.MODEL_ARCH.ERNIE4_5_MOE+ _experts: list[dict[str, Tensor]] | None = None++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self._experts = [{} for _ in range(self.block_count)]++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_expert_count(self.hparams["moe_num_experts"])+ self.gguf_writer.add_expert_used_count(self.hparams["moe_k"])+ self.gguf_writer.add_interleave_moe_layer_step(self.hparams["moe_layer_interval"])+ self.gguf_writer.add_leading_dense_block_count(self.hparams["moe_layer_start_index"])+ if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:+ self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)+ if (shared_expert_count := self.hparams.get('moe_num_shared_experts')) is not None:+ self.gguf_writer.add_expert_shared_count(shared_expert_count)+ if shared_expert_count > 0 and (shared_expert_intermediate_size := self.hparams.get('intermediate_size')+ ) is not None and (num_key_value_heads :=+ self.hparams.get('num_key_value_heads')) is not None:+ self.gguf_writer.add_expert_shared_feed_forward_length(+ shared_expert_intermediate_size // num_key_value_heads)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ # Modify correction bias name as in DeepseekV2+ if name.endswith("e_score_correction_bias"):+ name = name.replace("e_score_correction_bias", "e_score_correction.bias")++ # skip Multi-Token Prediction (MTP) layers (again, same as DeepseekV2)+ match = re.match(r"model.mtp_block.(\d+)", name)+ if match:+ return []++ # skip all other MTP tensors for now+ match = re.match(r"model.mtp_emb_norm.(\d+)", name)+ if match:+ return []++ match = re.match(r"model.mtp_hidden_norm.(\d+)", name)+ if match:+ return []++ match = re.match(r"model.mtp_linear_proj.(\d+)", name)+ if match:+ return []++ # process the experts separately+ if name.find("mlp.experts") != -1:+ n_experts = self.hparams["moe_num_experts"]+ assert bid is not None++ if self._experts is None:+ self._experts = [{} for _ in range(self.block_count)]++ self._experts[bid][name] = data_torch++ if len(self._experts[bid]) >= n_experts * 3:+ tensors: list[tuple[str, Tensor]] = []++ # merge the experts into a single 3d tensor+ for w_name in ["gate_proj", "up_proj", "down_proj"]:+ data: list[Tensor] = []++ for xid in range(n_experts):+ ename_to_retrieve = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"+ data.append(self._experts[bid][ename_to_retrieve])+ del self._experts[bid][ename_to_retrieve]++ data_torch = torch.stack(data, dim=0)+ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"+ new_name = self.map_tensor_name(merged_name)+ tensors.append((new_name, data_torch))++ return tensors+ else:+ return []+ return [(self.map_tensor_name(name), data_torch)]++ def prepare_tensors(self):+ super().prepare_tensors()++ if self._experts is not None:+ # flatten `list[dict[str, Tensor]]` into `list[str]`+ experts = [k for d in self._experts for k in d.keys()]+ if len(experts) > 0:+ raise ValueError(f"Unprocessed experts: {experts}")++@ModelBase.register("Qwen2VLModel","Qwen2VLForConditionalGeneration",@@ -3357,7 +3555,6 @@ class Qwen2VLModel(TextModel):del bid # unusedif name.startswith("thinker."):name = name.replace("thinker.", "")-if name.startswith("visual") or name.startswith("audio") or \name.startswith("talker") or name.startswith("token2wav"):# skip multimodal tensors@@ -3500,8 +3697,6 @@ class Qwen25OmniModel(Qwen2VLVisionModel):# transpose conv1 and conv2 biasdata_torch = data_torch.unsqueeze(-1)if "audio_bos_eos_token" in name:- # this tensor is left unused in transformers code- # https://github.com/huggingface/transformers/blob/6e3063422c4b1c014aa60c32b9254fd2902f0f28/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py#L1809return []return [(self.map_tensor_name(name), data_torch)]@@ -4035,6 +4230,175 @@ class PlamoModel(TextModel):return [(new_name, data_torch)][email protected]("Plamo2ForCausalLM", "PLaMo2ForCausalLM")+class Plamo2Model(TextModel):+ model_arch = gguf.MODEL_ARCH.PLAMO2++ def set_vocab(self):+ # PLaMo 2 uses a custom tokenizer with a .jsonl file+ # We need to handle this specially+ tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl"+ tokenizer_config_path = self.dir_model / "tokenizer_config.json"++ if not tokenizer_jsonl_path.is_file():+ raise FileNotFoundError(f"PLaMo 2 tokenizer file not found: {tokenizer_jsonl_path}")++ # Load tokenizer config+ with open(tokenizer_config_path, 'r', encoding='utf-8') as f:+ tokenizer_config = json.load(f)++ # Load tokens from JSONL file (actually a list format)+ tokens = []+ scores = []+ toktypes = []++ with open(tokenizer_jsonl_path, 'r', encoding='utf-8') as f:+ for line_num, line in enumerate(f):+ if line.strip():+ token_data = json.loads(line)+ # Format: [token, score, type, ?, ?, ?, ?]+ token = token_data[0].encode("utf-8")+ score = float(token_data[1])+ token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL"++ tokens.append(token)+ scores.append(score)++ # Map token type strings to GGUF token types+ if token_type_str == "UNKNOWN":+ toktypes.append(gguf.TokenType.UNKNOWN)+ elif token_type_str == "CONTROL":+ toktypes.append(gguf.TokenType.CONTROL)+ elif token_type_str == "BYTE":+ toktypes.append(gguf.TokenType.BYTE)+ else:+ # Check for PLaMo-2 special tokens+ token_str = token_data[0]+ if token_str.startswith("<|plamo:") and token_str.endswith("|>"):+ toktypes.append(gguf.TokenType.CONTROL)+ else:+ toktypes.append(gguf.TokenType.NORMAL)++ vocab_size = self.hparams["vocab_size"]+ if vocab_size > len(tokens):+ pad_count = vocab_size - len(tokens)+ logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")+ for i in range(1, pad_count + 1):+ tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))+ scores.append(-1000.0)+ toktypes.append(gguf.TokenType.UNUSED)++ # Use "plamo2" tokenizer type for PLaMo-2's custom Aho-Corasick tokenizer+ self.gguf_writer.add_tokenizer_model("plamo2")+ self.gguf_writer.add_tokenizer_pre("default")+ self.gguf_writer.add_token_list(tokens)+ self.gguf_writer.add_token_scores(scores)+ self.gguf_writer.add_token_types(toktypes)++ # Add special tokens from config+ if "bos_token" in tokenizer_config and tokenizer_config["bos_token"] is not None:+ token_id = tokens.index(tokenizer_config["bos_token"].encode("utf-8"))+ self.gguf_writer.add_bos_token_id(token_id)+ if "eos_token" in tokenizer_config and tokenizer_config["eos_token"] is not None:+ token_id = tokens.index(tokenizer_config["eos_token"].encode("utf-8"))+ self.gguf_writer.add_eos_token_id(token_id)+ if "pad_token" in tokenizer_config and tokenizer_config["pad_token"] is not None:+ token_id = tokens.index(tokenizer_config["pad_token"].encode("utf-8"))+ self.gguf_writer.add_pad_token_id(token_id)+ if "sep_token" in tokenizer_config and tokenizer_config["sep_token"] is not None:+ token_id = tokens.index(tokenizer_config["sep_token"].encode("utf-8"))+ self.gguf_writer.add_sep_token_id(token_id)+ if "unk_token" in tokenizer_config and tokenizer_config["unk_token"] is not None:+ token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8"))+ self.gguf_writer.add_unk_token_id(token_id)++ # Add <|plamo:op|> as EOT to ensure appropriate end of generation+ self.gguf_writer.add_eot_token_id(4)++ self.gguf_writer.add_add_space_prefix(False)++ def set_gguf_parameters(self):+ hparams = self.hparams+ block_count = hparams["num_hidden_layers"]+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])++ # Which layers are Mamba layers+ # PLaMo 2 uses mamba_step to indicate the pattern (e.g., 2 means every other layer)+ # This logic matches modeling_plamo.py's is_mamba function+ mamba_step = hparams.get("mamba_step", 2)+ mamba_enabled = hparams.get("mamba_enabled", True)+ mamba_layers = []++ if mamba_enabled:+ for i in range(block_count):+ if block_count <= (mamba_step // 2):+ # use attention in last layer+ is_mamba = (i != block_count - 1)+ else:+ is_mamba = (i % mamba_step) != (mamba_step // 2)+ if is_mamba:+ mamba_layers.append(0)+ else:+ mamba_layers.append(hparams.get("num_key_value_heads", 4))++ if mamba_layers:+ self.gguf_writer.add_head_count_kv(mamba_layers)++ self.gguf_writer.add_context_length(hparams.get("max_position_embeddings", 2048))+ self.gguf_writer.add_embedding_length(hparams.get("hidden_size", 4096))+ self.gguf_writer.add_block_count(block_count)+ self.gguf_writer.add_head_count(hparams.get("num_attention_heads", 32))+ self.gguf_writer.add_layer_norm_rms_eps(hparams.get("rms_norm_eps", 1e-06))+ self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 1000000.0))++ # Mamba parameters+ self.gguf_writer.add_ssm_state_size(hparams.get("mamba_d_state", 64))+ self.gguf_writer.add_ssm_conv_kernel(hparams.get("mamba_d_conv", 4))+ self.gguf_writer.add_ssm_time_step_rank(hparams.get("mamba_num_heads", 64))+ intermediate_size = hparams.get("mamba_num_heads", 64) * hparams.get("hidden_size_per_head", 128)+ self.gguf_writer.add_ssm_inner_size(intermediate_size)+ self.gguf_writer.add_ssm_group_count(0)++ # MLP feed forward parameters (for attention layers)+ self.gguf_writer.add_feed_forward_length(hparams.get("intermediate_size", 16384))+ self.gguf_writer.add_file_type(self.ftype)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ del bid # unused++ if name.endswith(".A_log"):+ data_torch = -torch.exp(data_torch)+ elif name.endswith(".dt_bias"):+ name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"+ elif name.endswith(".dt_norm_weight"):+ name = name.rpartition(".dt_norm_weight")[0] + ".dt_norm.weight"+ elif name.endswith(".B_norm_weight"):+ name = name.rpartition(".B_norm_weight")[0] + ".B_norm.weight"+ elif name.endswith(".C_norm_weight"):+ name = name.rpartition(".C_norm_weight")[0] + ".C_norm.weight"+ elif name.endswith(".k_weight"):+ name = name.rpartition(".k_weight")[0] + ".k.weight"+ elif name.endswith(".q_weight"):+ name = name.rpartition(".q_weight")[0] + ".q.weight"+ elif name.endswith(".conv1d.weight"):+ data_torch = torch.squeeze(data_torch) # remove (, 1, )+ assert data_torch.ndim == 2+ elif name.endswith(".pre_mixer_norm.weight"):+ data_torch += 1.0+ elif name.endswith(".post_mixer_norm.weight"):+ data_torch += 1.0 / 5+ elif name.endswith(".pre_mlp_norm.weight"):+ data_torch += 1.0+ elif name.endswith(".post_mlp_norm.weight"):+ data_torch += 1.0 / (5**1.5)+ elif name.endswith(".norm.weight"):+ data_torch += 1.0++ new_name = self.map_tensor_name(name)++ return [(new_name, data_torch)]++@ModelBase.register("CodeShellForCausalLM")class CodeShellModel(TextModel):model_arch = gguf.MODEL_ARCH.CODESHELL@@ -5454,6 +5818,10 @@ class Mamba2Model(TextModel):with open(dir_model / "config.json", "r", encoding="utf-8") as f:hparams = json.load(f)super().__init__(dir_model, *args, hparams=hparams, **kwargs)+ self.d_model = self.find_hparam(["hidden_size", "d_model", "dim"])+ self.d_inner = self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"],+ optional=True) or 2 * self.d_model+ self.n_group = self.find_hparam(["n_groups"], optional=True) or 1def set_vocab(self):vocab_size = self.hparams["vocab_size"]@@ -5477,30 +5845,29 @@ class Mamba2Model(TextModel):self._set_vocab_builtin("gpt-neox", vocab_size)def set_gguf_parameters(self):- d_model = self.find_hparam(["hidden_size", "d_model", "dim"])d_conv = self.find_hparam(["conv_kernel", "d_conv"], optional=True) or 4- d_inner = self.find_hparam(["intermediate_size", "d_inner"], optional=True) or 2 * d_modeld_state = self.find_hparam(["state_size", "d_state"], optional=True) or 128- head_dim = self.find_hparam(["head_dim"], optional=True) or 64- n_group = self.find_hparam(["n_groups"], optional=True) or 1+ head_dim = self.find_hparam(["mamba_d_head", "head_dim"], optional=True) or 64rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-5# Fail early for models which don't have a block expansion factor of 2# TODO: does this really matter?- assert d_inner == 2 * d_model- assert d_inner % head_dim == 0+ # skip the assertion for FalconH1 Model+ if self.model_arch != gguf.MODEL_ARCH.FALCON_H1:+ assert self.d_inner == 2 * self.d_model+ assert self.d_inner % head_dim == 0self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default- self.gguf_writer.add_embedding_length(d_model)+ self.gguf_writer.add_embedding_length(self.d_model)self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loadingself.gguf_writer.add_head_count(0) # unused, but seemingly required when loadingself.gguf_writer.add_block_count(self.block_count)self.gguf_writer.add_ssm_conv_kernel(d_conv)- self.gguf_writer.add_ssm_inner_size(d_inner)+ self.gguf_writer.add_ssm_inner_size(self.d_inner)self.gguf_writer.add_ssm_state_size(d_state)- self.gguf_writer.add_ssm_time_step_rank(d_inner // head_dim)- self.gguf_writer.add_ssm_group_count(n_group)+ self.gguf_writer.add_ssm_time_step_rank(self.d_inner // head_dim)+ self.gguf_writer.add_ssm_group_count(self.n_group)self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)self.gguf_writer.add_file_type(self.ftype)@@ -5525,10 +5892,7 @@ class Mamba2Model(TextModel):# (D is also unsqueezed, but for more straightforward broadcast internally)data_torch = data_torch.reshape((*data_torch.shape, 1))elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_NORM, bid):- d_model = self.find_hparam(["hidden_size", "d_model", "dim"])- d_inner = self.find_hparam(["intermediate_size", "d_inner"], optional=True) or 2 * d_model- n_group = self.hparams.get("n_groups", 1)- data_torch = data_torch.reshape((n_group, d_inner // n_group))+ data_torch = data_torch.reshape((self.n_group, self.d_inner // self.n_group))if name.endswith(".A_log"):logger.debug("A_log --> A ==> " + new_name)@@ -5537,51 +5901,165 @@ class Mamba2Model(TextModel):yield (new_name, data_torch)[email protected]("CohereForCausalLM")-class CommandR2Model(TextModel):- model_arch = gguf.MODEL_ARCH.COMMAND_R[email protected]("JambaForCausalLM")+class JambaModel(TextModel):+ model_arch = gguf.MODEL_ARCH.JAMBA- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)+ def get_vocab_base_pre(self, tokenizer) -> str:+ del tokenizer # unused- # max_position_embeddings = 8192 in config.json but model was actually- # trained on 128k context length- # aya-23 models don't have model_max_length specified- self.hparams["max_position_embeddings"] = self.find_hparam(["model_max_length", "max_position_embeddings"])+ return "gpt-2"++ def set_vocab(self):+ if (self.dir_model / "tokenizer.model").is_file():+ # Using Jamba's tokenizer.json causes errors on model load+ # (something about "byte not found in vocab"),+ # but there's a working tokenizer.model+ self._set_vocab_sentencepiece()+ else:+ # Some Jamba models only have a tokenizer.json, which works.+ self._set_vocab_gpt2()def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)+ d_model = self.find_hparam(["hidden_size", "mamba_d_model"])+ d_conv = self.find_hparam(["mamba_d_conv"], optional=True) or 4+ d_inner = self.hparams["mamba_expand"] * d_model+ d_state = self.find_hparam(["mamba_d_state"], optional=True) or 16+ dt_rank = self.find_hparam(["mamba_dt_rank"], optional=True) or -(d_model // -16)+ rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-6+ n_kv_head = self.hparams["num_key_value_heads"]+ attn_offset = self.hparams["attn_layer_offset"]+ attn_period = self.hparams["attn_layer_period"]+ n_kv_vec = [0 for _ in range(attn_offset)] + [+ n_kv_head if (i-attn_offset) % attn_period == 0 else 0 for i in range(attn_offset, self.block_count)+ ]+ self.gguf_writer.add_block_count(self.block_count)+ self.gguf_writer.add_context_length(self.find_hparam(["max_position_embeddings", "n_ctx"]))+ self.gguf_writer.add_embedding_length(d_model)+ self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])+ self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])+ self.gguf_writer.add_head_count_kv(n_kv_vec)+ self.gguf_writer.add_ssm_conv_kernel(d_conv)+ self.gguf_writer.add_ssm_inner_size(d_inner)+ self.gguf_writer.add_ssm_state_size(d_state)+ self.gguf_writer.add_ssm_time_step_rank(dt_rank)+ self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)+ self.gguf_writer.add_expert_count(self.hparams["num_experts"])+ self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])+ self.gguf_writer.add_file_type(self.ftype)[email protected]("Cohere2ForCausalLM")-class Cohere2Model(TextModel):- model_arch = gguf.MODEL_ARCH.COHERE2+ _experts: list[dict[str, Tensor]] | None = None- def set_gguf_parameters(self):- super().set_gguf_parameters()+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])- self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])- self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])+ # Mini-Jamba+ name = name.replace(".moe.", ".feed_forward.")+ if bid is not None:+ moe_offset = self.hparams["expert_layer_offset"]+ moe_period = self.hparams["expert_layer_period"]- rotary_pct = self.hparams["rotary_pct"]- hidden_size = self.hparams["hidden_size"]- num_attention_heads = self.hparams["num_attention_heads"]- self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size//num_attention_heads)))- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)+ if not (bid >= moe_offset and (bid-moe_offset) % moe_period == 0):+ name = name.replace(".experts.0.", ".")+ # process the experts separately+ if ".feed_forward.experts." in name:+ n_experts = self.hparams["num_experts"][email protected]("OlmoForCausalLM")[email protected]("OLMoForCausalLM")-class OlmoModel(TextModel):- model_arch = gguf.MODEL_ARCH.OLMO+ assert bid is not None- def set_gguf_parameters(self):- super().set_gguf_parameters()- self.gguf_writer.add_layer_norm_eps(1e-5)- clip_qkv = self.hparams.get("clip_qkv")+ if self._experts is None:+ self._experts = [{} for _ in range(self.block_count)]++ self._experts[bid][name] = data_torch++ if len(self._experts[bid]) >= n_experts * 3:++ # merge the experts into a single 3d tensor+ for wid in ["down_proj", "gate_proj", "up_proj"]:+ data: list[Tensor] = []++ for xid in range(n_experts):+ ename = f"model.layers.{bid}.feed_forward.experts.{xid}.{wid}.weight"+ data.append(self._experts[bid][ename])+ del self._experts[bid][ename]++ data_torch = torch.stack(data, dim=0)++ # using the same merged name as qwen2moe+ merged_name = f"model.layers.{bid}.mlp.experts.{wid}.weight"++ new_name = self.map_tensor_name(merged_name)++ yield new_name, data_torch+ return++ new_name = self.map_tensor_name(name)++ if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_CONV1D, bid):+ data_torch = data_torch.squeeze()++ if name.endswith(".A_log"):+ logger.debug("A_log --> A ==> " + new_name)+ data_torch = -torch.exp(data_torch)++ yield (new_name, data_torch)++ def prepare_tensors(self):+ super().prepare_tensors()++ if self._experts is not None:+ # flatten `list[dict[str, Tensor]]` into `list[str]`+ experts = [k for d in self._experts for k in d.keys()]+ if len(experts) > 0:+ raise ValueError(f"Unprocessed experts: {experts}")++[email protected]("CohereForCausalLM")+class CommandR2Model(TextModel):+ model_arch = gguf.MODEL_ARCH.COMMAND_R++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)++ # max_position_embeddings = 8192 in config.json but model was actually+ # trained on 128k context length+ # aya-23 models don't have model_max_length specified+ self.hparams["max_position_embeddings"] = self.find_hparam(["model_max_length", "max_position_embeddings"])++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)++[email protected]("Cohere2ForCausalLM")+class Cohere2Model(TextModel):+ model_arch = gguf.MODEL_ARCH.COHERE2++ def set_gguf_parameters(self):+ super().set_gguf_parameters()++ self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])+ self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])++ rotary_pct = self.hparams["rotary_pct"]+ hidden_size = self.hparams["hidden_size"]+ num_attention_heads = self.hparams["num_attention_heads"]+ self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size//num_attention_heads)))+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)++[email protected]("OlmoForCausalLM")[email protected]("OLMoForCausalLM")+class OlmoModel(TextModel):+ model_arch = gguf.MODEL_ARCH.OLMO++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_layer_norm_eps(1e-5)+ clip_qkv = self.hparams.get("clip_qkv")if clip_qkv is not None:self.gguf_writer.add_clamp_kqv(clip_qkv)@@ -5832,8 +6310,8 @@ class ArcticModel(TextModel):token_score = 0.0logger.info(- f"Setting added token {token_id} to '{token_content}' (type: {token_type},"- f" score: {token_score:.2f})"+ f"Setting added token {token_id} to '{token_content}'"+ f" (type: {token_type}, score: {token_score:.2f})")tokens[token_id] = token_content.encode("utf-8")toktypes[token_id] = token_type@@ -6006,7 +6484,58 @@ class DeepseekV2Model(TextModel):model_arch = gguf.MODEL_ARCH.DEEPSEEK2def set_vocab(self):- self._set_vocab_gpt2()+ try:+ self._set_vocab_gpt2()+ return+ except Exception:+ pass++ from transformers import AutoTokenizer+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)+ tokpre = self.get_vocab_base_pre(tokenizer)++ if tokpre == "kimi-k2":+ # Build merges list using the approach similar to HunYuanMoE+ merges = []+ vocab = {}+ mergeable_ranks = tokenizer.model._mergeable_ranks+ for token, rank in mergeable_ranks.items():+ vocab[QwenModel.token_bytes_to_string(token)] = rank+ if len(token) == 1:+ continue+ merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)+ if len(merged) == 2:+ merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))++ # Build token list+ vocab_size = self.hparams["vocab_size"]+ special_tokens = tokenizer.special_tokens+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}+ tokens: list[str] = []+ toktypes: list[int] = []++ for i in range(vocab_size):+ if i not in reverse_vocab:+ tokens.append(f"[PAD{i}]")+ toktypes.append(gguf.TokenType.UNUSED)+ else:+ token = reverse_vocab[i]+ tokens.append(token)+ if i in special_tokens.values():+ toktypes.append(gguf.TokenType.CONTROL)+ else:+ toktypes.append(gguf.TokenType.NORMAL)++ self.gguf_writer.add_tokenizer_model("gpt2")+ self.gguf_writer.add_tokenizer_pre(tokpre)+ self.gguf_writer.add_token_list(tokens)+ self.gguf_writer.add_token_types(toktypes)+ self.gguf_writer.add_token_merges(merges)++ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)+ special_vocab.add_to_gguf(self.gguf_writer)+ else:+ raise NotImplementedError(f"Deepseek pre-tokenizer {tokpre!r} is not supported yet!")def set_gguf_parameters(self):@@ -6705,8 +7234,7 @@ class ChatGLMModel(TextModel):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused- if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision.") \- or name.startswith("transformers.vision."):+ if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision."):return []name = name.removeprefix("transformer.")@@ -6823,6 +7351,78 @@ class ExaoneModel(TextModel):torch.tensor(rope_factors, dtype=torch.float32))[email protected]("Exaone4ForCausalLM")+class Exaone4Model(TextModel):+ model_arch = gguf.MODEL_ARCH.EXAONE4++ def set_vocab(self):+ tokens, toktypes, tokpre = self.get_vocab_base()+ self.gguf_writer.add_tokenizer_model("gpt2")+ self.gguf_writer.add_tokenizer_pre(tokpre)+ self.gguf_writer.add_token_list(tokens)+ self.gguf_writer.add_token_types(toktypes)++ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)+ special_vocab.add_to_gguf(self.gguf_writer)++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ hparams = self.hparams+ self.gguf_writer.add_vocab_size(hparams["vocab_size"])++ if hparams.get("sliding_window") is not None:+ self.gguf_writer.add_sliding_window(hparams["sliding_window"])+ if "layer_types" in hparams:+ self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]])+ elif "sliding_window_pattern" in hparams:+ sliding_window_pattern = []+ if isinstance(hparams["sliding_window_pattern"], str): # e.g. LLLG+ for i in range(hparams["num_hidden_layers"]):+ sliding_window_pattern.append(+ hparams["sliding_window_pattern"][i % len(hparams["sliding_window_pattern"])] == "L")+ if isinstance(hparams["sliding_window_pattern"], int): # e.g. 4+ for i in range(hparams["num_hidden_layers"]):+ sliding_window_pattern.append((i+1) % hparams["sliding_window_pattern"] != 0)+ if len(sliding_window_pattern) == hparams["num_hidden_layers"]:+ self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)++ rope_scaling = self.hparams.get("rope_scaling") or {}+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])++ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:+ if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):+ if rope_scaling.get("rope_type", '').lower() == "llama3":+ base = self.hparams.get("rope_theta", 10_000.0)+ if (dim := self.hparams.get("head_dim")) is None:+ dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]+ freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))++ factor = rope_scaling.get("factor", 16.0)+ low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)+ high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)+ old_context_len = self.hparams.get("original_max_position_embeddings", 8192)++ low_freq_wavelen = old_context_len / low_freq_factor+ high_freq_wavelen = old_context_len / high_freq_factor++ rope_factors = []+ for freq in freqs:+ wavelen = 2 * math.pi / freq+ if wavelen < high_freq_wavelen:+ rope_factors.append(1)+ elif wavelen > low_freq_wavelen:+ rope_factors.append(factor)+ else:+ smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)+ rope_factors.append(1 / ((1-smooth) / factor + smooth))++ yield (+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),+ torch.tensor(rope_factors, dtype=torch.float32))++@ModelBase.register("GraniteForCausalLM")class GraniteModel(LlamaModel):"""Conversion for IBM's GraniteForCausalLM"""@@ -6887,18 +7487,127 @@ class GraniteMoeModel(GraniteModel):(self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), up),]+ has_experts = bool(self.hparams.get('num_local_experts'))+if name.endswith("shared_mlp.input_linear.weight"):ffn_dim = self.hparams["shared_intermediate_size"]assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size"gate, up = data_torch.split(ffn_dim, dim=-2)+ if has_experts:+ return [+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_SHEXP, bid), gate),+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), up),+ ]return [- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_SHEXP, bid), gate),- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), up),+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), gate),+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), up),]+ if not has_experts and name.endswith("shared_mlp.output_linear.weight"):+ return [(self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN, bid), data_torch)]+return super().modify_tensors(data_torch, name, bid)[email protected]("GraniteMoeHybridForCausalLM", "BambaForCausalLM")+class GraniteHybridModel(Mamba2Model, GraniteMoeModel):+ """GraniteHybrid is a hybrid SSM + Attention model that uses Mamba2 SSM+ layers and optionally uses MoE w/ a shared expert"""+ model_arch = gguf.MODEL_ARCH.GRANITE_HYBRID+ undo_permute = True++ def __init__(self, *args, **kwargs):++ # Hybrid mamba models use a prefix for the mamba-specific params.+ # TODO: Extend this if the prefix(es) need to be configurable+ self.hparam_prefixes = ["mamba"]++ super().__init__(*args, **kwargs)++ # Lists of which layers use ssm vs attention+ self._attn_layers = self.get_attn_layers()+ self._ssm_layers = [i for i in range(self.block_count) if i not in self._attn_layers]++ # n_group and d_inner are used during reshape_tensors for mamba2+ self.d_model = self.find_hparam(["hidden_size", "d_model"])+ self.n_group = self.find_hparam(["n_groups"])+ self.d_inner = self.find_hparam(["expand"]) * self.d_model++ def get_attn_layers(self):+ # Explicit list of layer type names+ if layer_types := self.hparams.get("layer_types"):+ return [i for i, typ in enumerate(layer_types) if typ == "attention"]++ # Layer types indicated by index or period+ attn_layers = self.hparams.get("attn_layer_indices", [])+ if not attn_layers:+ attn_period = self.hparams.get("attn_layer_period")+ assert attn_period, "Didn't find attn_layer_indices or attn_layer_period"+ attn_offset = self.hparams.get("attn_layer_offset")+ assert attn_offset is not None, "No attention layer offset set with attn_layer_period"+ attn_layers = [i for i in range(self.block_count) if i % attn_period == attn_offset]+ return attn_layers++ def find_hparam(self, keys: Iterable[str], *args, **kwargs) -> Any:+ prefixed = []+ for pfx in self.hparam_prefixes:+ prefixed.extend("_".join([pfx, k]) for k in keys)+ keys = list(keys) + prefixed+ return Mamba2Model.find_hparam(self, keys, *args, **kwargs)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if (name.endswith("block_sparse_moe.input_linear.weight") or "shared_mlp" in name):+ return GraniteMoeModel.modify_tensors(self, data_torch, name, bid)++ # Determine whether this is a mamba layer or an attention layer+ if bid in self._ssm_layers:+ return Mamba2Model.modify_tensors(self, data_torch, name, bid)+ elif bid in self._attn_layers:+ return GraniteMoeModel.modify_tensors(self, data_torch, name, bid)+ return [(self.map_tensor_name(name), data_torch)]++ def set_gguf_parameters(self):+ """This method merges params from both parents and some that are+ specific to this model. The result is some duplication of how the params+ get set. The following warnings are expected during conversion:++ WARNING:Duplicated key name 'granitehybrid.attention.head_count_kv'+ WARNING:Duplicated key name 'granitehybrid.context_length'+ """+ GraniteMoeModel.set_gguf_parameters(self)++ ## Mamba mixer params ##+ self.gguf_writer.add_ssm_conv_kernel(self.find_hparam(["conv_kernel", "d_conv"]))+ self.gguf_writer.add_ssm_state_size(self.find_hparam(["state_size", "d_state"]))+ self.gguf_writer.add_ssm_group_count(self.n_group)+ self.gguf_writer.add_ssm_inner_size(self.d_inner)+ # NOTE: The mamba_dt_rank is _not_ the right field for how this is used+ # in llama.cpp+ self.gguf_writer.add_ssm_time_step_rank(self.find_hparam(["n_heads"]))++ ## Attention params ##+ head_count_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"])+ head_count_kv_vec = [head_count_kv if i in self._attn_layers else 0 for i in range(self.block_count)]+ if rope_dim := self.hparams.get("attn_rotary_emb"):+ self.gguf_writer.add_rope_dimension_count(rope_dim)+ self.gguf_writer.add_head_count_kv(head_count_kv_vec)++ ## If Bamba, use rope, otherwise don't+ use_rope = "BambaForCausalLM" in self.hparams["architectures"]+ self.gguf_writer.add_rope_scaling_finetuned(use_rope)+ if not use_rope:+ self.gguf_writer.add_context_length(2**20)++ ## Validation ##+ d_head = self.find_hparam(["d_head"], optional=True) or 64+ assert self.hparams.get("hidden_act") in [None, "silu"], "Only SILU activation supported"+ assert self.d_inner % d_head == 0, f"SSM inner size {self.d_inner} not a multiple of head dim {d_head}"++ def set_vocab(self):+ self.hparams["pad_vocab_size_multiple"] = 8+ Mamba2Model.set_vocab(self)++@ModelBase.register("BailingMoeForCausalLM")class BailingMoeModel(TextModel):model_arch = gguf.MODEL_ARCH.BAILINGMOE@@ -7054,8 +7763,8 @@ class UltravoxModel(TextModel):def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)raise NotImplementedError(- "Ultravox does not have text decoder. Instead, it uses Llama or other models for text. If you want to"- " get the audio encoder, please use --mmproj argument"+ "Ultravox does not have text decoder. Instead, it uses Llama or other models for text."+ " If you want to get the audio encoder, please use --mmproj argument")@@ -7110,6 +7819,317 @@ class UltravoxWhisperEncoderModel(WhisperEncoderModel):self.gguf_writer.add_audio_stack_factor(self.global_config["stack_factor"])[email protected]("FalconH1ForCausalLM")+class FalconH1Model(Mamba2Model):+ model_arch = gguf.MODEL_ARCH.FALCON_H1++ def __init__(self, *args, **kwargs):+ # Set the hparam prefixes for Falcon Mamba2+ self.hparam_prefixes = ["mamba"]++ # Initialize the base Mamba2Model+ super().__init__(*args, **kwargs)++ # Use Llama conversion for attention+ self._transformer_model_class = LlamaModel++ # n_group and d_inner are used during reshape_tensors for mamba2+ self.n_group = self.find_hparam(["n_groups"])+ self.d_inner = self.find_hparam(["mamba_d_ssm"])+ self.d_head = self.find_hparam(["d_head"])++ # Initialize any Falcon Mamba2 specific attributes+ self.has_attention = True # Falcon Mamba2 has attention components++ # Load Falcon-H1 multipliers from hyperparameters+ self.attention_in_multiplier = self.find_hparam(["attention_in_multiplier"], optional=True)+ self.attention_out_multiplier = self.find_hparam(["attention_out_multiplier"], optional=True)+ self.ssm_in_multiplier = self.find_hparam(["ssm_in_multiplier"], optional=True)+ self.ssm_out_multiplier = self.find_hparam(["ssm_out_multiplier"], optional=True)+ self.mlp_multipliers = self.find_hparam(["mlp_multipliers"], optional=True)+ self.ssm_multipliers = self.find_hparam(["ssm_multipliers"], optional=True)+ self.intermediate_size = self.find_hparam(["intermediate_size"])+ self.key_multiplier = self.find_hparam(["key_multiplier"], optional=True)++ def find_hparam(self, keys: Iterable[str], *args, **kwargs) -> Any:+ prefixed = []+ for pfx in self.hparam_prefixes:+ prefixed.extend("_".join([pfx, k]) for k in keys)+ keys = list(keys) + prefixed+ return super().find_hparam(keys, *args, **kwargs)++ def set_vocab(self):+ self._set_vocab_gpt2()++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ tensors = list(super().modify_tensors(data_torch, name, bid))+ tensor = tensors[0][1]++ if "down_proj" in name:+ tensor = tensor * self.mlp_multipliers[1]+ elif "gate_proj" in name:+ tensor = tensor * self.mlp_multipliers[0]+ elif "k_proj" in name:+ tensor = tensor * self.key_multiplier * self.attention_in_multiplier+ elif "q_proj" in name:+ tensor = tensor * self.attention_in_multiplier+ elif "v_proj" in name:+ tensor = tensor * self.attention_in_multiplier+ elif "o_proj" in name:+ tensor = tensor * self.attention_out_multiplier+ elif "out_proj" in name:+ tensor = tensor * self.ssm_out_multiplier+ elif "in_proj" in name:+ tensor = tensor * self.ssm_in_multiplier+ zxbcdt_multipliers = self.hparams["ssm_multipliers"]+ intermediate_size = self.hparams["mamba_d_ssm"]+ groups_time_state_size = self.hparams["mamba_n_groups"] * self.hparams["mamba_d_state"]+ tensor[:intermediate_size, :] *= zxbcdt_multipliers[0]+ tensor[intermediate_size:2 * intermediate_size, :] *= zxbcdt_multipliers[1]+ tensor[2 * intermediate_size:2*intermediate_size + groups_time_state_size, :] *= zxbcdt_multipliers[2]+ tensor[2*intermediate_size + groups_time_state_size:2*intermediate_size ++ 2*groups_time_state_size, :] *= zxbcdt_multipliers[3]+ tensor[2*intermediate_size + 2*groups_time_state_size:, :] *= zxbcdt_multipliers[4]+ elif "lm_head" in name:+ tensor = tensor * self.hparams["lm_head_multiplier"]+ elif "embed_tokens" in name:+ tensor = tensor * self.hparams["embedding_multiplier"]+ elif "mamba.norm" in name:+ tensor = tensor.reshape(self.n_group, self.d_inner // self.n_group)++ tensors = [(tensors[0][0], tensor)]+ return tensors++ def set_gguf_parameters(self):+ super().set_gguf_parameters()++ ## General Params ##+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])+ # Override some Mamba2 defaults+ self.gguf_writer.add_block_count(self.block_count)+ self.gguf_writer.add_context_length(self.hparams.get("max_position_embeddings", 0))+ self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])++ ## Attention params ##+ self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) # Override value 0 from Mamba2+ self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"])+ self.gguf_writer.add_key_length(self.hparams["head_dim"])+ self.gguf_writer.add_value_length(self.hparams["head_dim"])++ ## Validation ##+ assert self.hparams.get("hidden_act") in [None, "silu"], "Only SILU activation supported"+ assert self.d_inner % self.d_head == 0, \+ f"SSM inner size {self.d_inner} not a multiple of head dim {self.d_head}"++ # Add any other Falcon Mamba2 specific configuration+ self.gguf_writer.add_rope_freq_base(self.find_hparam(["rope_theta"]))++[email protected]("HunYuanMoEV1ForCausalLM")+class HunYuanMoEModel(TextModel):+ model_arch = gguf.MODEL_ARCH.HUNYUAN_MOE++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ # For handling tied embeddings+ self._tok_embd = None++ def set_vocab(self):+ from transformers import AutoTokenizer+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)++ # 1. Get the pre-tokenizer identifier hash+ tokpre = self.get_vocab_base_pre(tokenizer)++ # 2. Reverse-engineer the merges list from mergeable_ranks+ merges = []+ vocab = {}+ mergeable_ranks = tokenizer.mergeable_ranks+ for token, rank in mergeable_ranks.items():+ vocab[QwenModel.token_bytes_to_string(token)] = rank+ if len(token) == 1:+ continue+ merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)+ if len(merged) == 2: # todo this is an assert in Qwen, why?+ merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))++ # 3. Generate the tokens and toktypes lists+ vocab_size = self.hparams["vocab_size"]+ assert tokenizer.vocab_size == vocab_size+ special_tokens = tokenizer.special_tokens+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}+ tokens: list[str] = []+ toktypes: list[int] = []+ for i in range(vocab_size):+ if i not in reverse_vocab:+ tokens.append(f"[PAD{i}]")+ toktypes.append(gguf.TokenType.UNUSED)+ else:+ token = reverse_vocab[i]+ tokens.append(token)+ if i in special_tokens.values():+ toktypes.append(gguf.TokenType.CONTROL)+ else:+ toktypes.append(gguf.TokenType.NORMAL)++ # 4. Write all vocab-related fields to the GGUF writer+ self.gguf_writer.add_tokenizer_model("gpt2")+ self.gguf_writer.add_tokenizer_pre(tokpre)+ self.gguf_writer.add_token_list(tokens)+ self.gguf_writer.add_token_types(toktypes)+ self.gguf_writer.add_token_merges(merges)++ # 5. Add special tokens and chat templates+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)+ special_vocab.add_to_gguf(self.gguf_writer)+ # FIX for BOS token: Overwrite incorrect id read from config.json+ self.gguf_writer.add_bos_token_id(127959) # <|bos|>++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ hparams = self.hparams++ self.gguf_writer.add_expert_count(hparams["num_experts"])+ self.gguf_writer.add_expert_shared_feed_forward_length(hparams["intermediate_size"])++ moe_intermediate_size = hparams["moe_intermediate_size"]+ assert all(n == moe_intermediate_size[0] for n in moe_intermediate_size)+ self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size[0])++ moe_topk = hparams["moe_topk"]+ assert all(topk == moe_topk[0] for topk in moe_topk)+ self.gguf_writer.add_expert_used_count(moe_topk[0])++ moe_shared_expert = hparams["num_shared_expert"]+ assert all(n == moe_shared_expert[0] for n in moe_shared_expert)+ self.gguf_writer.add_expert_shared_count(moe_shared_expert[0])++ # Rope+ rope_scaling = hparams.get("rope_scaling", {})+ if rope_scaling.get("type") == "dynamic":+ alpha = rope_scaling.get("alpha", 1000)+ base = hparams.get("rope_theta", 10000.0)+ dim = (hparams["hidden_size"] // hparams["num_attention_heads"]) # 128+ scaled_base = base * (alpha**(dim / (dim-2))) # 10000 * (1000 ** (128 / 126)) = 11158839.9251+ self.gguf_writer.add_rope_freq_base(scaled_base)+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)+ self.gguf_writer.add_rope_scaling_factor(1)+ # There is no consistent way to calculate ctx from alpha, and the config is incorrectly set to 32k+ self.gguf_writer.add_rope_scaling_orig_ctx_len(256 * 1024) # 256k context length+ self.gguf_writer.add_context_length(256 * 1024) # 256k context length++ assert alpha == 1000 and base == 10000.0 and dim == 128 \+ and self.hparams["max_position_embeddings"] in [32 * 1024, 256 * 1024] , \+ "HunYuan dynamic RoPE scaling assumptions changed, please update the logic or context length manually"++ _experts: list[dict[str, Tensor]] | None = None++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if name == "model.embed_tokens.weight":+ self._tok_embd = data_torch.clone()++ if name == "lm_head.weight":+ if self.hparams.get("tie_word_embeddings", False):+ logger.info("Skipping tied output layer 'lm_head.weight'")+ return []++ if name.find("mlp.experts") != -1:+ n_experts = self.hparams["num_experts"]+ assert bid is not None++ if self._experts is None:+ self._experts = [{} for _ in range(self.block_count)]++ self._experts[bid][name] = data_torch++ if len(self._experts[bid]) >= n_experts * 3:+ # merge the experts into a single 3d tensor+ tensors: list[tuple[str, Tensor]] = []+ for w_name in ["down_proj", "gate_proj", "up_proj"]:+ data: list[Tensor] = []++ for xid in range(n_experts):+ ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"+ data.append(self._experts[bid][ename])+ del self._experts[bid][ename]++ data_torch = torch.stack(data, dim=0)+ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"+ new_name = self.map_tensor_name(merged_name)+ tensors.append((new_name, data_torch))++ return tensors+ else:+ return []++ return [(self.map_tensor_name(name), data_torch)]++ def prepare_tensors(self):+ super().prepare_tensors()+ if self._experts is not None:+ experts = [k for d in self._experts for k in d.keys()]+ if len(experts) > 0:+ raise ValueError(f"Unprocessed experts: {experts}")++[email protected]("SmolLM3ForCausalLM")+class SmolLM3Model(LlamaModel):+ model_arch = gguf.MODEL_ARCH.SMOLLM3++ def set_vocab(self):+ super().set_vocab()+ # remove unsupported array slicing in chat template+ # ref: https://huggingface.co/ggml-org/SmolLM3-3B-GGUF/discussions/1+ from transformers import AutoTokenizer+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model)+ if tokenizer.chat_template is not None:+ chat_template = tokenizer.chat_template.replace("[:]", "")+ self.gguf_writer.add_chat_template(chat_template)++[email protected]("Lfm2ForCausalLM")[email protected]("LFM2ForCausalLM")+class LFM2Model(TextModel):+ model_arch = gguf.MODEL_ARCH.LFM2++ def _add_feed_forward_length(self):+ ff_dim = self.hparams["block_ff_dim"]++ auto_adjust_ff_dim = self.hparams["block_auto_adjust_ff_dim"]+ ff_dim = self.hparams["block_ff_dim"]+ ffn_dim_multiplier = self.hparams["block_ffn_dim_multiplier"]+ multiple_of = self.hparams["block_multiple_of"]++ if auto_adjust_ff_dim:+ ff_dim = int(2 * ff_dim / 3)+ # custom dim factor multiplier+ if ffn_dim_multiplier is not None:+ ff_dim = int(ffn_dim_multiplier * ff_dim)+ ff_dim = multiple_of * ((ff_dim+multiple_of-1) // multiple_of)++ self.gguf_writer.add_feed_forward_length(ff_dim)++ def set_gguf_parameters(self):+ # set num_key_value_heads only for attention layers+ self.hparams["num_key_value_heads"] = [+ self.hparams["num_key_value_heads"] if layer_type == "full_attention" else 0+ for layer_type in self.hparams["layer_types"]+ ]++ super().set_gguf_parameters()+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])+ self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams["norm_eps"])+ self._add_feed_forward_length()++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ # conv op requires 2d tensor+ if 'conv.conv' in name:+ data_torch = data_torch.squeeze(1)++ return [(self.map_tensor_name(name), data_torch)]++###### CONVERSION LOGIC ######@@ -7184,105 +8204,6 @@ class LazyTorchTensor(gguf.LazyBase):return cls._wrap_fn(func)(*args, **kwargs)-def parse_args() -> argparse.Namespace:- parser = argparse.ArgumentParser(description="Convert a huggingface model to a GGML compatible file")- parser.add_argument(- "--vocab-only",- action="store_true",- help="extract only the vocab",- )- parser.add_argument(- "--outfile",- type=Path,- help="path to write to; default: based on input. {ftype} will be replaced by the outtype.",- )- parser.add_argument(- "--outtype",- type=str,- choices=["f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"],- default="f16",- help=- "output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0"- " for ternary, and auto for the highest-fidelity 16-bit float type depending on the first loaded tensor type",- )- parser.add_argument(- "--bigendian",- action="store_true",- help="model is executed on big endian machine",- )- parser.add_argument(- "model",- type=str,- help="directory containing model file or huggingface repository ID (if --remote)",- nargs="?",- )- parser.add_argument(- "--use-temp-file",- action="store_true",- help="use the tempfile library while processing (helpful when running out of memory, process killed)",- )- parser.add_argument(- "--no-lazy",- action="store_true",- help="use more RAM by computing all outputs before writing (use in case lazy evaluation is broken)",- )- parser.add_argument(- "--model-name",- type=str,- default=None,- help="name of the model",- )- parser.add_argument(- "--verbose",- action="store_true",- help="increase output verbosity",- )- parser.add_argument(- "--split-max-tensors",- type=int,- default=0,- help="max tensors in each split",- )- parser.add_argument(- "--split-max-size",- type=str,- default="0",- help="max size per split N(M|G)",- )- parser.add_argument(- "--dry-run",- action="store_true",- help="only print out a split plan and exit, without writing any new files",- )- parser.add_argument(- "--no-tensor-first-split",- action="store_true",- help="do not add tensors to the first split (disabled by default)")- parser.add_argument("--metadata", type=Path, help="Specify the path for an authorship metadata override file")- parser.add_argument("--print-supported-models", action="store_true", help="Print the supported models")- parser.add_argument(- "--remote",- action="store_true",- help=- "(Experimental) Read safetensors file remotely without downloading to disk. Config and tokenizer files will"- " still be downloaded. To use this feature, you need to specify Hugging Face model repo name instead of a"- " local directory. For example: 'HuggingFaceTB/SmolLM2-1.7B-Instruct'. Note: To access gated repo, set"- " HF_TOKEN environment variable to your Hugging Face token.",- )- parser.add_argument(- "--mmproj",- action="store_true",- help=- "(Experimental) Export multimodal projector (mmproj) for vision models. This will only work on some"- " vision models. A prefix 'mmproj-' will be added to the output file name.",- )-- args = parser.parse_args()- if not args.print_supported_models and args.model is None:- parser.error("the following arguments are required: model")- return args--def split_str_to_n_bytes(split_str: str) -> int:if split_str.endswith("K"):n = int(split_str[:-1]) * 1000@@ -7320,103 +8241,4 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> starch = vision_config["architectures"][0]if arch is None:raise ValueError("Failed to detect model architecture")- return arch---def main() -> None:- args = parse_args()-- if args.print_supported_models:- logger.error("Supported models:")- ModelBase.print_registered_models()- sys.exit(0)-- if args.verbose:- logging.basicConfig(level=logging.DEBUG)- else:- logging.basicConfig(level=logging.INFO)-- if args.remote:- hf_repo_id = args.model- from huggingface_hub import snapshot_download- local_dir = snapshot_download(- repo_id=hf_repo_id, allow_patterns=["LICENSE", "*.json", "*.md", "*.txt", "tokenizer.model"])- dir_model = Path(local_dir)- logger.info(f"Downloaded config and tokenizer to {local_dir}")- else:- hf_repo_id = None- dir_model = Path(args.model)-- if not dir_model.is_dir():- logger.error(f'Error: {dir_model} is not a directory')- sys.exit(1)-- ftype_map: dict[str, gguf.LlamaFileType] = {- "f32": gguf.LlamaFileType.ALL_F32,- "f16": gguf.LlamaFileType.MOSTLY_F16,- "bf16": gguf.LlamaFileType.MOSTLY_BF16,- "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,- "tq1_0": gguf.LlamaFileType.MOSTLY_TQ1_0,- "tq2_0": gguf.LlamaFileType.MOSTLY_TQ2_0,- "auto": gguf.LlamaFileType.GUESSED,- }-- is_split = args.split_max_tensors > 0 or args.split_max_size != "0"- if args.use_temp_file and is_split:- logger.error("Error: Cannot use temp file when splitting")- sys.exit(1)-- if args.outfile is not None:- fname_out = args.outfile- elif hf_repo_id:- # if remote, use the model ID as the output file name- fname_out = Path("./" + hf_repo_id.replace("/", "-") + "-{ftype}.gguf")- else:- fname_out = dir_model-- logger.info(f"Loading model: {dir_model.name}")-- if args.mmproj:- if "mmproj" not in fname_out.name:- fname_out = ModelBase.add_prefix_to_filename(fname_out, "mmproj-")-- with torch.inference_mode():- output_type = ftype_map[args.outtype]- model_type = ModelType.MMPROJ if args.mmproj else ModelType.TEXT- hparams = ModelBase.load_hparams(dir_model)- model_architecture = get_model_architecture(hparams, model_type)- logger.info(f"Model architecture: {model_architecture}")- try:- model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)- except NotImplementedError:- logger.error(f"Model {model_architecture} is not supported")- sys.exit(1)-- model_instance = model_class(- dir_model,- output_type,- fname_out,- is_big_endian=args.bigendian,- use_temp_file=args.use_temp_file,- eager=args.no_lazy,- metadata_override=args.metadata,- model_name=args.model_name,- split_max_tensors=args.split_max_tensors,- split_max_size=split_str_to_n_bytes(args.split_max_size),- dry_run=args.dry_run,- small_first_shard=args.no_tensor_first_split,- remote_hf_model_id=hf_repo_id)-- if args.vocab_only:- logger.info("Exporting model vocab...")- model_instance.write_vocab()- logger.info(f"Model vocab successfully exported to {model_instance.fname_out}")- else:- logger.info("Exporting model...")- model_instance.write()- out_path = f"{model_instance.fname_out.parent}{os.sep}" if is_split else model_instance.fname_out- logger.info(f"Model successfully exported to {out_path}")---if __name__ == '__main__':- main()+ return arch\ No newline at end of filediff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex 7b71bca86..b377fdf28 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -16,6 +16,7 @@ import torchimport numpy as npfrom auto_round.export.export_to_gguf.config import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES+from auto_round.utils import get_reciprocalGGML_QUANT_TYPE = {}@@ -65,7 +66,7 @@ def ggml_quant(d_wmin = d_wmin.to(device) if d_wmin is not None else d_wminnew_data = quant_func(blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)-+assert new_data.shape[-1] == type_sizenew_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)@@ -87,11 +88,11 @@ def make_qx_quants(data, bits, rmse_type=0, qw=None):nmax = pow(2, bits - 1)imax = abs(data).argmax(axis=-1, keepdims=True)group_max = torch.take_along_dim(data, imax, dim=-1)- iscales = torch.where(group_max != 0, -nmax / group_max, 0)+ iscales = -nmax * get_reciprocal(group_max)if rmse_type == 0:L = (torch.round(iscales * data).clip(-nmax, nmax - 1) + nmax).to(torch.uint8)- scales = torch.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])+ scales = get_reciprocal(iscales).reshape(iscales.shape[:2])return scales, Lreturn_early = False@@ -112,9 +113,9 @@ def make_qx_quants(data, bits, rmse_type=0, qw=None):w = torch.sqrt(torch.abs(data))sumlx = torch.sum(w * data * L, dim=-1)suml2 = torch.sum(w * L * L, dim=-1)- scales = torch.where(suml2 != 0, sumlx / suml2, 0)+ scales = sumlx * get_reciprocal(suml2)if return_early:- iscales_inv = torch.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])+ iscales_inv = get_reciprocal(iscales).reshape(iscales.shape[:2])scales = torch.where(suml2 > 0, 0.5 * (scales + iscales_inv), iscales_inv)L = (L + nmax).to(torch.uint8)return scales, L@@ -122,7 +123,7 @@ def make_qx_quants(data, bits, rmse_type=0, qw=None):for _is in range(-9, 10):if _is == 0:continue- iscales = torch.where(group_max != 0, -(nmax + -0.1 * _is) / nmax, 0)+ iscales = -(nmax + -0.1 * _is) * get_reciprocal(group_max)tmp_L = torch.round(iscales * data).clip(-nmax, nmax - 1)sumlx = torch.sum(w * data * L, dim=-1)suml2 = torch.sum(w * L * L, dim=-1)@@ -140,7 +141,7 @@ def make_q3_quants(data, bits, do_rmse=False):nmax = pow(2, bits - 1)imax = abs(data).argmax(axis=-1, keepdims=True)group_max = torch.take_along_dim(data, imax, dim=-1)- iscale = torch.where(group_max != 0, -nmax / group_max, 0)+ iscale = -nmax * get_reciprocal(group_max)if do_rmse:L = torch.round(iscale * data).clip(-nmax, nmax - 1)w = torch.pow(data, 2)@@ -161,11 +162,10 @@ def make_q3_quants(data, bits, do_rmse=False):L[:,:, i][replace_idx] = new_L[replace_idx]sumlx = slxsuml2 = sl2- isuml2 = torch.where(suml2 == 0, 0, 1 / suml2)- return sumlx * isuml2, L.to(torch.uint8)+ return sumlx * get_reciprocal(suml2), L.to(torch.uint8)L = torch.round(iscale * data).clip(-nmax, nmax - 1) + nmax- scales = torch.where(iscale != 0, 1 / iscale, 0).reshape(iscale.shape[:2])+ scales = get_reciprocal(iscale).reshape(iscale.shape[:2])return scales, L.to(torch.uint8)@@ -199,7 +199,7 @@ def make_qkx2_quants(data, bits, weights=None, rmin=-1.0, rdelta=0.1, nstep=20,group_min[group_min > 0] = 0scale = (group_max - group_min) / nmax- iscale = torch.where(scale == 0, 0, 1 / scale)+ iscale = get_reciprocal(scale)l_values = torch.round(iscale * (data - group_min))L = torch.clip(l_values, 0, nmax).to(torch.uint8)@@ -213,7 +213,7 @@ def make_qkx2_quants(data, bits, weights=None, rmin=-1.0, rdelta=0.1, nstep=20,for step in range(nstep):new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)- new_iscale = torch.where(new_scale == 0, 0, 1 / new_scale)+ new_iscale = get_reciprocal(new_scale)l_values = torch.round(new_iscale * (data - group_min))Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)@@ -251,7 +251,7 @@ def make_qkx3_quants(data, bits, weights, rmin=-1.0, rdelta=0.1, nstep=20, use_mdef make_qp_quants(nmax, data, quant_weights):group_max = torch.max(data, dim=-1, keepdim=True)[0]scale = group_max / nmax- iscale = torch.where(scale == 0, 0, 1 / scale)+ iscale = get_reciprocal(scale)L = torch.round(iscale * data)diffs = data - scale * L@@ -261,7 +261,7 @@ def make_qp_quants(nmax, data, quant_weights):if _is == 0:continuescale_is = group_max / (0.1 * _is + nmax)- iscale_is = torch.where(scale == 0, 0, 1 / scale_is)+ iscale_is = get_reciprocal(scale_is)tmp_L = torch.round(iscale_is * data).clip(max=nmax)diffs = data - scale_is * tmp_L@@ -315,7 +315,7 @@ def q4_0_quant_block(blocks, scale=None, zp=None, **kwargs):imax = abs(blocks).argmax(axis=-1, keepdims=True)max = torch.take_along_dim(blocks, imax, axis=-1)d = max / -8- id = torch.where(d == 0, 0, 1 / d)+ id = get_reciprocal(d)qs = torch.trunc(blocks.to(torch.float64) * id.to(torch.float64) + 8.5).clip(0, 15).to(torch.uint8)@@ -338,7 +338,7 @@ def q4_1_quant_block(blocks, scale=None, zp=None, **kwargs):max = blocks.max(axis=-1, keepdims=True)[0]min = blocks.min(axis=-1, keepdims=True)[0]d = (max - min) / 15- id = torch.where(d == 0, 0, 1 / d)+ id = get_reciprocal(d)qs = torch.trunc((blocks - min) * id + 0.5).clip(0, 15).to(torch.uint8)@@ -361,7 +361,7 @@ def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):max = torch.take_along_dim(blocks, imax, dim=-1)d = max / -16- id = torch.where(d == 0, 0, 1 / d)+ id = get_reciprocal(d)n_blocks = blocks.shape[0]block_size = GGML_QUANT_SIZES["q5_0"][0]@@ -391,7 +391,7 @@ def q5_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):n_blocks = blocks.shape[0]block_size = GGML_QUANT_SIZES["q5_1"][0]- id = torch.where(d == 0, 0, 1 / d)+ id = get_reciprocal(d)q = torch.trunc((blocks - min) * id + 0.5).clip(0, 31).to(torch.uint8).cpu().numpy()qs = q.reshape((n_blocks, 2, block_size // 2))@@ -411,7 +411,7 @@ def q8_0_quant_block(blocks, scale=None, zp=None, **kwargs) -> np.ndarray:d = scale.reshape((-1, 1))else:d = torch.abs(blocks).max(dim=1, keepdim=True)[0] / 127- id = torch.where(d == 0, 0, 1 / d)+ id = get_reciprocal(d)qs = torch.clip(torch_roundf(blocks * id), -128, 127)@@ -434,17 +434,17 @@ def q2_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, imins = wmin.reshape((-1, QK_K // 16))output_d = d_scale.reshape(-1, 1).to(torch.float32)output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)- output_scale = torch.round(scales / output_d).clip(0, 15).to(torch.uint8)- output_scale |= torch.round(mins / output_dmin).clip(0, 15).to(torch.uint8) << 4+ output_scale = torch.round(scales * get_reciprocal(output_d)).clip(0, 15).to(torch.uint8)+ output_scale |= torch.round(mins * get_reciprocal(output_dmin)).clip(0, 15).to(torch.uint8) << 4all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 3).to(torch.uint8)elif original:scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]max_mins = torch.max(mins, dim=-1, keepdim=True)[0]- output_d = torch.where(max_scales > 0, max_scales / 15, 0)- output_dmin = torch.where(max_mins > 0, max_mins / 15., 0)- id_scales = torch.where(max_scales > 0, 15. / max_scales, 0)- id_mins = torch.where(max_mins > 0, 15. / max_mins, 0)+ output_d = (max_scales / 15).clamp(min=0)+ output_dmin = (max_mins / 15).clamp(min=0)+ id_scales = (15 * get_reciprocal(max_scales)).clamp(min=0)+ id_mins = (15 * get_reciprocal(max_mins)).clamp(min=0)replace_ids = (max_scales > 0).squeeze()output_scale = torch.zeros_like(scales).to(torch.uint8)@@ -474,8 +474,8 @@ def q2_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, imins = mins.reshape((-1, QK_K // 16))output_d = d_scale.reshape(-1, 1).to(torch.float32)output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)- output_scale = torch.round(scales / output_d).clip(0, 15).to(torch.uint8)- output_scale |= torch.round(mins / output_dmin).clip(0, 15).to(torch.uint8) << 4+ output_scale = torch.round(scales * get_reciprocal(output_d)).clip(0, 15).to(torch.uint8)+ output_scale |= torch.round(mins * get_reciprocal(output_dmin)).clip(0, 15).to(torch.uint8) << 4all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 3).to(torch.uint8)output_scale = output_scale.cpu().numpy()@@ -500,17 +500,17 @@ def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,if scale is not None:qdq_scale = scale.reshape(-1, QK_K // 16).to(torch.float32)dq_scale = d_scale.reshape(-1, 1).to(torch.float32)- all_L = (torch.round(blocks / qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)- q_scales_offset = torch.round(qdq_scale / dq_scale).clip(-32, 31) + 32+ all_L = (torch.round(blocks * get_reciprocal(qdq_scale.unsqueeze(-1))).clip(-4, 3) + 4).to(torch.uint8)+ q_scales_offset = torch.round(qdq_scale * get_reciprocal(dq_scale)).clip(-32, 31) + 32elif original: ## this is correctscales, _ = make_q3_quants(blocks, bits=3, do_rmse=True)scales_abs_max = abs(scales).argmax(dim=-1, keepdim=True)max_scales_mag = torch.take_along_dim(scales, scales_abs_max, dim=-1)- inverse_dq_scale = torch.where(max_scales_mag != 0, -32 / max_scales_mag, 0)- dq_scale = torch.where(inverse_dq_scale != 0, 1 / inverse_dq_scale, 0)+ inverse_dq_scale = -32 * get_reciprocal(max_scales_mag)+ dq_scale = get_reciprocal(inverse_dq_scale)qscale = torch.round(inverse_dq_scale * scales).clip(-32, 31)qdq_scale = dq_scale.to(torch.float32) * qscale- reverse_qdq_scale = torch.where(qdq_scale == 0, 0, 1 / qdq_scale)+ reverse_qdq_scale = get_reciprocal(qdq_scale)all_L = (torch.round(blocks * reverse_qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)q_scales_offset = torch.round(qdq_scale * inverse_dq_scale).clip(-32, 31) + 32else:@@ -521,8 +521,8 @@ def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,blocks = blocks.reshape((nb, QK_K // 16, 16))qdq_scale = scales.reshape((-1, QK_K // 16)).to(torch.float32)dq_scale = d_scale.reshape(-1, 1).to(torch.float32)- all_L = (torch.round(blocks / qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)- q_scales_offset = torch.round(qdq_scale / dq_scale).clip(-32, 31) + 32+ all_L = (torch.round(blocks * get_reciprocal(qdq_scale.unsqueeze(-1))).clip(-4, 3) + 4).to(torch.uint8)+ q_scales_offset = torch.round(qdq_scale * get_reciprocal(dq_scale)).clip(-32, 31) + 32q_scales_offset = q_scales_offset.cpu().numpy().astype(np.uint8)@@ -557,15 +557,16 @@ def q4_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, imins = wmin.reshape(-1, QK_K // 32)output_d = d_scale.reshape(-1, 1).to(torch.float32)output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)- q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)- q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)- all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 15).to(torch.uint8)+ q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)+ all_L = torch.round(+ (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 15).to(torch.uint8)elif original:scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]max_mins = torch.max(mins, dim=-1, keepdim=True)[0]- id_scales = torch.where(max_scales > 0, 63. / max_scales, 0)- id_mins = torch.where(max_mins > 0, 63. / max_mins, 0)+ id_scales = (63 * get_reciprocal(max_scales)).clamp(min=0)+ id_mins = (63 * get_reciprocal(max_mins)).clamp(min=0)output_d = max_scales / 63output_dmin = max_mins / 63q_scales = torch.round(id_scales * scales).clip(0, 63).to(torch.uint8)@@ -588,9 +589,10 @@ def q4_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, imins = mins.reshape((-1, QK_K // 32))output_d = d_scale.reshape(-1, 1).to(torch.float32)output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)- q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)- q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)- all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 15).to(torch.uint8)+ q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)+ all_L = torch.round(+ (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 15).to(torch.uint8)output_scale[:, :4] = q_scales[:, :4]output_scale[:, 4:8] = q_mins[:, :4]@@ -625,15 +627,16 @@ def q5_k_quant_block(mins = wmin.reshape(-1, QK_K // 32)output_d = d_scale.reshape(-1, 1).to(torch.float32)output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)- q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)- q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)- all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 31).to(torch.uint8)+ q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)+ all_L = torch.round(+ (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 31).to(torch.uint8)elif original:scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]max_mins = torch.max(mins, dim=-1, keepdim=True)[0]- id_scales = torch.where(max_scales > 0, 63. / max_scales, 0)- id_mins = torch.where(max_mins > 0, 63. / max_mins, 0)+ id_scales = (63 * get_reciprocal(max_scales)).clamp(min=0)+ id_mins = (63 * get_reciprocal(max_mins)).clamp(min=0)output_d = max_scales / 63output_dmin = max_mins / 63q_scales = torch.round(id_scales * scales).clip(0, 63).to(torch.uint8)@@ -657,9 +660,10 @@ def q5_k_quant_block(mins = mins.reshape((-1, QK_K // 32))output_d = d_scale.reshape(-1, 1).to(torch.float32)output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)- q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)- q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)- all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 31).to(torch.uint8)+ q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)+ all_L = torch.round(+ (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 31).to(torch.uint8)output_scale[:, :4] = q_scales[:, :4]output_scale[:, 4:8] = q_mins[:, :4]@@ -692,14 +696,14 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,if scale is not None:scales = scale.reshape(-1, QK_K // 16)output_d = d_scale.reshape(-1, 1).to(torch.float32)- output_scale = torch.round(scales / output_d).clip(max=127).to(torch.int8)- all_L = torch.round(blocks / scales.unsqueeze(-1) + 32).clip(0, 63).to(torch.uint8)+ output_scale = torch.round(scales * get_reciprocal(output_d)).clip(max=127).to(torch.int8)+ all_L = torch.round(blocks * get_reciprocal(scales.unsqueeze(-1)) + 32).clip(0, 63).to(torch.uint8)elif original:scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)imax = abs(scales).argmax(dim=-1, keepdim=True)max_scales = torch.take_along_dim(scales, imax, dim=-1)- iscales = torch.where(max_scales != 0, -128 / max_scales, 0)- output_d = torch.where(iscales != 0, 1 / iscales, -1)+ iscales = -128 * get_reciprocal(max_scales)+ output_d = get_reciprocal(iscales)output_scale = torch.round(iscales * scales).clip(max=127).to(torch.int8)d_tmp = output_d * output_scale.to(torch.float32)replace_ids = d_tmp != 0@@ -713,8 +717,8 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,blocks = blocks.reshape((nb, QK_K // 16, 16))scales = scales.reshape((-1, QK_K // 16))output_d = d_scale.reshape(-1, 1).to(torch.float32)- output_scale = torch.round(scales / output_d).clip(max=127).to(torch.int8)- all_L = torch.round(blocks / scales.unsqueeze(-1) + 32).clip(0, 63).to(torch.uint8)+ output_scale = torch.round(scales * get_reciprocal(output_d)).clip(max=127).to(torch.int8)+ all_L = torch.round(blocks * get_reciprocal(scales.unsqueeze(-1)) + 32).clip(0, 63).to(torch.uint8)tmp_L = all_L.reshape(nb, 4, 64) & 0xFoutput_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex efa2579d3..d237035e5 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -546,7 +546,7 @@ def tune(args):if gguf_format in file:gguf_file = file- logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")+ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")if eval_model_dtype == "float32" or eval_model_dtype == "auto":logger.warning("set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"@@ -636,7 +636,7 @@ def eval(args):tasks, model_args, device_str = _eval_init(args.tasks, args.model, args.device, args.disable_trust_remote_code, args.eval_model_dtype)- # load after _eval_int in order to make sure import torch after set CUDA_VISBILE_DEVICES+ # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICESfrom auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_modelfrom auto_round.utils import logger@@ -659,7 +659,7 @@ def eval(args):from lm_eval.utils import make_table # pylint: disable=E0401tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)- logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")+ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")if eval_model_dtype == "float32" or eval_model_dtype == "auto":logger.warning("set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"@@ -728,7 +728,7 @@ def eval_task_by_task(eval_model_dtype = get_model_dtype(eval_model_dtype)if is_gguf_file:tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)- logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")+ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")if eval_model_dtype == "float32" or eval_model_dtype == "auto":logger.warning("set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 6038802a6..ac0f9b039 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1929,3 +1929,10 @@ def clean_module_parameter(submodule, parameter):submodule._buffers[parameter] = Noneelse:submodule._parameters[parameter] = None++def get_reciprocal(tensor):+ if torch.dtype is torch.float16:+ tensor = torch.sign(tensor) * torch.clamp(torch.abs(tensor), min=1e-5)+ else:+ tensor = torch.where(torch.abs(tensor) < 1e-30, 0, tensor)+ return torch.where(tensor != 0, 1 / tensor, torch.zeros_like(tensor))\ No newline at end of file