Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

update gguf convert file and fix bug of permute bug (#679)

Heng Guo4e1b81f9ff5c
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index b1e7bd0b3..49d16562f 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -828,10 +828,10 @@ class AutoRound(object):
if not hasattr(module, "imatrix"):
module.imatrix = squared
- module.imatrix_cnt = 1
+ module.imatrix_cnt = input.shape[0]
else:
module.imatrix += squared
- module.imatrix_cnt += 1
+ module.imatrix_cnt += input.shape[0]
hook_handles = []
for name, module in model.named_modules():
@@ -885,33 +885,36 @@ class AutoRound(object):
cnt = 1
cnt += 1
except RuntimeError as e:
- try:
- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- import accelerate
- accelerate.hooks.remove_hook_from_submodules(model)
- # Fallback: out-of-memory → try CPU blockwise quantization
- logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")
- model = model.to("cpu")
- clear_memory()
- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
- except Exception:
- # Final fallback: warn and use CPU-only quantization
- logger.warning("Fallback to CPU. "
- "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`.")
- model = model.to("cpu")
- clear_memory()
- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- import accelerate
- accelerate.hooks.remove_hook_from_submodules(model)
-
- orig_device = self.device
- self.device = "cpu"
- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
- self.device = orig_device
- finally:
- # Always remove hooks
- for hook in hooks:
- hook.remove()
+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
+ try:
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+ accelerate.hooks.remove_hook_from_submodules(model)
+ # Fallback: out-of-memory → try CPU blockwise quantization
+ logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")
+ model = model.to("cpu")
+ clear_memory()
+ self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ except Exception:
+ # Final fallback: warn and use CPU-only quantization
+ logger.warning("Fallback to CPU. "
+ "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`.")
+ model = model.to("cpu")
+ clear_memory()
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+ accelerate.hooks.remove_hook_from_submodules(model)
+
+ orig_device = self.device
+ self.device = "cpu"
+ self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ self.device = orig_device
+ finally:
+ # Always remove hooks
+ for hook in hooks:
+ hook.remove()
+ else:
+ raise
# Move back to CPU and free memory
model.to("cpu")
@@ -1029,7 +1032,7 @@ class AutoRound(object):
# Step 2: Try quantization on GPU first, fall back to CPU if OOM
# if only export gguf, using gguf-packing instead of rtn
- if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and self.disable_opt_rtn:
+ if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:
m.scale = None
m.zp = None
else:
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index ae016c914..eba6db810 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -15,6 +15,7 @@
import torch
from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad, logger
from auto_round.data_type.register import register_dtype
+from auto_round.utils import get_reciprocal
@register_dtype("int_sym_dq")
@@ -69,7 +70,7 @@ def quant_tensor_sym_dq(
scale = scale.view(-1, 1)
zp = torch.full_like(scale, maxq) # pylint: disable=E1130
- int_w = torch.where(scale != 0, round_ste(tensor / scale + v), 0)
+ int_w = round_ste(tensor * get_reciprocal(scale) + v)
q = torch.clamp(int_w + zp, 0, 2 ** bits - 1)
qdq_result = (scale * (q - zp)).to(tensor.dtype)
qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
@@ -140,7 +141,7 @@ def double_quant_tensor(tensor, bits):
scale = wmax / maxq
scale = scale.view(-1, 1)
# inverse_scale = torch.where(scale == 0, 0, 1 / scale)
- inverse_scale = torch.where(wmax > 0, maxq / wmax, 0).view(-1, 1)
+ inverse_scale = (maxq * get_reciprocal(wmax)).clamp(min=0).view(-1, 1)
qdq_tensor = torch.clamp(round_ste(tensor * inverse_scale), max=maxq) * scale
return qdq_tensor, scale
@@ -151,7 +152,7 @@ def double_quant_tensor_sym(tensor, bits):
imax = abs(tensor).argmax(axis=-1, keepdims=True)
wmax = torch.take_along_dim(tensor, imax, dim=-1)
scale = wmax / -maxq
- inverse_scale = torch.where(scale == 0, 0, 1 / scale) ##1e-40
+ inverse_scale = get_reciprocal(scale)
qdq_tensor = torch.clip((round_ste(tensor * inverse_scale)), -maxq, maxq - 1) * scale
return qdq_tensor, scale
@@ -161,7 +162,7 @@ def make_qp_quants(nmax, data, quant_weights):
quant_weights = quant_weights.to(torch.float32)
group_max = torch.max(data, dim=-1, keepdim=True)[0]
scale = group_max / nmax
- iscale = torch.where(scale == 0, 0, 1 / scale)
+ iscale = get_reciprocal(scale)
L = torch.round(iscale * data)
diffs = data - scale * L
@@ -171,7 +172,7 @@ def make_qp_quants(nmax, data, quant_weights):
if _is == 0:
continue
scale_is = group_max / (0.1 * _is + nmax)
- iscale_is = torch.where(scale_is == 0, 0, 1 / scale_is)
+ iscale_is = get_reciprocal(scale_is)
tmp_L = torch.round(iscale_is * data).clip(max=nmax)
diffs = data - scale_is * tmp_L
@@ -328,11 +329,11 @@ def quant_tensor_gguf_asym_dq(
use_mad=params["use_mad"], weights=quant_weights
)
scale = scale.to(scale_dtype)
- scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)
+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)
scale = scale.reshape(-1, super_group_size)
wmin = wmin.reshape(-1, super_group_size)
scale, d_scale = double_quant_tensor(scale, super_bits)
- wmin = torch.where(torch.abs(wmin) < 1e-30, 0, wmin)
+ wmin = torch.where(torch.abs(wmin) < 1e-30, torch.zeros_like(wmin), wmin)
wmin, d_wmin = double_quant_tensor(wmin, super_bits)
wmin = wmin.view(-1, 1)
scale = scale.view(-1, 1)
@@ -386,7 +387,7 @@ def quant_tensor_gguf_asym_dq(
use_mad=params["use_mad"], weights=quant_weights
)
scale = scale.to(scale_dtype)
- scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)
+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)
nmax = 2 ** super_bits - 1
scale = scale.reshape(-1, super_group_size)
wmin = wmin_0.reshape(-1, super_group_size)
@@ -399,7 +400,7 @@ def quant_tensor_gguf_asym_dq(
d_wmin = d_wmin.unsqueeze(-1)
scale = (d_scale * q_scale).view(-1, 1)
wmin = (d_wmin * q_wmin).view(-1, 1)
- inverse_scale = torch.where(scale == 0, 0, 1 / scale)
+ inverse_scale = get_reciprocal(scale)
int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)
qdq_result = (scale * int_w - wmin).to(orig_dtype)
@@ -436,7 +437,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u
# scale = 1 / ((maxq - minq) / (rmax - rmin + 1e-8))
scale = (rmax - rmin) / (maxq - minq)
- iscale = torch.where(scale == 0, 0, 1 / scale)
+ iscale = get_reciprocal(scale)
# quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)
quant_data = torch.clamp(torch.round(iscale * (data - rmin)), minq, maxq)
diff = scale * quant_data + rmin - data
@@ -447,7 +448,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u
factor = rrmin + rdelta * is_ + maxq - minq
# iscale_new = factor / (rmax - rmin + 1e-8)
scale_new = (rmax - rmin) / factor
- iscale_new = torch.where(scale_new == 0, 0, 1 / scale_new)
+ iscale_new = get_reciprocal(scale_new)
quant_data_new = torch.clamp(torch.round(iscale_new * (data - rmin)), minq, maxq)
mul_weights_quant_data = weights * quant_data_new
@@ -460,7 +461,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u
this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D
this_min[this_min > 0] = 0
this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]
- reverse_this_scale = torch.where(this_scale == 0, 0, 1 / this_scale)
+ reverse_this_scale = get_reciprocal(this_scale)
quant_data = torch.clamp(torch.round(reverse_this_scale * (data - this_min)), minq, maxq)
diff = this_scale * quant_data + this_min - data
@@ -569,13 +570,13 @@ def quant_tensor_gguf_sym_dq(
quant_weights[mean_replace_index, :] = tmp_quant_weights[mean_replace_index, :]
scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)
- scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)
+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)
# conduct double quant
scale, d_scale = double_quant_tensor_sym(scale, super_bits)
scale = scale.unsqueeze(-1)
zp = torch.full_like(scale, maxq) # pylint: disable=E1130
- inverse_scale = torch.where(scale == 0, 0, 1.0 / scale)
+ inverse_scale = get_reciprocal(scale)
int_w = torch.round(tensor * inverse_scale).clip(-maxq, maxq - 1) + maxq
qdq_result = (scale * (int_w - zp)).to(orig_dtype)
qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 9ddd55585..034b86590 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -356,6 +356,7 @@ class OriModel:
gguf.MODEL_TENSOR.POS_EMBD,
gguf.MODEL_TENSOR.TOKEN_TYPES,
gguf.MODEL_TENSOR.SSM_CONV1D,
+ gguf.MODEL_TENSOR.SHORTCONV_CONV,
gguf.MODEL_TENSOR.TIME_MIX_FIRST,
gguf.MODEL_TENSOR.TIME_MIX_W1,
gguf.MODEL_TENSOR.TIME_MIX_W2,
@@ -648,8 +649,7 @@ class ModelBase(OriModel):
if hasattr(module, attr) and getattr(module, attr) is not None:
attr_tensor = getattr(module, attr)
ori_shape = attr_tensor.shape
- # attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)[0][1]
- attr_tensor = self.modify_tensors(attr_tensor, modify_name, bid)[0][1]
+ attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)[0][1]
attr_tensor = attr_tensor.reshape(ori_shape)
setattr(module, attr, attr_tensor)
scale = module.scale if hasattr(module, "scale") else None
@@ -1136,10 +1136,10 @@ class TextModel(ModelBase):
)
seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))
- seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))
+ seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>")) # deepseek-coder
# TODO: should these be marked as UNUSED instead? (maybe not)
- seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">"))
+ seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">")) # gemma{,-2}
return seems_special
@@ -1149,7 +1149,7 @@ class TextModel(ModelBase):
toktypes: list[int] = []
from transformers import AutoTokenizer
- tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))
assert max(tokenizer.vocab.values()) < vocab_size
@@ -1183,7 +1183,7 @@ class TextModel(ModelBase):
else:
# NOTE: this was added for Gemma.
# Encoding and decoding the tokens above isn't sufficient for this case.
- token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")
+ token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")
toktypes.append(gguf.TokenType.USER_DEFINED)
else:
toktypes.append(gguf.TokenType.NORMAL)
@@ -1214,6 +1214,36 @@ class TextModel(ModelBase):
# NOTE: if you get an error here, you need to update the convert_hf_to_gguf_update.py script
# or pull the latest version of the model from Huggingface
# don't edit the hashes manually!
+ if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b":
+ # ref: https://huggingface.co/THUDM/glm-4-9b-chat
+ res = "chatglm-bpe"
+ if chkhsh == "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516":
+ # ref: https://huggingface.co/THUDM/glm-4-9b-chat
+ res = "chatglm-bpe"
+ if chkhsh == "a1336059768a55c99a734006ffb02203cd450fed003e9a71886c88acf24fdbc2":
+ # ref: https://huggingface.co/THUDM/glm-4-9b-hf
+ res = "glm4"
+ if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":
+ # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0
+ res = "minerva-7b"
+ if chkhsh == "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664":
+ # ref: https://huggingface.co/tencent/Hunyuan-A13B-Instruct
+ res = "hunyuan"
+ if chkhsh == "a6b57017d60e6edb4d88ecc2845188e0eb333a70357e45dcc9b53964a73bbae6":
+ # ref: https://huggingface.co/tiiuae/Falcon-H1-0.5B-Base
+ res = "falcon-h1"
+ if chkhsh == "60476e1243776c4fb1b993dbd7a5f15ac22f83c80afdf425fa5ae01c8d44ef86":
+ # ref: https://huggingface.co/tiiuae/Falcon-H1-1B-Base
+ res = "falcon-h1"
+ if chkhsh == "3eda48b4c4dc7de733d1a8b3e3b4a85243dbbf704da2ee9d42c6beced8897896":
+ # ref: https://huggingface.co/tiiuae/Falcon-H1-7B-Base
+ res = "falcon-h1"
+ if chkhsh == "48f8e02c0359c0bbdd82f26909171fac1c18a457bb47573ed1fe3bbb2c1cfd4b":
+ # ref: https://huggingface.co/tiiuae/Falcon-H1-34B-Base
+ res = "falcon-h1"
+ if chkhsh == "81212dc7cdb7e0c1074ca62c5aeab0d43c9f52b8a737be7b12a777c953027890":
+ # ref: https://huggingface.co/moonshotai/Kimi-K2-Base
+ res = "kimi-k2"
if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":
# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
res = "llama-bpe"
@@ -1349,18 +1379,18 @@ class TextModel(ModelBase):
if chkhsh == "d5f1dd6f980fec569fb218a81a7658ac45fc56b38c5a0adeb1c232fbe04ef5ec":
# ref: https://huggingface.co/ByteDance-Seed/Seed-Coder-8B-Base
res = "seed-coder"
- if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b":
- # ref: https://huggingface.co/THUDM/glm-4-9b-chat
- res = "chatglm-bpe"
- if chkhsh == "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516":
- # ref: https://huggingface.co/THUDM/glm-4-9b-chat
- res = "chatglm-bpe"
- if chkhsh == "a1336059768a55c99a734006ffb02203cd450fed003e9a71886c88acf24fdbc2":
- # ref: https://huggingface.co/THUDM/glm-4-9b-hf
- res = "glm4"
- if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":
- # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0
- res = "minerva-7b"
+ if chkhsh == "b0a6b1c0bd5998ebd9df08611efde34a4ff03faed45ae09c43e6b31ebd4b94cf":
+ # ref: https://huggingface.co/skt/A.X-4.0
+ res = "a.x-4.0"
+ if chkhsh == "f6791d196f87ce6b56a7d234be618e0d58f8cda3549416635b2bebcd22cd95c4":
+ # ref: https://huggingface.co/K-intelligence/Midm-2.0-Base-Instruct
+ res = "midm-2.0"
+ if chkhsh == "169bf0296a13c4d9b7672313f749eb36501d931022de052aad6e36f2bf34dd51":
+ # ref: https://huggingface.co/LiquidAI/LFM2-Tokenizer
+ res = "lfm2"
+ if chkhsh == "2085e1638f6c377a0aa4ead21b27bb4cb941bf800df86ed391011769c1758dfb":
+ # ref: https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B
+ res = "exaone4"
if res is None:
logger.warning("\n")
@@ -1607,7 +1637,14 @@ class TextModel(ModelBase):
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_types(toktypes)
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)
- special_vocab.chat_template = "rwkv-world"
+ if special_vocab.chat_template is None:
+ template_path = Path(__file__).parent / "models" / "templates" / "llama-cpp-rwkv-world.jinja"
+ if template_path.is_file():
+ with open(template_path, "r", encoding="utf-8") as f:
+ template = f.read()
+ else:
+ template = "rwkv-world"
+ special_vocab.chat_template = template
# hack: Add '\n\n' as the EOT token to make it chat normally
special_vocab._set_special_token("eot", 261)
# hack: Override these as they have already been set (incorrectly)
@@ -3290,6 +3327,76 @@ class Qwen2Model(TextModel):
yield from super().modify_tensors(data_torch, name, bid)
[email protected]("DreamModel")
+class DreamModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.DREAM
+
+ def get_vocab_base(self) -> tuple[list[str], list[int], str]:
+ tokens: list[str] = []
+ toktypes: list[int] = []
+
+ from transformers import AutoTokenizer
+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+
+ vocab_dict = tokenizer.get_vocab()
+ vocab_size = self.hparams.get("vocab_size", len(vocab_dict))
+ assert max(vocab_dict.values()) < vocab_size
+
+ tokpre = self.get_vocab_base_pre(tokenizer)
+
+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()}
+ added_vocab = tokenizer.get_added_vocab()
+
+ for i in range(vocab_size):
+ if i not in reverse_vocab:
+ tokens.append(f"[PAD{i}]")
+ toktypes.append(gguf.TokenType.UNUSED)
+ elif reverse_vocab[i] in added_vocab:
+ tokens.append(reverse_vocab[i])
+ # Check if it's a special token - treat special tokens as CONTROL tokens
+ if hasattr(tokenizer, 'added_tokens_decoder') and i in tokenizer.added_tokens_decoder:
+ if tokenizer.added_tokens_decoder[i].special:
+ toktypes.append(gguf.TokenType.CONTROL)
+ else:
+ toktypes.append(gguf.TokenType.USER_DEFINED)
+ else:
+ # Fallback: treat all added vocab as control tokens for special tokens like <|im_start|>
+ toktypes.append(gguf.TokenType.CONTROL)
+ else:
+ tokens.append(reverse_vocab[i])
+ toktypes.append(gguf.TokenType.NORMAL)
+
+ return tokens, toktypes, tokpre
+
+ def set_vocab(self):
+ try:
+ self._set_vocab_sentencepiece()
+ except FileNotFoundError:
+ self._set_vocab_gpt2()
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self._try_set_pooling_type()
+
+ # Dream models use non-causal attention for diffusion
+ self.gguf_writer.add_causal_attention(False)
+ # Handle RoPE scaling similar to Qwen2
+ rope_scaling = self.hparams.get("rope_scaling") or {}
+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
+ self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
+
+ # Add Dream-specific parameters
+ mask_token_id = self.hparams.get("mask_token_id")
+ if mask_token_id is not None:
+ self.gguf_writer.add_mask_token_id(mask_token_id)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Dream model tensors should be mapped directly since it's the base model
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("Ernie4_5_ForCausalLM")
class Ernie4_5Model(TextModel):
model_arch = gguf.MODEL_ARCH.ERNIE4_5
@@ -3303,7 +3410,8 @@ class Ernie4_5Model(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
num_heads = self.hparams["num_attention_heads"]
num_kv_heads = self.hparams["num_key_value_heads"]
- head_dim = self.hparams["head_dim"]
+ if (head_dim := self.hparams.get("head_dim")) is None:
+ head_dim = self.hparams["hidden_size"] // num_heads
if "ernie." in name:
name = name.replace("ernie.", "model.")
@@ -3332,6 +3440,96 @@ class Ernie4_5Model(TextModel):
return [(self.map_tensor_name(name), data_torch)]
[email protected]("Ernie4_5_MoeForCausalLM")
+class Ernie4_5MoeModel(Ernie4_5Model):
+ model_arch = gguf.MODEL_ARCH.ERNIE4_5_MOE
+ _experts: list[dict[str, Tensor]] | None = None
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._experts = [{} for _ in range(self.block_count)]
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_expert_count(self.hparams["moe_num_experts"])
+ self.gguf_writer.add_expert_used_count(self.hparams["moe_k"])
+ self.gguf_writer.add_interleave_moe_layer_step(self.hparams["moe_layer_interval"])
+ self.gguf_writer.add_leading_dense_block_count(self.hparams["moe_layer_start_index"])
+ if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
+ self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
+ if (shared_expert_count := self.hparams.get('moe_num_shared_experts')) is not None:
+ self.gguf_writer.add_expert_shared_count(shared_expert_count)
+ if shared_expert_count > 0 and (shared_expert_intermediate_size := self.hparams.get('intermediate_size')
+ ) is not None and (num_key_value_heads :=
+ self.hparams.get('num_key_value_heads')) is not None:
+ self.gguf_writer.add_expert_shared_feed_forward_length(
+ shared_expert_intermediate_size // num_key_value_heads)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Modify correction bias name as in DeepseekV2
+ if name.endswith("e_score_correction_bias"):
+ name = name.replace("e_score_correction_bias", "e_score_correction.bias")
+
+ # skip Multi-Token Prediction (MTP) layers (again, same as DeepseekV2)
+ match = re.match(r"model.mtp_block.(\d+)", name)
+ if match:
+ return []
+
+ # skip all other MTP tensors for now
+ match = re.match(r"model.mtp_emb_norm.(\d+)", name)
+ if match:
+ return []
+
+ match = re.match(r"model.mtp_hidden_norm.(\d+)", name)
+ if match:
+ return []
+
+ match = re.match(r"model.mtp_linear_proj.(\d+)", name)
+ if match:
+ return []
+
+ # process the experts separately
+ if name.find("mlp.experts") != -1:
+ n_experts = self.hparams["moe_num_experts"]
+ assert bid is not None
+
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) >= n_experts * 3:
+ tensors: list[tuple[str, Tensor]] = []
+
+ # merge the experts into a single 3d tensor
+ for w_name in ["gate_proj", "up_proj", "down_proj"]:
+ data: list[Tensor] = []
+
+ for xid in range(n_experts):
+ ename_to_retrieve = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"
+ data.append(self._experts[bid][ename_to_retrieve])
+ del self._experts[bid][ename_to_retrieve]
+
+ data_torch = torch.stack(data, dim=0)
+ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+ new_name = self.map_tensor_name(merged_name)
+ tensors.append((new_name, data_torch))
+
+ return tensors
+ else:
+ return []
+ return [(self.map_tensor_name(name), data_torch)]
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+
+ if self._experts is not None:
+ # flatten `list[dict[str, Tensor]]` into `list[str]`
+ experts = [k for d in self._experts for k in d.keys()]
+ if len(experts) > 0:
+ raise ValueError(f"Unprocessed experts: {experts}")
+
+
@ModelBase.register(
"Qwen2VLModel",
"Qwen2VLForConditionalGeneration",
@@ -3357,7 +3555,6 @@ class Qwen2VLModel(TextModel):
del bid # unused
if name.startswith("thinker."):
name = name.replace("thinker.", "")
-
if name.startswith("visual") or name.startswith("audio") or \
name.startswith("talker") or name.startswith("token2wav"):
# skip multimodal tensors
@@ -3500,8 +3697,6 @@ class Qwen25OmniModel(Qwen2VLVisionModel):
# transpose conv1 and conv2 bias
data_torch = data_torch.unsqueeze(-1)
if "audio_bos_eos_token" in name:
- # this tensor is left unused in transformers code
- # https://github.com/huggingface/transformers/blob/6e3063422c4b1c014aa60c32b9254fd2902f0f28/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py#L1809
return []
return [(self.map_tensor_name(name), data_torch)]
@@ -4035,6 +4230,175 @@ class PlamoModel(TextModel):
return [(new_name, data_torch)]
[email protected]("Plamo2ForCausalLM", "PLaMo2ForCausalLM")
+class Plamo2Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.PLAMO2
+
+ def set_vocab(self):
+ # PLaMo 2 uses a custom tokenizer with a .jsonl file
+ # We need to handle this specially
+ tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl"
+ tokenizer_config_path = self.dir_model / "tokenizer_config.json"
+
+ if not tokenizer_jsonl_path.is_file():
+ raise FileNotFoundError(f"PLaMo 2 tokenizer file not found: {tokenizer_jsonl_path}")
+
+ # Load tokenizer config
+ with open(tokenizer_config_path, 'r', encoding='utf-8') as f:
+ tokenizer_config = json.load(f)
+
+ # Load tokens from JSONL file (actually a list format)
+ tokens = []
+ scores = []
+ toktypes = []
+
+ with open(tokenizer_jsonl_path, 'r', encoding='utf-8') as f:
+ for line_num, line in enumerate(f):
+ if line.strip():
+ token_data = json.loads(line)
+ # Format: [token, score, type, ?, ?, ?, ?]
+ token = token_data[0].encode("utf-8")
+ score = float(token_data[1])
+ token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL"
+
+ tokens.append(token)
+ scores.append(score)
+
+ # Map token type strings to GGUF token types
+ if token_type_str == "UNKNOWN":
+ toktypes.append(gguf.TokenType.UNKNOWN)
+ elif token_type_str == "CONTROL":
+ toktypes.append(gguf.TokenType.CONTROL)
+ elif token_type_str == "BYTE":
+ toktypes.append(gguf.TokenType.BYTE)
+ else:
+ # Check for PLaMo-2 special tokens
+ token_str = token_data[0]
+ if token_str.startswith("<|plamo:") and token_str.endswith("|>"):
+ toktypes.append(gguf.TokenType.CONTROL)
+ else:
+ toktypes.append(gguf.TokenType.NORMAL)
+
+ vocab_size = self.hparams["vocab_size"]
+ if vocab_size > len(tokens):
+ pad_count = vocab_size - len(tokens)
+ logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")
+ for i in range(1, pad_count + 1):
+ tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))
+ scores.append(-1000.0)
+ toktypes.append(gguf.TokenType.UNUSED)
+
+ # Use "plamo2" tokenizer type for PLaMo-2's custom Aho-Corasick tokenizer
+ self.gguf_writer.add_tokenizer_model("plamo2")
+ self.gguf_writer.add_tokenizer_pre("default")
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_scores(scores)
+ self.gguf_writer.add_token_types(toktypes)
+
+ # Add special tokens from config
+ if "bos_token" in tokenizer_config and tokenizer_config["bos_token"] is not None:
+ token_id = tokens.index(tokenizer_config["bos_token"].encode("utf-8"))
+ self.gguf_writer.add_bos_token_id(token_id)
+ if "eos_token" in tokenizer_config and tokenizer_config["eos_token"] is not None:
+ token_id = tokens.index(tokenizer_config["eos_token"].encode("utf-8"))
+ self.gguf_writer.add_eos_token_id(token_id)
+ if "pad_token" in tokenizer_config and tokenizer_config["pad_token"] is not None:
+ token_id = tokens.index(tokenizer_config["pad_token"].encode("utf-8"))
+ self.gguf_writer.add_pad_token_id(token_id)
+ if "sep_token" in tokenizer_config and tokenizer_config["sep_token"] is not None:
+ token_id = tokens.index(tokenizer_config["sep_token"].encode("utf-8"))
+ self.gguf_writer.add_sep_token_id(token_id)
+ if "unk_token" in tokenizer_config and tokenizer_config["unk_token"] is not None:
+ token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8"))
+ self.gguf_writer.add_unk_token_id(token_id)
+
+ # Add <|plamo:op|> as EOT to ensure appropriate end of generation
+ self.gguf_writer.add_eot_token_id(4)
+
+ self.gguf_writer.add_add_space_prefix(False)
+
+ def set_gguf_parameters(self):
+ hparams = self.hparams
+ block_count = hparams["num_hidden_layers"]
+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
+
+ # Which layers are Mamba layers
+ # PLaMo 2 uses mamba_step to indicate the pattern (e.g., 2 means every other layer)
+ # This logic matches modeling_plamo.py's is_mamba function
+ mamba_step = hparams.get("mamba_step", 2)
+ mamba_enabled = hparams.get("mamba_enabled", True)
+ mamba_layers = []
+
+ if mamba_enabled:
+ for i in range(block_count):
+ if block_count <= (mamba_step // 2):
+ # use attention in last layer
+ is_mamba = (i != block_count - 1)
+ else:
+ is_mamba = (i % mamba_step) != (mamba_step // 2)
+ if is_mamba:
+ mamba_layers.append(0)
+ else:
+ mamba_layers.append(hparams.get("num_key_value_heads", 4))
+
+ if mamba_layers:
+ self.gguf_writer.add_head_count_kv(mamba_layers)
+
+ self.gguf_writer.add_context_length(hparams.get("max_position_embeddings", 2048))
+ self.gguf_writer.add_embedding_length(hparams.get("hidden_size", 4096))
+ self.gguf_writer.add_block_count(block_count)
+ self.gguf_writer.add_head_count(hparams.get("num_attention_heads", 32))
+ self.gguf_writer.add_layer_norm_rms_eps(hparams.get("rms_norm_eps", 1e-06))
+ self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 1000000.0))
+
+ # Mamba parameters
+ self.gguf_writer.add_ssm_state_size(hparams.get("mamba_d_state", 64))
+ self.gguf_writer.add_ssm_conv_kernel(hparams.get("mamba_d_conv", 4))
+ self.gguf_writer.add_ssm_time_step_rank(hparams.get("mamba_num_heads", 64))
+ intermediate_size = hparams.get("mamba_num_heads", 64) * hparams.get("hidden_size_per_head", 128)
+ self.gguf_writer.add_ssm_inner_size(intermediate_size)
+ self.gguf_writer.add_ssm_group_count(0)
+
+ # MLP feed forward parameters (for attention layers)
+ self.gguf_writer.add_feed_forward_length(hparams.get("intermediate_size", 16384))
+ self.gguf_writer.add_file_type(self.ftype)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ del bid # unused
+
+ if name.endswith(".A_log"):
+ data_torch = -torch.exp(data_torch)
+ elif name.endswith(".dt_bias"):
+ name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"
+ elif name.endswith(".dt_norm_weight"):
+ name = name.rpartition(".dt_norm_weight")[0] + ".dt_norm.weight"
+ elif name.endswith(".B_norm_weight"):
+ name = name.rpartition(".B_norm_weight")[0] + ".B_norm.weight"
+ elif name.endswith(".C_norm_weight"):
+ name = name.rpartition(".C_norm_weight")[0] + ".C_norm.weight"
+ elif name.endswith(".k_weight"):
+ name = name.rpartition(".k_weight")[0] + ".k.weight"
+ elif name.endswith(".q_weight"):
+ name = name.rpartition(".q_weight")[0] + ".q.weight"
+ elif name.endswith(".conv1d.weight"):
+ data_torch = torch.squeeze(data_torch) # remove (, 1, )
+ assert data_torch.ndim == 2
+ elif name.endswith(".pre_mixer_norm.weight"):
+ data_torch += 1.0
+ elif name.endswith(".post_mixer_norm.weight"):
+ data_torch += 1.0 / 5
+ elif name.endswith(".pre_mlp_norm.weight"):
+ data_torch += 1.0
+ elif name.endswith(".post_mlp_norm.weight"):
+ data_torch += 1.0 / (5**1.5)
+ elif name.endswith(".norm.weight"):
+ data_torch += 1.0
+
+ new_name = self.map_tensor_name(name)
+
+ return [(new_name, data_torch)]
+
+
@ModelBase.register("CodeShellForCausalLM")
class CodeShellModel(TextModel):
model_arch = gguf.MODEL_ARCH.CODESHELL
@@ -5454,6 +5818,10 @@ class Mamba2Model(TextModel):
with open(dir_model / "config.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
+ self.d_model = self.find_hparam(["hidden_size", "d_model", "dim"])
+ self.d_inner = self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"],
+ optional=True) or 2 * self.d_model
+ self.n_group = self.find_hparam(["n_groups"], optional=True) or 1
def set_vocab(self):
vocab_size = self.hparams["vocab_size"]
@@ -5477,30 +5845,29 @@ class Mamba2Model(TextModel):
self._set_vocab_builtin("gpt-neox", vocab_size)
def set_gguf_parameters(self):
- d_model = self.find_hparam(["hidden_size", "d_model", "dim"])
d_conv = self.find_hparam(["conv_kernel", "d_conv"], optional=True) or 4
- d_inner = self.find_hparam(["intermediate_size", "d_inner"], optional=True) or 2 * d_model
d_state = self.find_hparam(["state_size", "d_state"], optional=True) or 128
- head_dim = self.find_hparam(["head_dim"], optional=True) or 64
- n_group = self.find_hparam(["n_groups"], optional=True) or 1
+ head_dim = self.find_hparam(["mamba_d_head", "head_dim"], optional=True) or 64
rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-5
# Fail early for models which don't have a block expansion factor of 2
# TODO: does this really matter?
- assert d_inner == 2 * d_model
- assert d_inner % head_dim == 0
+ # skip the assertion for FalconH1 Model
+ if self.model_arch != gguf.MODEL_ARCH.FALCON_H1:
+ assert self.d_inner == 2 * self.d_model
+ assert self.d_inner % head_dim == 0
self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default
- self.gguf_writer.add_embedding_length(d_model)
+ self.gguf_writer.add_embedding_length(self.d_model)
self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loading
self.gguf_writer.add_head_count(0) # unused, but seemingly required when loading
self.gguf_writer.add_block_count(self.block_count)
self.gguf_writer.add_ssm_conv_kernel(d_conv)
- self.gguf_writer.add_ssm_inner_size(d_inner)
+ self.gguf_writer.add_ssm_inner_size(self.d_inner)
self.gguf_writer.add_ssm_state_size(d_state)
- self.gguf_writer.add_ssm_time_step_rank(d_inner // head_dim)
- self.gguf_writer.add_ssm_group_count(n_group)
+ self.gguf_writer.add_ssm_time_step_rank(self.d_inner // head_dim)
+ self.gguf_writer.add_ssm_group_count(self.n_group)
self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)
self.gguf_writer.add_file_type(self.ftype)
@@ -5525,10 +5892,7 @@ class Mamba2Model(TextModel):
# (D is also unsqueezed, but for more straightforward broadcast internally)
data_torch = data_torch.reshape((*data_torch.shape, 1))
elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_NORM, bid):
- d_model = self.find_hparam(["hidden_size", "d_model", "dim"])
- d_inner = self.find_hparam(["intermediate_size", "d_inner"], optional=True) or 2 * d_model
- n_group = self.hparams.get("n_groups", 1)
- data_torch = data_torch.reshape((n_group, d_inner // n_group))
+ data_torch = data_torch.reshape((self.n_group, self.d_inner // self.n_group))
if name.endswith(".A_log"):
logger.debug("A_log --> A ==> " + new_name)
@@ -5537,51 +5901,165 @@ class Mamba2Model(TextModel):
yield (new_name, data_torch)
[email protected]("CohereForCausalLM")
-class CommandR2Model(TextModel):
- model_arch = gguf.MODEL_ARCH.COMMAND_R
[email protected]("JambaForCausalLM")
+class JambaModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.JAMBA
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
+ def get_vocab_base_pre(self, tokenizer) -> str:
+ del tokenizer # unused
- # max_position_embeddings = 8192 in config.json but model was actually
- # trained on 128k context length
- # aya-23 models don't have model_max_length specified
- self.hparams["max_position_embeddings"] = self.find_hparam(["model_max_length", "max_position_embeddings"])
+ return "gpt-2"
+
+ def set_vocab(self):
+ if (self.dir_model / "tokenizer.model").is_file():
+ # Using Jamba's tokenizer.json causes errors on model load
+ # (something about "byte not found in vocab"),
+ # but there's a working tokenizer.model
+ self._set_vocab_sentencepiece()
+ else:
+ # Some Jamba models only have a tokenizer.json, which works.
+ self._set_vocab_gpt2()
def set_gguf_parameters(self):
- super().set_gguf_parameters()
- self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+ d_model = self.find_hparam(["hidden_size", "mamba_d_model"])
+ d_conv = self.find_hparam(["mamba_d_conv"], optional=True) or 4
+ d_inner = self.hparams["mamba_expand"] * d_model
+ d_state = self.find_hparam(["mamba_d_state"], optional=True) or 16
+ dt_rank = self.find_hparam(["mamba_dt_rank"], optional=True) or -(d_model // -16)
+ rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-6
+ n_kv_head = self.hparams["num_key_value_heads"]
+ attn_offset = self.hparams["attn_layer_offset"]
+ attn_period = self.hparams["attn_layer_period"]
+ n_kv_vec = [0 for _ in range(attn_offset)] + [
+ n_kv_head if (i-attn_offset) % attn_period == 0 else 0 for i in range(attn_offset, self.block_count)
+ ]
+ self.gguf_writer.add_block_count(self.block_count)
+ self.gguf_writer.add_context_length(self.find_hparam(["max_position_embeddings", "n_ctx"]))
+ self.gguf_writer.add_embedding_length(d_model)
+ self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])
+ self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])
+ self.gguf_writer.add_head_count_kv(n_kv_vec)
+ self.gguf_writer.add_ssm_conv_kernel(d_conv)
+ self.gguf_writer.add_ssm_inner_size(d_inner)
+ self.gguf_writer.add_ssm_state_size(d_state)
+ self.gguf_writer.add_ssm_time_step_rank(dt_rank)
+ self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)
+ self.gguf_writer.add_expert_count(self.hparams["num_experts"])
+ self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])
+ self.gguf_writer.add_file_type(self.ftype)
[email protected]("Cohere2ForCausalLM")
-class Cohere2Model(TextModel):
- model_arch = gguf.MODEL_ARCH.COHERE2
+ _experts: list[dict[str, Tensor]] | None = None
- def set_gguf_parameters(self):
- super().set_gguf_parameters()
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
- self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])
- self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
- self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
+ # Mini-Jamba
+ name = name.replace(".moe.", ".feed_forward.")
+ if bid is not None:
+ moe_offset = self.hparams["expert_layer_offset"]
+ moe_period = self.hparams["expert_layer_period"]
- rotary_pct = self.hparams["rotary_pct"]
- hidden_size = self.hparams["hidden_size"]
- num_attention_heads = self.hparams["num_attention_heads"]
- self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size//num_attention_heads)))
- self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+ if not (bid >= moe_offset and (bid-moe_offset) % moe_period == 0):
+ name = name.replace(".experts.0.", ".")
+ # process the experts separately
+ if ".feed_forward.experts." in name:
+ n_experts = self.hparams["num_experts"]
[email protected]("OlmoForCausalLM")
[email protected]("OLMoForCausalLM")
-class OlmoModel(TextModel):
- model_arch = gguf.MODEL_ARCH.OLMO
+ assert bid is not None
- def set_gguf_parameters(self):
- super().set_gguf_parameters()
- self.gguf_writer.add_layer_norm_eps(1e-5)
- clip_qkv = self.hparams.get("clip_qkv")
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) >= n_experts * 3:
+
+ # merge the experts into a single 3d tensor
+ for wid in ["down_proj", "gate_proj", "up_proj"]:
+ data: list[Tensor] = []
+
+ for xid in range(n_experts):
+ ename = f"model.layers.{bid}.feed_forward.experts.{xid}.{wid}.weight"
+ data.append(self._experts[bid][ename])
+ del self._experts[bid][ename]
+
+ data_torch = torch.stack(data, dim=0)
+
+ # using the same merged name as qwen2moe
+ merged_name = f"model.layers.{bid}.mlp.experts.{wid}.weight"
+
+ new_name = self.map_tensor_name(merged_name)
+
+ yield new_name, data_torch
+ return
+
+ new_name = self.map_tensor_name(name)
+
+ if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_CONV1D, bid):
+ data_torch = data_torch.squeeze()
+
+ if name.endswith(".A_log"):
+ logger.debug("A_log --> A ==> " + new_name)
+ data_torch = -torch.exp(data_torch)
+
+ yield (new_name, data_torch)
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+
+ if self._experts is not None:
+ # flatten `list[dict[str, Tensor]]` into `list[str]`
+ experts = [k for d in self._experts for k in d.keys()]
+ if len(experts) > 0:
+ raise ValueError(f"Unprocessed experts: {experts}")
+
+
[email protected]("CohereForCausalLM")
+class CommandR2Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.COMMAND_R
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ # max_position_embeddings = 8192 in config.json but model was actually
+ # trained on 128k context length
+ # aya-23 models don't have model_max_length specified
+ self.hparams["max_position_embeddings"] = self.find_hparam(["model_max_length", "max_position_embeddings"])
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])
+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+
+
[email protected]("Cohere2ForCausalLM")
+class Cohere2Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.COHERE2
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])
+ self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
+
+ rotary_pct = self.hparams["rotary_pct"]
+ hidden_size = self.hparams["hidden_size"]
+ num_attention_heads = self.hparams["num_attention_heads"]
+ self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size//num_attention_heads)))
+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+
+
[email protected]("OlmoForCausalLM")
[email protected]("OLMoForCausalLM")
+class OlmoModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.OLMO
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_layer_norm_eps(1e-5)
+ clip_qkv = self.hparams.get("clip_qkv")
if clip_qkv is not None:
self.gguf_writer.add_clamp_kqv(clip_qkv)
@@ -5832,8 +6310,8 @@ class ArcticModel(TextModel):
token_score = 0.0
logger.info(
- f"Setting added token {token_id} to '{token_content}' (type: {token_type},"
- f" score: {token_score:.2f})"
+ f"Setting added token {token_id} to '{token_content}'"
+ f" (type: {token_type}, score: {token_score:.2f})"
)
tokens[token_id] = token_content.encode("utf-8")
toktypes[token_id] = token_type
@@ -6006,7 +6484,58 @@ class DeepseekV2Model(TextModel):
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
def set_vocab(self):
- self._set_vocab_gpt2()
+ try:
+ self._set_vocab_gpt2()
+ return
+ except Exception:
+ pass
+
+ from transformers import AutoTokenizer
+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ tokpre = self.get_vocab_base_pre(tokenizer)
+
+ if tokpre == "kimi-k2":
+ # Build merges list using the approach similar to HunYuanMoE
+ merges = []
+ vocab = {}
+ mergeable_ranks = tokenizer.model._mergeable_ranks
+ for token, rank in mergeable_ranks.items():
+ vocab[QwenModel.token_bytes_to_string(token)] = rank
+ if len(token) == 1:
+ continue
+ merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)
+ if len(merged) == 2:
+ merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
+
+ # Build token list
+ vocab_size = self.hparams["vocab_size"]
+ special_tokens = tokenizer.special_tokens
+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
+ tokens: list[str] = []
+ toktypes: list[int] = []
+
+ for i in range(vocab_size):
+ if i not in reverse_vocab:
+ tokens.append(f"[PAD{i}]")
+ toktypes.append(gguf.TokenType.UNUSED)
+ else:
+ token = reverse_vocab[i]
+ tokens.append(token)
+ if i in special_tokens.values():
+ toktypes.append(gguf.TokenType.CONTROL)
+ else:
+ toktypes.append(gguf.TokenType.NORMAL)
+
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+ self.gguf_writer.add_token_merges(merges)
+
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)
+ special_vocab.add_to_gguf(self.gguf_writer)
+ else:
+ raise NotImplementedError(f"Deepseek pre-tokenizer {tokpre!r} is not supported yet!")
def set_gguf_parameters(self):
@@ -6705,8 +7234,7 @@ class ChatGLMModel(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
del bid # unused
- if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision.") \
- or name.startswith("transformers.vision."):
+ if name.endswith(".rotary_pos_emb.inv_freq") or name.startswith("model.vision."):
return []
name = name.removeprefix("transformer.")
@@ -6823,6 +7351,78 @@ class ExaoneModel(TextModel):
torch.tensor(rope_factors, dtype=torch.float32))
[email protected]("Exaone4ForCausalLM")
+class Exaone4Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.EXAONE4
+
+ def set_vocab(self):
+ tokens, toktypes, tokpre = self.get_vocab_base()
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ special_vocab.add_to_gguf(self.gguf_writer)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hparams = self.hparams
+ self.gguf_writer.add_vocab_size(hparams["vocab_size"])
+
+ if hparams.get("sliding_window") is not None:
+ self.gguf_writer.add_sliding_window(hparams["sliding_window"])
+ if "layer_types" in hparams:
+ self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]])
+ elif "sliding_window_pattern" in hparams:
+ sliding_window_pattern = []
+ if isinstance(hparams["sliding_window_pattern"], str): # e.g. LLLG
+ for i in range(hparams["num_hidden_layers"]):
+ sliding_window_pattern.append(
+ hparams["sliding_window_pattern"][i % len(hparams["sliding_window_pattern"])] == "L")
+ if isinstance(hparams["sliding_window_pattern"], int): # e.g. 4
+ for i in range(hparams["num_hidden_layers"]):
+ sliding_window_pattern.append((i+1) % hparams["sliding_window_pattern"] != 0)
+ if len(sliding_window_pattern) == hparams["num_hidden_layers"]:
+ self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)
+
+ rope_scaling = self.hparams.get("rope_scaling") or {}
+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:
+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):
+ if rope_scaling.get("rope_type", '').lower() == "llama3":
+ base = self.hparams.get("rope_theta", 10_000.0)
+ if (dim := self.hparams.get("head_dim")) is None:
+ dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
+ freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))
+
+ factor = rope_scaling.get("factor", 16.0)
+ low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)
+ high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)
+ old_context_len = self.hparams.get("original_max_position_embeddings", 8192)
+
+ low_freq_wavelen = old_context_len / low_freq_factor
+ high_freq_wavelen = old_context_len / high_freq_factor
+
+ rope_factors = []
+ for freq in freqs:
+ wavelen = 2 * math.pi / freq
+ if wavelen < high_freq_wavelen:
+ rope_factors.append(1)
+ elif wavelen > low_freq_wavelen:
+ rope_factors.append(factor)
+ else:
+ smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)
+ rope_factors.append(1 / ((1-smooth) / factor + smooth))
+
+ yield (
+ self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),
+ torch.tensor(rope_factors, dtype=torch.float32))
+
+
@ModelBase.register("GraniteForCausalLM")
class GraniteModel(LlamaModel):
"""Conversion for IBM's GraniteForCausalLM"""
@@ -6887,18 +7487,127 @@ class GraniteMoeModel(GraniteModel):
(self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), up),
]
+ has_experts = bool(self.hparams.get('num_local_experts'))
+
if name.endswith("shared_mlp.input_linear.weight"):
ffn_dim = self.hparams["shared_intermediate_size"]
assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size"
gate, up = data_torch.split(ffn_dim, dim=-2)
+ if has_experts:
+ return [
+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_SHEXP, bid), gate),
+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), up),
+ ]
return [
- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_SHEXP, bid), gate),
- (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), up),
+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, bid), gate),
+ (self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, bid), up),
]
+ if not has_experts and name.endswith("shared_mlp.output_linear.weight"):
+ return [(self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN, bid), data_torch)]
+
return super().modify_tensors(data_torch, name, bid)
[email protected]("GraniteMoeHybridForCausalLM", "BambaForCausalLM")
+class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
+ """GraniteHybrid is a hybrid SSM + Attention model that uses Mamba2 SSM
+ layers and optionally uses MoE w/ a shared expert"""
+ model_arch = gguf.MODEL_ARCH.GRANITE_HYBRID
+ undo_permute = True
+
+ def __init__(self, *args, **kwargs):
+
+ # Hybrid mamba models use a prefix for the mamba-specific params.
+ # TODO: Extend this if the prefix(es) need to be configurable
+ self.hparam_prefixes = ["mamba"]
+
+ super().__init__(*args, **kwargs)
+
+ # Lists of which layers use ssm vs attention
+ self._attn_layers = self.get_attn_layers()
+ self._ssm_layers = [i for i in range(self.block_count) if i not in self._attn_layers]
+
+ # n_group and d_inner are used during reshape_tensors for mamba2
+ self.d_model = self.find_hparam(["hidden_size", "d_model"])
+ self.n_group = self.find_hparam(["n_groups"])
+ self.d_inner = self.find_hparam(["expand"]) * self.d_model
+
+ def get_attn_layers(self):
+ # Explicit list of layer type names
+ if layer_types := self.hparams.get("layer_types"):
+ return [i for i, typ in enumerate(layer_types) if typ == "attention"]
+
+ # Layer types indicated by index or period
+ attn_layers = self.hparams.get("attn_layer_indices", [])
+ if not attn_layers:
+ attn_period = self.hparams.get("attn_layer_period")
+ assert attn_period, "Didn't find attn_layer_indices or attn_layer_period"
+ attn_offset = self.hparams.get("attn_layer_offset")
+ assert attn_offset is not None, "No attention layer offset set with attn_layer_period"
+ attn_layers = [i for i in range(self.block_count) if i % attn_period == attn_offset]
+ return attn_layers
+
+ def find_hparam(self, keys: Iterable[str], *args, **kwargs) -> Any:
+ prefixed = []
+ for pfx in self.hparam_prefixes:
+ prefixed.extend("_".join([pfx, k]) for k in keys)
+ keys = list(keys) + prefixed
+ return Mamba2Model.find_hparam(self, keys, *args, **kwargs)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if (name.endswith("block_sparse_moe.input_linear.weight") or "shared_mlp" in name):
+ return GraniteMoeModel.modify_tensors(self, data_torch, name, bid)
+
+ # Determine whether this is a mamba layer or an attention layer
+ if bid in self._ssm_layers:
+ return Mamba2Model.modify_tensors(self, data_torch, name, bid)
+ elif bid in self._attn_layers:
+ return GraniteMoeModel.modify_tensors(self, data_torch, name, bid)
+ return [(self.map_tensor_name(name), data_torch)]
+
+ def set_gguf_parameters(self):
+ """This method merges params from both parents and some that are
+ specific to this model. The result is some duplication of how the params
+ get set. The following warnings are expected during conversion:
+
+ WARNING:Duplicated key name 'granitehybrid.attention.head_count_kv'
+ WARNING:Duplicated key name 'granitehybrid.context_length'
+ """
+ GraniteMoeModel.set_gguf_parameters(self)
+
+ ## Mamba mixer params ##
+ self.gguf_writer.add_ssm_conv_kernel(self.find_hparam(["conv_kernel", "d_conv"]))
+ self.gguf_writer.add_ssm_state_size(self.find_hparam(["state_size", "d_state"]))
+ self.gguf_writer.add_ssm_group_count(self.n_group)
+ self.gguf_writer.add_ssm_inner_size(self.d_inner)
+ # NOTE: The mamba_dt_rank is _not_ the right field for how this is used
+ # in llama.cpp
+ self.gguf_writer.add_ssm_time_step_rank(self.find_hparam(["n_heads"]))
+
+ ## Attention params ##
+ head_count_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"])
+ head_count_kv_vec = [head_count_kv if i in self._attn_layers else 0 for i in range(self.block_count)]
+ if rope_dim := self.hparams.get("attn_rotary_emb"):
+ self.gguf_writer.add_rope_dimension_count(rope_dim)
+ self.gguf_writer.add_head_count_kv(head_count_kv_vec)
+
+ ## If Bamba, use rope, otherwise don't
+ use_rope = "BambaForCausalLM" in self.hparams["architectures"]
+ self.gguf_writer.add_rope_scaling_finetuned(use_rope)
+ if not use_rope:
+ self.gguf_writer.add_context_length(2**20)
+
+ ## Validation ##
+ d_head = self.find_hparam(["d_head"], optional=True) or 64
+ assert self.hparams.get("hidden_act") in [None, "silu"], "Only SILU activation supported"
+ assert self.d_inner % d_head == 0, f"SSM inner size {self.d_inner} not a multiple of head dim {d_head}"
+
+ def set_vocab(self):
+ self.hparams["pad_vocab_size_multiple"] = 8
+ Mamba2Model.set_vocab(self)
+
+
@ModelBase.register("BailingMoeForCausalLM")
class BailingMoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.BAILINGMOE
@@ -7054,8 +7763,8 @@ class UltravoxModel(TextModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
raise NotImplementedError(
- "Ultravox does not have text decoder. Instead, it uses Llama or other models for text. If you want to"
- " get the audio encoder, please use --mmproj argument"
+ "Ultravox does not have text decoder. Instead, it uses Llama or other models for text."
+ " If you want to get the audio encoder, please use --mmproj argument"
)
@@ -7110,6 +7819,317 @@ class UltravoxWhisperEncoderModel(WhisperEncoderModel):
self.gguf_writer.add_audio_stack_factor(self.global_config["stack_factor"])
[email protected]("FalconH1ForCausalLM")
+class FalconH1Model(Mamba2Model):
+ model_arch = gguf.MODEL_ARCH.FALCON_H1
+
+ def __init__(self, *args, **kwargs):
+ # Set the hparam prefixes for Falcon Mamba2
+ self.hparam_prefixes = ["mamba"]
+
+ # Initialize the base Mamba2Model
+ super().__init__(*args, **kwargs)
+
+ # Use Llama conversion for attention
+ self._transformer_model_class = LlamaModel
+
+ # n_group and d_inner are used during reshape_tensors for mamba2
+ self.n_group = self.find_hparam(["n_groups"])
+ self.d_inner = self.find_hparam(["mamba_d_ssm"])
+ self.d_head = self.find_hparam(["d_head"])
+
+ # Initialize any Falcon Mamba2 specific attributes
+ self.has_attention = True # Falcon Mamba2 has attention components
+
+ # Load Falcon-H1 multipliers from hyperparameters
+ self.attention_in_multiplier = self.find_hparam(["attention_in_multiplier"], optional=True)
+ self.attention_out_multiplier = self.find_hparam(["attention_out_multiplier"], optional=True)
+ self.ssm_in_multiplier = self.find_hparam(["ssm_in_multiplier"], optional=True)
+ self.ssm_out_multiplier = self.find_hparam(["ssm_out_multiplier"], optional=True)
+ self.mlp_multipliers = self.find_hparam(["mlp_multipliers"], optional=True)
+ self.ssm_multipliers = self.find_hparam(["ssm_multipliers"], optional=True)
+ self.intermediate_size = self.find_hparam(["intermediate_size"])
+ self.key_multiplier = self.find_hparam(["key_multiplier"], optional=True)
+
+ def find_hparam(self, keys: Iterable[str], *args, **kwargs) -> Any:
+ prefixed = []
+ for pfx in self.hparam_prefixes:
+ prefixed.extend("_".join([pfx, k]) for k in keys)
+ keys = list(keys) + prefixed
+ return super().find_hparam(keys, *args, **kwargs)
+
+ def set_vocab(self):
+ self._set_vocab_gpt2()
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ tensors = list(super().modify_tensors(data_torch, name, bid))
+ tensor = tensors[0][1]
+
+ if "down_proj" in name:
+ tensor = tensor * self.mlp_multipliers[1]
+ elif "gate_proj" in name:
+ tensor = tensor * self.mlp_multipliers[0]
+ elif "k_proj" in name:
+ tensor = tensor * self.key_multiplier * self.attention_in_multiplier
+ elif "q_proj" in name:
+ tensor = tensor * self.attention_in_multiplier
+ elif "v_proj" in name:
+ tensor = tensor * self.attention_in_multiplier
+ elif "o_proj" in name:
+ tensor = tensor * self.attention_out_multiplier
+ elif "out_proj" in name:
+ tensor = tensor * self.ssm_out_multiplier
+ elif "in_proj" in name:
+ tensor = tensor * self.ssm_in_multiplier
+ zxbcdt_multipliers = self.hparams["ssm_multipliers"]
+ intermediate_size = self.hparams["mamba_d_ssm"]
+ groups_time_state_size = self.hparams["mamba_n_groups"] * self.hparams["mamba_d_state"]
+ tensor[:intermediate_size, :] *= zxbcdt_multipliers[0]
+ tensor[intermediate_size:2 * intermediate_size, :] *= zxbcdt_multipliers[1]
+ tensor[2 * intermediate_size:2*intermediate_size + groups_time_state_size, :] *= zxbcdt_multipliers[2]
+ tensor[2*intermediate_size + groups_time_state_size:2*intermediate_size +
+ 2*groups_time_state_size, :] *= zxbcdt_multipliers[3]
+ tensor[2*intermediate_size + 2*groups_time_state_size:, :] *= zxbcdt_multipliers[4]
+ elif "lm_head" in name:
+ tensor = tensor * self.hparams["lm_head_multiplier"]
+ elif "embed_tokens" in name:
+ tensor = tensor * self.hparams["embedding_multiplier"]
+ elif "mamba.norm" in name:
+ tensor = tensor.reshape(self.n_group, self.d_inner // self.n_group)
+
+ tensors = [(tensors[0][0], tensor)]
+ return tensors
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ ## General Params ##
+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
+ # Override some Mamba2 defaults
+ self.gguf_writer.add_block_count(self.block_count)
+ self.gguf_writer.add_context_length(self.hparams.get("max_position_embeddings", 0))
+ self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])
+
+ ## Attention params ##
+ self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) # Override value 0 from Mamba2
+ self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"])
+ self.gguf_writer.add_key_length(self.hparams["head_dim"])
+ self.gguf_writer.add_value_length(self.hparams["head_dim"])
+
+ ## Validation ##
+ assert self.hparams.get("hidden_act") in [None, "silu"], "Only SILU activation supported"
+ assert self.d_inner % self.d_head == 0, \
+ f"SSM inner size {self.d_inner} not a multiple of head dim {self.d_head}"
+
+ # Add any other Falcon Mamba2 specific configuration
+ self.gguf_writer.add_rope_freq_base(self.find_hparam(["rope_theta"]))
+
+
[email protected]("HunYuanMoEV1ForCausalLM")
+class HunYuanMoEModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.HUNYUAN_MOE
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # For handling tied embeddings
+ self._tok_embd = None
+
+ def set_vocab(self):
+ from transformers import AutoTokenizer
+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+
+ # 1. Get the pre-tokenizer identifier hash
+ tokpre = self.get_vocab_base_pre(tokenizer)
+
+ # 2. Reverse-engineer the merges list from mergeable_ranks
+ merges = []
+ vocab = {}
+ mergeable_ranks = tokenizer.mergeable_ranks
+ for token, rank in mergeable_ranks.items():
+ vocab[QwenModel.token_bytes_to_string(token)] = rank
+ if len(token) == 1:
+ continue
+ merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)
+ if len(merged) == 2: # todo this is an assert in Qwen, why?
+ merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
+
+ # 3. Generate the tokens and toktypes lists
+ vocab_size = self.hparams["vocab_size"]
+ assert tokenizer.vocab_size == vocab_size
+ special_tokens = tokenizer.special_tokens
+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
+ tokens: list[str] = []
+ toktypes: list[int] = []
+ for i in range(vocab_size):
+ if i not in reverse_vocab:
+ tokens.append(f"[PAD{i}]")
+ toktypes.append(gguf.TokenType.UNUSED)
+ else:
+ token = reverse_vocab[i]
+ tokens.append(token)
+ if i in special_tokens.values():
+ toktypes.append(gguf.TokenType.CONTROL)
+ else:
+ toktypes.append(gguf.TokenType.NORMAL)
+
+ # 4. Write all vocab-related fields to the GGUF writer
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+ self.gguf_writer.add_token_merges(merges)
+
+ # 5. Add special tokens and chat templates
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)
+ special_vocab.add_to_gguf(self.gguf_writer)
+ # FIX for BOS token: Overwrite incorrect id read from config.json
+ self.gguf_writer.add_bos_token_id(127959) # <|bos|>
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hparams = self.hparams
+
+ self.gguf_writer.add_expert_count(hparams["num_experts"])
+ self.gguf_writer.add_expert_shared_feed_forward_length(hparams["intermediate_size"])
+
+ moe_intermediate_size = hparams["moe_intermediate_size"]
+ assert all(n == moe_intermediate_size[0] for n in moe_intermediate_size)
+ self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size[0])
+
+ moe_topk = hparams["moe_topk"]
+ assert all(topk == moe_topk[0] for topk in moe_topk)
+ self.gguf_writer.add_expert_used_count(moe_topk[0])
+
+ moe_shared_expert = hparams["num_shared_expert"]
+ assert all(n == moe_shared_expert[0] for n in moe_shared_expert)
+ self.gguf_writer.add_expert_shared_count(moe_shared_expert[0])
+
+ # Rope
+ rope_scaling = hparams.get("rope_scaling", {})
+ if rope_scaling.get("type") == "dynamic":
+ alpha = rope_scaling.get("alpha", 1000)
+ base = hparams.get("rope_theta", 10000.0)
+ dim = (hparams["hidden_size"] // hparams["num_attention_heads"]) # 128
+ scaled_base = base * (alpha**(dim / (dim-2))) # 10000 * (1000 ** (128 / 126)) = 11158839.9251
+ self.gguf_writer.add_rope_freq_base(scaled_base)
+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+ self.gguf_writer.add_rope_scaling_factor(1)
+ # There is no consistent way to calculate ctx from alpha, and the config is incorrectly set to 32k
+ self.gguf_writer.add_rope_scaling_orig_ctx_len(256 * 1024) # 256k context length
+ self.gguf_writer.add_context_length(256 * 1024) # 256k context length
+
+ assert alpha == 1000 and base == 10000.0 and dim == 128 \
+ and self.hparams["max_position_embeddings"] in [32 * 1024, 256 * 1024] , \
+ "HunYuan dynamic RoPE scaling assumptions changed, please update the logic or context length manually"
+
+ _experts: list[dict[str, Tensor]] | None = None
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name == "model.embed_tokens.weight":
+ self._tok_embd = data_torch.clone()
+
+ if name == "lm_head.weight":
+ if self.hparams.get("tie_word_embeddings", False):
+ logger.info("Skipping tied output layer 'lm_head.weight'")
+ return []
+
+ if name.find("mlp.experts") != -1:
+ n_experts = self.hparams["num_experts"]
+ assert bid is not None
+
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) >= n_experts * 3:
+ # merge the experts into a single 3d tensor
+ tensors: list[tuple[str, Tensor]] = []
+ for w_name in ["down_proj", "gate_proj", "up_proj"]:
+ data: list[Tensor] = []
+
+ for xid in range(n_experts):
+ ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"
+ data.append(self._experts[bid][ename])
+ del self._experts[bid][ename]
+
+ data_torch = torch.stack(data, dim=0)
+ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+ new_name = self.map_tensor_name(merged_name)
+ tensors.append((new_name, data_torch))
+
+ return tensors
+ else:
+ return []
+
+ return [(self.map_tensor_name(name), data_torch)]
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+ if self._experts is not None:
+ experts = [k for d in self._experts for k in d.keys()]
+ if len(experts) > 0:
+ raise ValueError(f"Unprocessed experts: {experts}")
+
+
[email protected]("SmolLM3ForCausalLM")
+class SmolLM3Model(LlamaModel):
+ model_arch = gguf.MODEL_ARCH.SMOLLM3
+
+ def set_vocab(self):
+ super().set_vocab()
+ # remove unsupported array slicing in chat template
+ # ref: https://huggingface.co/ggml-org/SmolLM3-3B-GGUF/discussions/1
+ from transformers import AutoTokenizer
+ tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ if tokenizer.chat_template is not None:
+ chat_template = tokenizer.chat_template.replace("[:]", "")
+ self.gguf_writer.add_chat_template(chat_template)
+
+
[email protected]("Lfm2ForCausalLM")
[email protected]("LFM2ForCausalLM")
+class LFM2Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.LFM2
+
+ def _add_feed_forward_length(self):
+ ff_dim = self.hparams["block_ff_dim"]
+
+ auto_adjust_ff_dim = self.hparams["block_auto_adjust_ff_dim"]
+ ff_dim = self.hparams["block_ff_dim"]
+ ffn_dim_multiplier = self.hparams["block_ffn_dim_multiplier"]
+ multiple_of = self.hparams["block_multiple_of"]
+
+ if auto_adjust_ff_dim:
+ ff_dim = int(2 * ff_dim / 3)
+ # custom dim factor multiplier
+ if ffn_dim_multiplier is not None:
+ ff_dim = int(ffn_dim_multiplier * ff_dim)
+ ff_dim = multiple_of * ((ff_dim+multiple_of-1) // multiple_of)
+
+ self.gguf_writer.add_feed_forward_length(ff_dim)
+
+ def set_gguf_parameters(self):
+ # set num_key_value_heads only for attention layers
+ self.hparams["num_key_value_heads"] = [
+ self.hparams["num_key_value_heads"] if layer_type == "full_attention" else 0
+ for layer_type in self.hparams["layer_types"]
+ ]
+
+ super().set_gguf_parameters()
+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
+ self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])
+ self.gguf_writer.add_layer_norm_rms_eps(self.hparams["norm_eps"])
+ self._add_feed_forward_length()
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # conv op requires 2d tensor
+ if 'conv.conv' in name:
+ data_torch = data_torch.squeeze(1)
+
+ return [(self.map_tensor_name(name), data_torch)]
+
+
###### CONVERSION LOGIC ######
@@ -7184,105 +8204,6 @@ class LazyTorchTensor(gguf.LazyBase):
return cls._wrap_fn(func)(*args, **kwargs)
-def parse_args() -> argparse.Namespace:
- parser = argparse.ArgumentParser(description="Convert a huggingface model to a GGML compatible file")
- parser.add_argument(
- "--vocab-only",
- action="store_true",
- help="extract only the vocab",
- )
- parser.add_argument(
- "--outfile",
- type=Path,
- help="path to write to; default: based on input. {ftype} will be replaced by the outtype.",
- )
- parser.add_argument(
- "--outtype",
- type=str,
- choices=["f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"],
- default="f16",
- help=
- "output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0"
- " for ternary, and auto for the highest-fidelity 16-bit float type depending on the first loaded tensor type",
- )
- parser.add_argument(
- "--bigendian",
- action="store_true",
- help="model is executed on big endian machine",
- )
- parser.add_argument(
- "model",
- type=str,
- help="directory containing model file or huggingface repository ID (if --remote)",
- nargs="?",
- )
- parser.add_argument(
- "--use-temp-file",
- action="store_true",
- help="use the tempfile library while processing (helpful when running out of memory, process killed)",
- )
- parser.add_argument(
- "--no-lazy",
- action="store_true",
- help="use more RAM by computing all outputs before writing (use in case lazy evaluation is broken)",
- )
- parser.add_argument(
- "--model-name",
- type=str,
- default=None,
- help="name of the model",
- )
- parser.add_argument(
- "--verbose",
- action="store_true",
- help="increase output verbosity",
- )
- parser.add_argument(
- "--split-max-tensors",
- type=int,
- default=0,
- help="max tensors in each split",
- )
- parser.add_argument(
- "--split-max-size",
- type=str,
- default="0",
- help="max size per split N(M|G)",
- )
- parser.add_argument(
- "--dry-run",
- action="store_true",
- help="only print out a split plan and exit, without writing any new files",
- )
- parser.add_argument(
- "--no-tensor-first-split",
- action="store_true",
- help="do not add tensors to the first split (disabled by default)")
- parser.add_argument("--metadata", type=Path, help="Specify the path for an authorship metadata override file")
- parser.add_argument("--print-supported-models", action="store_true", help="Print the supported models")
- parser.add_argument(
- "--remote",
- action="store_true",
- help=
- "(Experimental) Read safetensors file remotely without downloading to disk. Config and tokenizer files will"
- " still be downloaded. To use this feature, you need to specify Hugging Face model repo name instead of a"
- " local directory. For example: 'HuggingFaceTB/SmolLM2-1.7B-Instruct'. Note: To access gated repo, set"
- " HF_TOKEN environment variable to your Hugging Face token.",
- )
- parser.add_argument(
- "--mmproj",
- action="store_true",
- help=
- "(Experimental) Export multimodal projector (mmproj) for vision models. This will only work on some"
- " vision models. A prefix 'mmproj-' will be added to the output file name.",
- )
-
- args = parser.parse_args()
- if not args.print_supported_models and args.model is None:
- parser.error("the following arguments are required: model")
- return args
-
-
def split_str_to_n_bytes(split_str: str) -> int:
if split_str.endswith("K"):
n = int(split_str[:-1]) * 1000
@@ -7320,103 +8241,4 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
arch = vision_config["architectures"][0]
if arch is None:
raise ValueError("Failed to detect model architecture")
- return arch
-
-
-def main() -> None:
- args = parse_args()
-
- if args.print_supported_models:
- logger.error("Supported models:")
- ModelBase.print_registered_models()
- sys.exit(0)
-
- if args.verbose:
- logging.basicConfig(level=logging.DEBUG)
- else:
- logging.basicConfig(level=logging.INFO)
-
- if args.remote:
- hf_repo_id = args.model
- from huggingface_hub import snapshot_download
- local_dir = snapshot_download(
- repo_id=hf_repo_id, allow_patterns=["LICENSE", "*.json", "*.md", "*.txt", "tokenizer.model"])
- dir_model = Path(local_dir)
- logger.info(f"Downloaded config and tokenizer to {local_dir}")
- else:
- hf_repo_id = None
- dir_model = Path(args.model)
-
- if not dir_model.is_dir():
- logger.error(f'Error: {dir_model} is not a directory')
- sys.exit(1)
-
- ftype_map: dict[str, gguf.LlamaFileType] = {
- "f32": gguf.LlamaFileType.ALL_F32,
- "f16": gguf.LlamaFileType.MOSTLY_F16,
- "bf16": gguf.LlamaFileType.MOSTLY_BF16,
- "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,
- "tq1_0": gguf.LlamaFileType.MOSTLY_TQ1_0,
- "tq2_0": gguf.LlamaFileType.MOSTLY_TQ2_0,
- "auto": gguf.LlamaFileType.GUESSED,
- }
-
- is_split = args.split_max_tensors > 0 or args.split_max_size != "0"
- if args.use_temp_file and is_split:
- logger.error("Error: Cannot use temp file when splitting")
- sys.exit(1)
-
- if args.outfile is not None:
- fname_out = args.outfile
- elif hf_repo_id:
- # if remote, use the model ID as the output file name
- fname_out = Path("./" + hf_repo_id.replace("/", "-") + "-{ftype}.gguf")
- else:
- fname_out = dir_model
-
- logger.info(f"Loading model: {dir_model.name}")
-
- if args.mmproj:
- if "mmproj" not in fname_out.name:
- fname_out = ModelBase.add_prefix_to_filename(fname_out, "mmproj-")
-
- with torch.inference_mode():
- output_type = ftype_map[args.outtype]
- model_type = ModelType.MMPROJ if args.mmproj else ModelType.TEXT
- hparams = ModelBase.load_hparams(dir_model)
- model_architecture = get_model_architecture(hparams, model_type)
- logger.info(f"Model architecture: {model_architecture}")
- try:
- model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)
- except NotImplementedError:
- logger.error(f"Model {model_architecture} is not supported")
- sys.exit(1)
-
- model_instance = model_class(
- dir_model,
- output_type,
- fname_out,
- is_big_endian=args.bigendian,
- use_temp_file=args.use_temp_file,
- eager=args.no_lazy,
- metadata_override=args.metadata,
- model_name=args.model_name,
- split_max_tensors=args.split_max_tensors,
- split_max_size=split_str_to_n_bytes(args.split_max_size),
- dry_run=args.dry_run,
- small_first_shard=args.no_tensor_first_split,
- remote_hf_model_id=hf_repo_id)
-
- if args.vocab_only:
- logger.info("Exporting model vocab...")
- model_instance.write_vocab()
- logger.info(f"Model vocab successfully exported to {model_instance.fname_out}")
- else:
- logger.info("Exporting model...")
- model_instance.write()
- out_path = f"{model_instance.fname_out.parent}{os.sep}" if is_split else model_instance.fname_out
- logger.info(f"Model successfully exported to {out_path}")
-
-
-if __name__ == '__main__':
- main()
+ return arch
\ No newline at end of file
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index 7b71bca86..b377fdf28 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -16,6 +16,7 @@ import torch
import numpy as np
from auto_round.export.export_to_gguf.config import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES
+from auto_round.utils import get_reciprocal
GGML_QUANT_TYPE = {}
@@ -65,7 +66,7 @@ def ggml_quant(
d_wmin = d_wmin.to(device) if d_wmin is not None else d_wmin
new_data = quant_func(
blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)
-
+
assert new_data.shape[-1] == type_size
new_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)
@@ -87,11 +88,11 @@ def make_qx_quants(data, bits, rmse_type=0, qw=None):
nmax = pow(2, bits - 1)
imax = abs(data).argmax(axis=-1, keepdims=True)
group_max = torch.take_along_dim(data, imax, dim=-1)
- iscales = torch.where(group_max != 0, -nmax / group_max, 0)
+ iscales = -nmax * get_reciprocal(group_max)
if rmse_type == 0:
L = (torch.round(iscales * data).clip(-nmax, nmax - 1) + nmax).to(torch.uint8)
- scales = torch.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])
+ scales = get_reciprocal(iscales).reshape(iscales.shape[:2])
return scales, L
return_early = False
@@ -112,9 +113,9 @@ def make_qx_quants(data, bits, rmse_type=0, qw=None):
w = torch.sqrt(torch.abs(data))
sumlx = torch.sum(w * data * L, dim=-1)
suml2 = torch.sum(w * L * L, dim=-1)
- scales = torch.where(suml2 != 0, sumlx / suml2, 0)
+ scales = sumlx * get_reciprocal(suml2)
if return_early:
- iscales_inv = torch.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])
+ iscales_inv = get_reciprocal(iscales).reshape(iscales.shape[:2])
scales = torch.where(suml2 > 0, 0.5 * (scales + iscales_inv), iscales_inv)
L = (L + nmax).to(torch.uint8)
return scales, L
@@ -122,7 +123,7 @@ def make_qx_quants(data, bits, rmse_type=0, qw=None):
for _is in range(-9, 10):
if _is == 0:
continue
- iscales = torch.where(group_max != 0, -(nmax + -0.1 * _is) / nmax, 0)
+ iscales = -(nmax + -0.1 * _is) * get_reciprocal(group_max)
tmp_L = torch.round(iscales * data).clip(-nmax, nmax - 1)
sumlx = torch.sum(w * data * L, dim=-1)
suml2 = torch.sum(w * L * L, dim=-1)
@@ -140,7 +141,7 @@ def make_q3_quants(data, bits, do_rmse=False):
nmax = pow(2, bits - 1)
imax = abs(data).argmax(axis=-1, keepdims=True)
group_max = torch.take_along_dim(data, imax, dim=-1)
- iscale = torch.where(group_max != 0, -nmax / group_max, 0)
+ iscale = -nmax * get_reciprocal(group_max)
if do_rmse:
L = torch.round(iscale * data).clip(-nmax, nmax - 1)
w = torch.pow(data, 2)
@@ -161,11 +162,10 @@ def make_q3_quants(data, bits, do_rmse=False):
L[:,:, i][replace_idx] = new_L[replace_idx]
sumlx = slx
suml2 = sl2
- isuml2 = torch.where(suml2 == 0, 0, 1 / suml2)
- return sumlx * isuml2, L.to(torch.uint8)
+ return sumlx * get_reciprocal(suml2), L.to(torch.uint8)
L = torch.round(iscale * data).clip(-nmax, nmax - 1) + nmax
- scales = torch.where(iscale != 0, 1 / iscale, 0).reshape(iscale.shape[:2])
+ scales = get_reciprocal(iscale).reshape(iscale.shape[:2])
return scales, L.to(torch.uint8)
@@ -199,7 +199,7 @@ def make_qkx2_quants(data, bits, weights=None, rmin=-1.0, rdelta=0.1, nstep=20,
group_min[group_min > 0] = 0
scale = (group_max - group_min) / nmax
- iscale = torch.where(scale == 0, 0, 1 / scale)
+ iscale = get_reciprocal(scale)
l_values = torch.round(iscale * (data - group_min))
L = torch.clip(l_values, 0, nmax).to(torch.uint8)
@@ -213,7 +213,7 @@ def make_qkx2_quants(data, bits, weights=None, rmin=-1.0, rdelta=0.1, nstep=20,
for step in range(nstep):
new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)
- new_iscale = torch.where(new_scale == 0, 0, 1 / new_scale)
+ new_iscale = get_reciprocal(new_scale)
l_values = torch.round(new_iscale * (data - group_min))
Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)
@@ -251,7 +251,7 @@ def make_qkx3_quants(data, bits, weights, rmin=-1.0, rdelta=0.1, nstep=20, use_m
def make_qp_quants(nmax, data, quant_weights):
group_max = torch.max(data, dim=-1, keepdim=True)[0]
scale = group_max / nmax
- iscale = torch.where(scale == 0, 0, 1 / scale)
+ iscale = get_reciprocal(scale)
L = torch.round(iscale * data)
diffs = data - scale * L
@@ -261,7 +261,7 @@ def make_qp_quants(nmax, data, quant_weights):
if _is == 0:
continue
scale_is = group_max / (0.1 * _is + nmax)
- iscale_is = torch.where(scale == 0, 0, 1 / scale_is)
+ iscale_is = get_reciprocal(scale_is)
tmp_L = torch.round(iscale_is * data).clip(max=nmax)
diffs = data - scale_is * tmp_L
@@ -315,7 +315,7 @@ def q4_0_quant_block(blocks, scale=None, zp=None, **kwargs):
imax = abs(blocks).argmax(axis=-1, keepdims=True)
max = torch.take_along_dim(blocks, imax, axis=-1)
d = max / -8
- id = torch.where(d == 0, 0, 1 / d)
+ id = get_reciprocal(d)
qs = torch.trunc(blocks.to(torch.float64) * id.to(torch.float64) + 8.5).clip(0, 15).to(torch.uint8)
@@ -338,7 +338,7 @@ def q4_1_quant_block(blocks, scale=None, zp=None, **kwargs):
max = blocks.max(axis=-1, keepdims=True)[0]
min = blocks.min(axis=-1, keepdims=True)[0]
d = (max - min) / 15
- id = torch.where(d == 0, 0, 1 / d)
+ id = get_reciprocal(d)
qs = torch.trunc((blocks - min) * id + 0.5).clip(0, 15).to(torch.uint8)
@@ -361,7 +361,7 @@ def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
max = torch.take_along_dim(blocks, imax, dim=-1)
d = max / -16
- id = torch.where(d == 0, 0, 1 / d)
+ id = get_reciprocal(d)
n_blocks = blocks.shape[0]
block_size = GGML_QUANT_SIZES["q5_0"][0]
@@ -391,7 +391,7 @@ def q5_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
n_blocks = blocks.shape[0]
block_size = GGML_QUANT_SIZES["q5_1"][0]
- id = torch.where(d == 0, 0, 1 / d)
+ id = get_reciprocal(d)
q = torch.trunc((blocks - min) * id + 0.5).clip(0, 31).to(torch.uint8).cpu().numpy()
qs = q.reshape((n_blocks, 2, block_size // 2))
@@ -411,7 +411,7 @@ def q8_0_quant_block(blocks, scale=None, zp=None, **kwargs) -> np.ndarray:
d = scale.reshape((-1, 1))
else:
d = torch.abs(blocks).max(dim=1, keepdim=True)[0] / 127
- id = torch.where(d == 0, 0, 1 / d)
+ id = get_reciprocal(d)
qs = torch.clip(torch_roundf(blocks * id), -128, 127)
@@ -434,17 +434,17 @@ def q2_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, i
mins = wmin.reshape((-1, QK_K // 16))
output_d = d_scale.reshape(-1, 1).to(torch.float32)
output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
- output_scale = torch.round(scales / output_d).clip(0, 15).to(torch.uint8)
- output_scale |= torch.round(mins / output_dmin).clip(0, 15).to(torch.uint8) << 4
+ output_scale = torch.round(scales * get_reciprocal(output_d)).clip(0, 15).to(torch.uint8)
+ output_scale |= torch.round(mins * get_reciprocal(output_dmin)).clip(0, 15).to(torch.uint8) << 4
all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 3).to(torch.uint8)
elif original:
scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0]
max_mins = torch.max(mins, dim=-1, keepdim=True)[0]
- output_d = torch.where(max_scales > 0, max_scales / 15, 0)
- output_dmin = torch.where(max_mins > 0, max_mins / 15., 0)
- id_scales = torch.where(max_scales > 0, 15. / max_scales, 0)
- id_mins = torch.where(max_mins > 0, 15. / max_mins, 0)
+ output_d = (max_scales / 15).clamp(min=0)
+ output_dmin = (max_mins / 15).clamp(min=0)
+ id_scales = (15 * get_reciprocal(max_scales)).clamp(min=0)
+ id_mins = (15 * get_reciprocal(max_mins)).clamp(min=0)
replace_ids = (max_scales > 0).squeeze()
output_scale = torch.zeros_like(scales).to(torch.uint8)
@@ -474,8 +474,8 @@ def q2_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, i
mins = mins.reshape((-1, QK_K // 16))
output_d = d_scale.reshape(-1, 1).to(torch.float32)
output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
- output_scale = torch.round(scales / output_d).clip(0, 15).to(torch.uint8)
- output_scale |= torch.round(mins / output_dmin).clip(0, 15).to(torch.uint8) << 4
+ output_scale = torch.round(scales * get_reciprocal(output_d)).clip(0, 15).to(torch.uint8)
+ output_scale |= torch.round(mins * get_reciprocal(output_dmin)).clip(0, 15).to(torch.uint8) << 4
all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 3).to(torch.uint8)
output_scale = output_scale.cpu().numpy()
@@ -500,17 +500,17 @@ def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,
if scale is not None:
qdq_scale = scale.reshape(-1, QK_K // 16).to(torch.float32)
dq_scale = d_scale.reshape(-1, 1).to(torch.float32)
- all_L = (torch.round(blocks / qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)
- q_scales_offset = torch.round(qdq_scale / dq_scale).clip(-32, 31) + 32
+ all_L = (torch.round(blocks * get_reciprocal(qdq_scale.unsqueeze(-1))).clip(-4, 3) + 4).to(torch.uint8)
+ q_scales_offset = torch.round(qdq_scale * get_reciprocal(dq_scale)).clip(-32, 31) + 32
elif original: ## this is correct
scales, _ = make_q3_quants(blocks, bits=3, do_rmse=True)
scales_abs_max = abs(scales).argmax(dim=-1, keepdim=True)
max_scales_mag = torch.take_along_dim(scales, scales_abs_max, dim=-1)
- inverse_dq_scale = torch.where(max_scales_mag != 0, -32 / max_scales_mag, 0)
- dq_scale = torch.where(inverse_dq_scale != 0, 1 / inverse_dq_scale, 0)
+ inverse_dq_scale = -32 * get_reciprocal(max_scales_mag)
+ dq_scale = get_reciprocal(inverse_dq_scale)
qscale = torch.round(inverse_dq_scale * scales).clip(-32, 31)
qdq_scale = dq_scale.to(torch.float32) * qscale
- reverse_qdq_scale = torch.where(qdq_scale == 0, 0, 1 / qdq_scale)
+ reverse_qdq_scale = get_reciprocal(qdq_scale)
all_L = (torch.round(blocks * reverse_qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)
q_scales_offset = torch.round(qdq_scale * inverse_dq_scale).clip(-32, 31) + 32
else:
@@ -521,8 +521,8 @@ def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,
blocks = blocks.reshape((nb, QK_K // 16, 16))
qdq_scale = scales.reshape((-1, QK_K // 16)).to(torch.float32)
dq_scale = d_scale.reshape(-1, 1).to(torch.float32)
- all_L = (torch.round(blocks / qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)
- q_scales_offset = torch.round(qdq_scale / dq_scale).clip(-32, 31) + 32
+ all_L = (torch.round(blocks * get_reciprocal(qdq_scale.unsqueeze(-1))).clip(-4, 3) + 4).to(torch.uint8)
+ q_scales_offset = torch.round(qdq_scale * get_reciprocal(dq_scale)).clip(-32, 31) + 32
q_scales_offset = q_scales_offset.cpu().numpy().astype(np.uint8)
@@ -557,15 +557,16 @@ def q4_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, i
mins = wmin.reshape(-1, QK_K // 32)
output_d = d_scale.reshape(-1, 1).to(torch.float32)
output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
- q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)
- q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)
- all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 15).to(torch.uint8)
+ q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)
+ q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)
+ all_L = torch.round(
+ (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 15).to(torch.uint8)
elif original:
scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0]
max_mins = torch.max(mins, dim=-1, keepdim=True)[0]
- id_scales = torch.where(max_scales > 0, 63. / max_scales, 0)
- id_mins = torch.where(max_mins > 0, 63. / max_mins, 0)
+ id_scales = (63 * get_reciprocal(max_scales)).clamp(min=0)
+ id_mins = (63 * get_reciprocal(max_mins)).clamp(min=0)
output_d = max_scales / 63
output_dmin = max_mins / 63
q_scales = torch.round(id_scales * scales).clip(0, 63).to(torch.uint8)
@@ -588,9 +589,10 @@ def q4_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, i
mins = mins.reshape((-1, QK_K // 32))
output_d = d_scale.reshape(-1, 1).to(torch.float32)
output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
- q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)
- q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)
- all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 15).to(torch.uint8)
+ q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)
+ q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)
+ all_L = torch.round(
+ (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 15).to(torch.uint8)
output_scale[:, :4] = q_scales[:, :4]
output_scale[:, 4:8] = q_mins[:, :4]
@@ -625,15 +627,16 @@ def q5_k_quant_block(
mins = wmin.reshape(-1, QK_K // 32)
output_d = d_scale.reshape(-1, 1).to(torch.float32)
output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
- q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)
- q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)
- all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 31).to(torch.uint8)
+ q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)
+ q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)
+ all_L = torch.round(
+ (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 31).to(torch.uint8)
elif original:
scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0]
max_mins = torch.max(mins, dim=-1, keepdim=True)[0]
- id_scales = torch.where(max_scales > 0, 63. / max_scales, 0)
- id_mins = torch.where(max_mins > 0, 63. / max_mins, 0)
+ id_scales = (63 * get_reciprocal(max_scales)).clamp(min=0)
+ id_mins = (63 * get_reciprocal(max_mins)).clamp(min=0)
output_d = max_scales / 63
output_dmin = max_mins / 63
q_scales = torch.round(id_scales * scales).clip(0, 63).to(torch.uint8)
@@ -657,9 +660,10 @@ def q5_k_quant_block(
mins = mins.reshape((-1, QK_K // 32))
output_d = d_scale.reshape(-1, 1).to(torch.float32)
output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
- q_scales = torch.round(scales / output_d).clip(0, 63).to(torch.uint8)
- q_mins = torch.round(mins / output_dmin).clip(0, 63).to(torch.uint8)
- all_L = torch.round((blocks + mins.unsqueeze(-1)) / scales.unsqueeze(-1)).clip(0, 31).to(torch.uint8)
+ q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)
+ q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)
+ all_L = torch.round(
+ (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 31).to(torch.uint8)
output_scale[:, :4] = q_scales[:, :4]
output_scale[:, 4:8] = q_mins[:, :4]
@@ -692,14 +696,14 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,
if scale is not None:
scales = scale.reshape(-1, QK_K // 16)
output_d = d_scale.reshape(-1, 1).to(torch.float32)
- output_scale = torch.round(scales / output_d).clip(max=127).to(torch.int8)
- all_L = torch.round(blocks / scales.unsqueeze(-1) + 32).clip(0, 63).to(torch.uint8)
+ output_scale = torch.round(scales * get_reciprocal(output_d)).clip(max=127).to(torch.int8)
+ all_L = torch.round(blocks * get_reciprocal(scales.unsqueeze(-1)) + 32).clip(0, 63).to(torch.uint8)
elif original:
scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)
imax = abs(scales).argmax(dim=-1, keepdim=True)
max_scales = torch.take_along_dim(scales, imax, dim=-1)
- iscales = torch.where(max_scales != 0, -128 / max_scales, 0)
- output_d = torch.where(iscales != 0, 1 / iscales, -1)
+ iscales = -128 * get_reciprocal(max_scales)
+ output_d = get_reciprocal(iscales)
output_scale = torch.round(iscales * scales).clip(max=127).to(torch.int8)
d_tmp = output_d * output_scale.to(torch.float32)
replace_ids = d_tmp != 0
@@ -713,8 +717,8 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,
blocks = blocks.reshape((nb, QK_K // 16, 16))
scales = scales.reshape((-1, QK_K // 16))
output_d = d_scale.reshape(-1, 1).to(torch.float32)
- output_scale = torch.round(scales / output_d).clip(max=127).to(torch.int8)
- all_L = torch.round(blocks / scales.unsqueeze(-1) + 32).clip(0, 63).to(torch.uint8)
+ output_scale = torch.round(scales * get_reciprocal(output_d)).clip(max=127).to(torch.int8)
+ all_L = torch.round(blocks * get_reciprocal(scales.unsqueeze(-1)) + 32).clip(0, 63).to(torch.uint8)
tmp_L = all_L.reshape(nb, 4, 64) & 0xF
output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index efa2579d3..d237035e5 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -546,7 +546,7 @@ def tune(args):
if gguf_format in file:
gguf_file = file
- logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")
+ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")
if eval_model_dtype == "float32" or eval_model_dtype == "auto":
logger.warning(
"set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
@@ -636,7 +636,7 @@ def eval(args):
tasks, model_args, device_str = _eval_init(
args.tasks, args.model, args.device, args.disable_trust_remote_code, args.eval_model_dtype)
- # load after _eval_int in order to make sure import torch after set CUDA_VISBILE_DEVICES
+ # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES
from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model
from auto_round.utils import logger
@@ -659,7 +659,7 @@ def eval(args):
from lm_eval.utils import make_table # pylint: disable=E0401
tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
- logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")
+ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")
if eval_model_dtype == "float32" or eval_model_dtype == "auto":
logger.warning(
"set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
@@ -728,7 +728,7 @@ def eval_task_by_task(
eval_model_dtype = get_model_dtype(eval_model_dtype)
if is_gguf_file:
tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
- logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")
+ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")
if eval_model_dtype == "float32" or eval_model_dtype == "auto":
logger.warning(
"set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 6038802a6..ac0f9b039 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1929,3 +1929,10 @@ def clean_module_parameter(submodule, parameter):
submodule._buffers[parameter] = None
else:
submodule._parameters[parameter] = None
+
+def get_reciprocal(tensor):
+ if torch.dtype is torch.float16:
+ tensor = torch.sign(tensor) * torch.clamp(torch.abs(tensor), min=1e-5)
+ else:
+ tensor = torch.where(torch.abs(tensor) < 1e-30, 0, tensor)
+ return torch.where(tensor != 0, 1 / tensor, torch.zeros_like(tensor))
\ No newline at end of file