Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gguf bug of moe models and lmhead/embedding bits setting regression (#628)

Heng Guo20eed5b6c469
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index fc3f74b80..25f832b6b 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -430,7 +430,7 @@ class AutoRound(object):
if "mx_fp" in self.data_type and self.group_size != 32:
logger.warning("mx_fp should only support group_size of 32 in real deployment")
- if "nv_fp" in self.data_type and (self.group_size != 32 or self.group_size!=16):
+ if "nv_fp" in self.data_type and (self.group_size != 32 or self.group_size != 16):
logger.warning("mx_fp should only support group_size of 16/32 in real deployment")
if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
@@ -466,7 +466,7 @@ class AutoRound(object):
raise ValueError("gguf format is not compatible with other formats, please choose only one of them")
if has_gguf and self.iters != 0 and self.bits != 3:
logger.warning(
- "We recommend setting `iters=0` when exporting to GGUF format,"
+ "`iters=0` is recommended when exporting to GGUF format except for bits 3,"
" as we have optimized the RTN method for this case."
" We are likely to release new algorithm for certain configurations in the future."
)
@@ -498,8 +498,8 @@ class AutoRound(object):
"You can also try to increase the model_max_length to avoid this issue.")
self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)
- if self.group_size==0 and "fp8" not in self.data_type:
- logger.warning("group_size of 0 is not supported for data_type other than fp8 ")
+ if self.group_size == 0 and "fp8" not in self.data_type:
+ logger.warning("group_size of 0 is not supported for data_type other than fp8 ")
def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):
"""Quantizes the model and saves it in the specified format(s).
@@ -539,7 +539,7 @@ class AutoRound(object):
if not ("gguf" in format_ or "fake" in format_):
only_gguf = False
break
- if len(formats)==1 and "fake" == formats[0]:
+ if len(formats) == 1 and "fake" == formats[0]:
only_gguf = False
if only_gguf:
self.scale_dtype = torch.float32
@@ -691,8 +691,8 @@ class AutoRound(object):
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
from accelerate.big_modeling import dispatch_model
dispatch_model(model, model.hf_device_map)
- else:
- model = self.model.to(self.device)
+
+
def register_act_hook(model):
def get_act_max_hook(module, input, output):
@@ -716,6 +716,7 @@ class AutoRound(object):
hooks = register_act_hook(self.model)
try:
+ model = self.model.to(self.device)
cnt = 0
for data in self.dataloader:
cnt += data["input_ids"].shape[0]
@@ -724,9 +725,15 @@ class AutoRound(object):
if cnt > self.nsamples:
break
except RuntimeError as e:
- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
- logger.warning("out of vram, fallback to cpu, pleaser use more gpus by setting `--device 0,1,2,3`")
- model = self.model.to("cpu")
+ try:
+ model=self.model.to("cpu")
+ clear_memory()
+ logger.warning("out of vram, fallback to blockwise way to get imatrix, this may affect accuracy")
+ self.model_forward_blockwise()
+ except Exception as e:
+ model=self.model.to("cpu")
+ clear_memory()
+ logger.warning("fallback to cpu, pleaser use more gpus to get imatrix, by setting `--device 0,1,2,3`")
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
accelerate.hooks.remove_hook_from_submodules(
self.model) ##self.model.hf_device_map has not been changed
@@ -737,8 +744,6 @@ class AutoRound(object):
model(**data)
if cnt > self.nsamples:
break
- else:
- raise
for hook in hooks:
hook.remove()
@@ -776,7 +781,8 @@ class AutoRound(object):
keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]
self.layer_config[embedding_name] = self.layer_config.get(embedding_name, {})
for key in keys:
- if key in self.layer_config[embedding_name]:
+ if key in self.layer_config[embedding_name]:
+ self.layer_config["fixed_by_user"] = True
continue
self.layer_config[embedding_name][key] = getattr(config, "get")(key)
setattr(get_module(self.model, embedding_name), key, config.get(key))
@@ -796,7 +802,8 @@ class AutoRound(object):
keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]
self.layer_config[lm_head_name] = self.layer_config.get(lm_head_name, {})
for key in keys:
- if key in self.layer_config[lm_head_name]:
+ if key in self.layer_config[lm_head_name] and self.layer_config[lm_head_name].get("fixed_by_user",
+ False):
continue
self.layer_config[lm_head_name][key] = getattr(config, "get")(key)
setattr(get_module(self.model, lm_head_name), key, config.get(key))
@@ -866,15 +873,87 @@ class AutoRound(object):
self.quantized = True
return self.model, self.layer_config
- def quantize(self):
- """Quantize the model and return the quantized model along with layer configurations.
- the entry of AutoRound.
+ def model_forward_blockwise(self):
+ if bool(self.quant_block_list):
+ all_blocks = self.quant_block_list
+ else:
+ all_blocks = get_block_names(self.model)
+ if len(all_blocks) == 0:
+ raise ValueError("could not find any blocks, please check the model or quant_block_list")
+
+ ##ignore layers
+ all_first_block_names = [block[0] for block in all_blocks]
+ all_inputs = self.cache_inter_data(
+ all_first_block_names, self.nsamples)
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ accelerate.hooks.remove_hook_from_submodules(self.model) ##self.model.hf_device_map has not been changed
+
+ for block_names in all_blocks:
+ inputs = all_inputs[block_names[0]]
+ all_inputs.pop(block_names[0])
+ keys = inputs.keys()
+ input_id_str = [key for key in keys if key.startswith('hidden_state')]
+ if len(input_id_str) != 1:
+ raise RuntimeError(f"hidden_states arg mismatch error,"
+ "please raise an issue in https://github.com/intel/auto-round/issues")
+ inputs["input_ids"] = inputs.pop(input_id_str[0], None)
+
+ clear_memory(self.inputs)
+
+ if "input_ids" in inputs.keys():
+ total_samples = len(inputs["input_ids"])
+ if total_samples < self.batch_size:
+ self.batch_size = total_samples
+ logger.warning(f"force the train batch size to {total_samples}")
+
+ input_ids = inputs["input_ids"]
+ inputs.pop("input_ids", None)
+ for block_name in block_names:
+ input_others = inputs
+ clear_memory()
+ input_ids = to_device(input_ids, self.cache_device)
+ input_others = to_device(input_others, self.cache_device)
+ ## as in the calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
+ tmp_dtype = self.amp_dtype if self.amp else torch.float32
+ for i in range(len(input_ids)):
+ input_ids[i] = input_ids[i].to(tmp_dtype)
+
+ for key in input_others.keys():
+ if isinstance(input_others[key], torch.Tensor) and (
+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16
+ ):
+ input_others[key] = input_others[key].to(tmp_dtype)
+ elif isinstance(input_others[key], list):
+ for i in range(len(input_others[key])):
+ to_dtype(input_others[key][i], tmp_dtype)
+
+ block = get_module(self.model, block_name)
+
+ if self.device_map is not None:
+ from accelerate import dispatch_model
+ for n, m in block.named_modules():
+ if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
+ continue
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+ hook = AlignDevicesHook(m.tuning_device, io_same_device=True)
+ add_hook_to_module(m, hook, True)
+
+ input_ids = self.get_block_outputs(block, input_ids, input_others,
+ self.batch_size * self.infer_bs_coeff,
+ self.device,
+ self.cache_device)
+
+ if self.device_map is not None:
+ accelerate.hooks.remove_hook_from_submodules(
+ block)
+ mv_module_from_gpu(block, self.low_cpu_mem_usage)
+
+ def quantize(self):
+ """Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.
Returns:
The quantized model and layer configurations.
"""
- ## TODO add common check
-
self._check_compatibility()
self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config)
if not hasattr(self, "formats"):
@@ -1104,7 +1183,7 @@ class AutoRound(object):
# Skip unsupported types
supported_types = self.supported_types
- if not isinstance(m,supported_types):
+ if not isinstance(m, supported_types):
continue
all_supported_layer_names.append(n)
@@ -1122,8 +1201,8 @@ class AutoRound(object):
for match_name in matched_names:
layer_config[match_name] = val
else:
- tmp_m = get_module(self.model,name)
- if not isinstance(tmp_m, torch.nn.Embedding): ##TODO not good code style
+ tmp_m = get_module(self.model, name)
+ if not isinstance(tmp_m, torch.nn.Embedding): ##TODO not good code style
raise ValueError(f"key {name} in layer_config is invalid, please have a double check")
has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)
@@ -1145,6 +1224,7 @@ class AutoRound(object):
for key in keys:
if key not in layer_config[n].keys():
layer_config[n][key] = getattr(self, key)
+ layer_config[n]["fixed_by_user"] = True
# If the layer is not in the config and not part of a quantization block,
# use default configuration and set specific values
else:
@@ -1161,7 +1241,7 @@ class AutoRound(object):
# If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block
if (n not in layers_in_blocks and check_to_quantized(layer_config[n])
- and not isinstance(m,torch.nn.Embedding)):
+ and not isinstance(m, torch.nn.Embedding)):
has_qlayer_outside_block = True
in_features, out_features = get_layer_features(m)
@@ -1355,7 +1435,7 @@ class AutoRound(object):
logger.info("switch to cpu to cache block inputs")
if (self.has_qlayer_outside_block or
self.__class__.__name__ == "AutoRoundMLLM"):
- logger.warning("We strongly recommend using more GPUs in calibration."
+ logger.warning("we strongly recommend using more GPUs in calibration."
" Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy.")
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
accelerate.hooks.remove_hook_from_submodules(
@@ -2072,7 +2152,7 @@ class AutoRound(object):
save_quantized_as_format = EXPORT_FORMAT.get(format)
if "gptq" in format and not self.sym:
logger.warning(
- "The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
+ "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
" particularly for 2-bit quantization and smaller models."
" We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
"the AutoRound format(2/3/4/8 bits)."
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index 0ab57d9b9..96d8bf7da 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -139,7 +139,8 @@ def double_quant_tensor(tensor, bits):
wmax = torch.clamp(tensor.max(-1)[0], min=0)
scale = wmax / maxq
scale = scale.view(-1, 1)
- inverse_scale = torch.where(scale == 0, 0, 1 / scale)
+ # inverse_scale = torch.where(scale == 0, 0, 1 / scale)
+ inverse_scale = torch.where(wmax > 0, maxq / wmax, 0).view(-1, 1)
qdq_tensor = torch.clamp(round_ste(tensor * inverse_scale), max=maxq) * scale
return qdq_tensor, scale
@@ -244,26 +245,26 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_
scale = ((wmax - wmin) / maxq).to(scale_dtype)
scale = torch.clamp(scale, min=q_scale_thresh)
scale = scale.view(-1, super_group_size)
- wmin_m = -wmin # pylint: disable=E1130
- wmin_m = wmin_m.view(-1, super_group_size)
+ wmin = -wmin # pylint: disable=E1130
+ wmin = wmin.view(-1, super_group_size)
##conduct double quant
scale, d_scale = double_quant_tensor(scale, super_bits)
- wmin_m, d_wmin_m = double_quant_tensor(wmin_m, super_bits)
+ wmin, d_wmin = double_quant_tensor(wmin, super_bits)
scale = scale.view(-1, 1)
scale = torch.clamp(scale, q_scale_thresh)
d_scale = torch.clamp(d_scale, q_scale_thresh)
- d_wmin_m = torch.clamp(d_wmin_m, q_scale_thresh)
- wmin_m = wmin_m.view(-1, 1)
+ d_wmin = torch.clamp(d_wmin, q_scale_thresh)
+ wmin = wmin.view(-1, 1)
- int_w = round_ste((tensor + wmin_m) / scale + v)
+ int_w = round_ste((tensor + wmin) / scale + v)
q = torch.clamp(int_w, 0, maxq)
- qdq_result = (scale * q - wmin_m).to(tensor.dtype)
+ qdq_result = (scale * q - wmin).to(tensor.dtype)
qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
- # zp = round_ste(wmin_m / scale) # remove this later
- return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}
+ # zp = round_ste(wmin / scale) # remove this later
+ return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}
@register_dtype("rtn_int_asym_dq")
@@ -324,18 +325,18 @@ def quant_tensor_gguf_asym_dq(
av_x = torch.sqrt(sigma2)
quant_weights = torch.abs(tensor) + av_x
params = search_kwargs[bits]
- scale, wmin_m = iterative_wls_quant_search(
+ scale, wmin = iterative_wls_quant_search(
tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],
use_mad=params["use_mad"], weights=quant_weights
)
scale = scale.to(scale_dtype)
scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)
scale = scale.reshape(-1, super_group_size)
- wmin = wmin_m.reshape(-1, super_group_size)
+ wmin = wmin.reshape(-1, super_group_size)
scale, d_scale = double_quant_tensor(scale, super_bits)
wmin = torch.where(torch.abs(wmin) < 1e-30, 0, wmin)
- wmin_m, d_wmin_m = double_quant_tensor(wmin, super_bits)
- wmin_m = wmin_m.view(-1, 1)
+ wmin, d_wmin = double_quant_tensor(wmin, super_bits)
+ wmin = wmin.view(-1, 1)
scale = scale.view(-1, 1)
else:
imatrix = imatrix.to(tensor.device)
@@ -359,7 +360,7 @@ def quant_tensor_gguf_asym_dq(
params = search_kwargs[bits]
- scale, wmin_m_0 = iterative_wls_quant_search(
+ scale, wmin_0 = iterative_wls_quant_search(
tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],
use_mad=params["use_mad"], weights=quant_weights
)
@@ -367,21 +368,22 @@ def quant_tensor_gguf_asym_dq(
scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)
nmax = 2 ** super_bits - 1
scale = scale.reshape(-1, super_group_size)
- wmin_m = wmin_m_0.reshape(-1, super_group_size)
+ wmin = wmin_0.reshape(-1, super_group_size)
sum_quant_weights = quant_weights.sum(-1, keepdim=True).reshape(-1, super_group_size)
d_scale, q_scale = make_qp_quants(nmax, scale, sum_quant_weights)
- d_wmin_m, q_wmin_m = make_qp_quants(nmax, wmin_m, sum_quant_weights)
+ d_wmin, q_wmin = make_qp_quants(nmax, wmin, sum_quant_weights)
+
d_scale = d_scale.unsqueeze(-1)
- d_wmin_m = d_wmin_m.unsqueeze(-1)
+ d_wmin = d_wmin.unsqueeze(-1)
scale = (d_scale * q_scale).view(-1, 1)
- wmin_m = (d_wmin_m * q_wmin_m).view(-1, 1)
+ wmin = (d_wmin * q_wmin).view(-1, 1)
inverse_scale = torch.where(scale == 0, 0, 1 / scale)
- int_w = torch.clamp(round_ste((tensor + wmin_m) * inverse_scale + v), 0, maxq)
- qdq_result = (scale * int_w - wmin_m).to(orig_dtype)
+ int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)
+ qdq_result = (scale * int_w - wmin).to(orig_dtype)
qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
- return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}
+ return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}
def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None):
@@ -411,29 +413,38 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u
sum_w = torch.sum(weights, dim=1, keepdim=True)
sum_x = torch.sum(weights * data, dim=1, keepdim=True)
- scale = 1 / ((maxq - minq) / (rmax - rmin + 1e-8))
- quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)
+ # scale = 1 / ((maxq - minq) / (rmax - rmin + 1e-8))
+ scale = (rmax - rmin) / (maxq - minq)
+ iscale = torch.where(scale == 0, 0, 1 / scale)
+ # quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)
+ quant_data = torch.clamp(torch.round(iscale * (data - rmin)), minq, maxq)
diff = scale * quant_data + rmin - data
best_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=1, keepdim=True)
for is_ in range(nstep):
factor = rrmin + rdelta * is_ + maxq - minq
- iscale_new = factor / (rmax - rmin + 1e-8)
+ # iscale_new = factor / (rmax - rmin + 1e-8)
+ scale_new = (rmax - rmin) / factor
+ iscale_new = torch.where(scale_new == 0, 0, 1 / scale_new)
quant_data_new = torch.clamp(torch.round(iscale_new * (data - rmin)), minq, maxq)
mul_weights_quant_data = weights * quant_data_new
- sum_l = torch.sum(mul_weights_quant_data, dim=1, keepdim=True)
- sum_l2 = torch.sum(mul_weights_quant_data * quant_data_new, dim=1, keepdim=True)
- sum_xl = torch.sum(mul_weights_quant_data * data, dim=1, keepdim=True)
+ sum_l = torch.sum(mul_weights_quant_data, dim=-1, keepdim=True)
+ sum_l2 = torch.sum(mul_weights_quant_data * quant_data_new, dim=-1, keepdim=True)
+ sum_xl = torch.sum(mul_weights_quant_data * data, dim=-1, keepdim=True)
D = sum_w * sum_l2 - sum_l ** 2
this_scale = (sum_w * sum_xl - sum_x * sum_l) / D
this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D
+ this_min[this_min > 0] = 0
+ this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]
+ reverse_this_scale = torch.where(this_scale == 0, 0, 1 / this_scale)
- quant_data = torch.clamp(torch.round((1 / this_scale) * (data - this_min)), minq, maxq)
+ quant_data = torch.clamp(torch.round(reverse_this_scale * (data - this_min)), minq, maxq)
diff = this_scale * quant_data + this_min - data
- mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=1, keepdim=True)
+ # diff = this_scale * quant_data_new + this_min - data
+ mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=-1, keepdim=True)
idx_to_replace = torch.where((mad < best_mad) & (D > 0))[0]
best_mad[idx_to_replace] = mad[idx_to_replace]
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index b0dbcd7b9..4a3b180c4 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -77,7 +77,6 @@ class SentencePieceTokenTypes(IntEnum):
AnyModel = TypeVar("AnyModel", bound="type[Model]")
-
class OriModel:
_model_classes: dict[str, type[Model]] = {}
@@ -1130,6 +1129,30 @@ class Model(OriModel):
# would require using decorated functions instead of simply defining the property
if "model_arch" not in cls.__dict__:
raise TypeError(f"Missing property 'model_arch' for {cls.__name__!r}")
+
+ def get_moe_name(self, name, new_name):
+ type_mapping = {
+ "FFN_GATE_EXP": ["gate_proj", "w1", "linear"],
+ "FFN_DOWN_EXP": ["down_proj", "w2", "linear_1"],
+ "FFN_UP_EXP": ["up_proj", "w3", "linear_v"],
+ }
+ nums = re.findall('\d+', name)
+ if len(nums) != 2:
+ return name
+ name_tmp = name[:-len(".weight")].replace(f'.{nums[1]}', '')
+ new_name_tmp = new_name[:-len(".weight")]
+ if self.tensor_map.get_type(name_tmp) == self.tensor_map.get_type(new_name_tmp):
+ return name
+
+ tensor_type = self.tensor_map.get_type(new_name_tmp).name
+ experts_name = name_tmp.split(".")[-1]
+ for k, v in type_mapping.items():
+ if experts_name in v:
+ idx = v.index(experts_name)
+ name = name.replace(experts_name, type_mapping[tensor_type][idx])
+ return name
+ return name
+
def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
for name, tensor in self.model.named_parameters():
@@ -1144,7 +1167,7 @@ class Model(OriModel):
if hasattr(module, "scale"):
if hasattr(self, "permute"):
bs = module.weight.shape[0]
- for attr in ["scale", "zp", "w_d_scale", "w_d_wmin_m", "w_wmin_m"]:
+ for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:
if hasattr(module, attr) and getattr(module, attr) is not None:
attr_tensor = getattr(module, attr)
ori_shape = attr_tensor.shape
@@ -1164,17 +1187,17 @@ class Model(OriModel):
zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp
if data_qtype.name.lower().endswith("_k"):
d_scale = module.w_d_scale.to(torch.float32)
- d_wmin_m = module.w_d_wmin_m.to(torch.float32) if hasattr(module, "w_d_wmin_m") else None
- wmin_m = module.w_wmin_m.to(torch.float32) if hasattr(module, "w_wmin_m") else None
+ d_wmin = module.w_d_wmin.to(torch.float32) if hasattr(module, "w_d_wmin") else None
+ wmin = module.w_wmin.to(torch.float32) if hasattr(module, "w_wmin") else None
data = ggml_quant_gpu(
data_torch,
data_qtype.name.lower(),
scale,
zp,
- wmin_m=wmin_m,
+ wmin=wmin,
d_scale=d_scale,
- d_wmin_m=d_wmin_m,
+ d_wmin=d_wmin,
device=device
)
else:
@@ -1199,6 +1222,8 @@ class Model(OriModel):
def get_qtype_by_layer_config(self, layer_config, name, data_qtype):
name = name[:-len('.weight')]
if name not in layer_config or layer_config[name]['bits'] >= 16:
+ if isinstance(data_qtype, bool):
+ return gguf.GGMLQuantizationType.F32
return data_qtype
bits = layer_config[name]['bits']
super_bits = layer_config[name]["super_bits"]
@@ -1315,6 +1340,9 @@ class Model(OriModel):
data_qtype = gguf.GGMLQuantizationType.F16
# data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type
+ # get name by new_name (for experts),
+ name = self.get_moe_name(name, new_name)
+
# get data_qtype by layer_config
data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)
@@ -1384,6 +1412,8 @@ class Model(OriModel):
shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"
# n_dims is implicit in the shape
+ if isinstance(data_qtype, bool):
+ breakpoint()
logger.info(
f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}"
f" --> {data_qtype.name}, shape = {shape_str}")
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index c5633e996..3fcc3f842 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -28,22 +28,22 @@ def register_qtype(name):
return register
-def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, imatrix=None,
+def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, imatrix=None,
device="cuda"):
block_size, type_size = GGML_QUANT_SIZES[ggml_type]
data = data.to(torch.float32).to(device)
scale = scale.to(device) if scale is not None else scale
zp = zp.to(device) if zp is not None else zp
- wmin_m = wmin_m.to(device) if wmin_m is not None else wmin_m
+ wmin = wmin.to(device) if wmin is not None else wmin
d_scale = d_scale.to(device) if d_scale is not None else d_scale
- d_wmin_m = d_wmin_m.to(device) if d_wmin_m is not None else d_wmin_m
+ d_wmin = d_wmin.to(device) if d_wmin is not None else d_wmin
shape = data.shape
n_blocks = data.nelement() // block_size
blocks = data.reshape((n_blocks, block_size))
quant_func = GGML_QUANT_TYPE[ggml_type]
- new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m, imatrix=imatrix)
+ new_data = quant_func(blocks, scale, zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix)
assert new_data.shape[-1] == type_size
new_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)
@@ -404,28 +404,30 @@ def q8_0_quant_block(blocks, scale=None, zp=None, **kwargs) -> np.ndarray:
@register_qtype("q2_k")
-def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):
+def q2_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):
nb = blocks.shape[0]
blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)
if scale is not None:
scales = scale.reshape((-1, QK_K // 16))
- mins = wmin_m.reshape((-1, QK_K // 16))
+ mins = wmin.reshape((-1, QK_K // 16))
output_d = d_scale.reshape(-1, 1).to(torch.float32)
- output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)
+ output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
inv_scales = torch.where(output_d == 0, 0, 1 / output_d)
- inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
- max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)
- max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
+ inv_mins = torch.where(d_wmin == 0, 0, 1 / output_dmin)
+ max_scales = torch.max(scales, dim=-1, keepdim=True)[0]
+ max_mins = torch.max(mins, dim=-1, keepdim=True)[0]
all_L = torch.zeros_like(blocks, dtype=torch.uint8)
else:
scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0]
max_mins = torch.max(mins, dim=-1, keepdim=True)[0]
+ output_d = torch.where(max_scales > 0, max_scales / 15, 0)
+ output_dmin = torch.where(max_mins > 0, max_mins / 15., 0)
inv_scales = torch.where(max_scales > 0, 15. / max_scales, 0)
inv_mins = torch.where(max_mins > 0, 15. / max_mins, 0)
-
+
replace_ids = (max_scales > 0).squeeze()
output_scale = torch.zeros_like(scales).to(torch.uint8)
output_scale[replace_ids] = torch.round(
@@ -434,10 +436,6 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
replace_ids = (max_mins > 0).squeeze()
output_scale[replace_ids] |= torch.round(
inv_mins[replace_ids] * mins[replace_ids]).clip(0, 15).to(torch.uint8) << 4
- if d_scale is None:
- output_d = torch.where(max_scales > 0, max_scales / 15, 0)
- if d_wmin_m is None:
- output_dmin = torch.where(max_mins > 0, max_mins / 15., 0)
d_tmp = output_d * (output_scale & 0xF)
dm_tmp = output_dmin * (output_scale >> 4)
@@ -446,7 +444,6 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
all_L[replace_ids] = torch.round(
(blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 3).to(
torch.uint8)
- all_L = np.clip(all_L.cpu().numpy(), 0, 3).astype(np.uint8)
output_scale = output_scale.cpu().numpy()
all_L = all_L.reshape(-1, 4, 32)
@@ -455,14 +452,13 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
output_dmin = output_dmin.cpu().numpy()
output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
- output_qs = output_qs.reshape((nb, QK_K // 4))
+ output_qs = output_qs.cpu().numpy().astype(np.uint8).reshape((nb, QK_K // 4))
# [scale, qs, d, dmin]
return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)
-
@register_qtype("q3_k")
-def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):
+def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):
nb = blocks.shape[0]
blocks = blocks.reshape(nb, QK_K // 16, 16)
@@ -498,13 +494,12 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
all_L = (torch.round(blocks * reverse_qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(
torch.uint8)
- all_L = all_L.cpu().numpy()
- output_hmask = np.bitwise_or.reduce(
- all_L.reshape(nb, 8, 32) >> 2 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1) # pylint: disable=E1121
- all_L = np.where(all_L > 3, all_L - 4, all_L)
+ output_hmask = all_L.reshape(nb, 8, 32) >> 2 << torch.arange(8, device=all_L.device).reshape(1, 8, 1)
+ output_hmask = np.bitwise_or.reduce(output_hmask.cpu().numpy(), axis=1, dtype=np.uint8) # pylint: disable=E1121
+ all_L = torch.where(all_L > 3, all_L - 4, all_L)
- output_qs = np.bitwise_or.reduce(
- all_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121
+ output_qs = all_L.reshape(nb, 2, 4, 32) << torch.tensor([0, 2, 4, 6], device=all_L.device).reshape(1, 1, 4, 1)
+ output_qs = np.bitwise_or.reduce(output_qs.cpu().numpy(), axis=2, dtype=np.uint8) # pylint: disable=E1121
output_qs = output_qs.reshape(nb, 64).astype(np.uint8)
dq_scale = dq_scale.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)
@@ -514,20 +509,19 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
@register_qtype("q4_k")
-def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):
+def q4_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):
nb = blocks.shape[0]
blocks = blocks.reshape((nb, QK_K // 32, 32))
- output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
- output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)
+ output_scale = torch.empty((nb, K_SCALE_SIZE), dtype=torch.uint8, device=blocks.device)
if scale is not None:
scales = scale.reshape(-1, QK_K // 32)
- mins = wmin_m.reshape(-1, QK_K // 32)
+ mins = wmin.reshape(-1, QK_K // 32)
output_d = d_scale.reshape(-1, 1).to(torch.float32)
- output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)
+ output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
inv_scales = torch.where(output_d == 0, 0, 1 / output_d)
- inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
+ inv_mins = torch.where(d_wmin == 0, 0, 1 / output_dmin)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)
max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
all_L = torch.zeros_like(blocks, dtype=torch.uint8)
@@ -542,18 +536,18 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)
inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)
- q_scales = torch.round(inv_scales * scales).clip(0, 63)
- q_mins = torch.round(inv_mins * mins).clip(0, 63)
+ q_scales = torch.round(inv_scales * scales).clip(0, 63).to(torch.uint8)
+ q_mins = torch.round(inv_mins * mins).clip(0, 63).to(torch.uint8)
if d_scale is None:
output_d = max_scales / 63
- if d_wmin_m is None:
+ if d_wmin is None:
output_dmin = max_mins / 63
d_tmp = output_d * q_scales
dm_tmp = output_dmin * q_mins
- q_scales = q_scales.cpu().numpy().astype(np.uint8)
- q_mins = q_mins.cpu().numpy().astype(np.uint8)
+ # q_scales = q_scales.cpu().numpy().astype(np.uint8)
+ # q_mins = q_mins.cpu().numpy().astype(np.uint8)
output_scale[:, :4] = q_scales[:, :4]
output_scale[:, 4:8] = q_mins[:, :4]
@@ -565,32 +559,32 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
all_L[replace_ids] = torch.round(
(blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(
0, 15).to(torch.uint8)
- all_L = np.clip(all_L.cpu().numpy(), 0, 15).astype(np.uint8)
output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)
output_d = output_d.cpu().numpy()
output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
output_dmin = output_dmin.cpu().numpy()
output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
- output_qs = output_qs.reshape(nb, QK_K // 2)
+ output_qs = output_qs.reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)
+ output_scale = output_scale.cpu().numpy().astype(np.uint8)
# [d, dmin, scale, qs]
return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)
@register_qtype("q5_k")
-def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):
+def q5_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):
nb = blocks.shape[0]
blocks = blocks.reshape((nb, QK_K // 32, 32))
- output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
+ output_scale = torch.empty((nb, K_SCALE_SIZE), dtype=torch.uint8, device=blocks.device)
if scale is not None:
scales = scale.reshape(-1, QK_K // 32)
- mins = wmin_m.reshape(-1, QK_K // 32)
+ mins = wmin.reshape(-1, QK_K // 32)
output_d = d_scale.reshape(-1, 1).to(torch.float32)
- output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)
+ output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
inv_scales = torch.where(output_d == 0, 0, 1 / output_d)
- inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
+ inv_mins = torch.where(d_wmin == 0, 0, 1 / output_dmin)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)
max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
all_L = torch.zeros_like(blocks, dtype=torch.uint8)
@@ -600,20 +594,20 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
max_mins = torch.max(mins, dim=-1, keepdim=True)[0]
inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)
inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)
-
- q_scales = torch.round(inv_scales * scales).clip(0, 63)
- q_mins = torch.round(inv_mins * mins).clip(0, 63)
+
+ q_scales = torch.round(inv_scales * scales).clip(0, 63).to(torch.uint8)
+ q_mins = torch.round(inv_mins * mins).clip(0, 63).to(torch.uint8)
if d_scale is None:
output_d = max_scales / 63
- if d_wmin_m is None:
+ if d_wmin is None:
output_dmin = max_mins / 63
d_tmp = output_d * q_scales
dm_tmp = output_dmin * q_mins
- q_scales = q_scales.cpu().numpy().astype(np.uint8)
- q_mins = q_mins.cpu().numpy().astype(np.uint8)
+ # q_scales = q_scales.cpu().numpy().astype(np.uint8)
+ # q_mins = q_mins.cpu().numpy().astype(np.uint8)
output_scale[:, :4] = q_scales[:, :4]
output_scale[:, 4:8] = q_mins[:, :4]
@@ -625,23 +619,27 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
all_L[replace_ids] = torch.round(
(blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) \
/ d_tmp[replace_ids].unsqueeze(-1)).clip(0, 31).to(torch.uint8)
- all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 31) # (nb, 8, 32)
+ # all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 31) # (nb, 8, 32)
+ all_L = torch.clamp(all_L, 0, 31)
output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)
+ output_qh = all_L >> 4 << torch.arange(8, device=all_L.device).reshape(1, 8, 1)
output_qh = np.bitwise_or.reduce(
- all_L >> 4 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1).astype(np.uint8) # pylint: disable=E1121
+ output_qh.cpu().numpy(), axis=1, dtype=np.uint8).astype(np.uint8) # pylint: disable=E1121
output_d = output_d.cpu().numpy()
output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
output_dmin = output_dmin.cpu().numpy()
output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
- output_qs = output_qs.reshape(nb, QK_K // 2)
+ output_qs = output_qs.reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)
+ output_scale = output_scale.cpu().numpy().astype(np.uint8)
+
# [d, dmin, scale, qh, qs]
return np.concatenate([output_d, output_dmin, output_scale, output_qh, output_qs], axis=-1)
@register_qtype("q6_k")
-def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):
+def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):
nb = blocks.shape[0]
blocks = blocks.reshape((nb, QK_K // 16, 16))
@@ -664,12 +662,14 @@ def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
all_L[replace_ids] = torch.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32) \
.clip(0, 63).to(torch.uint8)
- all_L = all_L.cpu().numpy()
+ # all_L = all_L.cpu().numpy()
tmp_L = all_L.reshape(nb, 4, 64) & 0xF
- output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2)
+ output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)
+ ouptut_qh = (all_L >> 4).reshape(nb, 2, 4, 32) << torch.tensor(
+ [0, 2, 4, 6], device=all_L.device).reshape(1, 1, 4, 1)
output_qh = np.bitwise_or.reduce(
- (all_L >> 4).reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), # pylint: disable=E1121
- axis=2).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121
+ ouptut_qh.cpu().numpy(), # pylint: disable=E1121
+ axis=2, dtype=np.uint8).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121
output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 4a2781a82..a923a37ec 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1584,6 +1584,20 @@ def _get_digital_in_layer_name(layer_name):
else:
return None
+def _search_gguf_type(gguf_type):
+ if gguf_type in GGUF_INNER_CONFIG:
+ return gguf_type
+ pattern = re.compile("gguf:q([0-9]{1,})_[01k]")
+ bits = re.search(pattern, gguf_type)
+ if not bits:
+ raise KeyError(f"{gguf_type} is not a correct gguf type, please check")
+
+ for suffix in ["_0", "_1", "_k"]:
+ if gguf_type.endswith(suffix):
+ continue
+ if (tmp_type := re.sub("_[01k]", suffix, gguf_type)) in GGUF_INNER_CONFIG:
+ return tmp_type
+ return None
##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129
def get_layer_config_by_gguf_format(layer_config, gguf_format, model):
@@ -1651,17 +1665,30 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):
input_features = layer.weight.shape[-1]
i_layer = _get_digital_in_layer_name(layer_name)
+ if lm_head_name is not None and layer_name == lm_head_name:
+ target_bits = int(re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]['lm_head']).group(1))
+ if isinstance(layer, torch.nn.Embedding):
+ target_bits = int(
+ re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]['embedding']).group(1))
+
gguf_name = tensor_map.get_name(layer_name)
- if target_bits is not None and "bits" in config and config["bits"] != target_bits:
- bits_index = 6
+ bits_index = 6
+ if config.get("fixed_by_user", False):
+ if "bits" not in config:
+ logger.warning(
+ f"Setting layer_config requires providing bits, {layer_name} has not bits,"
+ f" using bits={target_bits} instead.")
+ new_type = new_type[:bits_index] + target_bits + new_type[bits_index + 1:]
+ else:
+ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1:]
+ new_type = _search_gguf_type(new_type)
+ if new_type is None:
+ raise ValueError(f"invalid bit setting for {layer_name}")
+ elif (target_bits is not None and
+ "bits" in config and config["bits"] != target_bits):
new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1:]
- if not new_type in GGUF_INNER_CONFIG:
- for tmp_type in (new_type[:bits_index + 1] + "_k", new_type[:bits_index + 1] + "_1",
- new_type[:bits_index + 1] + "_0"):
- if tmp_type in GGUF_INNER_CONFIG:
- new_type = tmp_type
- break
- if not new_type in GGUF_INNER_CONFIG:
+ new_type = _search_gguf_type(new_type)
+ if new_type is None:
raise ValueError(f"invalid bit setting for {layer_name}")
elif lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:
if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:
@@ -1845,4 +1872,3 @@ def get_gguf_qtype_by_layer_config(layer_config):
if bits == 8 and sym and group_size == 32:
return gguf.GGMLQuantizationType.Q8_0
raise ValueError(f"Unknown layer config")
-
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index b58d3acc2..6a5192fdd 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -237,6 +237,39 @@ class TestGGUF(unittest.TestCase):
shutil.rmtree("./saved", ignore_errors=True)
+ def test_q4_k_m(self):
+ model_name = "Qwen/Qwen2.5-7B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ layer_config = {
+ "lm_head": {'bits': 4, 'group_size': 32, 'sym': False, 'data_type': 'int_asym_dq', 'scale_dtype': torch.float32, 'super_bits': 6, 'super_group_size': 8, 'in_blocks': False},
+ "model.embed_tokens": {'bits': 6, 'group_size': 32, 'super_bits': 6, 'super_group_size': 8},
+ "model.layers.12.mlp.gate_proj": {'bits': 3},
+ "model.layers.10.mlp.gate_proj": {'bits': 8},
+ }
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ layer_config=layer_config,
+ iters=0,
+ seqlen=1,
+ dataset=self.llm_dataloader,
+ disable_opt_rtn=True
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")
+ self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16)
+ self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "int_sym_dq")
+ self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "int_asym_dq")
+ self.assertEqual(autoround.model.model.layers[0].self_attn.v_proj.bits, 6)
+ self.assertEqual(autoround.model.model.layers[12].self_attn.v_proj.bits, 4)
+ self.assertEqual(autoround.model.model.embed_tokens.bits, 6)
+ self.assertEqual(autoround.model.model.embed_tokens.group_size, 16)
+ self.assertEqual(autoround.model.model.layers[12].mlp.gate_proj.bits, 3)
+ self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8)
+ self.assertEqual(autoround.layer_config['model.layers.10.mlp.gate_proj']['mostly'], "gguf:q8_0")
+ shutil.rmtree("./saved", ignore_errors=True)
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.py
index 022508213..405596c89 100644
--- a/test/test_cuda/test_gguf.py
+++ b/test/test_cuda/test_gguf.py
@@ -171,33 +171,5 @@ class TestAutoRound(unittest.TestCase):
self.assertGreater(result['results']['piqa']['acc,none'], 0.55)
shutil.rmtree("./saved", ignore_errors=True)
- @require_gguf
- def test_q4_k_m(self):
- model_name = "/models/Qwen2.5-1.5B-Instruct"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- bits, group_size, sym, super_group_size, super_bits = 4, 32, False, 8, 6
- autoround = AutoRound(
- model,
- tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=0,
- seqlen=1,
- dataset=self.llm_dataloader,
- data_type="int_asym_dq",
- super_group_size=super_group_size,
- super_bits=super_bits
- )
- quantized_model_path = "./saved"
- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")
- self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16)
- self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "gguf_int_sym_dq")
- self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "gguf_int_asym_dq")
- self.assertEqual(autoround.model.model.layers[0].self_attn.v_proj.bits, 6)
- self.assertEqual(autoround.model.model.layers[12].self_attn.v_proj.bits, 4)
- shutil.rmtree("./saved", ignore_errors=True)
-
if __name__ == "__main__":
unittest.main()
\ No newline at end of file