Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support customized mixed bits for gguf (#615)

Wenhua Chenge06bdbc31936
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 70eb63393..c2adb83ae 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -472,7 +472,7 @@ class AutoRound(object):
if self.data_type == "int" and hasattr(self, "formats") and (
"auto_round" in self.formats or "auto_gptq" in self.formats or "auto_awq" in self.formats):
for n, m in self.model.named_modules():
- if isinstance(m, self.supported_types) :
+ if isinstance(m, self.supported_types):
if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
self.layer_config[n] = {"bits": 16}
logger.info(
@@ -533,6 +533,8 @@ class AutoRound(object):
if not ("gguf" in format_ or "fake" in format_):
only_gguf = False
break
+ if "fake" == formats:
+ only_gguf = False
if only_gguf:
self.scale_dtype = torch.float32
logger.info(f"change `scale_dtype` to `torch.float32`")
@@ -712,19 +714,22 @@ class AutoRound(object):
model(**data)
if cnt > self.nsamples:
break
- except:
- logger.warning("out of vram, fallback to cpu, pleaser use more gpus by setting `--device 0,1,2,3`")
- model = self.model.to("cpu")
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(
- self.model) ##self.model.hf_device_map has not been changed
- cnt = 0
- for data in self.dataloader:
- cnt += data["input_ids"].shape[0]
- data = to_device(data, self.model.device)
- model(**data)
- if cnt > self.nsamples:
- break
+ except RuntimeError as e:
+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
+ logger.warning("out of vram, fallback to cpu, pleaser use more gpus by setting `--device 0,1,2,3`")
+ model = self.model.to("cpu")
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ accelerate.hooks.remove_hook_from_submodules(
+ self.model) ##self.model.hf_device_map has not been changed
+ cnt = 0
+ for data in self.dataloader:
+ cnt += data["input_ids"].shape[0]
+ data = to_device(data, self.model.device)
+ model(**data)
+ if cnt > self.nsamples:
+ break
+ else:
+ raise
for hook in hooks:
hook.remove()
@@ -830,6 +835,8 @@ class AutoRound(object):
m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False,
enable_round_tuning=False)
m = m.unwrapper({})
+ else:
+ raise
if self.low_gpu_mem_usage:
clear_memory()
if self.is_packing_immediate:
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index e5ec0b54a..4fde404ad 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -129,15 +129,15 @@ def make_q3_quants(data, bits, do_rmse=False):
if len(suml2.shape) == 2:
suml2 = suml2.unsqueeze(-1)
slx = sumlx - w * data * L
- repalce_idx = slx > 0
+ replace_idx = slx > 0
sl2 = suml2 - w * L * L
new_L = torch.round(data * sl2 / slx).clip(-nmax, nmax - 1)
- tmp_repalce_idx = repalce_idx & (new_L != L)
- slx[tmp_repalce_idx] += w[tmp_repalce_idx] * data[tmp_repalce_idx] * new_L[tmp_repalce_idx]
- sl2[tmp_repalce_idx] += w[tmp_repalce_idx] * new_L[tmp_repalce_idx] * new_L[tmp_repalce_idx]
- repalce_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)
+ tmp_replace_idx = replace_idx & (new_L != L)
+ slx[tmp_replace_idx] += w[tmp_replace_idx] * data[tmp_replace_idx] * new_L[tmp_replace_idx]
+ sl2[tmp_replace_idx] += w[tmp_replace_idx] * new_L[tmp_replace_idx] * new_L[tmp_replace_idx]
+ replace_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)
- L[repalce_idx] = new_L[repalce_idx]
+ L[replace_idx] = new_L[replace_idx]
sumlx = slx
suml2 = sl2
@@ -417,7 +417,7 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
tmp_inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
all_L = torch.clip(
- torch.round((blocks + mins.reshape(*mins.shape, 1)) / tmp_inverse_scale.reshape(*scales.shape, 1)), 0,
+ torch.round((blocks + mins.reshape(*mins.shape, 1)) * tmp_inverse_scale.reshape(*scales.shape, 1)), 0,
3).to(torch.uint8)
else:
scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)
@@ -469,10 +469,11 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
if scale is not None:
- scales = scale.reshape(-1, QK_K // 16)
+ scales = scale.reshape(-1, QK_K // 16).to(torch.float32)
output_d = d_scale.reshape(-1, 1).to(torch.float32)
iscales = torch.where(output_d == 0, 0, 1 / output_d)
- all_L = torch.round(blocks / scales.unsqueeze(-1)).clip(-4, 3) + 4
+ inverse_scale = torch.where(scales==0, 0, 1.0 / scales)
+ all_L = torch.round(blocks*inverse_scale.unsqueeze(-1)).clip(-4, 3) + 4
all_L = all_L.to(torch.uint8)
else:
scales, all_L = make_q3_quants(blocks, bits=3, do_rmse=True)
@@ -525,7 +526,8 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)
max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).clip(0, 15).to(
+ inverse_scale = torch.where(scales==0,0,1.0/scales)
+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(0, 15).to(
torch.uint8)
else:
scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)
@@ -585,7 +587,8 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)
max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)
- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).clip(0, 31).to(
+ inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1))*inverse_scale.reshape(*scales.shape, 1)).clip(0, 31).to(
torch.uint8)
else:
scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)
@@ -643,7 +646,8 @@ def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
scales = scale.reshape(-1, QK_K // 16)
output_d = d_scale.reshape(-1, 1).to(torch.float32)
iscales = torch.where(output_d == 0, 0, 1 / output_d)
- all_L = torch.round(blocks / scales.reshape(*scales.shape, 1)).clip(0, 63).to(torch.uint8)
+ inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)
+ all_L = (torch.round(blocks * inverse_scale).clip(-32, 31)+32).to(torch.uint8)
else:
scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)
imax = abs(scales).argmax(dim=-1, keepdim=True)
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 40d13e19b..3007aa0dc 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1243,13 +1243,7 @@ def _gguf_args_check(args_or_ar, format_str=None):
logger.warning(
f"format {format} does not support for {', '.join(unsupport_list)},"
f" reset to {', '.join(reset_list)}.")
- if not isinstance(args_or_ar, argparse.Namespace) and len(unsupport_list) > 0:
- for layer_name in args_or_ar.layer_config:
- if args_or_ar.layer_config[layer_name]['bits'] >= 16:
- continue
- for k in args_or_ar.layer_config[layer_name]:
- if hasattr(args_or_ar, k):
- args_or_ar.layer_config[layer_name][k] = getattr(args_or_ar, k)
+# Removed obsolete commented-out block for improved readability and maintainability.
return args_or_ar
@@ -1625,6 +1619,9 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):
i_attention_wv = 0
i_ffn_down = 0
layer_config_copy = copy.deepcopy(layer_config)
+ target_bits = None
+ if inner_gguf_format.startswith("gguf:q") and len(inner_gguf_format) >= 7 and (inner_gguf_format[6]).isdigit():
+ target_bits = int(inner_gguf_format[6])
for layer_name, config in layer_config_copy.items():
if not check_to_quantized(config):
@@ -1638,8 +1635,18 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):
i_layer = _get_digital_in_layer_name(layer_name)
gguf_name = tensor_map.get_name(layer_name)
-
- if lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:
+ if target_bits is not None and "bits" in config and config["bits"] != target_bits:
+ bits_index = 6
+ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1:]
+ if not new_type in GGUF_INNER_CONFIG:
+ for tmp_type in (new_type[:bits_index + 1] + "_k", new_type[:bits_index + 1] + "_1",
+ new_type[:bits_index + 1] + "_0"):
+ if tmp_type in GGUF_INNER_CONFIG:
+ new_type = tmp_type
+ break
+ if not new_type in GGUF_INNER_CONFIG:
+ raise ValueError(f"invalid bit setting for {layer_name}")
+ elif lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:
if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:
new_type = "gguf:q8_0"
elif "lm_head" in GGUF_CONFIG[target_gguf_format]:
@@ -1751,7 +1758,8 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):
new_type = "gguf:q5_k"
elif target_gguf_format == "gguf:q5_k_m":
new_type = "gguf:q5_k"
- if input_features % block_size != 0:
+ new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]
+ if input_features % new_block_size != 0:
if new_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):
new_type = "gguf:q5_0"
elif new_type == "gguf:q5_k":