PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support customized mixed bits for gguf (#615)
Wenhua Chenge06bdbc31936
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 70eb63393..c2adb83ae 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -472,7 +472,7 @@ class AutoRound(object):if self.data_type == "int" and hasattr(self, "formats") and ("auto_round" in self.formats or "auto_gptq" in self.formats or "auto_awq" in self.formats):for n, m in self.model.named_modules():- if isinstance(m, self.supported_types) :+ if isinstance(m, self.supported_types):if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:self.layer_config[n] = {"bits": 16}logger.info(@@ -533,6 +533,8 @@ class AutoRound(object):if not ("gguf" in format_ or "fake" in format_):only_gguf = Falsebreak+ if "fake" == formats:+ only_gguf = Falseif only_gguf:self.scale_dtype = torch.float32logger.info(f"change `scale_dtype` to `torch.float32`")@@ -712,19 +714,22 @@ class AutoRound(object):model(**data)if cnt > self.nsamples:break- except:- logger.warning("out of vram, fallback to cpu, pleaser use more gpus by setting `--device 0,1,2,3`")- model = self.model.to("cpu")- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- accelerate.hooks.remove_hook_from_submodules(- self.model) ##self.model.hf_device_map has not been changed- cnt = 0- for data in self.dataloader:- cnt += data["input_ids"].shape[0]- data = to_device(data, self.model.device)- model(**data)- if cnt > self.nsamples:- break+ except RuntimeError as e:+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ logger.warning("out of vram, fallback to cpu, pleaser use more gpus by setting `--device 0,1,2,3`")+ model = self.model.to("cpu")+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(+ self.model) ##self.model.hf_device_map has not been changed+ cnt = 0+ for data in self.dataloader:+ cnt += data["input_ids"].shape[0]+ data = to_device(data, self.model.device)+ model(**data)+ if cnt > self.nsamples:+ break+ else:+ raisefor hook in hooks:hook.remove()@@ -830,6 +835,8 @@ class AutoRound(object):m = WrapperLinear(m, enable_minmax_tuning=False, enable_norm_bias_tuning=False,enable_round_tuning=False)m = m.unwrapper({})+ else:+ raiseif self.low_gpu_mem_usage:clear_memory()if self.is_packing_immediate:diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex e5ec0b54a..4fde404ad 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -129,15 +129,15 @@ def make_q3_quants(data, bits, do_rmse=False):if len(suml2.shape) == 2:suml2 = suml2.unsqueeze(-1)slx = sumlx - w * data * L- repalce_idx = slx > 0+ replace_idx = slx > 0sl2 = suml2 - w * L * Lnew_L = torch.round(data * sl2 / slx).clip(-nmax, nmax - 1)- tmp_repalce_idx = repalce_idx & (new_L != L)- slx[tmp_repalce_idx] += w[tmp_repalce_idx] * data[tmp_repalce_idx] * new_L[tmp_repalce_idx]- sl2[tmp_repalce_idx] += w[tmp_repalce_idx] * new_L[tmp_repalce_idx] * new_L[tmp_repalce_idx]- repalce_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)+ tmp_replace_idx = replace_idx & (new_L != L)+ slx[tmp_replace_idx] += w[tmp_replace_idx] * data[tmp_replace_idx] * new_L[tmp_replace_idx]+ sl2[tmp_replace_idx] += w[tmp_replace_idx] * new_L[tmp_replace_idx] * new_L[tmp_replace_idx]+ replace_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)- L[repalce_idx] = new_L[repalce_idx]+ L[replace_idx] = new_L[replace_idx]sumlx = slxsuml2 = sl2@@ -417,7 +417,7 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmax_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)tmp_inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)all_L = torch.clip(- torch.round((blocks + mins.reshape(*mins.shape, 1)) / tmp_inverse_scale.reshape(*scales.shape, 1)), 0,+ torch.round((blocks + mins.reshape(*mins.shape, 1)) * tmp_inverse_scale.reshape(*scales.shape, 1)), 0,3).to(torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)@@ -469,10 +469,11 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleoutput_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)if scale is not None:- scales = scale.reshape(-1, QK_K // 16)+ scales = scale.reshape(-1, QK_K // 16).to(torch.float32)output_d = d_scale.reshape(-1, 1).to(torch.float32)iscales = torch.where(output_d == 0, 0, 1 / output_d)- all_L = torch.round(blocks / scales.unsqueeze(-1)).clip(-4, 3) + 4+ inverse_scale = torch.where(scales==0, 0, 1.0 / scales)+ all_L = torch.round(blocks*inverse_scale.unsqueeze(-1)).clip(-4, 3) + 4all_L = all_L.to(torch.uint8)else:scales, all_L = make_q3_quants(blocks, bits=3, do_rmse=True)@@ -525,7 +526,8 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_winv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).clip(0, 15).to(+ inverse_scale = torch.where(scales==0,0,1.0/scales)+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) * inverse_scale.reshape(*scales.shape, 1)).clip(0, 15).to(torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)@@ -585,7 +587,8 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_winv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)- all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).clip(0, 31).to(+ inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1))*inverse_scale.reshape(*scales.shape, 1)).clip(0, 31).to(torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)@@ -643,7 +646,8 @@ def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scalescales = scale.reshape(-1, QK_K // 16)output_d = d_scale.reshape(-1, 1).to(torch.float32)iscales = torch.where(output_d == 0, 0, 1 / output_d)- all_L = torch.round(blocks / scales.reshape(*scales.shape, 1)).clip(0, 63).to(torch.uint8)+ inverse_scale = torch.where(scales == 0, 0, 1.0 / scales)+ all_L = (torch.round(blocks * inverse_scale).clip(-32, 31)+32).to(torch.uint8)else:scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)imax = abs(scales).argmax(dim=-1, keepdim=True)diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 40d13e19b..3007aa0dc 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1243,13 +1243,7 @@ def _gguf_args_check(args_or_ar, format_str=None):logger.warning(f"format {format} does not support for {', '.join(unsupport_list)},"f" reset to {', '.join(reset_list)}.")- if not isinstance(args_or_ar, argparse.Namespace) and len(unsupport_list) > 0:- for layer_name in args_or_ar.layer_config:- if args_or_ar.layer_config[layer_name]['bits'] >= 16:- continue- for k in args_or_ar.layer_config[layer_name]:- if hasattr(args_or_ar, k):- args_or_ar.layer_config[layer_name][k] = getattr(args_or_ar, k)+# Removed obsolete commented-out block for improved readability and maintainability.return args_or_ar@@ -1625,6 +1619,9 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):i_attention_wv = 0i_ffn_down = 0layer_config_copy = copy.deepcopy(layer_config)+ target_bits = None+ if inner_gguf_format.startswith("gguf:q") and len(inner_gguf_format) >= 7 and (inner_gguf_format[6]).isdigit():+ target_bits = int(inner_gguf_format[6])for layer_name, config in layer_config_copy.items():if not check_to_quantized(config):@@ -1638,8 +1635,18 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):i_layer = _get_digital_in_layer_name(layer_name)gguf_name = tensor_map.get_name(layer_name)-- if lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:+ if target_bits is not None and "bits" in config and config["bits"] != target_bits:+ bits_index = 6+ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1:]+ if not new_type in GGUF_INNER_CONFIG:+ for tmp_type in (new_type[:bits_index + 1] + "_k", new_type[:bits_index + 1] + "_1",+ new_type[:bits_index + 1] + "_0"):+ if tmp_type in GGUF_INNER_CONFIG:+ new_type = tmp_type+ break+ if not new_type in GGUF_INNER_CONFIG:+ raise ValueError(f"invalid bit setting for {layer_name}")+ elif lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:new_type = "gguf:q8_0"elif "lm_head" in GGUF_CONFIG[target_gguf_format]:@@ -1751,7 +1758,8 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):new_type = "gguf:q5_k"elif target_gguf_format == "gguf:q5_k_m":new_type = "gguf:q5_k"- if input_features % block_size != 0:+ new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]+ if input_features % new_block_size != 0:if new_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):new_type = "gguf:q5_0"elif new_type == "gguf:q5_k":