PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[GGUF STEP4] fix search bug and improve packing & eval speed (#545)
Heng Guo4b85b512a009
diff --git a/README.md b/README.mdindex 2c72f6e7c..dffcd2135 100644--- a/README.md+++ b/README.md@@ -32,7 +32,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).<div align="left">## What's New-* [2025/04] AutoRound supports some recipes for Qwen3 series, please refer to [Qwen3-8B-sym-recipe](./docs/Qwen3-8B-sym-recipe.md) and [Qwen3-14B-sym-recipe](./docs/Qwen3-14B-sym-recipe.md) for more details.+* [2025/04] AutoRound provides some recipes for Qwen3 series, please refer to [Qwen3-8B-sym-recipe](./docs/Qwen3-8B-sym-recipe.md) and [Qwen3-14B-sym-recipe](./docs/Qwen3-14B-sym-recipe.md) for more details.* [2025/04] AutoRound has been integrated into Transformers. You can run models in the AutoRound format directly with Transformers versions later than 4.51.3.* [2025/03] The INT2-mixed R1 model (~200GB) retains 97.9% accuracy. Checkout [OPEA/DeepSeek-R1-int2-mixed-sym-inc](https://huggingface.co/OPEA/DeepSeek-R1-int2-mixed-sym-inc).diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 6b4649545..d1c5af73f 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -23,7 +23,9 @@ def run_eval():device=args.device,tasks=args.tasks,batch_size=args.eval_bs,- trust_remote_code=not args.disable_trust_remote_code)+ trust_remote_code=not args.disable_trust_remote_code,+ eval_model_dtype=args.eval_model_dtype+ )else:from auto_round.script.llm import evaleval(args)diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex b0b4d5dd2..82f8645d6 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -25,17 +25,19 @@ from lm_eval.models.huggingface import HFLMdef simple_evaluate_user_model(user_model,tokenizer,- batch_size: Optional[int] = None,- max_batch_size: Optional[int] = None,+ batch_size: Optional[int] = 1,+ max_batch_size: Optional[int] = 64,+ eval_model_dtype = "auto",**kwargs):- hflm = HFLM(pretrained=user_model, tokenizer=tokenizer, batch_size=batch_size, max_batch_size=max_batch_size)- return lm_simple_evaluate(- model=hflm,- model_args=None,+ hflm = HFLM(+ pretrained=user_model,+ tokenizer=tokenizer,batch_size=batch_size,max_batch_size=max_batch_size,- **kwargs)+ dtype=eval_model_dtype)+ return lm_simple_evaluate(+ model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, **kwargs)def simple_evaluate(diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 94e77adc4..22c282ae1 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -54,7 +54,8 @@ import torch# autoround importfrom auto_round.utils import logger, LazyImport-from .quant import ggml_quant+from .quant_cpu import ggml_quant_cpu+from .quant_gpu import ggml_quant_gpugguf = LazyImport("gguf")@@ -1134,6 +1135,49 @@ class Model(OriModel):def get_tensors(self) -> Iterator[tuple[str, Tensor]]:for name, tensor in self.model.named_parameters():yield name, tensor++ def _quant_data(self, data_torch, data_qtype, name, bid):+ from auto_round.utils import get_module+ suffix = '.weight'+ if suffix in name:+ layer_name = name[:-len(suffix)]+ module = get_module(self.model, layer_name)+ if hasattr(module, "scale"):+ if hasattr(self, "permute"):+ bs = module.scale.shape[0]+ for attr in ["scale", "zp", "w_d_scale", "w_d_wmin_m", "w_wmin_m"]:+ if hasattr(module, attr) and getattr(module, attr) is not None:+ attr_tensor = getattr(module, attr)+ ori_shape = attr_tensor.shape+ attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), name, bid)[0][1]+ attr_tensor = attr_tensor.reshape(ori_shape)+ setattr(module, attr, attr_tensor)+ scale = module.scale+ zp = module.zp if hasattr(module, "zp") else None+ ggml_quant = ggml_quant_gpu if torch.cuda.is_available() else ggml_quant_cpu++ if data_qtype.name.lower().endswith("_k"):+ d_scale = module.w_d_scale.to(torch.float32)+ d_wmin_m = module.w_d_wmin_m.to(torch.float32)+ wmin_m = module.w_wmin_m.to(torch.float32)+ data = ggml_quant(+ data_torch,+ data_qtype.name.lower(),+ scale,+ zp,+ wmin_m=wmin_m,+ d_scale=d_scale,+ d_wmin_m=d_wmin_m+ )+ else:+ data = ggml_quant(data_torch, data_qtype.name.lower(), scale, zp)+ else:+ data_qtype = gguf.GGMLQuantizationType.F32+ data = data_torch.squeeze().cpu().numpy()+ else:+ data_qtype = gguf.GGMLQuantizationType.F32+ data = data_torch.squeeze().cpu().numpy()+ return data, data_qtypedef prepare_tensors(self):max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")@@ -1225,67 +1269,21 @@ class Model(OriModel):# data_qtype = gguf.GGMLQuantizationType.F16# data = gguf.quants.quantize(data, data_qtype)- # Quant here- def _quant_data(data, data_qtype):- from auto_round.utils import get_module- suffix = '.weight'- if suffix in name:- layer_name = name[:-len(suffix)]- module = get_module(self.model, layer_name)- if hasattr(module, "scale"):-- if hasattr(self, "permute"):- bs = module.scale.shape[0]- for attr in ["scale", "zp", "w_d_scale", "w_d_wmin_m", "w_wmin_m"]:- if hasattr(module, attr) and getattr(module, attr) is not None:- attr_tensor = getattr(module, attr)- ori_shape = attr_tensor.shape- attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), name, bid)[0][1]- attr_tensor = attr_tensor.reshape(ori_shape)- setattr(module, attr, attr_tensor)-- scale = module.scale-- if isinstance(scale, torch.Tensor):- scale = scale.numpy()- zp = module.zp if hasattr(module, "zp") else None- if isinstance(zp, torch.Tensor):- zp = zp.numpy()- if data_qtype.name.lower().endswith("_k"):- d_scale = module.w_d_scale.numpy()- d_wmin_m = module.w_d_wmin_m.numpy()- wmin_m = module.w_wmin_m.numpy()- data = ggml_quant(- data,- data_qtype.name.lower(),- scale,- zp,- wmin_m=wmin_m,- d_scale=d_scale,- d_wmin_m=d_wmin_m)- else:- data = ggml_quant(data, data_qtype.name.lower(), scale, zp)- else:- data_qtype = gguf.GGMLQuantizationType.F32- else:- data_qtype = gguf.GGMLQuantizationType.F32- return data, data_qtype-# for MOE model- if len(data.shape) == 3:+ if len(data_torch.shape) == 3:new_data = []- for idx, arr in enumerate(data):+ for idx, arr in enumerate(data_torch):arr_name = name.split('.')for i in range(len(arr_name) - 1, -1, -1):- if arr_name[i].isdecimal() and int(arr_name[i]) == (data.shape[0] - 1):+ if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):arr_name[i] = str(idx)arr_name = ".".join(arr_name)- arr, data_qtype = _quant_data(arr, data_qtype)+ arr, data_qtype = self._quant_data(arr, data_qtype, name, bid)new_data.append(arr)data = np.array(new_data)del new_dataelse:- data, data_qtype = _quant_data(data, data_qtype)+ data, data_qtype = self._quant_data(data_torch, data_qtype, name, bid)shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shapediff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 485f187f5..81a56616b 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -39,6 +39,9 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):"""Export the model to gguf format."""+ if output_dir is not None and os.path.exists(output_dir):+ logger.warning(f"{output_dir} already exists, this may cause model conflict")+st = time.time()model = kwargs["model"]diff --git a/auto_round/export/export_to_gguf/quant.py b/auto_round/export/export_to_gguf/quant_cpu.pysimilarity index 52%rename from auto_round/export/export_to_gguf/quant.pyrename to auto_round/export/export_to_gguf/quant_cpu.pyindex ca75ad4f1..24b20e94b 100644--- a/auto_round/export/export_to_gguf/quant.py+++ b/auto_round/export/export_to_gguf/quant_cpu.py@@ -39,7 +39,15 @@ def register_block(name):return register-def ggml_quant(data: np.array, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+def ggml_quant_cpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ import torch+ data = data.squeeze().cpu().numpy() if isinstance(data, torch.Tensor) else data+ scale = scale.numpy() if isinstance(scale, torch.Tensor) else scale+ zp = zp.numpy() if isinstance(zp, torch.Tensor) else zp+ wmin_m = wmin_m.numpy() if isinstance(wmin_m, torch.Tensor) else wmin_m+ d_scale = d_scale.numpy() if isinstance(d_scale, torch.Tensor) else d_scale+ d_wmin_m = d_wmin_m.numpy() if isinstance(d_wmin_m, torch.Tensor) else d_wmin_m+block_size, type_size = GGML_QUANT_SIZES[ggml_type]data = data.astype(np.float32, copy=False)@@ -52,6 +60,7 @@ def ggml_quant(data: np.array, ggml_type, scale=None, zp=None, wmin_m=None, d_scelse:worker = 0+ worker = 0if worker > 0:n_groups = (data.shape[0] // worker) or 1blocks = np.array_split(blocks, n_groups, axis=0)@@ -73,6 +82,7 @@ def ggml_quant(data: np.array, ggml_type, scale=None, zp=None, wmin_m=None, d_scelse:quant_func = GGML_QUANT_BLOCK[ggml_type]if ggml_type.endswith("_k"):+ # 0.5588 vs 63.05new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)else:new_data = quant_func(blocks, scale, zp)@@ -165,216 +175,181 @@ def q8_0_quant_block(blocks: np.array, scale=None, zp=None) -> np.ndarray:return np.concatenate([d, qs], axis=1)+def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):+ assert bits in [2, 4], f"bits={bits} is not supported"+ nmax = bits ** 2 -1+ # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K+ if len(data.shape) == 2:+ data_shape = (-1, 8, 32) if bits == 4 else (-1, 16, 16)+ data = data.reshape(data_shape)+ sum_x2 = np.sum(np.power(data, 2), axis=-1, keepdims=True)+ av_x = np.sqrt(sum_x2 / 32)+ weight = np.abs(data) + av_x++ group_min = np.min(data, axis=-1, keepdims=True)+ group_max = np.max(data, axis=-1, keepdims=True)-def make_qkx2_quants(data, weight, nmax, group_size, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):- group_min = np.min(data)- group_max = np.max(data)+ the_mins = -group_min- sum_w = np.sum(weight)- sum_x = np.sum(weight * data)+ sum_w = np.sum(weight, axis=-1, keepdims=True)+ sum_x = np.sum(weight * data, axis=-1, keepdims=True)- if group_min > 0:- group_min = 0- if group_min == group_max:- L = np.zeros(group_size, dtype=np.uint8)- the_min = -group_min- return 0.0, L, the_min+ group_min[group_min > 0] = 0- iscale = nmax / (group_max-group_min)- scale = 1 / iscale- L = np.zeros(group_size, dtype=np.uint8)+ scale = (group_max - group_min) / nmax+ iscale = np.where(scale == 0, 0, 1 /scale)- l_values = np.round(iscale * (data-group_max))+ l_values = np.round(iscale * (data - group_min))L = np.clip(l_values, 0, nmax).astype(np.uint8)- diffs = scale*L + group_min - data++ diffs = scale * L + group_min - datadiffs = np.abs(diffs) if use_mad else diffs**2- best_mad = np.sum(weight * (diffs))+ best_mad = np.sum(weight * diffs, axis=-1, keepdims=True)if nstep < 1:- the_min = -group_min- return scale, L, the_min+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])- Laux = []for step in range(nstep):- iscale = (rmin + rdelta*step + nmax) / (group_max-group_min)- l_values = np.round(iscale * (data-group_min))+ new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)+ new_iscale = np.where(new_scale == 0, 0, 1 /new_scale)++ l_values = np.round(new_iscale * (data - group_min))Laux = np.clip(l_values, 0, nmax).astype(np.uint8)- sum_l = np.sum(weight * Laux)- sum_l2 = np.sum(weight * Laux**2)- sum_xl = np.sum(weight * Laux * data)+ sum_l = np.sum(weight * Laux, axis=-1, keepdims=True)+ sum_l2 = np.sum(weight * Laux**2, axis=-1, keepdims=True)+ sum_xl = np.sum(weight * Laux * data, axis=-1, keepdims=True)- D = sum_w*sum_l2 - sum_l*sum_l- if D > 0:- this_scale = (sum_w*sum_xl - sum_x*sum_l) / D- this_min = (sum_l2*sum_x - sum_l*sum_xl) / D- if this_min > 0:- this_min = 0- this_scale = sum_xl / sum_l2+ D = sum_w * sum_l2 - sum_l * sum_l+ replace_idx = D > 0- diffs = this_scale*Laux + this_min - data- mad = np.sum(weight * diffs**2)+ this_scale = (sum_w * sum_xl - sum_x * sum_l) / D+ this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D+ this_min[this_min > 0] = 0+ this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]- if mad < best_mad:- L = Laux.copy()- best_mad = mad- scale = this_scale- group_min = this_min+ diffs = this_scale * Laux + this_min - data+ diffs = np.abs(diffs) if use_mad else diffs**2+ mad = np.sum(weight * diffs, axis=-1, keepdims=True)- the_min = -group_min- return scale, L, the_min+ replace_idx &= mad < best_mad+ best_mad[replace_idx] = mad[replace_idx]+ L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]+ scale[replace_idx] = this_scale[replace_idx]+ group_min[replace_idx] = this_min[replace_idx]++ the_mins = -group_min+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])@register_block("q2_k")def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):nb = blocks.shape[0]output_scale = np.empty((nb, 16), dtype=np.uint8)- output_d = np.empty(nb, dtype=np.float32)- output_dmin = np.empty(nb, dtype=np.float32)output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)- blocks = blocks.reshape((nb, QK_K // 16, 16))- weight = np.abs(blocks)- scales = np.empty(QK_K // 16, dtype=np.float32)- mins = np.empty((QK_K // 16), dtype=np.float32)+ blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)if scale is not None:- scale = scale.reshape((-1, QK_K // 16))- wmin_m = wmin_m.reshape((-1, QK_K // 16))- output_d = d_scale.astype(np.float32)- output_dmin = d_wmin_m.astype(np.float32)+ scales = scale.reshape((-1, QK_K // 16))+ mins = wmin_m.reshape((-1, QK_K // 16))+ output_d = d_scale.astype(np.float32) if len(d_scale.shape) == 2 \+ else d_scale.reshape(-1, 1).astype(np.float32)+ output_dmin = d_wmin_m.astype(np.float32) if len(d_wmin_m.shape) == 2 \+ else d_wmin_m.reshape(-1, 1).astype(np.float32)inv_scales = np.where(d_scale == 0, 0, 1 / output_d)inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)-- # for i in tqdm(range(nb), desc="packing layer"):- for i in range(nb):- all_L = np.empty(blocks[i].shape, dtype=np.uint8)-- if scale is not None:- scales = scale[i]- mins = wmin_m[i]- inv_scale = inv_scales[i]- inv_min = inv_mins[i]- max_scale = max(scales)- max_min = max(mins)- all_L = np.round((blocks[i] + mins.reshape(-1, 1)) / scales.reshape(-1, 1)).astype(np.uint8)- else:- for j in range(QK_K // 16):- tmp_scale, tmp_l, the_min = make_qkx2_quants(- blocks[i][j], weight[i][j], nmax=3, group_size=16, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)- all_L[j] = tmp_l- scales[j] = tmp_scale- mins[j] = the_min-- max_scale = max(scales)- max_min = max(mins)- inv_scale = 15. / max_scale- inv_min = 15. / max_min-- if max_scale > 0:- output_scale[i] = np.round(inv_scale * scales).astype(np.uint8)- output_d[i] = max_scale / 15.- else:- output_scale[i] = np.zeros_like(scales).astype(np.uint8)- output_d[i] = 0.- if max_min > 0:- output_scale[i] |= (np.round(inv_min * mins).astype(np.uint8) << 4)- output_dmin[i] = max_min / 15.- else:- output_dmin[i] = 0.-- d_tmp = output_d[i] * (output_scale[i] & 0xF)- dm_tmp = output_dmin[i] * (output_scale[i] >> 4)- for j in range(QK_K // 16):- if d_tmp[j] == 0.:- continue- else:- all_L[j] = np.round((blocks[i][j] + dm_tmp[j]) / d_tmp[j]).astype(np.uint8)- all_L = np.clip(all_L, 0, 3)- output_qs[i] = all_L[::4] | (all_L[1::4] << 2) | (all_L[2::4] << 4) | (all_L[3::4] << 6)-+ max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)+ max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)+ all_L = np.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).astype(np.uint8)+ else:+ scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)+ max_scales = np.max(scales, axis=-1, keepdims=True)+ max_mins = np.max(mins, axis=-1, keepdims=True)+ inv_scales = np.where(max_scales > 0, 15. / max_scales, 0)+ inv_mins = np.where(max_mins > 0, 15. / max_mins, 0)++ replace_ids = (max_scales > 0).reshape(-1)+ output_scale = np.zeros_like(scales).astype(np.uint8)+ output_scale[replace_ids] = np.round(inv_scales * scales).astype(np.uint8)++ replace_ids = (max_mins > 0).reshape(-1)+ output_scale[replace_ids] |= np.round(inv_mins * mins).astype(np.uint8) << 4++ if d_scale is None:+ output_d = np.where(max_scales > 0, max_scales / 15, 0)+ if d_wmin_m is None:+ output_dmin = np.where(max_mins > 0 ,max_mins / 15., 0)++ d_tmp = output_d * (output_scale & 0xF)+ dm_tmp = output_dmin * (output_scale >> 4)++ replace_ids = d_tmp != 0+ all_L[replace_ids] = np.round(+ (blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(+ np.uint8)+ all_L = np.clip(all_L, 0, 3)++ output_qs = all_L[:,::4] | (all_L[:,1::4] << 2) | (all_L[:,2::4] << 4) | (all_L[:,3::4] << 6)output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)output_qs = output_qs.reshape((nb, QK_K // 4))# [scale, qs, d, dmin]return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)-+@register_block("q4_k")def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):nb = blocks.shape[0]+ blocks = blocks.reshape((nb, QK_K // 32, 32)) # (nb, 8, 32)+output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)- output_d = np.empty(nb, dtype=np.float32)- output_dmin = np.empty(nb, dtype=np.float32)output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)- blocks = blocks.reshape((nb, QK_K // 32, 32))- sum_x2 = np.sum(np.power(blocks, 2), axis=-1)- av_x = np.sqrt(sum_x2 / 32)- weight = blocks + av_x.reshape((*av_x.shape, 1))- scales = np.empty(QK_K // 32, dtype=np.float32)- mins = np.empty(QK_K // 32, dtype=np.float32)-if scale is not None:- scale = scale.reshape(-1, QK_K // 32)- wmin_m = wmin_m.reshape(-1, QK_K // 32)- output_d = d_scale.astype(np.float32)- output_dmin = d_wmin_m.astype(np.float32)+ scales = scale.reshape(-1, QK_K // 32)+ mins = wmin_m.reshape(-1, QK_K // 32)+ output_d = d_scale.astype(np.float32) if len(d_scale.shape) == 2 \+ else d_scale.reshape(-1, 1).astype(np.float32)+ output_dmin = d_wmin_m.astype(np.float32) if len(d_wmin_m.shape) == 2 \+ else d_wmin_m.reshape(-1, 1).astype(np.float32)inv_scales = np.where(d_scale == 0, 0, 1 / output_d)inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)-- # for i in tqdm(range(nb), desc="packing layer"):- for i in range(nb):- all_L = np.empty(blocks[i].shape, dtype=np.uint8)-- if scale is not None:- scales = scale[i]- mins = wmin_m[i]- inv_scale = inv_scales[i]- inv_min = inv_mins[i]- max_scale = max(scales)- max_min = max(mins)- all_L = np.round((blocks[i] + mins.reshape(-1, 1)) / scales.reshape(-1, 1)).astype(np.uint8)- else:- for j in range(QK_K // 32):- tmp_scale, tmp_l, the_min = make_qkx2_quants(- blocks[i][j], weight[i][j], nmax=15, group_size=32, rmin=-1, rdelta=0.1, nstep=20)- all_L[j] = tmp_l- scales[j] = tmp_scale- mins[j] = the_min-- max_scale = max(scales)- max_min = max(mins)- inv_scale = 63. / max_scale if max_scale > 0 else 0.- inv_min = 63. / max_min if max_min > 0 else 0.-- ls = np.round(inv_scale * scales).astype(np.uint8)- lm = np.round(inv_min * mins).astype(np.uint8)-- output_scale[i][:4] = ls[:4]- output_scale[i][4:8] = lm[:4]-- output_scale[i][8:] = (ls[4:] & 0xF) | ((lm[4:] & 0xF) << 4)- output_scale[i][:4] |= ((ls[4:] >> 4) << 6)- output_scale[i][4:8] |= ((lm[4:] >> 4) << 6)-- if d_scale is None:- output_d[i] = max_scale / 63- if d_wmin_m is None:- output_dmin[i] = max_min / 63-- d_tmp = output_d[i] * ls- dm_tmp = output_dmin[i] * lm-- for j in range(d_tmp.size):- if d_tmp[j] == 0.:- continue- else:- all_L[j] = np.round((blocks[i][j] + dm_tmp[j]) / d_tmp[j]).astype(np.uint8)-- all_L = np.clip(all_L, 0, 15)- output_qs[i] = all_L[::2] | (all_L[1::2] << 4)+ max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)+ max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)+ all_L = np.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).astype(np.uint8)+ else:+ scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)+ max_scales = np.max(scales, axis=-1, keepdims=True)+ max_mins = np.max(mins, axis=-1, keepdims=True)+ inv_scales = np.where(max_scales > 0, 63. / max_scales, 0)+ inv_mins = np.where(max_mins > 0, 63. / max_mins, 0)++ q_scales = np.round(inv_scales * scales).astype(np.uint8)+ q_mins = np.round(inv_mins * mins).astype(np.uint8)++ output_scale[:,:4] = q_scales[:,:4]+ output_scale[:,4:8] = q_mins[:,:4]++ output_scale[:,8:] = (q_scales[:,4:] & 0xF) | ((q_mins[:,4:] & 0xF) << 4)+ output_scale[:,:4] |= ((q_scales[:,4:] >> 4) << 6)+ output_scale[:,4:8] |= ((q_mins[:,4:] >> 4) << 6)++ if d_scale is None:+ output_d = max_scales / 63+ if d_wmin_m is None:+ output_dmin = max_mins / 63++ d_tmp = output_d * q_scales+ dm_tmp = output_dmin * q_mins++ replace_ids = d_tmp != 0+ all_L[replace_ids] = np.round(+ (blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(+ np.uint8)+ all_L = np.clip(all_L, 0, 15)+ output_qs = all_L[:,::2] | (all_L[:,1::2] << 4)output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)diff --git a/auto_round/export/export_to_gguf/quant_gpu.py b/auto_round/export/export_to_gguf/quant_gpu.pynew file mode 100644index 000000000..7d669bfda--- /dev/null+++ b/auto_round/export/export_to_gguf/quant_gpu.py@@ -0,0 +1,331 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch+import numpy as np++QK_K = 256+K_SCALE_SIZE = 12+GGML_QUANT_SIZES = {+ "bf16": (1, 2),+ "q4_0": (32, 2 + 16),+ "q4_1": (32, 2 + 2 + 16),+ "q4_k": (256, 2 + 2 + QK_K//2 + 12),+ "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),+ "q8_0": (32, 2 + 32)+}++GGML_QUANT_BLOCK = {}+++++def register_block(name):++ def register(cls):+ GGML_QUANT_BLOCK[name] = cls+ return cls++ return register+++def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, device="cuda"):+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]++ data = data.to(torch.float32).to(device)+ scale = scale.to(device) if scale is not None else scale+ zp = zp.to(device) if zp is not None else zp+ wmin_m = wmin_m.to(device) if wmin_m is not None else wmin_m+ d_scale = d_scale.to(device) if d_scale is not None else d_scale+ d_wmin_m = d_wmin_m.to(device) if d_wmin_m is not None else d_wmin_m++ shape = data.shape+ n_blocks = data.nelement() // block_size+ blocks = data.reshape((n_blocks, block_size))++ quant_func = GGML_QUANT_BLOCK[ggml_type]+ new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)++ assert new_data.shape[-1] == type_size+ new_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)+ return new_data+++def torch_roundf(n):+ a = torch.abs(n)+ floored = torch.floor(a)+ b = floored + torch.floor(2 * (a-floored))+ return torch.sign(n) * b+++@register_block("bf16")+def bf16_quant_block(blocks, scale=None, zp=None, **kwargs):+ n = blocks.view(torch.uint32)+ # force nan to quiet+ n = torch.where((n & 0x7fffffff) > 0x7f800000, (n & 0xffff0000) | (64 << 16), n)+ # round to nearest even+ n = n.to(torch.uint64) + (0x7fff + ((n >> 16) & 1)) >> 16+ return n.cpu().numpy().astype(np.uint16).view(np.uint8)+++@register_block("q4_0")+def q4_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):+ if scale is not None:+ d = scale.reshape((-1, 1))+ else:+ max = torch.max(blocks, axis=-1, keepdims=True)[0]+ d = max / -8+ id = torch.where(d == 0, 0, 1 / d)++ qs = torch.trunc(blocks.to(torch.float64) * id.to(torch.float64) + 8.5).to(torch.uint8).clip(0, 15)++ n_blocks = blocks.shape[0]+ block_size = GGML_QUANT_SIZES["q4_0"][0]+ qs = qs.reshape((n_blocks, 2, block_size // 2)).cpu().numpy()+ qs = qs[..., 0, :] | (qs[..., 1, :] << 4)++ d = d.cpu().numpy().astype(np.float16).view(np.uint8)++ return np.concatenate([d, qs], axis=-1)+++@register_block("q4_1")+def q4_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):+ if scale is not None:+ d = scale.reshape((-1, 1))+ min = zp.reshape((-1, 1)) * d * -1+ else:+ max = blocks.max(axis=-1, keepdims=True)[0]+ min = blocks.min(axis=-1, keepdims=True)[0]+ d = (max-min) / 15+ id = torch.where(d == 0, 0, 1 / d)++ qs = torch.trunc((blocks-min) * id + 0.5).to(torch.uint8).clip(0, 15)++ n_blocks = blocks.shape[0]+ block_size = GGML_QUANT_SIZES["q4_1"][0]+ qs = qs.reshape((n_blocks, 2, block_size // 2)).cpu().numpy()+ qs = qs[..., 0, :] | (qs[..., 1, :] << np.uint8(4))++ d = d.cpu().numpy().astype(np.float16).view(np.uint8)+ m = min.cpu().numpy().astype(np.float16).view(np.uint8)+ return np.concatenate([d, m, qs], axis=-1)+++@register_block("q8_0")+def q8_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs) -> np.ndarray:+ if scale is not None:+ d = scale.reshape((-1, 1))+ else:+ d = torch.abs(blocks).max(axis=1, keepdims=True)[0] / 127+ id = torch.where(d == 0, 0, 1 / d)++ qs = torch_roundf(blocks * id)++ # (n_blocks, 2)+ d = d.cpu().numpy().astype(np.float16).view(np.uint8)+ # (n_blocks, block_size)+ qs = qs.cpu().numpy().astype(np.int8).view(np.uint8)++ return np.concatenate([d, qs], axis=1)+++def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):+ assert bits in [2, 4], f"bits={bits} is not supported"+ nmax = bits ** 2 -1+ # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K+ if len(data.shape) == 2:+ data_shape = (-1, 8, 32) if bits == 4 else (-1, 16, 16)+ data = data.reshape(data_shape)+ sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)+ av_x = torch.sqrt(sum_x2 / 32)+ weight = torch.abs(data) + av_x++ group_min = torch.min(data, axis=-1, keepdims=True)[0]+ group_max = torch.max(data, axis=-1, keepdims=True)[0]++ the_mins = -group_min++ sum_w = torch.sum(weight, axis=-1, keepdims=True)+ sum_x = torch.sum(weight * data, axis=-1, keepdims=True)++ group_min[group_min > 0] = 0++ scale = (group_max - group_min) / nmax+ iscale = torch.where(scale == 0, 0, 1 /scale)++ l_values = torch.round(iscale * (data - group_min))+ L = torch.clip(l_values, 0, nmax).to(torch.uint8)++ diffs = scale * L + group_min - data+ diffs = torch.abs(diffs) if use_mad else diffs**2+ best_mad = torch.sum(weight * diffs, axis=-1, keepdims=True)++ if nstep < 1:+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])++ for step in range(nstep):+ new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)+ new_iscale = torch.where(new_scale == 0, 0, 1 /new_scale)++ l_values = torch.round(new_iscale * (data - group_min))+ Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)++ sum_l = torch.sum(weight * Laux, axis=-1, keepdims=True)+ sum_l2 = torch.sum(weight * Laux**2, axis=-1, keepdims=True)+ sum_xl = torch.sum(weight * Laux * data, axis=-1, keepdims=True)++ D = sum_w * sum_l2 - sum_l * sum_l+ replace_idx = D > 0++ this_scale = (sum_w * sum_xl - sum_x * sum_l) / D+ this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D+ this_min[this_min > 0] = 0+ this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]++ diffs = this_scale * Laux + this_min - data+ diffs = torch.abs(diffs) if use_mad else diffs**2+ mad = torch.sum(weight * diffs, axis=-1, keepdims=True)++ replace_idx &= mad < best_mad+ best_mad[replace_idx] = mad[replace_idx]+ L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]+ scale[replace_idx] = this_scale[replace_idx]+ group_min[replace_idx] = this_min[replace_idx]++ the_mins = -group_min+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])+++@register_block("q2_k")+def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ nb = blocks.shape[0]+ output_scale = np.empty((nb, 16), dtype=np.uint8)+ output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)++ blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)++ if scale is not None:+ scales = scale.reshape((-1, QK_K // 16))+ mins = wmin_m.reshape((-1, QK_K // 16))+ output_d = d_scale.to(torch.float32) if len(d_scale.shape) == 2 \+ else d_scale.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin_m.to(torch.float32) if len(d_wmin_m.shape) == 2 \+ else d_wmin_m.reshape(-1, 1).to(torch.float32)+ inv_scales = torch.where(d_scale == 0, 0, 1 / output_d)+ inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).to(torch.uint8)+ else:+ scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0]+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0]+ inv_scales = torch.where(max_scales > 0, 15. / max_scales, 0)+ inv_mins = torch.where(max_mins > 0, 15. / max_mins, 0)++ replace_ids = (max_scales > 0).squeeze()+ output_scale = torch.zeros_like(scales).to(torch.uint8)+ output_scale[replace_ids] = torch.round(inv_scales * scales).to(torch.uint8)++ replace_ids = (max_mins > 0).squeeze()+ output_scale[replace_ids] |= torch.round(inv_mins * mins).to(torch.uint8) << 4++ if d_scale is None:+ output_d = torch.where(max_scales > 0, max_scales / 15, 0)+ if d_wmin_m is None:+ output_dmin = torch.where(max_mins > 0 ,max_mins / 15., 0)++ d_tmp = output_d * (output_scale & 0xF)+ dm_tmp = output_dmin * (output_scale >> 4)++ replace_ids = d_tmp != 0+ all_L[replace_ids] = torch.round(+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(+ torch.uint8)+ all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 3)++ output_scale = output_scale.cpu().numpy()+ output_qs = all_L[:,::4] | (all_L[:,1::4] << 2) | (all_L[:,2::4] << 4) | (all_L[:,3::4] << 6)+ output_d = output_d.cpu().numpy()+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_dmin = output_dmin.cpu().numpy()+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_qs = output_qs.reshape((nb, QK_K // 4))++ # [scale, qs, d, dmin]+ return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)+++@register_block("q4_k")+def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ nb = blocks.shape[0]+ blocks = blocks.reshape((nb, QK_K // 32, 32))++ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)+ output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)++ if scale is not None:+ scales = scale.reshape(-1, QK_K // 32)+ mins = wmin_m.reshape(-1, QK_K // 32)+ output_d = d_scale.to(torch.float32) if len(d_scale.shape) == 2 \+ else d_scale.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin_m.to(torch.float32) if len(d_wmin_m.shape) == 2 \+ else d_wmin_m.reshape(-1, 1).to(torch.float32)+ inv_scales = torch.where(d_scale == 0, 0, 1 / output_d)+ inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).to(torch.uint8)+ else:+ scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0]+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0]+ inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)+ inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)++ q_scales = torch.round(inv_scales * scales)+ q_mins = torch.round(inv_mins * mins)++ if d_scale is None:+ output_d = max_scales / 63+ if d_wmin_m is None:+ output_dmin = max_mins / 63++ d_tmp = output_d * q_scales+ dm_tmp = output_dmin * q_mins++ q_scales = q_scales.cpu().numpy().astype(np.uint8)+ q_mins = q_mins.cpu().numpy().astype(np.uint8)+ output_scale[:,:4] = q_scales[:,:4]+ output_scale[:,4:8] = q_mins[:,:4]++ output_scale[:,8:] = (q_scales[:,4:] & 0xF) | ((q_mins[:,4:] & 0xF) << 4)+ output_scale[:,:4] |= ((q_scales[:,4:] >> 4) << 6)+ output_scale[:,4:8] |= ((q_mins[:,4:] >> 4) << 6)++ replace_ids = d_tmp != 0+ all_L[replace_ids] = torch.round(+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)+ all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 15)+ output_qs = all_L[:,::2] | (all_L[:,1::2] << 4)++ output_d = output_d.cpu().numpy()+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_dmin = output_dmin.cpu().numpy()+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_qs = output_qs.reshape(nb, QK_K // 2)++ # [d, dmin, scale, qs]+ return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex b3685d9f6..5a9594f61 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -34,6 +34,7 @@ from auto_round.utils import (get_fp_layer_names,clear_memory,get_device_and_parallelism,+ get_model_dtype,set_cuda_visible_devices)os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"@@ -192,6 +193,12 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant.")+ self.add_argument(+ "--eval_model_dtype",+ default=None,+ type=str,+ help="the torch_dytpe to load the model for evaluation.")+class EvalArgumentParser(argparse.ArgumentParser):@@ -218,6 +225,11 @@ class EvalArgumentParser(argparse.ArgumentParser):"--disable_trust_remote_code", action='store_true', help="whether to disable trust_remote_code")self.add_argument("--eval_bs", "--bs", "--batch_size", default=None, type=int, help="batch size in evaluation")self.add_argument("--eval_task_by_task", action='store_true', help="whether to eval task by task.")+ self.add_argument(+ "--eval_model_dtype",+ default=None,+ type=str,+ help="the torch_dytpe to load the model for evaluation.")def setup_parser():@@ -282,7 +294,7 @@ def setup_light_parser():"--lr", default=5e-3, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically")args = parser.parse_args()-+return args@@ -519,51 +531,84 @@ def tune(args):eval_gguf_model = Truebreak+ import time+ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")if args.act_bits <= 8 or eval_gguf_model:if eval_gguf_model:# gguf floder only contains one filefor file in os.listdir(eval_folder):gguf_file = file- user_model = AutoModelForCausalLM.from_pretrained(- eval_folder, gguf_file=gguf_file, device_map="auto" if use_auto_mapping else None)++ logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":+ logger.warning(+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"+ " but may affect accuracy."+ )+ model = AutoModelForCausalLM.from_pretrained(+ eval_folder, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)+ model.eval()tokenizer = AutoTokenizer.from_pretrained(eval_folder, gguf_file=gguf_file)else:if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:from accelerate.big_modeling import dispatch_modeldispatch_model(model, model.hf_device_map)- user_model = modelelse:device_str = detect_device(device_str)- user_model = model.to(device_str)+ model = model.to(device_str)+ if model.dtype != eval_model_dtype and eval_model_dtype != "auto":+ model.to(getattr(torch, eval_model_dtype))if args.eval_task_by_task:eval_task_by_task(- user_model, tokenizer=tokenizer, device=device_str, tasks=args.tasks, batch_size=args.eval_bs)+ model,+ tokenizer=tokenizer,+ device=device_str,+ tasks=args.tasks,+ batch_size=args.eval_bs,+ eval_model_dtype=eval_model_dtype)else:if args.eval_bs is None or args.eval_bs == "auto":logger.warning("This API does not support auto currently, reset eval_bs to 16")args.eval_bs = 16from auto_round.eval.evaluation import simple_evaluate_user_model+ st = time.time()res = simple_evaluate_user_model(- user_model, tokenizer, tasks=tasks, batch_size=args.eval_bs, device=device_str)+ model,+ tokenizer,+ tasks=tasks,+ batch_size=args.eval_bs,+ device=device_str,+ eval_model_dtype=eval_model_dtype)print(make_table(res))+ print("evaluation running time=", time.time() - st)else:if args.eval_task_by_task:- eval_task_by_task(eval_folder, device=device_str, tasks=args.tasks, batch_size=args.eval_bs)+ eval_task_by_task(+ eval_folder,+ device=device_str,+ tasks=args.tasks,+ batch_size=args.eval_bs,+ eval_model_dtype=eval_model_dtype)else:from auto_round.eval.evaluation import simple_evaluatetasks, model_args, device_str = _eval_init(- args.tasks, eval_folder, args.device, args.disable_trust_remote_code)+ args.tasks, eval_folder, args.device, args.disable_trust_remote_code, dtype=eval_model_dtype)+ st = time.time()res = simple_evaluate(model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs)print(make_table(res))+ print("evaluation running time=", time.time() - st)-def _eval_init(tasks, model_path, device, disable_trust_remote_code=False):+def _eval_init(tasks, model_path, device, disable_trust_remote_code=False, dtype="auto"):set_cuda_visible_devices(device)device_str, parallelism = get_device_and_parallelism(device)- model_args = f'pretrained={model_path},trust_remote_code={not disable_trust_remote_code}' # ,add_bos_token={True}+ if dtype != "auto":+ dtype = get_model_dtype(model_dtype=dtype)+ # ,add_bos_token={True}+ model_args = f'pretrained={model_path},trust_remote_code={not disable_trust_remote_code},dtype={dtype}'if parallelism:model_args += ",parallelize=True"if isinstance(tasks, str):@@ -572,19 +617,66 @@ def _eval_init(tasks, model_path, device, disable_trust_remote_code=False):def eval(args):- tasks, model_args, device_str = _eval_init(args.tasks, args.model, args.device, args.disable_trust_remote_code)+ import time+ tasks, model_args, device_str = _eval_init(+ args.tasks, args.model, args.device, args.disable_trust_remote_code, args.eval_model_dtype)# load after _eval_int in order to make sure import torch after set CUDA_VISBILE_DEVICES- from auto_round.eval.evaluation import simple_evaluate+ from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model+ from auto_round.utils import logger- res = simple_evaluate(model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs)+ is_gguf_file = False+ if os.path.isfile(args.model) and args.model.endswith(".gguf"):+ is_gguf_file = True+ gguf_file = os.path.basename(args.model)+ model = os.path.dirname(args.model)+ else:+ for file in os.listdir(args.model):+ if file.endswith(".gguf"):+ is_gguf_file = True+ gguf_file = file+ eval_model_dtype = get_model_dtype(args.eval_model_dtype)+ if is_gguf_file:+ import torch+ from transformers import AutoTokenizer, AutoModelForCausalLM+ from lm_eval.utils import make_table # pylint: disable=E0401+ tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)- from lm_eval.utils import make_table # pylint: disable=E0401- print(make_table(res))+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":+ logger.warning(+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"+ " but may affect accuracy."+ )+ model = AutoModelForCausalLM.from_pretrained(+ model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)+ model.eval()+ if (batch_size := args.eval_bs) is None:+ batch_size = "auto:8"+ st = time.time()+ res = simple_evaluate_user_model(+ model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str)+ print(make_table(res))+ print("evaluation running time=", time.time() - st)+ else:+ st = time.time()+ res = simple_evaluate(+ model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs)+ from lm_eval.utils import make_table # pylint: disable=E0401+ print(make_table(res))+ print("evaluation running time=", time.time() - st)def eval_task_by_task(- model, device=None, tasks=None, tokenizer=None, batch_size=None, max_batch_size=64, trust_remote_code=True):+ model,+ device=None,+ tasks=None,+ tokenizer=None,+ batch_size=None,+ max_batch_size=64,+ trust_remote_code=True,+ eval_model_dtype=None,+ retry_times=3):set_cuda_visible_devices(device)device_str, parallelism = get_device_and_parallelism(device)@@ -597,7 +689,7 @@ def eval_task_by_task(from auto_round import AutoRoundConfig # pylint: disable=E0611if batch_size is None:- batch_size = "auto"+ batch_size = "auto:8"is_gguf_file = Falseif not isinstance(model, str):parallelism = False@@ -611,9 +703,20 @@ def eval_task_by_task(if file.endswith(".gguf"):is_gguf_file = Truegguf_file = file+ eval_model_dtype = get_model_dtype(eval_model_dtype)if is_gguf_file:tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)- model = AutoModelForCausalLM.from_pretrained(model, gguf_file=gguf_file, device_map="auto")+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":+ logger.warning(+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"+ " but may affect accuracy."+ )++ model = AutoModelForCausalLM.from_pretrained(+ model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)+ model.eval()+ parallelism=Falsehflm = HFLM(pretrained=model,tokenizer=tokenizer,@@ -621,7 +724,9 @@ def eval_task_by_task(batch_size=batch_size,max_batch_size=max_batch_size,parallelize=parallelism,- trust_remote_code=trust_remote_code)+ trust_remote_code=trust_remote_code,+ dtype=eval_model_dtype+ )if isinstance(tasks, str):tasks = tasks.replace(" ", "").split(",")@@ -629,23 +734,32 @@ def eval_task_by_task(from lm_eval.utils import make_table # pylint: disable=E0401res_all = {}res_keys = ["results", "versions", "n-shot", "higher_is_better"]+ import time+ st = time.time()for task in tasks:- try:- res = lm_simple_evaluate(model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size)- except RuntimeError as e:- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):- try:- logger.warning("Out of memory, reset batch_size to 1 and re-try.")- res = lm_simple_evaluate(model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1)- except Exception as e:+ while retry_times:+ try:+ res = lm_simple_evaluate(+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size)+ break+ except Exception as e:+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ ori_batch_sizes = hflm.batch_sizes if hflm.batch_sizes else {"0": 64}+ try:+ for k, v in hflm.batch_sizes.items():+ hflm.batch_sizes[k] = max(v // 2, 1)+ logger.warning(+ f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")+ res = lm_simple_evaluate(+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1)+ hflm.batch_sizes = ori_batch_sizes+ except Exception as e:+ traceback.print_exc()+ pass+ else:traceback.print_exc()- continue- else:- traceback.print_exc()- continue- except Exception as e:- traceback.print_exc()- continue+ break+ retry_times -= 1if not res_all:res_all = res@@ -654,3 +768,4 @@ def eval_task_by_task(res_all[key].update(res[key])print(make_table(res_all))+ print("total eval time:", time.time() - st)diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 00db00aae..394bfc10e 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1185,6 +1185,13 @@ def _gguf_args_check(args):pre_dq_format = ""for format in GGUF_CONFIG:if format in formats:+ try:+ from auto_round.export.export_to_gguf.convert import Model+ except:+ raise ImportError(+ f"Please use the latest gguf-py for {format}, you can use the following command to install it:\n"+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install .")+ sys.exit(-1)if re.search(pattern, format):if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:logger.error(f"Cannot eport {pre_dq_format} and {format} at the same time.")@@ -1221,7 +1228,7 @@ def _gguf_args_check(args):setattr(args, k, v)if len(unsupport_list) > 0:logger.error(- f"format {format} not support for {', '.join(unsupport_list)},"+ f"format {format} does not support for {', '.join(unsupport_list)},"f" reset to {', '.join(reset_list)}.")logger.info(f"export format {format}, sym = {not args.asym}, group_size = {args.group_size}")@@ -1439,3 +1446,18 @@ def get_shared_keys(model):shared_keys = shared_cache_keysshared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())return shared_keys+++def get_model_dtype(model_dtype, default="auto"):+ if model_dtype is None or model_dtype == "auto":+ model_dtype = default+ elif model_dtype in ["bf16", "bfloat16"]:+ model_dtype = "bfloat16"+ elif model_dtype in ["f16", "float16", "fp16"]:+ model_dtype = "float16"+ elif model_dtype in ["f32", "float32", "fp32"]:+ model_dtype = "float32"+ else:+ logger.warning(f"Unable to identify model_dtype {model_dtype}, reset to default model_dtype {default}")+ model_dtype = default+ return model_dtype\ No newline at end of filediff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 9ffadaebd..2be872828 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -100,7 +100,7 @@ CPU, Intel GPU, HPU,and CUDA for both quantization and inference.This setting provides the best accuracy in most scenarios but is 4–5× slower than the standard AutoRound recipe. It is especially recommended for 2-bit quantization and is a good choice if sufficient resources are available.-- ```bash+ ```bashauto-round-best --model facebook/opt-125m --bits 4 --group_size 128 --format "auto_gptq,auto_awq,auto_round"```