Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[GGUF STEP4] fix search bug and improve packing & eval speed (#545)

Heng Guo4b85b512a009
diff --git a/README.md b/README.md
index 2c72f6e7c..dffcd2135 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
<div align="left">
## What's New
-* [2025/04] AutoRound supports some recipes for Qwen3 series, please refer to [Qwen3-8B-sym-recipe](./docs/Qwen3-8B-sym-recipe.md) and [Qwen3-14B-sym-recipe](./docs/Qwen3-14B-sym-recipe.md) for more details.
+* [2025/04] AutoRound provides some recipes for Qwen3 series, please refer to [Qwen3-8B-sym-recipe](./docs/Qwen3-8B-sym-recipe.md) and [Qwen3-14B-sym-recipe](./docs/Qwen3-14B-sym-recipe.md) for more details.
* [2025/04] AutoRound has been integrated into Transformers. You can run models in the AutoRound format directly with Transformers versions later than 4.51.3.
* [2025/03] The INT2-mixed R1 model (~200GB) retains 97.9% accuracy. Check
out [OPEA/DeepSeek-R1-int2-mixed-sym-inc](https://huggingface.co/OPEA/DeepSeek-R1-int2-mixed-sym-inc).
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 6b4649545..d1c5af73f 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -23,7 +23,9 @@ def run_eval():
device=args.device,
tasks=args.tasks,
batch_size=args.eval_bs,
- trust_remote_code=not args.disable_trust_remote_code)
+ trust_remote_code=not args.disable_trust_remote_code,
+ eval_model_dtype=args.eval_model_dtype
+ )
else:
from auto_round.script.llm import eval
eval(args)
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index b0b4d5dd2..82f8645d6 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -25,17 +25,19 @@ from lm_eval.models.huggingface import HFLM
def simple_evaluate_user_model(
user_model,
tokenizer,
- batch_size: Optional[int] = None,
- max_batch_size: Optional[int] = None,
+ batch_size: Optional[int] = 1,
+ max_batch_size: Optional[int] = 64,
+ eval_model_dtype = "auto",
**kwargs
):
- hflm = HFLM(pretrained=user_model, tokenizer=tokenizer, batch_size=batch_size, max_batch_size=max_batch_size)
- return lm_simple_evaluate(
- model=hflm,
- model_args=None,
+ hflm = HFLM(
+ pretrained=user_model,
+ tokenizer=tokenizer,
batch_size=batch_size,
max_batch_size=max_batch_size,
- **kwargs)
+ dtype=eval_model_dtype)
+ return lm_simple_evaluate(
+ model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, **kwargs)
def simple_evaluate(
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 94e77adc4..22c282ae1 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -54,7 +54,8 @@ import torch
# autoround import
from auto_round.utils import logger, LazyImport
-from .quant import ggml_quant
+from .quant_cpu import ggml_quant_cpu
+from .quant_gpu import ggml_quant_gpu
gguf = LazyImport("gguf")
@@ -1134,6 +1135,49 @@ class Model(OriModel):
def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
for name, tensor in self.model.named_parameters():
yield name, tensor
+
+ def _quant_data(self, data_torch, data_qtype, name, bid):
+ from auto_round.utils import get_module
+ suffix = '.weight'
+ if suffix in name:
+ layer_name = name[:-len(suffix)]
+ module = get_module(self.model, layer_name)
+ if hasattr(module, "scale"):
+ if hasattr(self, "permute"):
+ bs = module.scale.shape[0]
+ for attr in ["scale", "zp", "w_d_scale", "w_d_wmin_m", "w_wmin_m"]:
+ if hasattr(module, attr) and getattr(module, attr) is not None:
+ attr_tensor = getattr(module, attr)
+ ori_shape = attr_tensor.shape
+ attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), name, bid)[0][1]
+ attr_tensor = attr_tensor.reshape(ori_shape)
+ setattr(module, attr, attr_tensor)
+ scale = module.scale
+ zp = module.zp if hasattr(module, "zp") else None
+ ggml_quant = ggml_quant_gpu if torch.cuda.is_available() else ggml_quant_cpu
+
+ if data_qtype.name.lower().endswith("_k"):
+ d_scale = module.w_d_scale.to(torch.float32)
+ d_wmin_m = module.w_d_wmin_m.to(torch.float32)
+ wmin_m = module.w_wmin_m.to(torch.float32)
+ data = ggml_quant(
+ data_torch,
+ data_qtype.name.lower(),
+ scale,
+ zp,
+ wmin_m=wmin_m,
+ d_scale=d_scale,
+ d_wmin_m=d_wmin_m
+ )
+ else:
+ data = ggml_quant(data_torch, data_qtype.name.lower(), scale, zp)
+ else:
+ data_qtype = gguf.GGMLQuantizationType.F32
+ data = data_torch.squeeze().cpu().numpy()
+ else:
+ data_qtype = gguf.GGMLQuantizationType.F32
+ data = data_torch.squeeze().cpu().numpy()
+ return data, data_qtype
def prepare_tensors(self):
max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")
@@ -1225,67 +1269,21 @@ class Model(OriModel):
# data_qtype = gguf.GGMLQuantizationType.F16
# data = gguf.quants.quantize(data, data_qtype)
- # Quant here
- def _quant_data(data, data_qtype):
- from auto_round.utils import get_module
- suffix = '.weight'
- if suffix in name:
- layer_name = name[:-len(suffix)]
- module = get_module(self.model, layer_name)
- if hasattr(module, "scale"):
-
- if hasattr(self, "permute"):
- bs = module.scale.shape[0]
- for attr in ["scale", "zp", "w_d_scale", "w_d_wmin_m", "w_wmin_m"]:
- if hasattr(module, attr) and getattr(module, attr) is not None:
- attr_tensor = getattr(module, attr)
- ori_shape = attr_tensor.shape
- attr_tensor = self.modify_tensors(attr_tensor.reshape(bs, -1), name, bid)[0][1]
- attr_tensor = attr_tensor.reshape(ori_shape)
- setattr(module, attr, attr_tensor)
-
- scale = module.scale
-
- if isinstance(scale, torch.Tensor):
- scale = scale.numpy()
- zp = module.zp if hasattr(module, "zp") else None
- if isinstance(zp, torch.Tensor):
- zp = zp.numpy()
- if data_qtype.name.lower().endswith("_k"):
- d_scale = module.w_d_scale.numpy()
- d_wmin_m = module.w_d_wmin_m.numpy()
- wmin_m = module.w_wmin_m.numpy()
- data = ggml_quant(
- data,
- data_qtype.name.lower(),
- scale,
- zp,
- wmin_m=wmin_m,
- d_scale=d_scale,
- d_wmin_m=d_wmin_m)
- else:
- data = ggml_quant(data, data_qtype.name.lower(), scale, zp)
- else:
- data_qtype = gguf.GGMLQuantizationType.F32
- else:
- data_qtype = gguf.GGMLQuantizationType.F32
- return data, data_qtype
-
# for MOE model
- if len(data.shape) == 3:
+ if len(data_torch.shape) == 3:
new_data = []
- for idx, arr in enumerate(data):
+ for idx, arr in enumerate(data_torch):
arr_name = name.split('.')
for i in range(len(arr_name) - 1, -1, -1):
- if arr_name[i].isdecimal() and int(arr_name[i]) == (data.shape[0] - 1):
+ if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):
arr_name[i] = str(idx)
arr_name = ".".join(arr_name)
- arr, data_qtype = _quant_data(arr, data_qtype)
+ arr, data_qtype = self._quant_data(arr, data_qtype, name, bid)
new_data.append(arr)
data = np.array(new_data)
del new_data
else:
- data, data_qtype = _quant_data(data, data_qtype)
+ data, data_qtype = self._quant_data(data_torch, data_qtype, name, bid)
shape = gguf.quant_shape_from_byte_shape(
data.shape, data_qtype) if data.dtype == np.uint8 else data.shape
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index 485f187f5..81a56616b 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -39,6 +39,9 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {
def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):
"""Export the model to gguf format."""
+ if output_dir is not None and os.path.exists(output_dir):
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
+
st = time.time()
model = kwargs["model"]
diff --git a/auto_round/export/export_to_gguf/quant.py b/auto_round/export/export_to_gguf/quant_cpu.py
similarity index 52%
rename from auto_round/export/export_to_gguf/quant.py
rename to auto_round/export/export_to_gguf/quant_cpu.py
index ca75ad4f1..24b20e94b 100644
--- a/auto_round/export/export_to_gguf/quant.py
+++ b/auto_round/export/export_to_gguf/quant_cpu.py
@@ -39,7 +39,15 @@ def register_block(name):
return register
-def ggml_quant(data: np.array, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+def ggml_quant_cpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ import torch
+ data = data.squeeze().cpu().numpy() if isinstance(data, torch.Tensor) else data
+ scale = scale.numpy() if isinstance(scale, torch.Tensor) else scale
+ zp = zp.numpy() if isinstance(zp, torch.Tensor) else zp
+ wmin_m = wmin_m.numpy() if isinstance(wmin_m, torch.Tensor) else wmin_m
+ d_scale = d_scale.numpy() if isinstance(d_scale, torch.Tensor) else d_scale
+ d_wmin_m = d_wmin_m.numpy() if isinstance(d_wmin_m, torch.Tensor) else d_wmin_m
+
block_size, type_size = GGML_QUANT_SIZES[ggml_type]
data = data.astype(np.float32, copy=False)
@@ -52,6 +60,7 @@ def ggml_quant(data: np.array, ggml_type, scale=None, zp=None, wmin_m=None, d_sc
else:
worker = 0
+ worker = 0
if worker > 0:
n_groups = (data.shape[0] // worker) or 1
blocks = np.array_split(blocks, n_groups, axis=0)
@@ -73,6 +82,7 @@ def ggml_quant(data: np.array, ggml_type, scale=None, zp=None, wmin_m=None, d_sc
else:
quant_func = GGML_QUANT_BLOCK[ggml_type]
if ggml_type.endswith("_k"):
+ # 0.5588 vs 63.05
new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)
else:
new_data = quant_func(blocks, scale, zp)
@@ -165,216 +175,181 @@ def q8_0_quant_block(blocks: np.array, scale=None, zp=None) -> np.ndarray:
return np.concatenate([d, qs], axis=1)
+def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
+ assert bits in [2, 4], f"bits={bits} is not supported"
+ nmax = bits ** 2 -1
+ # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K
+ if len(data.shape) == 2:
+ data_shape = (-1, 8, 32) if bits == 4 else (-1, 16, 16)
+ data = data.reshape(data_shape)
+ sum_x2 = np.sum(np.power(data, 2), axis=-1, keepdims=True)
+ av_x = np.sqrt(sum_x2 / 32)
+ weight = np.abs(data) + av_x
+
+ group_min = np.min(data, axis=-1, keepdims=True)
+ group_max = np.max(data, axis=-1, keepdims=True)
-def make_qkx2_quants(data, weight, nmax, group_size, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
- group_min = np.min(data)
- group_max = np.max(data)
+ the_mins = -group_min
- sum_w = np.sum(weight)
- sum_x = np.sum(weight * data)
+ sum_w = np.sum(weight, axis=-1, keepdims=True)
+ sum_x = np.sum(weight * data, axis=-1, keepdims=True)
- if group_min > 0:
- group_min = 0
- if group_min == group_max:
- L = np.zeros(group_size, dtype=np.uint8)
- the_min = -group_min
- return 0.0, L, the_min
+ group_min[group_min > 0] = 0
- iscale = nmax / (group_max-group_min)
- scale = 1 / iscale
- L = np.zeros(group_size, dtype=np.uint8)
+ scale = (group_max - group_min) / nmax
+ iscale = np.where(scale == 0, 0, 1 /scale)
- l_values = np.round(iscale * (data-group_max))
+ l_values = np.round(iscale * (data - group_min))
L = np.clip(l_values, 0, nmax).astype(np.uint8)
- diffs = scale*L + group_min - data
+
+ diffs = scale * L + group_min - data
diffs = np.abs(diffs) if use_mad else diffs**2
- best_mad = np.sum(weight * (diffs))
+ best_mad = np.sum(weight * diffs, axis=-1, keepdims=True)
if nstep < 1:
- the_min = -group_min
- return scale, L, the_min
+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
- Laux = []
for step in range(nstep):
- iscale = (rmin + rdelta*step + nmax) / (group_max-group_min)
- l_values = np.round(iscale * (data-group_min))
+ new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)
+ new_iscale = np.where(new_scale == 0, 0, 1 /new_scale)
+
+ l_values = np.round(new_iscale * (data - group_min))
Laux = np.clip(l_values, 0, nmax).astype(np.uint8)
- sum_l = np.sum(weight * Laux)
- sum_l2 = np.sum(weight * Laux**2)
- sum_xl = np.sum(weight * Laux * data)
+ sum_l = np.sum(weight * Laux, axis=-1, keepdims=True)
+ sum_l2 = np.sum(weight * Laux**2, axis=-1, keepdims=True)
+ sum_xl = np.sum(weight * Laux * data, axis=-1, keepdims=True)
- D = sum_w*sum_l2 - sum_l*sum_l
- if D > 0:
- this_scale = (sum_w*sum_xl - sum_x*sum_l) / D
- this_min = (sum_l2*sum_x - sum_l*sum_xl) / D
- if this_min > 0:
- this_min = 0
- this_scale = sum_xl / sum_l2
+ D = sum_w * sum_l2 - sum_l * sum_l
+ replace_idx = D > 0
- diffs = this_scale*Laux + this_min - data
- mad = np.sum(weight * diffs**2)
+ this_scale = (sum_w * sum_xl - sum_x * sum_l) / D
+ this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D
+ this_min[this_min > 0] = 0
+ this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]
- if mad < best_mad:
- L = Laux.copy()
- best_mad = mad
- scale = this_scale
- group_min = this_min
+ diffs = this_scale * Laux + this_min - data
+ diffs = np.abs(diffs) if use_mad else diffs**2
+ mad = np.sum(weight * diffs, axis=-1, keepdims=True)
- the_min = -group_min
- return scale, L, the_min
+ replace_idx &= mad < best_mad
+ best_mad[replace_idx] = mad[replace_idx]
+ L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]
+ scale[replace_idx] = this_scale[replace_idx]
+ group_min[replace_idx] = this_min[replace_idx]
+
+ the_mins = -group_min
+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
@register_block("q2_k")
def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
nb = blocks.shape[0]
output_scale = np.empty((nb, 16), dtype=np.uint8)
- output_d = np.empty(nb, dtype=np.float32)
- output_dmin = np.empty(nb, dtype=np.float32)
output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)
- blocks = blocks.reshape((nb, QK_K // 16, 16))
- weight = np.abs(blocks)
- scales = np.empty(QK_K // 16, dtype=np.float32)
- mins = np.empty((QK_K // 16), dtype=np.float32)
+ blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)
if scale is not None:
- scale = scale.reshape((-1, QK_K // 16))
- wmin_m = wmin_m.reshape((-1, QK_K // 16))
- output_d = d_scale.astype(np.float32)
- output_dmin = d_wmin_m.astype(np.float32)
+ scales = scale.reshape((-1, QK_K // 16))
+ mins = wmin_m.reshape((-1, QK_K // 16))
+ output_d = d_scale.astype(np.float32) if len(d_scale.shape) == 2 \
+ else d_scale.reshape(-1, 1).astype(np.float32)
+ output_dmin = d_wmin_m.astype(np.float32) if len(d_wmin_m.shape) == 2 \
+ else d_wmin_m.reshape(-1, 1).astype(np.float32)
inv_scales = np.where(d_scale == 0, 0, 1 / output_d)
inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)
-
- # for i in tqdm(range(nb), desc="packing layer"):
- for i in range(nb):
- all_L = np.empty(blocks[i].shape, dtype=np.uint8)
-
- if scale is not None:
- scales = scale[i]
- mins = wmin_m[i]
- inv_scale = inv_scales[i]
- inv_min = inv_mins[i]
- max_scale = max(scales)
- max_min = max(mins)
- all_L = np.round((blocks[i] + mins.reshape(-1, 1)) / scales.reshape(-1, 1)).astype(np.uint8)
- else:
- for j in range(QK_K // 16):
- tmp_scale, tmp_l, the_min = make_qkx2_quants(
- blocks[i][j], weight[i][j], nmax=3, group_size=16, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)
- all_L[j] = tmp_l
- scales[j] = tmp_scale
- mins[j] = the_min
-
- max_scale = max(scales)
- max_min = max(mins)
- inv_scale = 15. / max_scale
- inv_min = 15. / max_min
-
- if max_scale > 0:
- output_scale[i] = np.round(inv_scale * scales).astype(np.uint8)
- output_d[i] = max_scale / 15.
- else:
- output_scale[i] = np.zeros_like(scales).astype(np.uint8)
- output_d[i] = 0.
- if max_min > 0:
- output_scale[i] |= (np.round(inv_min * mins).astype(np.uint8) << 4)
- output_dmin[i] = max_min / 15.
- else:
- output_dmin[i] = 0.
-
- d_tmp = output_d[i] * (output_scale[i] & 0xF)
- dm_tmp = output_dmin[i] * (output_scale[i] >> 4)
- for j in range(QK_K // 16):
- if d_tmp[j] == 0.:
- continue
- else:
- all_L[j] = np.round((blocks[i][j] + dm_tmp[j]) / d_tmp[j]).astype(np.uint8)
- all_L = np.clip(all_L, 0, 3)
- output_qs[i] = all_L[::4] | (all_L[1::4] << 2) | (all_L[2::4] << 4) | (all_L[3::4] << 6)
-
+ max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)
+ max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)
+ all_L = np.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).astype(np.uint8)
+ else:
+ scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)
+ max_scales = np.max(scales, axis=-1, keepdims=True)
+ max_mins = np.max(mins, axis=-1, keepdims=True)
+ inv_scales = np.where(max_scales > 0, 15. / max_scales, 0)
+ inv_mins = np.where(max_mins > 0, 15. / max_mins, 0)
+
+ replace_ids = (max_scales > 0).reshape(-1)
+ output_scale = np.zeros_like(scales).astype(np.uint8)
+ output_scale[replace_ids] = np.round(inv_scales * scales).astype(np.uint8)
+
+ replace_ids = (max_mins > 0).reshape(-1)
+ output_scale[replace_ids] |= np.round(inv_mins * mins).astype(np.uint8) << 4
+
+ if d_scale is None:
+ output_d = np.where(max_scales > 0, max_scales / 15, 0)
+ if d_wmin_m is None:
+ output_dmin = np.where(max_mins > 0 ,max_mins / 15., 0)
+
+ d_tmp = output_d * (output_scale & 0xF)
+ dm_tmp = output_dmin * (output_scale >> 4)
+
+ replace_ids = d_tmp != 0
+ all_L[replace_ids] = np.round(
+ (blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(
+ np.uint8)
+ all_L = np.clip(all_L, 0, 3)
+
+ output_qs = all_L[:,::4] | (all_L[:,1::4] << 2) | (all_L[:,2::4] << 4) | (all_L[:,3::4] << 6)
output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
output_qs = output_qs.reshape((nb, QK_K // 4))
# [scale, qs, d, dmin]
return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)
-
+
@register_block("q4_k")
def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
nb = blocks.shape[0]
+ blocks = blocks.reshape((nb, QK_K // 32, 32)) # (nb, 8, 32)
+
output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
- output_d = np.empty(nb, dtype=np.float32)
- output_dmin = np.empty(nb, dtype=np.float32)
output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)
- blocks = blocks.reshape((nb, QK_K // 32, 32))
- sum_x2 = np.sum(np.power(blocks, 2), axis=-1)
- av_x = np.sqrt(sum_x2 / 32)
- weight = blocks + av_x.reshape((*av_x.shape, 1))
- scales = np.empty(QK_K // 32, dtype=np.float32)
- mins = np.empty(QK_K // 32, dtype=np.float32)
-
if scale is not None:
- scale = scale.reshape(-1, QK_K // 32)
- wmin_m = wmin_m.reshape(-1, QK_K // 32)
- output_d = d_scale.astype(np.float32)
- output_dmin = d_wmin_m.astype(np.float32)
+ scales = scale.reshape(-1, QK_K // 32)
+ mins = wmin_m.reshape(-1, QK_K // 32)
+ output_d = d_scale.astype(np.float32) if len(d_scale.shape) == 2 \
+ else d_scale.reshape(-1, 1).astype(np.float32)
+ output_dmin = d_wmin_m.astype(np.float32) if len(d_wmin_m.shape) == 2 \
+ else d_wmin_m.reshape(-1, 1).astype(np.float32)
inv_scales = np.where(d_scale == 0, 0, 1 / output_d)
inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)
-
- # for i in tqdm(range(nb), desc="packing layer"):
- for i in range(nb):
- all_L = np.empty(blocks[i].shape, dtype=np.uint8)
-
- if scale is not None:
- scales = scale[i]
- mins = wmin_m[i]
- inv_scale = inv_scales[i]
- inv_min = inv_mins[i]
- max_scale = max(scales)
- max_min = max(mins)
- all_L = np.round((blocks[i] + mins.reshape(-1, 1)) / scales.reshape(-1, 1)).astype(np.uint8)
- else:
- for j in range(QK_K // 32):
- tmp_scale, tmp_l, the_min = make_qkx2_quants(
- blocks[i][j], weight[i][j], nmax=15, group_size=32, rmin=-1, rdelta=0.1, nstep=20)
- all_L[j] = tmp_l
- scales[j] = tmp_scale
- mins[j] = the_min
-
- max_scale = max(scales)
- max_min = max(mins)
- inv_scale = 63. / max_scale if max_scale > 0 else 0.
- inv_min = 63. / max_min if max_min > 0 else 0.
-
- ls = np.round(inv_scale * scales).astype(np.uint8)
- lm = np.round(inv_min * mins).astype(np.uint8)
-
- output_scale[i][:4] = ls[:4]
- output_scale[i][4:8] = lm[:4]
-
- output_scale[i][8:] = (ls[4:] & 0xF) | ((lm[4:] & 0xF) << 4)
- output_scale[i][:4] |= ((ls[4:] >> 4) << 6)
- output_scale[i][4:8] |= ((lm[4:] >> 4) << 6)
-
- if d_scale is None:
- output_d[i] = max_scale / 63
- if d_wmin_m is None:
- output_dmin[i] = max_min / 63
-
- d_tmp = output_d[i] * ls
- dm_tmp = output_dmin[i] * lm
-
- for j in range(d_tmp.size):
- if d_tmp[j] == 0.:
- continue
- else:
- all_L[j] = np.round((blocks[i][j] + dm_tmp[j]) / d_tmp[j]).astype(np.uint8)
-
- all_L = np.clip(all_L, 0, 15)
- output_qs[i] = all_L[::2] | (all_L[1::2] << 4)
+ max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)
+ max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)
+ all_L = np.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).astype(np.uint8)
+ else:
+ scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)
+ max_scales = np.max(scales, axis=-1, keepdims=True)
+ max_mins = np.max(mins, axis=-1, keepdims=True)
+ inv_scales = np.where(max_scales > 0, 63. / max_scales, 0)
+ inv_mins = np.where(max_mins > 0, 63. / max_mins, 0)
+
+ q_scales = np.round(inv_scales * scales).astype(np.uint8)
+ q_mins = np.round(inv_mins * mins).astype(np.uint8)
+
+ output_scale[:,:4] = q_scales[:,:4]
+ output_scale[:,4:8] = q_mins[:,:4]
+
+ output_scale[:,8:] = (q_scales[:,4:] & 0xF) | ((q_mins[:,4:] & 0xF) << 4)
+ output_scale[:,:4] |= ((q_scales[:,4:] >> 4) << 6)
+ output_scale[:,4:8] |= ((q_mins[:,4:] >> 4) << 6)
+
+ if d_scale is None:
+ output_d = max_scales / 63
+ if d_wmin_m is None:
+ output_dmin = max_mins / 63
+
+ d_tmp = output_d * q_scales
+ dm_tmp = output_dmin * q_mins
+
+ replace_ids = d_tmp != 0
+ all_L[replace_ids] = np.round(
+ (blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(
+ np.uint8)
+ all_L = np.clip(all_L, 0, 15)
+ output_qs = all_L[:,::2] | (all_L[:,1::2] << 4)
output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
diff --git a/auto_round/export/export_to_gguf/quant_gpu.py b/auto_round/export/export_to_gguf/quant_gpu.py
new file mode 100644
index 000000000..7d669bfda
--- /dev/null
+++ b/auto_round/export/export_to_gguf/quant_gpu.py
@@ -0,0 +1,331 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import numpy as np
+
+QK_K = 256
+K_SCALE_SIZE = 12
+GGML_QUANT_SIZES = {
+ "bf16": (1, 2),
+ "q4_0": (32, 2 + 16),
+ "q4_1": (32, 2 + 2 + 16),
+ "q4_k": (256, 2 + 2 + QK_K//2 + 12),
+ "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),
+ "q8_0": (32, 2 + 32)
+}
+
+GGML_QUANT_BLOCK = {}
+
+
+
+
+def register_block(name):
+
+ def register(cls):
+ GGML_QUANT_BLOCK[name] = cls
+ return cls
+
+ return register
+
+
+def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, device="cuda"):
+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]
+
+ data = data.to(torch.float32).to(device)
+ scale = scale.to(device) if scale is not None else scale
+ zp = zp.to(device) if zp is not None else zp
+ wmin_m = wmin_m.to(device) if wmin_m is not None else wmin_m
+ d_scale = d_scale.to(device) if d_scale is not None else d_scale
+ d_wmin_m = d_wmin_m.to(device) if d_wmin_m is not None else d_wmin_m
+
+ shape = data.shape
+ n_blocks = data.nelement() // block_size
+ blocks = data.reshape((n_blocks, block_size))
+
+ quant_func = GGML_QUANT_BLOCK[ggml_type]
+ new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)
+
+ assert new_data.shape[-1] == type_size
+ new_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)
+ return new_data
+
+
+def torch_roundf(n):
+ a = torch.abs(n)
+ floored = torch.floor(a)
+ b = floored + torch.floor(2 * (a-floored))
+ return torch.sign(n) * b
+
+
+@register_block("bf16")
+def bf16_quant_block(blocks, scale=None, zp=None, **kwargs):
+ n = blocks.view(torch.uint32)
+ # force nan to quiet
+ n = torch.where((n & 0x7fffffff) > 0x7f800000, (n & 0xffff0000) | (64 << 16), n)
+ # round to nearest even
+ n = n.to(torch.uint64) + (0x7fff + ((n >> 16) & 1)) >> 16
+ return n.cpu().numpy().astype(np.uint16).view(np.uint8)
+
+
+@register_block("q4_0")
+def q4_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
+ if scale is not None:
+ d = scale.reshape((-1, 1))
+ else:
+ max = torch.max(blocks, axis=-1, keepdims=True)[0]
+ d = max / -8
+ id = torch.where(d == 0, 0, 1 / d)
+
+ qs = torch.trunc(blocks.to(torch.float64) * id.to(torch.float64) + 8.5).to(torch.uint8).clip(0, 15)
+
+ n_blocks = blocks.shape[0]
+ block_size = GGML_QUANT_SIZES["q4_0"][0]
+ qs = qs.reshape((n_blocks, 2, block_size // 2)).cpu().numpy()
+ qs = qs[..., 0, :] | (qs[..., 1, :] << 4)
+
+ d = d.cpu().numpy().astype(np.float16).view(np.uint8)
+
+ return np.concatenate([d, qs], axis=-1)
+
+
+@register_block("q4_1")
+def q4_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
+ if scale is not None:
+ d = scale.reshape((-1, 1))
+ min = zp.reshape((-1, 1)) * d * -1
+ else:
+ max = blocks.max(axis=-1, keepdims=True)[0]
+ min = blocks.min(axis=-1, keepdims=True)[0]
+ d = (max-min) / 15
+ id = torch.where(d == 0, 0, 1 / d)
+
+ qs = torch.trunc((blocks-min) * id + 0.5).to(torch.uint8).clip(0, 15)
+
+ n_blocks = blocks.shape[0]
+ block_size = GGML_QUANT_SIZES["q4_1"][0]
+ qs = qs.reshape((n_blocks, 2, block_size // 2)).cpu().numpy()
+ qs = qs[..., 0, :] | (qs[..., 1, :] << np.uint8(4))
+
+ d = d.cpu().numpy().astype(np.float16).view(np.uint8)
+ m = min.cpu().numpy().astype(np.float16).view(np.uint8)
+ return np.concatenate([d, m, qs], axis=-1)
+
+
+@register_block("q8_0")
+def q8_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs) -> np.ndarray:
+ if scale is not None:
+ d = scale.reshape((-1, 1))
+ else:
+ d = torch.abs(blocks).max(axis=1, keepdims=True)[0] / 127
+ id = torch.where(d == 0, 0, 1 / d)
+
+ qs = torch_roundf(blocks * id)
+
+ # (n_blocks, 2)
+ d = d.cpu().numpy().astype(np.float16).view(np.uint8)
+ # (n_blocks, block_size)
+ qs = qs.cpu().numpy().astype(np.int8).view(np.uint8)
+
+ return np.concatenate([d, qs], axis=1)
+
+
+def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
+ assert bits in [2, 4], f"bits={bits} is not supported"
+ nmax = bits ** 2 -1
+ # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K
+ if len(data.shape) == 2:
+ data_shape = (-1, 8, 32) if bits == 4 else (-1, 16, 16)
+ data = data.reshape(data_shape)
+ sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)
+ av_x = torch.sqrt(sum_x2 / 32)
+ weight = torch.abs(data) + av_x
+
+ group_min = torch.min(data, axis=-1, keepdims=True)[0]
+ group_max = torch.max(data, axis=-1, keepdims=True)[0]
+
+ the_mins = -group_min
+
+ sum_w = torch.sum(weight, axis=-1, keepdims=True)
+ sum_x = torch.sum(weight * data, axis=-1, keepdims=True)
+
+ group_min[group_min > 0] = 0
+
+ scale = (group_max - group_min) / nmax
+ iscale = torch.where(scale == 0, 0, 1 /scale)
+
+ l_values = torch.round(iscale * (data - group_min))
+ L = torch.clip(l_values, 0, nmax).to(torch.uint8)
+
+ diffs = scale * L + group_min - data
+ diffs = torch.abs(diffs) if use_mad else diffs**2
+ best_mad = torch.sum(weight * diffs, axis=-1, keepdims=True)
+
+ if nstep < 1:
+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
+
+ for step in range(nstep):
+ new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)
+ new_iscale = torch.where(new_scale == 0, 0, 1 /new_scale)
+
+ l_values = torch.round(new_iscale * (data - group_min))
+ Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)
+
+ sum_l = torch.sum(weight * Laux, axis=-1, keepdims=True)
+ sum_l2 = torch.sum(weight * Laux**2, axis=-1, keepdims=True)
+ sum_xl = torch.sum(weight * Laux * data, axis=-1, keepdims=True)
+
+ D = sum_w * sum_l2 - sum_l * sum_l
+ replace_idx = D > 0
+
+ this_scale = (sum_w * sum_xl - sum_x * sum_l) / D
+ this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D
+ this_min[this_min > 0] = 0
+ this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]
+
+ diffs = this_scale * Laux + this_min - data
+ diffs = torch.abs(diffs) if use_mad else diffs**2
+ mad = torch.sum(weight * diffs, axis=-1, keepdims=True)
+
+ replace_idx &= mad < best_mad
+ best_mad[replace_idx] = mad[replace_idx]
+ L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]
+ scale[replace_idx] = this_scale[replace_idx]
+ group_min[replace_idx] = this_min[replace_idx]
+
+ the_mins = -group_min
+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
+
+
+@register_block("q2_k")
+def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ nb = blocks.shape[0]
+ output_scale = np.empty((nb, 16), dtype=np.uint8)
+ output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)
+
+ blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)
+
+ if scale is not None:
+ scales = scale.reshape((-1, QK_K // 16))
+ mins = wmin_m.reshape((-1, QK_K // 16))
+ output_d = d_scale.to(torch.float32) if len(d_scale.shape) == 2 \
+ else d_scale.reshape(-1, 1).to(torch.float32)
+ output_dmin = d_wmin_m.to(torch.float32) if len(d_wmin_m.shape) == 2 \
+ else d_wmin_m.reshape(-1, 1).to(torch.float32)
+ inv_scales = torch.where(d_scale == 0, 0, 1 / output_d)
+ inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)
+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)
+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).to(torch.uint8)
+ else:
+ scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)
+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0]
+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0]
+ inv_scales = torch.where(max_scales > 0, 15. / max_scales, 0)
+ inv_mins = torch.where(max_mins > 0, 15. / max_mins, 0)
+
+ replace_ids = (max_scales > 0).squeeze()
+ output_scale = torch.zeros_like(scales).to(torch.uint8)
+ output_scale[replace_ids] = torch.round(inv_scales * scales).to(torch.uint8)
+
+ replace_ids = (max_mins > 0).squeeze()
+ output_scale[replace_ids] |= torch.round(inv_mins * mins).to(torch.uint8) << 4
+
+ if d_scale is None:
+ output_d = torch.where(max_scales > 0, max_scales / 15, 0)
+ if d_wmin_m is None:
+ output_dmin = torch.where(max_mins > 0 ,max_mins / 15., 0)
+
+ d_tmp = output_d * (output_scale & 0xF)
+ dm_tmp = output_dmin * (output_scale >> 4)
+
+ replace_ids = d_tmp != 0
+ all_L[replace_ids] = torch.round(
+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(
+ torch.uint8)
+ all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 3)
+
+ output_scale = output_scale.cpu().numpy()
+ output_qs = all_L[:,::4] | (all_L[:,1::4] << 2) | (all_L[:,2::4] << 4) | (all_L[:,3::4] << 6)
+ output_d = output_d.cpu().numpy()
+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_dmin = output_dmin.cpu().numpy()
+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_qs = output_qs.reshape((nb, QK_K // 4))
+
+ # [scale, qs, d, dmin]
+ return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)
+
+
+@register_block("q4_k")
+def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ nb = blocks.shape[0]
+ blocks = blocks.reshape((nb, QK_K // 32, 32))
+
+ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
+ output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)
+
+ if scale is not None:
+ scales = scale.reshape(-1, QK_K // 32)
+ mins = wmin_m.reshape(-1, QK_K // 32)
+ output_d = d_scale.to(torch.float32) if len(d_scale.shape) == 2 \
+ else d_scale.reshape(-1, 1).to(torch.float32)
+ output_dmin = d_wmin_m.to(torch.float32) if len(d_wmin_m.shape) == 2 \
+ else d_wmin_m.reshape(-1, 1).to(torch.float32)
+ inv_scales = torch.where(d_scale == 0, 0, 1 / output_d)
+ inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)
+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)
+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).to(torch.uint8)
+ else:
+ scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)
+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0]
+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0]
+ inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)
+ inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)
+
+ q_scales = torch.round(inv_scales * scales)
+ q_mins = torch.round(inv_mins * mins)
+
+ if d_scale is None:
+ output_d = max_scales / 63
+ if d_wmin_m is None:
+ output_dmin = max_mins / 63
+
+ d_tmp = output_d * q_scales
+ dm_tmp = output_dmin * q_mins
+
+ q_scales = q_scales.cpu().numpy().astype(np.uint8)
+ q_mins = q_mins.cpu().numpy().astype(np.uint8)
+ output_scale[:,:4] = q_scales[:,:4]
+ output_scale[:,4:8] = q_mins[:,:4]
+
+ output_scale[:,8:] = (q_scales[:,4:] & 0xF) | ((q_mins[:,4:] & 0xF) << 4)
+ output_scale[:,:4] |= ((q_scales[:,4:] >> 4) << 6)
+ output_scale[:,4:8] |= ((q_mins[:,4:] >> 4) << 6)
+
+ replace_ids = d_tmp != 0
+ all_L[replace_ids] = torch.round(
+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)
+ all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 15)
+ output_qs = all_L[:,::2] | (all_L[:,1::2] << 4)
+
+ output_d = output_d.cpu().numpy()
+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_dmin = output_dmin.cpu().numpy()
+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_qs = output_qs.reshape(nb, QK_K // 2)
+
+ # [d, dmin, scale, qs]
+ return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index b3685d9f6..5a9594f61 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -34,6 +34,7 @@ from auto_round.utils import (
get_fp_layer_names,
clear_memory,
get_device_and_parallelism,
+ get_model_dtype,
set_cuda_visible_devices)
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
@@ -192,6 +193,12 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument(
"--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant.")
+ self.add_argument(
+ "--eval_model_dtype",
+ default=None,
+ type=str,
+ help="the torch_dytpe to load the model for evaluation.")
+
class EvalArgumentParser(argparse.ArgumentParser):
@@ -218,6 +225,11 @@ class EvalArgumentParser(argparse.ArgumentParser):
"--disable_trust_remote_code", action='store_true', help="whether to disable trust_remote_code")
self.add_argument("--eval_bs", "--bs", "--batch_size", default=None, type=int, help="batch size in evaluation")
self.add_argument("--eval_task_by_task", action='store_true', help="whether to eval task by task.")
+ self.add_argument(
+ "--eval_model_dtype",
+ default=None,
+ type=str,
+ help="the torch_dytpe to load the model for evaluation.")
def setup_parser():
@@ -282,7 +294,7 @@ def setup_light_parser():
"--lr", default=5e-3, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically")
args = parser.parse_args()
-
+
return args
@@ -519,51 +531,84 @@ def tune(args):
eval_gguf_model = True
break
+ import time
+ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
if args.act_bits <= 8 or eval_gguf_model:
if eval_gguf_model:
# gguf floder only contains one file
for file in os.listdir(eval_folder):
gguf_file = file
- user_model = AutoModelForCausalLM.from_pretrained(
- eval_folder, gguf_file=gguf_file, device_map="auto" if use_auto_mapping else None)
+
+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")
+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":
+ logger.warning(
+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
+ " but may affect accuracy."
+ )
+ model = AutoModelForCausalLM.from_pretrained(
+ eval_folder, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)
+ model.eval()
tokenizer = AutoTokenizer.from_pretrained(eval_folder, gguf_file=gguf_file)
else:
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
from accelerate.big_modeling import dispatch_model
dispatch_model(model, model.hf_device_map)
- user_model = model
else:
device_str = detect_device(device_str)
- user_model = model.to(device_str)
+ model = model.to(device_str)
+ if model.dtype != eval_model_dtype and eval_model_dtype != "auto":
+ model.to(getattr(torch, eval_model_dtype))
if args.eval_task_by_task:
eval_task_by_task(
- user_model, tokenizer=tokenizer, device=device_str, tasks=args.tasks, batch_size=args.eval_bs)
+ model,
+ tokenizer=tokenizer,
+ device=device_str,
+ tasks=args.tasks,
+ batch_size=args.eval_bs,
+ eval_model_dtype=eval_model_dtype)
else:
if args.eval_bs is None or args.eval_bs == "auto":
logger.warning("This API does not support auto currently, reset eval_bs to 16")
args.eval_bs = 16
from auto_round.eval.evaluation import simple_evaluate_user_model
+ st = time.time()
res = simple_evaluate_user_model(
- user_model, tokenizer, tasks=tasks, batch_size=args.eval_bs, device=device_str)
+ model,
+ tokenizer,
+ tasks=tasks,
+ batch_size=args.eval_bs,
+ device=device_str,
+ eval_model_dtype=eval_model_dtype)
print(make_table(res))
+ print("evaluation running time=", time.time() - st)
else:
if args.eval_task_by_task:
- eval_task_by_task(eval_folder, device=device_str, tasks=args.tasks, batch_size=args.eval_bs)
+ eval_task_by_task(
+ eval_folder,
+ device=device_str,
+ tasks=args.tasks,
+ batch_size=args.eval_bs,
+ eval_model_dtype=eval_model_dtype)
else:
from auto_round.eval.evaluation import simple_evaluate
tasks, model_args, device_str = _eval_init(
- args.tasks, eval_folder, args.device, args.disable_trust_remote_code)
+ args.tasks, eval_folder, args.device, args.disable_trust_remote_code, dtype=eval_model_dtype)
+ st = time.time()
res = simple_evaluate(
model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs)
print(make_table(res))
+ print("evaluation running time=", time.time() - st)
-def _eval_init(tasks, model_path, device, disable_trust_remote_code=False):
+def _eval_init(tasks, model_path, device, disable_trust_remote_code=False, dtype="auto"):
set_cuda_visible_devices(device)
device_str, parallelism = get_device_and_parallelism(device)
- model_args = f'pretrained={model_path},trust_remote_code={not disable_trust_remote_code}' # ,add_bos_token={True}
+ if dtype != "auto":
+ dtype = get_model_dtype(model_dtype=dtype)
+ # ,add_bos_token={True}
+ model_args = f'pretrained={model_path},trust_remote_code={not disable_trust_remote_code},dtype={dtype}'
if parallelism:
model_args += ",parallelize=True"
if isinstance(tasks, str):
@@ -572,19 +617,66 @@ def _eval_init(tasks, model_path, device, disable_trust_remote_code=False):
def eval(args):
- tasks, model_args, device_str = _eval_init(args.tasks, args.model, args.device, args.disable_trust_remote_code)
+ import time
+ tasks, model_args, device_str = _eval_init(
+ args.tasks, args.model, args.device, args.disable_trust_remote_code, args.eval_model_dtype)
# load after _eval_int in order to make sure import torch after set CUDA_VISBILE_DEVICES
- from auto_round.eval.evaluation import simple_evaluate
+ from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model
+ from auto_round.utils import logger
- res = simple_evaluate(model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs)
+ is_gguf_file = False
+ if os.path.isfile(args.model) and args.model.endswith(".gguf"):
+ is_gguf_file = True
+ gguf_file = os.path.basename(args.model)
+ model = os.path.dirname(args.model)
+ else:
+ for file in os.listdir(args.model):
+ if file.endswith(".gguf"):
+ is_gguf_file = True
+ gguf_file = file
+ eval_model_dtype = get_model_dtype(args.eval_model_dtype)
+ if is_gguf_file:
+ import torch
+ from transformers import AutoTokenizer, AutoModelForCausalLM
+ from lm_eval.utils import make_table # pylint: disable=E0401
+ tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
- from lm_eval.utils import make_table # pylint: disable=E0401
- print(make_table(res))
+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")
+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":
+ logger.warning(
+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
+ " but may affect accuracy."
+ )
+ model = AutoModelForCausalLM.from_pretrained(
+ model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)
+ model.eval()
+ if (batch_size := args.eval_bs) is None:
+ batch_size = "auto:8"
+ st = time.time()
+ res = simple_evaluate_user_model(
+ model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str)
+ print(make_table(res))
+ print("evaluation running time=", time.time() - st)
+ else:
+ st = time.time()
+ res = simple_evaluate(
+ model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs)
+ from lm_eval.utils import make_table # pylint: disable=E0401
+ print(make_table(res))
+ print("evaluation running time=", time.time() - st)
def eval_task_by_task(
- model, device=None, tasks=None, tokenizer=None, batch_size=None, max_batch_size=64, trust_remote_code=True):
+ model,
+ device=None,
+ tasks=None,
+ tokenizer=None,
+ batch_size=None,
+ max_batch_size=64,
+ trust_remote_code=True,
+ eval_model_dtype=None,
+ retry_times=3):
set_cuda_visible_devices(device)
device_str, parallelism = get_device_and_parallelism(device)
@@ -597,7 +689,7 @@ def eval_task_by_task(
from auto_round import AutoRoundConfig # pylint: disable=E0611
if batch_size is None:
- batch_size = "auto"
+ batch_size = "auto:8"
is_gguf_file = False
if not isinstance(model, str):
parallelism = False
@@ -611,9 +703,20 @@ def eval_task_by_task(
if file.endswith(".gguf"):
is_gguf_file = True
gguf_file = file
+ eval_model_dtype = get_model_dtype(eval_model_dtype)
if is_gguf_file:
tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
- model = AutoModelForCausalLM.from_pretrained(model, gguf_file=gguf_file, device_map="auto")
+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")
+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":
+ logger.warning(
+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
+ " but may affect accuracy."
+ )
+
+ model = AutoModelForCausalLM.from_pretrained(
+ model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)
+ model.eval()
+ parallelism=False
hflm = HFLM(
pretrained=model,
tokenizer=tokenizer,
@@ -621,7 +724,9 @@ def eval_task_by_task(
batch_size=batch_size,
max_batch_size=max_batch_size,
parallelize=parallelism,
- trust_remote_code=trust_remote_code)
+ trust_remote_code=trust_remote_code,
+ dtype=eval_model_dtype
+ )
if isinstance(tasks, str):
tasks = tasks.replace(" ", "").split(",")
@@ -629,23 +734,32 @@ def eval_task_by_task(
from lm_eval.utils import make_table # pylint: disable=E0401
res_all = {}
res_keys = ["results", "versions", "n-shot", "higher_is_better"]
+ import time
+ st = time.time()
for task in tasks:
- try:
- res = lm_simple_evaluate(model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size)
- except RuntimeError as e:
- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
- try:
- logger.warning("Out of memory, reset batch_size to 1 and re-try.")
- res = lm_simple_evaluate(model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1)
- except Exception as e:
+ while retry_times:
+ try:
+ res = lm_simple_evaluate(
+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size)
+ break
+ except Exception as e:
+ if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
+ ori_batch_sizes = hflm.batch_sizes if hflm.batch_sizes else {"0": 64}
+ try:
+ for k, v in hflm.batch_sizes.items():
+ hflm.batch_sizes[k] = max(v // 2, 1)
+ logger.warning(
+ f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")
+ res = lm_simple_evaluate(
+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1)
+ hflm.batch_sizes = ori_batch_sizes
+ except Exception as e:
+ traceback.print_exc()
+ pass
+ else:
traceback.print_exc()
- continue
- else:
- traceback.print_exc()
- continue
- except Exception as e:
- traceback.print_exc()
- continue
+ break
+ retry_times -= 1
if not res_all:
res_all = res
@@ -654,3 +768,4 @@ def eval_task_by_task(
res_all[key].update(res[key])
print(make_table(res_all))
+ print("total eval time:", time.time() - st)
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 00db00aae..394bfc10e 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1185,6 +1185,13 @@ def _gguf_args_check(args):
pre_dq_format = ""
for format in GGUF_CONFIG:
if format in formats:
+ try:
+ from auto_round.export.export_to_gguf.convert import Model
+ except:
+ raise ImportError(
+ f"Please use the latest gguf-py for {format}, you can use the following command to install it:\n"
+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install .")
+ sys.exit(-1)
if re.search(pattern, format):
if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:
logger.error(f"Cannot eport {pre_dq_format} and {format} at the same time.")
@@ -1221,7 +1228,7 @@ def _gguf_args_check(args):
setattr(args, k, v)
if len(unsupport_list) > 0:
logger.error(
- f"format {format} not support for {', '.join(unsupport_list)},"
+ f"format {format} does not support for {', '.join(unsupport_list)},"
f" reset to {', '.join(reset_list)}.")
logger.info(f"export format {format}, sym = {not args.asym}, group_size = {args.group_size}")
@@ -1439,3 +1446,18 @@ def get_shared_keys(model):
shared_keys = shared_cache_keys
shared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())
return shared_keys
+
+
+def get_model_dtype(model_dtype, default="auto"):
+ if model_dtype is None or model_dtype == "auto":
+ model_dtype = default
+ elif model_dtype in ["bf16", "bfloat16"]:
+ model_dtype = "bfloat16"
+ elif model_dtype in ["f16", "float16", "fp16"]:
+ model_dtype = "float16"
+ elif model_dtype in ["f32", "float32", "fp32"]:
+ model_dtype = "float32"
+ else:
+ logger.warning(f"Unable to identify model_dtype {model_dtype}, reset to default model_dtype {default}")
+ model_dtype = default
+ return model_dtype
\ No newline at end of file
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 9ffadaebd..2be872828 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -100,7 +100,7 @@ CPU, Intel GPU, HPU,and CUDA for both quantization and inference.
This setting provides the best accuracy in most scenarios but is 4–5× slower than the standard AutoRound recipe. It is especially recommended for 2-bit quantization and is a good choice if sufficient resources are available.
-- ```bash
+ ```bash
auto-round-best --model facebook/opt-125m --bits 4 --group_size 128 --format "auto_gptq,auto_awq,auto_round"
```