Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

GGUF support step2: add naive Q2_KS and Q4_KS (#448)

Heng Guo12539ef5aab0
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 1bfca8b89..abd7750aa 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -162,6 +162,8 @@ class AutoRound(object):
enable_norm_bias_tuning: bool = False,
enable_torch_compile: bool = False,
device_map: Union[str, dict] = None,
+ super_bits: int = None,
+ super_group_size: int = None,
**kwargs,
):
self.quantized = False
@@ -223,6 +225,9 @@ class AutoRound(object):
self.batch_dim = None
self.infer_bs_coeff = 1
+ self.super_bits = super_bits
+ self.super_group_size = super_group_size
+
torch.set_printoptions(precision=3, sci_mode=True)
self.check_configs()
if self.act_bits <= 8 and self.amp_dtype == torch.float16:
@@ -523,7 +528,7 @@ class AutoRound(object):
"""
layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
keys = ["data_type", "bits", "group_size", "sym", "scale_dtype", "act_bits", "act_group_size", "act_sym",
- "act_dynamic", "act_data_type"]
+ "act_dynamic", "act_data_type", "super_bits", "super_group_size"]
for n, m in self.model.named_modules():
##delete keys to avoid conflict with the previous tuning
for key in keys:
@@ -1662,6 +1667,8 @@ class AutoRoundOPT(AutoRound):
enable_torch_compile: bool = False,
device_map: Union[str, dict] = None,
optimizer="AdamW",
+ super_bits: int = None,
+ super_group_size: int = None,
**kwargs,
):
super(AutoRoundOPT, self).__init__(
@@ -1702,6 +1709,8 @@ class AutoRoundOPT(AutoRound):
enable_norm_bias_tuning=enable_norm_bias_tuning,
enable_torch_compile=enable_torch_compile,
device_map=device_map,
+ super_bits=super_bits,
+ super_group_size=super_group_size,
**kwargs,
)
@@ -1836,6 +1845,8 @@ class AutoRoundAdam(AutoRoundOPT):
enable_torch_compile: bool = False,
device_map: Union[str, dict] = None,
optimizer="AdamW",
+ super_bits: int = None,
+ super_group_size: int = None,
**kwargs,
):
super(AutoRoundAdam, self).__init__(
@@ -1877,5 +1888,7 @@ class AutoRoundAdam(AutoRoundOPT):
enable_torch_compile=enable_torch_compile,
device_map=device_map,
optimizer=optimizer,
+ super_bits=super_bits,
+ super_group_size=super_group_size,
**kwargs,
)
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index 26e5a4d7a..899abf17a 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -65,7 +65,7 @@ def quant_tensor_sym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scal
## the values should be positive
def double_quant_tensor(tensor, bits, q_scale_thresh):
- maxq = 2 ** bits
+ maxq = 2 ** bits - 1
wmax = torch.clamp(tensor.max(-1)[0], min=0)
scale = torch.clamp(wmax / maxq, q_scale_thresh)
scale = scale.view(-1, 1)
@@ -75,7 +75,7 @@ def double_quant_tensor(tensor, bits, q_scale_thresh):
@register_dtype("int_asym_dq")
def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
- tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, super_group_size=32, super_bits=6,
+ tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, super_group_size=8, super_bits=6,
**kwargs):
"""Quantize and de-quantize tensor asymmetrically.
@@ -128,7 +128,7 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_
qdq_result = (scale * q - wmin_m).to(tensor.dtype)
qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
zp = round_ste(wmin_m / scale) # remove this later
- return qdq_result, scale, zp
+ return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}
@register_dtype("int_asym")
diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.py
new file mode 100644
index 000000000..2d7b1f163
--- /dev/null
+++ b/auto_round/export/export_to_gguf/config.py
@@ -0,0 +1,41 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GGUF_CONFIG = {}
+
+GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": False, "data_type": "int"}
+
+GGUF_CONFIG["gguf:q4_1"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": True, "data_type": "int"}
+
+GGUF_CONFIG["gguf:q4_k_s"] = {
+ "bits": 4,
+ "act_bits": 16,
+ "super_group_size": 8,
+ "super_bits": 6,
+ "group_size": 32,
+ "asym": True,
+ "data_type": "int_asym_dq"
+}
+
+GGUF_CONFIG["gguf:q2_k_s"] = {
+ "bits": 2,
+ "act_bits": 16,
+ "super_group_size": 16,
+ "super_bits": 4,
+ "group_size": 16,
+ "asym": True,
+ "data_type": "int_asym_dq"
+}
+
+GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "asym": False, "data_type": "int"}
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 9024947de..9d3c4b852 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -1253,16 +1253,16 @@ class Model(OriModel):
data_qtype = gguf.GGMLQuantizationType.F16
elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:
data_qtype = gguf.GGMLQuantizationType.BF16
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:
- # data_qtype = gguf.GGMLQuantizationType.Q8_0
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_TQ1_0:
- # data_qtype = gguf.GGMLQuantizationType.TQ1_0
- # elif self.ftype == gguf.LlamaFileType.MOSTLY_TQ2_0:
- # data_qtype = gguf.GGMLQuantizationType.TQ2_0
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:
+ data_qtype = gguf.GGMLQuantizationType.Q8_0
elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:
data_qtype = gguf.GGMLQuantizationType.Q4_0
elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:
data_qtype = gguf.GGMLQuantizationType.Q4_1
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:
+ data_qtype = gguf.GGMLQuantizationType.Q4_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:
+ data_qtype = gguf.GGMLQuantizationType.Q2_K
else:
raise ValueError(
f"Unknown file type: {self.ftype.name}")
@@ -1285,11 +1285,23 @@ class Model(OriModel):
scale = module.scale
if isinstance(scale, torch.Tensor):
scale = scale.numpy()
- zp = module.zp
+ zp = module.zp if hasattr(module, "zp") else None
if isinstance(zp, torch.Tensor):
zp = zp.numpy()
- data = ggml_quant(data, data_qtype.name.lower(), scale,
- zp)
+ if data_qtype.name.lower().endswith("_k"):
+ d_scale = module.w_d_scale.numpy()
+ d_wmin_m = module.w_d_wmin_m.numpy()
+ wmin_m = module.w_wmin_m.numpy()
+ data = ggml_quant(
+ data,
+ data_qtype.name.lower(),
+ scale,
+ zp,
+ wmin_m=wmin_m,
+ d_scale=d_scale,
+ d_wmin_m=d_wmin_m)
+ else:
+ data = ggml_quant(data, data_qtype.name.lower(), scale, zp)
else:
data_qtype = gguf.GGMLQuantizationType.F32
else:
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index 211e1f00e..e5af13201 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -19,6 +19,7 @@ import torch
from .convert import Model
from auto_round.utils import logger, LazyImport
from pathlib import Path
+import time
gguf = LazyImport("gguf")
@@ -30,12 +31,15 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {
"q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,
"q4_0": gguf.LlamaFileType.MOSTLY_Q4_0,
"q4_1": gguf.LlamaFileType.MOSTLY_Q4_1,
- "q4_k": gguf.LlamaFileType.MOSTLY_Q4_K_S,
+ "q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,
+ "q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,
+ "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,
"auto": gguf.LlamaFileType.GUESSED,
}
def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):
"""Export the model to gguf format."""
+ st = time.time()
model = kwargs["model"]
tokenizer = kwargs.get("tokenizer", None)
@@ -77,7 +81,8 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):
dry_run=False,
small_first_shard=False)
model_instance.write()
- logger.info(f"Model successfully exported to {model_instance.fname_out}")
+ rt = time.time() - st
+ logger.info(f"Model successfully exported to {model_instance.fname_out}, running time={rt}")
shutil.rmtree(tmp_work_dir, ignore_errors=True)
diff --git a/auto_round/export/export_to_gguf/quant.py b/auto_round/export/export_to_gguf/quant.py
index aa12b6ee3..1b0be3a80 100644
--- a/auto_round/export/export_to_gguf/quant.py
+++ b/auto_round/export/export_to_gguf/quant.py
@@ -13,38 +13,80 @@
# limitations under the License.
import numpy as np
+from tqdm import tqdm
+from concurrent.futures import ProcessPoolExecutor
QK_K = 256
+K_SCALE_SIZE = 12
GGML_QUANT_SIZES = {
"bf16": (1, 2),
"q4_0": (32, 2 + 16),
"q4_1": (32, 2 + 2 + 16),
- "q4_k": (256, 2 + 2 + QK_K // 2 + 12),
+ "q4_k": (256, 2 + 2 + QK_K//2 + 12),
+ "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),
+ "q8_0": (32, 2 + 32)
}
GGML_QUANT_BLOCK = {}
+
def register_block(name):
+
def register(cls):
GGML_QUANT_BLOCK[name] = cls
return cls
+
return register
-def ggml_quant(data: np.array, ggml_type, scale = None, zp = None):
+def ggml_quant(data: np.array, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, worker=16):
block_size, type_size = GGML_QUANT_SIZES[ggml_type]
data = data.astype(np.float32, copy=False)
shape = data.shape
n_blocks = data.size // block_size
blocks = data.reshape((n_blocks, block_size))
-
- new_data = GGML_QUANT_BLOCK[ggml_type](blocks, scale, zp)
+
+ if worker > 0:
+ n_groups = (data.shape[0] // worker) or 1
+ blocks = np.array_split(blocks, n_groups, axis=0)
+ scale = np.array_split(scale, n_groups, axis=0) if scale is not None else [None] * n_groups
+ zp = np.array_split(zp, n_groups, axis=0) if zp is not None else [None] * n_groups
+ wmin_m = np.array_split(wmin_m, n_groups, axis=0) if wmin_m is not None else [None] * n_groups
+ d_scale = np.array_split(d_scale, n_groups, axis=0) if d_scale is not None else [None] * n_groups
+ d_wmin_m = np.array_split(d_wmin_m, n_groups, axis=0) if d_wmin_m is not None else [None] * n_groups
+
+ quant_func = GGML_QUANT_BLOCK[ggml_type]
+ if ggml_type.endswith("_k"):
+ with ProcessPoolExecutor(worker) as executor:
+ result = executor.map(quant_func, blocks, scale, zp, wmin_m, d_scale, d_wmin_m)
+ new_data = np.array(list(result), dtype=np.uint8)
+ else:
+ with ProcessPoolExecutor(n_groups) as executor:
+ result = executor.map(quant_func, blocks, scale, zp)
+ new_data = np.array(list(result), dtype=np.uint8)
+ else:
+ quant_func = GGML_QUANT_BLOCK[ggml_type]
+ if ggml_type.endswith("_k"):
+ new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)
+ else:
+ new_data = quant_func(blocks, scale, zp)
+
+ assert new_data.dtype == np.uint8
+ assert new_data.shape[-1] == type_size
new_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)
return new_data
+
+def np_roundf(n: np.ndarray) -> np.ndarray:
+ a = abs(n)
+ floored = np.floor(a)
+ b = floored + np.floor(2 * (a-floored))
+ return np.sign(n) * b
+
+
@register_block("bf16")
-def bf16_quant_block(blocks: np.array, scale = None, zp = None):
+def bf16_quant_block(blocks: np.array, scale=None, zp=None):
n = blocks.view(np.uint32)
# force nan to quiet
n = np.where((n & 0x7fffffff) > 0x7f800000, (n & np.uint32(0xffff0000)) | np.uint32(64 << 16), n)
@@ -52,10 +94,11 @@ def bf16_quant_block(blocks: np.array, scale = None, zp = None):
n = (np.uint64(n) + (0x7fff + ((n >> 16) & 1))) >> 16
return n.astype(np.uint16).view(np.uint8)
+
@register_block("q4_0")
-def q4_0_quant_block(blocks: np.array, scale = None, zp = None):
+def q4_0_quant_block(blocks: np.array, scale=None, zp=None):
if scale is not None:
- d = scale.reshape((-1,1))
+ d = scale.reshape((-1, 1))
else:
imax = abs(blocks).argmax(axis=-1, keepdims=True)
max = np.take_along_axis(blocks, imax, axis=-1)
@@ -63,9 +106,8 @@ def q4_0_quant_block(blocks: np.array, scale = None, zp = None):
with np.errstate(divide="ignore"):
id = np.where(d == 0, 0, 1 / d)
- qs = np.trunc(
- (np.float64(blocks) * np.float64(id)) + np.float64(8.5),
- dtype=np.float32).astype(np.uint8).clip(0, 15)
+ qs = np.trunc((np.float64(blocks) * np.float64(id)) + np.float64(8.5),
+ dtype=np.float32).astype(np.uint8).clip(0, 15)
n_blocks = blocks.shape[0]
block_size = GGML_QUANT_SIZES["q4_0"][0]
@@ -78,18 +120,18 @@ def q4_0_quant_block(blocks: np.array, scale = None, zp = None):
@register_block("q4_1")
-def q4_1_quant_block(blocks: np.array, scale = None, zp = None):
+def q4_1_quant_block(blocks: np.array, scale=None, zp=None):
if scale is not None:
- d = scale.reshape((-1,1))
- min = zp.reshape((-1,1)) * d * -1
+ d = scale.reshape((-1, 1))
+ min = zp.reshape((-1, 1)) * d * -1
else:
max = blocks.max(axis=-1, keepdims=True)
min = blocks.min(axis=-1, keepdims=True)
- d = (max - min) / 15
+ d = (max-min) / 15
with np.errstate(divide="ignore"):
id = np.where(d == 0, 0, 1 / d)
- qs = np.trunc((blocks - min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 15)
+ qs = np.trunc((blocks-min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 15)
n_blocks = blocks.shape[0]
block_size = GGML_QUANT_SIZES["q4_1"][0]
@@ -99,3 +141,239 @@ def q4_1_quant_block(blocks: np.array, scale = None, zp = None):
d = d.astype(np.float16).view(np.uint8)
m = min.astype(np.float16).view(np.uint8)
return np.concatenate([d, m, qs], axis=-1)
+
+
+@register_block("q8_0")
+def q8_0_quant_block(blocks: np.array, scale=None, zp=None) -> np.ndarray:
+ if scale is not None:
+ d = scale.reshape((-1, 1))
+ else:
+ d = abs(blocks).max(axis=1, keepdims=True) / 127
+ with np.errstate(divide="ignore"):
+ id = np.where(d == 0, 0, 1 / d)
+ qs = np_roundf(blocks * id)
+
+ # (n_blocks, 2)
+ d = d.astype(np.float16).view(np.uint8)
+ # (n_blocks, block_size)
+ qs = qs.astype(np.int8).view(np.uint8)
+
+ return np.concatenate([d, qs], axis=1)
+
+
+def make_qkx2_quants(data, weight, nmax, group_size, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
+ group_min = np.min(data)
+ group_max = np.max(data)
+
+ sum_w = np.sum(weight)
+ sum_x = np.sum(weight * data)
+
+ if group_min > 0:
+ group_min = 0
+ if group_min == group_max:
+ L = np.zeros(group_size, dtype=np.uint8)
+ the_min = -group_min
+ return 0.0, L, the_min
+
+ iscale = nmax / (group_max-group_min)
+ scale = 1 / iscale
+ L = np.zeros(group_size, dtype=np.uint8)
+
+ l_values = np.round(iscale * (data-group_max))
+ L = np.clip(l_values, 0, nmax).astype(np.uint8)
+ diffs = scale*L + group_min - data
+ diffs = np.abs(diffs) if use_mad else diffs**2
+ best_mad = np.sum(weight * (diffs))
+
+ if nstep < 1:
+ the_min = -group_min
+ return scale, L, the_min
+
+ Laux = []
+ for step in range(nstep):
+ iscale = (rmin + rdelta*step + nmax) / (group_max-group_min)
+ l_values = np.round(iscale * (data-group_min))
+ Laux = np.clip(l_values, 0, nmax).astype(np.uint8)
+
+ sum_l = np.sum(weight * Laux)
+ sum_l2 = np.sum(weight * Laux**2)
+ sum_xl = np.sum(weight * Laux * data)
+
+ D = sum_w*sum_l2 - sum_l*sum_l
+ if D > 0:
+ this_scale = (sum_w*sum_xl - sum_x*sum_l) / D
+ this_min = (sum_l2*sum_x - sum_l*sum_xl) / D
+ if this_min > 0:
+ this_min = 0
+ this_scale = sum_xl / sum_l2
+
+ diffs = this_scale*Laux + this_min - data
+ mad = np.sum(weight * diffs**2)
+
+ if mad < best_mad:
+ L = Laux.copy()
+ best_mad = mad
+ scale = this_scale
+ group_min = this_min
+
+ the_min = -group_min
+ return scale, L, the_min
+
+
+@register_block("q2_k")
+def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ nb = blocks.shape[0]
+ output_scale = np.empty((nb, 16), dtype=np.uint8)
+ output_d = np.empty(nb, dtype=np.float32)
+ output_dmin = np.empty(nb, dtype=np.float32)
+ output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)
+
+ blocks = blocks.reshape((nb, QK_K // 16, 16))
+ weight = np.abs(blocks)
+ scales = np.empty(QK_K // 16, dtype=np.float32)
+ mins = np.empty((QK_K // 16), dtype=np.float32)
+
+ if scale is not None:
+ scale = scale.reshape((-1, QK_K // 16))
+ wmin_m = wmin_m.reshape((-1, QK_K // 16))
+ output_d = d_scale.astype(np.float32)
+ output_dmin = d_wmin_m.astype(np.float32)
+ inv_scales = np.where(d_scale == 0, 0, 1 / output_d)
+ inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)
+
+ # for i in tqdm(range(nb), desc="packing layer"):
+ for i in range(nb):
+ all_L = np.empty(blocks[i].shape, dtype=np.uint8)
+
+ if scale is not None:
+ scales = scale[i]
+ mins = wmin_m[i]
+ inv_scale = inv_scales[i]
+ inv_min = inv_mins[i]
+ max_scale = max(scales)
+ max_min = max(mins)
+ all_L = np.round((blocks[i] + mins.reshape(-1, 1)) / scales.reshape(-1, 1)).astype(np.uint8)
+ else:
+ for j in range(QK_K // 16):
+ tmp_scale, tmp_l, the_min = make_qkx2_quants(
+ blocks[i][j], weight[i][j], nmax=3, group_size=16, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)
+ all_L[j] = tmp_l
+ scales[j] = tmp_scale
+ mins[j] = the_min
+
+ max_scale = max(scales)
+ max_min = max(mins)
+ inv_scale = 15. / max_scale
+ inv_min = 15. / max_min
+
+ if max_scale > 0:
+ output_scale[i] = np.round(inv_scale * scales).astype(np.uint8)
+ output_d[i] = max_scale / 15.
+ else:
+ output_scale[i] = np.zeros_like(scales).astype(np.uint8)
+ output_d[i] = 0.
+ if max_min > 0:
+ output_scale[i] |= (np.round(inv_min * mins).astype(np.uint8) << 4)
+ output_dmin[i] = max_min / 15.
+ else:
+ output_dmin[i] = 0.
+
+ d_tmp = output_d[i] * (output_scale[i] & 0xF)
+ dm_tmp = output_dmin[i] * (output_scale[i] >> 4)
+ for j in range(QK_K // 16):
+ if d_tmp[j] == 0.:
+ continue
+ else:
+ all_L[j] = np.round((blocks[i][j] + dm_tmp[j]) / d_tmp[j]).astype(np.uint8)
+ all_L = np.clip(all_L, 0, 3)
+ output_qs[i] = all_L[::4] | (all_L[1::4] << 2) | (all_L[2::4] << 4) | (all_L[3::4] << 6)
+
+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_qs = output_qs.reshape((nb, QK_K // 4))
+
+ # [scale, qs, d, dmin]
+ return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)
+
+
+@register_block("q4_k")
+def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ nb = blocks.shape[0]
+ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
+ output_d = np.empty(nb, dtype=np.float32)
+ output_dmin = np.empty(nb, dtype=np.float32)
+ output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)
+
+ blocks = blocks.reshape((nb, QK_K // 32, 32))
+ sum_x2 = np.sum(np.power(blocks, 2), axis=-1)
+ av_x = np.sqrt(sum_x2 / 32)
+ weight = blocks + av_x.reshape((*av_x.shape, 1))
+ scales = np.empty(QK_K // 32, dtype=np.float32)
+ mins = np.empty(QK_K // 32, dtype=np.float32)
+
+ if scale is not None:
+ scale = scale.reshape(-1, QK_K // 32)
+ wmin_m = wmin_m.reshape(-1, QK_K // 32)
+ output_d = d_scale.astype(np.float32)
+ output_dmin = d_wmin_m.astype(np.float32)
+ inv_scales = np.where(d_scale == 0, 0, 1 / output_d)
+ inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)
+
+ # for i in tqdm(range(nb), desc="packing layer"):
+ for i in range(nb):
+ all_L = np.empty(blocks[i].shape, dtype=np.uint8)
+
+ if scale is not None:
+ scales = scale[i]
+ mins = wmin_m[i]
+ inv_scale = inv_scales[i]
+ inv_min = inv_mins[i]
+ max_scale = max(scales)
+ max_min = max(mins)
+ all_L = np.round((blocks[i] + mins.reshape(-1, 1)) / scales.reshape(-1, 1)).astype(np.uint8)
+ else:
+ for j in range(QK_K // 32):
+ tmp_scale, tmp_l, the_min = make_qkx2_quants(
+ blocks[i][j], weight[i][j], nmax=15, group_size=32, rmin=-1, rdelta=0.1, nstep=20)
+ all_L[j] = tmp_l
+ scales[j] = tmp_scale
+ mins[j] = the_min
+
+ max_scale = max(scales)
+ max_min = max(mins)
+ inv_scale = 63. / max_scale if max_scale > 0 else 0.
+ inv_min = 63. / max_min if max_min > 0 else 0.
+
+ ls = np.round(inv_scale * scales).astype(np.uint8)
+ lm = np.round(inv_min * mins).astype(np.uint8)
+
+ output_scale[i][:4] = ls[:4]
+ output_scale[i][4:8] = lm[:4]
+
+ output_scale[i][8:] = (ls[4:] & 0xF) | ((lm[4:] & 0xF) << 4)
+ output_scale[i][:4] |= ((ls[4:] >> 4) << 6)
+ output_scale[i][4:8] |= ((lm[4:] >> 4) << 6)
+
+ if d_scale is None:
+ output_d[i] = max_scale / 63
+ if d_wmin_m is None:
+ output_dmin[i] = max_min / 63
+
+ d_tmp = output_d[i] * ls
+ dm_tmp = output_dmin[i] * lm
+
+ for j in range(d_tmp.size):
+ if d_tmp[j] == 0.:
+ continue
+ else:
+ all_L[j] = np.round((blocks[i][j] + dm_tmp[j]) / d_tmp[j]).astype(np.uint8)
+
+ all_L = np.clip(all_L, 0, 15)
+ output_qs[i] = all_L[::2] | (all_L[1::2] << 4)
+
+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_qs = output_qs.reshape(nb, QK_K // 2)
+
+ # [d, dmin, scale, qs]
+ return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 9765fff0b..2efe4d499 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -186,6 +186,12 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")
+ self.add_argument(
+ "--super_group_size", default=None, type=int, help="the number of super group size when use double quant.")
+
+ self.add_argument(
+ "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant.")
+
class EvalArgumentParser(argparse.ArgumentParser):
@@ -304,24 +310,10 @@ def setup_eval_parser():
def _gguf_args_check(args):
from auto_round.utils import logger
-
- _GGUF_CONFIG = {
- "gguf:q4_0": {
- "bits": 4,
- "act_bits": 16,
- "group_size": 32,
- "asym": False,
- },
- "gguf:q4_1": {
- "bits": 4,
- "act_bits": 16,
- "group_size": 32,
- "asym": True,
- }
- }
+ from auto_round.export.export_to_gguf.config import GGUF_CONFIG
formats = args.format.lower().replace(' ', '').split(",")
- for format in _GGUF_CONFIG:
+ for format in GGUF_CONFIG:
if format in formats:
from pathlib import Path
from auto_round.export.export_to_gguf.convert import Model
@@ -333,8 +325,17 @@ def _gguf_args_check(args):
logger.error(f"Model {model_architecture} is not supported to export GGUF format")
sys.exit(1)
+ if format.endswith("_k") and ("hidden_size" in hparams and hparams["hidden_size"] % 256 !=0):
+ model_name = args.model.split('/')
+ model_name = model_name[-1] if model_name[-1] else model_name[-2]
+ hidden_size = hparams["hidden_size"]
+ logger.error(
+ f"Currently only support pure mode for format: {format}. "
+ f"{model_name} is not supported, cause hidden_size({hidden_size}) % 256 !=0")
+ sys.exit(-1)
+
unsupport_list, reset_list = [], []
- gguf_config = _GGUF_CONFIG[format]
+ gguf_config = GGUF_CONFIG[format]
for k, v in gguf_config.items():
if getattr(args, k) != v:
unsupport_list.append(f"{k}={getattr(args, k)}")
@@ -363,14 +364,17 @@ def tune(args):
from auto_round import AutoRoundConfig
from auto_round.utils import detect_device, get_library_version, detect_device_count
from auto_round.utils import logger
+ from auto_round.export.export_to_gguf.config import GGUF_CONFIG
tasks = args.tasks
if args.format is None:
args.format = "auto_round"
supported_formats = [
"auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_gptq:marlin",
- "gguf:q4_0", "gguf:q4_1", "itrex", "itrex_xpu", "fake"
+ "itrex", "itrex_xpu", "fake"
]
+ supported_formats.extend(list(GGUF_CONFIG.keys()))
+
formats = args.format.lower().replace(' ', '').split(",")
for format in formats:
if format not in supported_formats:
@@ -564,7 +568,10 @@ def tune(args):
enable_torch_compile=enable_torch_compile,
act_data_type=args.act_data_type,
act_dynamic=not args.disable_act_dynamic,
- device_map=args.device_map)
+ device_map=args.device_map,
+ super_group_size=args.super_group_size,
+ super_bits=args.super_bits,
+ )
model, _ = autoround.quantize()
model_name = args.model.rstrip("/")
if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
@@ -582,17 +589,18 @@ def tune(args):
format_list = args.format.replace(' ', '').split(',')
inplace = False if len(format_list) > 1 else True
eval_folder = None
+
+ eval_gguf_model = False
for format_ in format_list:
save_format_ = format_.replace(":", "-")
save_format_ = save_format_.replace("_", "-")
save_folder = f'{export_dir}-{save_format_}'
autoround.save_quantized(save_folder, format=format_, inplace=inplace)
if 'gguf' in format_:
- gguf_format = format_.upper().split(":")[-1]
- if not any([file.endswith(f"{gguf_format}.gguf") for file in os.listdir(save_folder)]):
+ if not any([file.endswith(".gguf") for file in os.listdir(save_folder)]):
logger.error(f"fail to export {format_}")
- else:
- eval_folder = save_folder
+ eval_gguf_model = True
+ eval_folder = save_folder
lm_eval_version = get_library_version("lm-eval")
@@ -604,7 +612,11 @@ def tune(args):
logger.info(f"Using lm-eval version {lm_eval_version}")
- if args.act_bits <= 8:
+ if args.act_bits <= 8 or eval_gguf_model:
+ if eval_gguf_model:
+ for file in os.listdir(eval_folder):
+ gguf_file = file
+ user_model = AutoModelForCausalLM.from_pretrained(save_folder, gguf_file=gguf_file, device_map="auto")
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
from accelerate.big_modeling import dispatch_model
@@ -659,7 +671,7 @@ def eval(args):
print(make_table(res))
-def eval_task_by_task(model, device, tasks, batch_size=None, max_batch_size=64, trust_remote_code=True):
+def eval_task_by_task(model, device, tasks, tokenizer=None, batch_size=None, max_batch_size=64, trust_remote_code=True):
set_cuda_visible_devices(device)
device_str, parallelism = get_device_and_parallelism(device)
@@ -668,14 +680,30 @@ def eval_task_by_task(model, device, tasks, batch_size=None, max_batch_size=64,
from auto_round.utils import logger
from lm_eval import simple_evaluate as lm_simple_evaluate
from lm_eval.models.huggingface import HFLM
+ from transformers import AutoModelForCausalLM, AutoTokenizer
# from auto_round import AutoRoundConfig
if batch_size is None:
batch_size = "auto"
+ is_gguf_file = False
if not isinstance(model, str):
parallelism = False
+ else:
+ if os.path.isfile(model) and model.endswith(".gguf"):
+ is_gguf_file = True
+ gguf_file = model
+ model = os.path.dirname(model)
+ else:
+ for file in os.listdir(model):
+ if file.endswith(".gguf"):
+ is_gguf_file = True
+ gguf_file = file
+ if is_gguf_file:
+ tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
+ model = AutoModelForCausalLM.from_pretrained(model, gguf_file=gguf_file, device_map="auto")
hflm = HFLM(
pretrained=model,
+ tokenizer=tokenizer,
device=device_str,
batch_size=batch_size,
max_batch_size=max_batch_size,
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 555618908..4e1a5d8a7 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -166,6 +166,11 @@ class WrapperLinear(torch.nn.Module):
weight = self.orig_layer.get_weight().to(self.device)
if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):
weight = weight.t()
+
+ quant_kwargs = {}
+ if hasattr(self.orig_layer, "super_bits"):
+ quant_kwargs["super_bits"] = self.orig_layer.super_bits
+ quant_kwargs["super_group_size"] = self.orig_layer.super_group_size
weight_q, scale, zp = self.weight_quant_func(
weight,
@@ -179,8 +184,8 @@ class WrapperLinear(torch.nn.Module):
tensor_max=self.weight_max,
data_type=self.data_type,
q_scale_thresh=self.q_scale_thresh,
-
- )
+ **quant_kwargs
+ )
weight_q = weight_q.to(weight.dtype)
if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):
weight_q = weight_q.t()
@@ -237,6 +242,7 @@ class WrapperLinear(torch.nn.Module):
self.orig_layer.to(self.device)
##unwrapper weight
qdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)
+
self.orig_layer.weight.data.copy_(qdq_weight)
self.orig_layer.weight.grad = None
@@ -244,19 +250,27 @@ class WrapperLinear(torch.nn.Module):
if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):
shape = qdq_weight.t().shape
- if zp is not None:
- zp = zp.reshape(shape[0], -1)
- if isinstance(scale, dict):
- for key in scale.keys():
- if key == "scale":
- name = key
- self.orig_layer.scale = scale[key].reshape(shape[0], -1).to("cpu")
+ def _set_dict_attr(attr_dict, attr_name):
+ for key in attr_dict.keys():
+ if key == attr_name:
+ setattr(self.orig_layer, attr_name, attr_dict[key].reshape(shape[0], -1).to("cpu"))
else:
name = "w_" + key
- setattr(self.orig_layer, name, scale[key].to("cpu"))
+ setattr(self.orig_layer, name, attr_dict[key].to("cpu"))
+
+ if isinstance(scale, dict):
+ _set_dict_attr(scale, "scale")
else:
self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")
- self.orig_layer.zp = zp.to("cpu") if zp is not None else None
+
+ if zp is not None:
+ if isinstance(zp, dict):
+ _set_dict_attr(zp, "zp")
+ else:
+ zp = zp.reshape(shape[0], -1)
+ self.orig_layer.zp = zp.to("cpu") if zp is not None else None
+ else:
+ self.orig_layer.zp = None
##unwrapper bias
if self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quant