Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Support for more gguf format and float zp for Q*_1 (#560)

Heng Guob38d19dc1039
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index 91509b257..59dace8f9 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -6,7 +6,7 @@ echo "set up UT env..."
export TQDM_MININTERVAL=60
export TQDM_POSITION=-1
pip install pytest-cov pytest-html
-pip install -r /auto-round/test/requirements.txt
+pip install -r /auto-round/test/requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu
pip list
# install latest gguf for ut test
git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install .
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 12577a29a..b527f16bf 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -52,6 +52,7 @@ from .utils import (
llm_load_model,
reset_params,
init_cache, check_skippable_keywords, get_shared_keys, supported_dtypes, infer_bits_by_data_type,
+ _gguf_args_check
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -225,7 +226,7 @@ class AutoRound(object):
self.act_dynamic = act_dynamic
self.act_data_type = act_data_type
if self.act_data_type is None:
- if data_type in supported_dtypes and self.act_bits <= 16:
+ if data_type in supported_dtypes and self.act_bits < 16:
self.act_data_type = data_type
logger.info(f"activation adopts {data_type}")
else:
@@ -294,7 +295,6 @@ class AutoRound(object):
"data_type",
"enable_quanted_input",
"enable_minmax_tuning",
- "data_type",
"seqlen",
"batch_size",
"scale_dtype",
@@ -452,7 +452,8 @@ class AutoRound(object):
ValueError: If an unsupported format is specified.
"""
# Validate and process the specified formats
- formats = format.replace(' ', '').split(',')
+ _gguf_args_check(self, format)
+ formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')
from auto_round.utils import supported_formats
for format_ in formats:
if format_ not in supported_formats:
@@ -1193,8 +1194,8 @@ class AutoRound(object):
if q_inputs is not None:
q_inputs[i] = q_inputs[i].to(layer.weight.dtype)
- wrapper_linear = WrapperLinear(layer, enable_minmax_tuning=self.enable_minmax_tuning, device=device).to(
- device)
+ wrapper_linear = WrapperLinear(
+ layer, enable_minmax_tuning=self.enable_minmax_tuning, device=device).to(device)
round_params = []
minmax_params = []
for key in wrapper_linear.params.keys():
@@ -1623,6 +1624,7 @@ class AutoRound(object):
Returns:
object: The compressed model object.
"""
+ format = format.replace("q*_", f"q{self.bits}_")
# only support to export afp8
if self.act_bits <= 8:
if "fp8" not in self.act_data_type or self.act_dynamic:
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index 396053648..a16a730f1 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -72,6 +72,73 @@ def double_quant_tensor(tensor, bits, q_scale_thresh):
qdq_tensor = torch.clamp(round_ste(tensor / scale), max=maxq) * scale
return qdq_tensor, scale
+def double_quant_tensor_sym(tensor, bits, q_scale_thresh):
+ maxq = 2 ** (bits - 1)
+ imax = abs(tensor).argmax(axis=-1, keepdims=True)
+ wmax = torch.take_along_dim(tensor, imax, axis=-1)
+ scale = wmax / -maxq
+ qdq_tensor = torch.where(scale != 0, round_ste(tensor / scale), 0).clip(-maxq, maxq -1) * scale
+ return qdq_tensor, scale
+
+@register_dtype("int_sym_dq")
+def quant_tensor_sym_dq(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ min_scale=1.0,
+ max_scale=1.0,
+ scale_dtype=torch.float16,
+ tensor_min=None,
+ tensor_max=None,
+ q_scale_thresh=1e-5,
+ super_group_size=16,
+ super_bits=6,
+ **kwargs):
+ """Quantize and de-quantize tensor asymmetrically. full range, credict goes to llamacpp community
+
+ Args:
+ tensor: Tensor containing the tensor to be quantized
+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: Number of elements to share scale for quantization
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for tensor
+ max_scale: Maximum scale coefficient for tensor
+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.
+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
+
+ Returns:
+ Quantized and de-quantized tensor, scale, zero-point
+ """
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+
+ maxq = 2 ** (bits - 1)
+ if tensor_min is None or tensor_max is None:
+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)
+ else:
+ wmin_tmp = tensor_min
+ wmax_tmp = tensor_max
+
+ wmin_abs = -(wmin_tmp * min_scale) # pylint: disable=E1130
+ wmax_abs = wmax_tmp * max_scale
+ max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)
+ scale = (max_v / maxq).to(scale_dtype)
+ scale = torch.where(scale < 0, torch.clamp(scale, max=-q_scale_thresh), torch.clamp(scale, min=q_scale_thresh))
+ scale = scale.view(-1, super_group_size)
+
+ #conduct double quant
+ scale, d_scale = double_quant_tensor_sym(scale, super_bits, q_scale_thresh)
+
+ scale = scale.view(-1, 1)
+ zp = torch.full_like(scale, maxq) # pylint: disable=E1130
+ int_w = round_ste(tensor / scale + v)
+ q = torch.clamp(int_w + zp, 0, 2 ** bits - 1)
+ qdq_result = (scale * (q - zp)).to(tensor.dtype)
+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_result, {"scale": scale, "d_scale": d_scale}, zp
@register_dtype("int_asym_dq")
def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
@@ -177,6 +244,62 @@ def quant_tensor_asym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_sca
return qdq_result, scale, zp
+@register_dtype("int_asym_float_zp")
+def quant_tensor_asym_float_zp(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ min_scale=1.0,
+ max_scale=1.0,
+ scale_dtype=torch.float16,
+ tensor_min=None,
+ tensor_max=None,
+ q_scale_thresh=1e-5,
+ **kwargs):
+ """Quantize and de-quantize tensor asymmetrically.
+
+ Args:
+ tensor: Tensor containing the tensor to be quantized
+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: Number of elements to share scale for quantization
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for tensor
+ max_scale: Maximum scale coefficient for tensor
+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.
+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
+
+ Returns:
+ Quantized and de-quantized tensor, scale, zero-point
+ """
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ maxq = 2 ** bits - 1
+ if tensor_min is None or tensor_max is None:
+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)
+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)
+ else:
+ wmin_tmp = tensor_min
+ wmax_tmp = tensor_max
+ if isinstance(min_scale, torch.Tensor):
+ wmin = wmin_tmp * min_scale
+ wmax = wmax_tmp * max_scale
+ else:
+ wmin = wmin_tmp
+ wmax = wmax_tmp
+ scale = ((wmax - wmin) / maxq).to(scale_dtype)
+ scale = torch.clamp(scale, min=q_scale_thresh)
+ zp = -wmin / scale # pylint: disable=E1130
+ scale = scale.unsqueeze(dim=-1)
+ zp = zp.unsqueeze(dim=-1)
+ int_w = round_ste(tensor / scale + v)
+ q = torch.clamp(int_w + zp, 0, maxq)
+ qdq_result = (scale * (q - zp)).to(tensor.dtype)
+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_result, scale, zp
+
+
@register_dtype("int_sym_gptq")
def quant_tensor_sym_gptq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
tensor_min=None,
diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.py
index 2d7b1f163..8cc3f85e2 100644
--- a/auto_round/export/export_to_gguf/config.py
+++ b/auto_round/export/export_to_gguf/config.py
@@ -14,20 +14,31 @@
GGUF_CONFIG = {}
-GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": False, "data_type": "int"}
+GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}
-GGUF_CONFIG["gguf:q4_1"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": True, "data_type": "int"}
-
-GGUF_CONFIG["gguf:q4_k_s"] = {
+GGUF_CONFIG["gguf:q4_1"] = {
"bits": 4,
"act_bits": 16,
- "super_group_size": 8,
- "super_bits": 6,
"group_size": 32,
"asym": True,
- "data_type": "int_asym_dq"
+ "sym": False,
+ "data_type": "int_asym_float_zp"
+}
+
+GGUF_CONFIG["gguf:q5_0"] = {"bits": 5, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}
+
+GGUF_CONFIG["gguf:q5_1"] = {
+ "bits": 5,
+ "act_bits": 16,
+ "group_size": 32,
+ "asym": True,
+ "sym": False,
+ "data_type": "int_asym_float_zp"
}
+GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}
+
+
GGUF_CONFIG["gguf:q2_k_s"] = {
"bits": 2,
"act_bits": 16,
@@ -35,7 +46,50 @@ GGUF_CONFIG["gguf:q2_k_s"] = {
"super_bits": 4,
"group_size": 16,
"asym": True,
+ "sym": False,
"data_type": "int_asym_dq"
}
-GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "asym": False, "data_type": "int"}
+GGUF_CONFIG["gguf:q3_k_s"] = {
+ "bits": 3,
+ "act_bits": 16,
+ "super_group_size": 16,
+ "super_bits": 6,
+ "group_size": 16,
+ "asym": False,
+ "sym": True,
+ "data_type": "int_sym_dq"
+}
+
+GGUF_CONFIG["gguf:q4_k_s"] = {
+ "bits": 4,
+ "act_bits": 16,
+ "super_group_size": 8,
+ "super_bits": 6,
+ "group_size": 32,
+ "asym": True,
+ "sym": False,
+ "data_type": "int_asym_dq"
+}
+
+GGUF_CONFIG["gguf:q5_k_s"] = {
+ "bits": 5,
+ "act_bits": 16,
+ "super_group_size": 8,
+ "super_bits": 6,
+ "group_size": 32,
+ "asym": True,
+ "sym": False,
+ "data_type": "int_asym_dq"
+}
+
+GGUF_CONFIG["gguf:q6_k"] = GGUF_CONFIG["gguf:q6_k_s"] = {
+ "bits": 6,
+ "act_bits": 16,
+ "super_group_size": 16,
+ "super_bits": 8,
+ "group_size": 16,
+ "asym": False,
+ "sym": True,
+ "data_type": "int_sym_dq"
+}
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 22c282ae1..6cdd5efd3 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -1155,11 +1155,12 @@ class Model(OriModel):
scale = module.scale
zp = module.zp if hasattr(module, "zp") else None
ggml_quant = ggml_quant_gpu if torch.cuda.is_available() else ggml_quant_cpu
+ # ggml_quant = ggml_quant_cpu
if data_qtype.name.lower().endswith("_k"):
d_scale = module.w_d_scale.to(torch.float32)
- d_wmin_m = module.w_d_wmin_m.to(torch.float32)
- wmin_m = module.w_wmin_m.to(torch.float32)
+ d_wmin_m = module.w_d_wmin_m.to(torch.float32) if hasattr(module, "w_d_wmin_m") else None
+ wmin_m = module.w_wmin_m.to(torch.float32) if hasattr(module, "w_wmin_m") else None
data = ggml_quant(
data_torch,
data_qtype.name.lower(),
@@ -1255,10 +1256,20 @@ class Model(OriModel):
data_qtype = gguf.GGMLQuantizationType.Q4_0
elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:
data_qtype = gguf.GGMLQuantizationType.Q4_1
- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:
- data_qtype = gguf.GGMLQuantizationType.Q4_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:
+ data_qtype = gguf.GGMLQuantizationType.Q5_0
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:
+ data_qtype = gguf.GGMLQuantizationType.Q5_1
elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:
data_qtype = gguf.GGMLQuantizationType.Q2_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S:
+ data_qtype = gguf.GGMLQuantizationType.Q3_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:
+ data_qtype = gguf.GGMLQuantizationType.Q4_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S:
+ data_qtype = gguf.GGMLQuantizationType.Q5_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:
+ data_qtype = gguf.GGMLQuantizationType.Q6_K
else:
raise ValueError(f"Unknown file type: {self.ftype.name}")
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index 81a56616b..5d6d3459c 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -23,7 +23,6 @@ import time
gguf = LazyImport("gguf")
-
FTYPE_MAP: dict[str, gguf.LlamaFileType] = {
"f32": gguf.LlamaFileType.ALL_F32,
"f16": gguf.LlamaFileType.MOSTLY_F16,
@@ -31,9 +30,15 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {
"q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,
"q4_0": gguf.LlamaFileType.MOSTLY_Q4_0,
"q4_1": gguf.LlamaFileType.MOSTLY_Q4_1,
- "q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,
- "q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,
+ "q5_0": gguf.LlamaFileType.MOSTLY_Q5_0,
+ "q5_1": gguf.LlamaFileType.MOSTLY_Q5_1,
"q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,
+ "q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,
+ "q3_k_s": gguf.LlamaFileType.MOSTLY_Q3_K_S,
+ "q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,
+ "q5_k_s": gguf.LlamaFileType.MOSTLY_Q5_K_S,
+ "q6_k": gguf.LlamaFileType.MOSTLY_Q6_K,
+ "q6_k_s": gguf.LlamaFileType.MOSTLY_Q6_K,
"auto": gguf.LlamaFileType.GUESSED,
}
diff --git a/auto_round/export/export_to_gguf/quant_cpu.py b/auto_round/export/export_to_gguf/quant_cpu.py
index 24b20e94b..563284017 100644
--- a/auto_round/export/export_to_gguf/quant_cpu.py
+++ b/auto_round/export/export_to_gguf/quant_cpu.py
@@ -13,27 +13,15 @@
# limitations under the License.
import numpy as np
-from tqdm import tqdm
from concurrent.futures import ProcessPoolExecutor
+from .utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES
-QK_K = 256
-K_SCALE_SIZE = 12
-GGML_QUANT_SIZES = {
- "bf16": (1, 2),
- "q4_0": (32, 2 + 16),
- "q4_1": (32, 2 + 2 + 16),
- "q4_k": (256, 2 + 2 + QK_K//2 + 12),
- "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),
- "q8_0": (32, 2 + 32)
-}
+GGML_QUANT_TYPE = {}
-GGML_QUANT_BLOCK = {}
-
-
-def register_block(name):
+def register_qtype(name):
def register(cls):
- GGML_QUANT_BLOCK[name] = cls
+ GGML_QUANT_TYPE[name] = cls
return cls
return register
@@ -55,37 +43,8 @@ def ggml_quant_cpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=No
n_blocks = data.size // block_size
blocks = data.reshape((n_blocks, block_size))
- if ggml_type.endswith("_k"):
- worker = 16
- else:
- worker = 0
-
- worker = 0
- if worker > 0:
- n_groups = (data.shape[0] // worker) or 1
- blocks = np.array_split(blocks, n_groups, axis=0)
- scale = np.array_split(scale, n_groups, axis=0) if scale is not None else [None] * n_groups
- zp = np.array_split(zp, n_groups, axis=0) if zp is not None else [None] * n_groups
- wmin_m = np.array_split(wmin_m, n_groups, axis=0) if wmin_m is not None else [None] * n_groups
- d_scale = np.array_split(d_scale, n_groups, axis=0) if d_scale is not None else [None] * n_groups
- d_wmin_m = np.array_split(d_wmin_m, n_groups, axis=0) if d_wmin_m is not None else [None] * n_groups
-
- quant_func = GGML_QUANT_BLOCK[ggml_type]
- if ggml_type.endswith("_k"):
- with ProcessPoolExecutor(worker) as executor:
- result = executor.map(quant_func, blocks, scale, zp, wmin_m, d_scale, d_wmin_m)
- new_data = np.array(list(result), dtype=np.uint8)
- else:
- with ProcessPoolExecutor(n_groups) as executor:
- result = executor.map(quant_func, blocks, scale, zp)
- new_data = np.array(list(result), dtype=np.uint8)
- else:
- quant_func = GGML_QUANT_BLOCK[ggml_type]
- if ggml_type.endswith("_k"):
- # 0.5588 vs 63.05
- new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)
- else:
- new_data = quant_func(blocks, scale, zp)
+ quant_func = GGML_QUANT_TYPE[ggml_type]
+ new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)
assert new_data.dtype == np.uint8
assert new_data.shape[-1] == type_size
@@ -100,8 +59,170 @@ def np_roundf(n: np.ndarray) -> np.ndarray:
return np.sign(n) * b
-@register_block("bf16")
-def bf16_quant_block(blocks: np.array, scale=None, zp=None):
+def make_qx_quant(data, bits, rmse_type=0, qw = None):
+ nmax = pow(2, bits - 1)
+ imax = abs(data).argmax(axis=-1, keepdims=True)
+ group_max = np.take_along_axis(data, imax, axis=-1)
+ iscales = np.where(group_max != 0, -nmax / group_max, 0)
+
+ if rmse_type == 0:
+ L = (np.round(iscales * data).clip(-nmax, nmax-1) + nmax).astype(np.uint8)
+ scales = np.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])
+ return scales, L
+
+ return_early = False
+ if rmse_type < 0:
+ rmse_type = -rmse_type
+ return_early = True
+
+ L = np.round(iscales * data).clip(-nmax, nmax-1)
+ if qw is not None:
+ w = qw
+ elif rmse_type == 1:
+ w = data ** 2
+ elif rmse_type == 2:
+ w = 1
+ elif rmse_type == 3:
+ w = np.fabs(data)
+ else:
+ w = np.sqrt(np.fabs(data))
+ sumlx = np.sum(w * data * L, axis=-1)
+ suml2 = np.sum(w * L * L, axis=-1)
+ scales = np.where(suml2 !=0, sumlx / suml2, 0)
+ if return_early:
+ iscales_inv = np.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])
+ scales = np.where(suml2 > 0, 0.5 * (scales + iscales_inv), iscales_inv)
+ L = (L + nmax).astype(np.uint8)
+ return scales, L
+ best = scales * sumlx
+ for _is in range(-9, 10):
+ if _is == 0:
+ continue
+ iscales = np.where(group_max != 0, -(nmax + -0.1 * _is) / nmax, 0)
+ tmp_L = np.round(iscales * data).clip(-nmax, nmax - 1)
+ sumlx = np.sum(w * data * L, axis=-1)
+ suml2 = np.sum(w * L * L, axis=-1)
+
+ replace_id = (suml2 > 0) & (sumlx*sumlx > best*suml2)
+ L[replace_id] = tmp_L[replace_id]
+ scales[replace_id] = sumlx[replace_id] / suml2[replace_id]
+ best[replace_id] = scales[replace_id] * sumlx[replace_id]
+
+ L = (L + nmax).astype(np.uint8)
+ return scales, L
+
+
+def make_q3_quants(data, bits, do_rmse=False):
+ # data shape (nb, 16, 16)
+ nmax = pow(2, bits - 1)
+ imax = abs(data).argmax(axis=-1, keepdims=True)
+ group_max = np.take_along_axis(data, imax, axis=-1)
+ iscale = np.where(group_max != 0, -nmax / group_max, 0)
+ if do_rmse:
+ L = np.round(iscale * data).clip(-nmax, nmax - 1)
+ w = np.power(data, 2)
+ sumlx = np.sum(w * data * L, axis=-1)
+ suml2 = np.sum(w * L * L, axis=-1)
+
+ for itry in range(5):
+ # w = np.power(data, 2)
+ if len(sumlx.shape) == 2:
+ sumlx = sumlx.reshape(*sumlx.shape, 1)
+ if len(suml2.shape) == 2:
+ suml2 = suml2.reshape(*suml2.shape, 1)
+ slx = sumlx - w*data*L
+ repalce_idx = slx > 0
+ sl2 = suml2 - w*L*L
+ tmp_L = np.where(slx > 0, data * sl2 / slx, 0)
+ new_L = np.round(tmp_L).clip(-nmax, nmax - 1)
+ tmp_repalce_idx = repalce_idx & (new_L != L)
+ slx[tmp_repalce_idx] += w[tmp_repalce_idx] * data[tmp_repalce_idx] * new_L[tmp_repalce_idx]
+ sl2[tmp_repalce_idx] += w[tmp_repalce_idx] * new_L[tmp_repalce_idx] * new_L[tmp_repalce_idx]
+ repalce_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)
+
+ L[repalce_idx] = new_L[repalce_idx]
+ sumlx = slx
+ suml2 = sl2
+
+ L = np.round(iscale * data).clip(-nmax, nmax - 1) + nmax
+ L = L.astype(np.uint8)
+ scales = np.where(iscale != 0, 1 / iscale, 0).reshape(iscale.shape[:2])
+ return scales, L
+
+
+def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
+ # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K
+ nmax = pow(2, bits) - 1
+ if len(data.shape) == 2:
+ if bits in [4, 5]:
+ data_shape = (-1, 8, 32)
+ elif bits in [2, 3]:
+ data_shape = (-1, 16, 16)
+ else:
+ raise NotImplementedError(f"bits = {bits} is not supported")
+ data = data.reshape(data_shape)
+ sum_x2 = np.sum(np.power(data, 2), axis=-1, keepdims=True)
+ av_x = np.sqrt(sum_x2 / 32)
+ weight = np.abs(data) + av_x
+
+ group_min = np.min(data, axis=-1, keepdims=True)
+ group_max = np.max(data, axis=-1, keepdims=True)
+
+ the_mins = -group_min
+
+ sum_w = np.sum(weight, axis=-1, keepdims=True)
+ sum_x = np.sum(weight * data, axis=-1, keepdims=True)
+
+ group_min[group_min > 0] = 0
+
+ scale = (group_max-group_min) / nmax
+ iscale = np.where(scale == 0, 0, 1 / scale)
+
+ l_values = np.round(iscale * (data-group_min))
+ L = np.clip(l_values, 0, nmax).astype(np.uint8)
+
+ diffs = scale*L + group_min - data
+ diffs = np.abs(diffs) if use_mad else diffs**2
+ best_mad = np.sum(weight * diffs, axis=-1, keepdims=True)
+
+ if nstep < 1:
+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
+
+ for step in range(nstep):
+ new_scale = (group_max-group_min) / (rmin + rdelta*step + nmax)
+ new_iscale = np.where(new_scale == 0, 0, 1 / new_scale)
+
+ l_values = np.round(new_iscale * (data-group_min))
+ Laux = np.clip(l_values, 0, nmax).astype(np.uint8)
+
+ sum_l = np.sum(weight * Laux, axis=-1, keepdims=True)
+ sum_l2 = np.sum(weight * Laux**2, axis=-1, keepdims=True)
+ sum_xl = np.sum(weight * Laux * data, axis=-1, keepdims=True)
+
+ D = sum_w*sum_l2 - sum_l*sum_l
+ replace_idx = D > 0
+
+ this_scale = (sum_w*sum_xl - sum_x*sum_l) / D
+ this_min = (sum_l2*sum_x - sum_l*sum_xl) / D
+ this_min[this_min > 0] = 0
+ this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]
+
+ diffs = this_scale*Laux + this_min - data
+ diffs = np.abs(diffs) if use_mad else diffs**2
+ mad = np.sum(weight * diffs, axis=-1, keepdims=True)
+
+ replace_idx &= mad < best_mad
+ best_mad[replace_idx] = mad[replace_idx]
+ L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]
+ scale[replace_idx] = this_scale[replace_idx]
+ group_min[replace_idx] = this_min[replace_idx]
+
+ the_mins = -group_min
+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
+
+
+@register_qtype("bf16")
+def bf16_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
n = blocks.view(np.uint32)
# force nan to quiet
n = np.where((n & 0x7fffffff) > 0x7f800000, (n & np.uint32(0xffff0000)) | np.uint32(64 << 16), n)
@@ -110,8 +231,8 @@ def bf16_quant_block(blocks: np.array, scale=None, zp=None):
return n.astype(np.uint16).view(np.uint8)
-@register_block("q4_0")
-def q4_0_quant_block(blocks: np.array, scale=None, zp=None):
+@register_qtype("q4_0")
+def q4_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
if scale is not None:
d = scale.reshape((-1, 1))
else:
@@ -134,8 +255,8 @@ def q4_0_quant_block(blocks: np.array, scale=None, zp=None):
return np.concatenate([d, qs], axis=-1)
-@register_block("q4_1")
-def q4_1_quant_block(blocks: np.array, scale=None, zp=None):
+@register_qtype("q4_1")
+def q4_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
if scale is not None:
d = scale.reshape((-1, 1))
min = zp.reshape((-1, 1)) * d * -1
@@ -158,106 +279,93 @@ def q4_1_quant_block(blocks: np.array, scale=None, zp=None):
return np.concatenate([d, m, qs], axis=-1)
-@register_block("q8_0")
-def q8_0_quant_block(blocks: np.array, scale=None, zp=None) -> np.ndarray:
+@register_qtype("q5_0")
+def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
if scale is not None:
d = scale.reshape((-1, 1))
else:
- d = abs(blocks).max(axis=1, keepdims=True) / 127
+ imax = abs(blocks).argmax(axis=-1, keepdims=True)
+ max = np.take_along_axis(blocks, imax, axis=-1)
+ d = max / -16
+
+ n_blocks = blocks.shape[0]
+ block_size = GGML_QUANT_SIZES["q5_0"][0]
with np.errstate(divide="ignore"):
id = np.where(d == 0, 0, 1 / d)
- qs = np_roundf(blocks * id)
-
- # (n_blocks, 2)
- d = d.astype(np.float16).view(np.uint8)
- # (n_blocks, block_size)
- qs = qs.astype(np.int8).view(np.uint8)
-
- return np.concatenate([d, qs], axis=1)
-
-def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
- assert bits in [2, 4], f"bits={bits} is not supported"
- nmax = bits ** 2 -1
- # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K
- if len(data.shape) == 2:
- data_shape = (-1, 8, 32) if bits == 4 else (-1, 16, 16)
- data = data.reshape(data_shape)
- sum_x2 = np.sum(np.power(data, 2), axis=-1, keepdims=True)
- av_x = np.sqrt(sum_x2 / 32)
- weight = np.abs(data) + av_x
+ # FIXME: Q5_0's reference rounding is cursed and depends on FMA
+ q = np.trunc((np.float64(blocks) * np.float64(id)) + np.float64(16.5),
+ dtype=np.float32).astype(np.uint8).clip(0, 31)
- group_min = np.min(data, axis=-1, keepdims=True)
- group_max = np.max(data, axis=-1, keepdims=True)
+ qs = q.reshape((n_blocks, 2, block_size // 2))
+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))
- the_mins = -group_min
+ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)
- sum_w = np.sum(weight, axis=-1, keepdims=True)
- sum_x = np.sum(weight * data, axis=-1, keepdims=True)
+ d = d.astype(np.float16).view(np.uint8)
- group_min[group_min > 0] = 0
+ return np.concatenate([d, qh, qs], axis=-1)
- scale = (group_max - group_min) / nmax
- iscale = np.where(scale == 0, 0, 1 /scale)
- l_values = np.round(iscale * (data - group_min))
- L = np.clip(l_values, 0, nmax).astype(np.uint8)
+@register_qtype("q5_1")
+def q5_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
+ if scale is not None:
+ d = scale.reshape((-1, 1))
+ min = zp.reshape((-1, 1)) * d * -1
+ else:
+ max = blocks.max(axis=-1, keepdims=True)
+ min = blocks.min(axis=-1, keepdims=True)
+ d = (max-min) / 31
- diffs = scale * L + group_min - data
- diffs = np.abs(diffs) if use_mad else diffs**2
- best_mad = np.sum(weight * diffs, axis=-1, keepdims=True)
+ n_blocks = blocks.shape[0]
+ block_size = GGML_QUANT_SIZES["q5_1"][0]
- if nstep < 1:
- return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
+ with np.errstate(divide="ignore"):
+ id = np.where(d == 0, 0, 1 / d)
+ q = np.trunc((blocks-min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 31)
- for step in range(nstep):
- new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)
- new_iscale = np.where(new_scale == 0, 0, 1 /new_scale)
+ qs = q.reshape((n_blocks, 2, block_size // 2))
+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))
- l_values = np.round(new_iscale * (data - group_min))
- Laux = np.clip(l_values, 0, nmax).astype(np.uint8)
+ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)
- sum_l = np.sum(weight * Laux, axis=-1, keepdims=True)
- sum_l2 = np.sum(weight * Laux**2, axis=-1, keepdims=True)
- sum_xl = np.sum(weight * Laux * data, axis=-1, keepdims=True)
+ d = d.astype(np.float16).view(np.uint8)
+ m = min.astype(np.float16).view(np.uint8)
- D = sum_w * sum_l2 - sum_l * sum_l
- replace_idx = D > 0
+ return np.concatenate([d, m, qh, qs], axis=-1)
- this_scale = (sum_w * sum_xl - sum_x * sum_l) / D
- this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D
- this_min[this_min > 0] = 0
- this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]
- diffs = this_scale * Laux + this_min - data
- diffs = np.abs(diffs) if use_mad else diffs**2
- mad = np.sum(weight * diffs, axis=-1, keepdims=True)
+@register_qtype("q8_0")
+def q8_0_quant_block(blocks: np.array, scale=None, zp=None) -> np.ndarray:
+ if scale is not None:
+ d = scale.reshape((-1, 1))
+ else:
+ d = abs(blocks).max(axis=1, keepdims=True) / 127
+ with np.errstate(divide="ignore"):
+ id = np.where(d == 0, 0, 1 / d)
+ qs = np_roundf(blocks * id)
- replace_idx &= mad < best_mad
- best_mad[replace_idx] = mad[replace_idx]
- L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]
- scale[replace_idx] = this_scale[replace_idx]
- group_min[replace_idx] = this_min[replace_idx]
+ # (n_blocks, 2)
+ d = d.astype(np.float16).view(np.uint8)
+ # (n_blocks, block_size)
+ qs = qs.astype(np.int8).view(np.uint8)
- the_mins = -group_min
- return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
+ return np.concatenate([d, qs], axis=1)
-@register_block("q2_k")
+@register_qtype("q2_k")
def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
nb = blocks.shape[0]
output_scale = np.empty((nb, 16), dtype=np.uint8)
output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)
- blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)
+ blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)
if scale is not None:
scales = scale.reshape((-1, QK_K // 16))
mins = wmin_m.reshape((-1, QK_K // 16))
- output_d = d_scale.astype(np.float32) if len(d_scale.shape) == 2 \
- else d_scale.reshape(-1, 1).astype(np.float32)
- output_dmin = d_wmin_m.astype(np.float32) if len(d_wmin_m.shape) == 2 \
- else d_wmin_m.reshape(-1, 1).astype(np.float32)
- inv_scales = np.where(d_scale == 0, 0, 1 / output_d)
+ output_d = d_scale.reshape(-1, 1).astype(np.float32)
+ output_dmin = d_wmin_m.reshape(-1, 1).astype(np.float32)
+ inv_scales = np.where(output_d == 0, 0, 1 / output_d)
inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)
max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)
max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)
@@ -268,7 +376,7 @@ def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
max_mins = np.max(mins, axis=-1, keepdims=True)
inv_scales = np.where(max_scales > 0, 15. / max_scales, 0)
inv_mins = np.where(max_mins > 0, 15. / max_mins, 0)
-
+
replace_ids = (max_scales > 0).reshape(-1)
output_scale = np.zeros_like(scales).astype(np.uint8)
output_scale[replace_ids] = np.round(inv_scales * scales).astype(np.uint8)
@@ -279,8 +387,8 @@ def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
if d_scale is None:
output_d = np.where(max_scales > 0, max_scales / 15, 0)
if d_wmin_m is None:
- output_dmin = np.where(max_mins > 0 ,max_mins / 15., 0)
-
+ output_dmin = np.where(max_mins > 0, max_mins / 15., 0)
+
d_tmp = output_d * (output_scale & 0xF)
dm_tmp = output_dmin * (output_scale >> 4)
@@ -290,19 +398,73 @@ def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
np.uint8)
all_L = np.clip(all_L, 0, 3)
- output_qs = all_L[:,::4] | (all_L[:,1::4] << 2) | (all_L[:,2::4] << 4) | (all_L[:,3::4] << 6)
+ output_qs = all_L[:, ::4] | (all_L[:, 1::4] << 2) | (all_L[:, 2::4] << 4) | (all_L[:, 3::4] << 6)
output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
output_qs = output_qs.reshape((nb, QK_K // 4))
# [scale, qs, d, dmin]
return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)
-
-@register_block("q4_k")
+
+@register_qtype("q3_k")
+def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ nb = blocks.shape[0]
+ blocks = blocks.reshape((nb, QK_K // 16, 16))
+
+ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
+
+ if scale is not None:
+ scales = scale.reshape(-1, QK_K // 16)
+ output_d = d_scale.reshape(-1, 1).astype(np.float32)
+ iscales = np.where(output_d == 0, 0, 1 / output_d)
+ all_L = np.round(blocks / scales.reshape(*scales.shape, 1)).clip(-4, 3) + 4
+ all_L = all_L.astype(np.uint8)
+ else:
+ scales, all_L = make_q3_quants(blocks, bits=3, do_rmse=False)
+ imax = abs(scales).argmax(axis=-1, keepdims=True)
+ max_scales = np.take_along_axis(scales, imax, axis=-1)
+ iscales = np.where(max_scales != 0, -32 / max_scales, 0)
+ output_d = np.where(iscales != 0, 1 / iscales, 0)
+
+ q_scales = np.round(iscales * scales).clip(-32, 31) + 32
+ q_scales = q_scales.astype(np.uint8)
+ output_scale[:, :8] = (q_scales[:, :8] & 0xF) | ((q_scales[:, 8:] & 0xF) << 4)
+ hmask = q_scales >> 4
+ output_scale[:, 8:] = hmask[:, :4] | hmask[:, 4:8] << 2 | hmask[:, 8:12] << 4 | hmask[:, 12:] << 6
+
+
+ # lscales, hscales = np.hsplit(output_scale, [8])
+ # lscales = lscales.reshape((nb, 1, 8)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 2, 1))
+ # lscales = lscales.reshape((nb, 16))
+ # hscales = hscales.reshape((nb, 1, 4)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 4, 1))
+ # hscales = hscales.reshape((nb, 16))
+ # sc = (lscales & np.uint8(0x0F)) | ((hscales & np.uint8(0x03)) << np.uint8(4))
+ # sc = (sc.astype(np.int8) - np.int8(32)).astype(np.float32)
+ sc = np.round(iscales * scales).clip(-32, 31)
+
+ d = output_d.astype(np.float32) * sc
+ replace_idx = (d != 0)
+
+ all_L[replace_idx] = np.round(blocks[replace_idx] / d[replace_idx].reshape(-1, 1)).clip(-4, 3) + 4
+
+ output_hmask = np.bitwise_or.reduce(
+ all_L.reshape(nb, 8, 32) >> 2 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1) # pylint: disable=E1121
+ all_L = np.where(all_L > 3, all_L - 4, all_L)
+
+ output_qs = np.bitwise_or.reduce(
+ all_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121
+
+ output_qs = output_qs.reshape(nb, 64).astype(np.uint8)
+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ # [hmask, qs, scale, d]
+ return np.concatenate([output_hmask, output_qs, output_scale, output_d], axis=-1)
+
+
+@register_qtype("q4_k")
def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
nb = blocks.shape[0]
- blocks = blocks.reshape((nb, QK_K // 32, 32)) # (nb, 8, 32)
+ blocks = blocks.reshape(nb, QK_K // 32, 32) # (nb, 8, 32)
output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)
@@ -310,11 +472,9 @@ def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
if scale is not None:
scales = scale.reshape(-1, QK_K // 32)
mins = wmin_m.reshape(-1, QK_K // 32)
- output_d = d_scale.astype(np.float32) if len(d_scale.shape) == 2 \
- else d_scale.reshape(-1, 1).astype(np.float32)
- output_dmin = d_wmin_m.astype(np.float32) if len(d_wmin_m.shape) == 2 \
- else d_wmin_m.reshape(-1, 1).astype(np.float32)
- inv_scales = np.where(d_scale == 0, 0, 1 / output_d)
+ output_d = d_scale.reshape(-1, 1).astype(np.float32)
+ output_dmin = d_wmin_m.reshape(-1, 1).astype(np.float32)
+ inv_scales = np.where(output_d == 0, 0, 1 / output_d)
inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)
max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)
max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)
@@ -326,15 +486,15 @@ def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
inv_scales = np.where(max_scales > 0, 63. / max_scales, 0)
inv_mins = np.where(max_mins > 0, 63. / max_mins, 0)
- q_scales = np.round(inv_scales * scales).astype(np.uint8)
- q_mins = np.round(inv_mins * mins).astype(np.uint8)
+ q_scales = np.round(inv_scales * scales).astype(np.uint8).clip(0, 63)
+ q_mins = np.round(inv_mins * mins).astype(np.uint8).clip(0, 63)
- output_scale[:,:4] = q_scales[:,:4]
- output_scale[:,4:8] = q_mins[:,:4]
+ output_scale[:, :4] = q_scales[:, :4]
+ output_scale[:, 4:8] = q_mins[:, :4]
- output_scale[:,8:] = (q_scales[:,4:] & 0xF) | ((q_mins[:,4:] & 0xF) << 4)
- output_scale[:,:4] |= ((q_scales[:,4:] >> 4) << 6)
- output_scale[:,4:8] |= ((q_mins[:,4:] >> 4) << 6)
+ output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)
+ output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)
+ output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)
if d_scale is None:
output_d = max_scales / 63
@@ -349,7 +509,7 @@ def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
(blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(
np.uint8)
all_L = np.clip(all_L, 0, 15)
- output_qs = all_L[:,::2] | (all_L[:,1::2] << 4)
+ output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)
output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
@@ -357,3 +517,99 @@ def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
# [d, dmin, scale, qs]
return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)
+
+
+@register_qtype("q5_k")
+def q5_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ nb = blocks.shape[0]
+ blocks = blocks.reshape((nb, QK_K // 32, 32))
+
+ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
+ output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)
+ output_qh = np.empty((nb, QK_K // 8), dtype=np.uint8)
+
+ if scale is not None:
+ scales = scale.reshape(-1, QK_K // 32)
+ mins = wmin_m.reshape(-1, QK_K // 32)
+ output_d = d_scale.reshape(-1, 1).astype(np.float32)
+ output_dmin = d_wmin_m.reshape(-1, 1).astype(np.float32)
+ inv_scales = np.where(output_d == 0, 0, 1 / output_d)
+ inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)
+ max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)
+ max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)
+ all_L = np.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).astype(np.uint8)
+ else:
+ scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)
+ max_scales = np.max(scales, axis=-1, keepdims=True)
+ max_mins = np.max(mins, axis=-1, keepdims=True)
+ inv_scales = np.where(max_scales > 0, 63. / max_scales, 0)
+ inv_mins = np.where(max_mins > 0, 63. / max_mins, 0)
+
+ q_scales = np.round(inv_scales * scales).astype(np.uint8).clip(0, 63)
+ q_mins = np.round(inv_mins * mins).astype(np.uint8).clip(0, 63)
+
+ output_scale[:, :4] = q_scales[:, :4]
+ output_scale[:, 4:8] = q_mins[:, :4]
+
+ output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)
+ output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)
+ output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)
+
+ if d_scale is None:
+ output_d = max_scales / 63
+ if d_wmin_m is None:
+ output_dmin = max_mins / 63
+
+ d_tmp = output_d * q_scales
+ dm_tmp = output_dmin * q_mins
+
+ replace_ids = d_tmp != 0
+ all_L[replace_ids] = np.round(
+ (blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(
+ np.uint8)
+ all_L = np.clip(all_L, 0, 31) # (nb, 8, 32)
+
+ output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)
+ output_qh = np.bitwise_or.reduce(
+ all_L >> 4 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1).astype(np.uint8) # pylint: disable=E1121
+
+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_qs = output_qs.reshape(nb, QK_K // 2)
+ # [d, dmin, scale, qh, qs]
+ return np.concatenate([output_d, output_dmin, output_scale, output_qh, output_qs], axis=-1)
+
+@register_qtype("q6_k")
+def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ nb = blocks.shape[0]
+ blocks = blocks.reshape((nb, QK_K // 16, 16))
+
+ if scale is not None:
+ scales = scale.reshape(-1, QK_K // 16)
+ output_d = d_scale.reshape(-1, 1).astype(np.float32)
+ iscales = np.where(output_d == 0, 0, 1 / output_d)
+ all_L = (np.round(blocks / scales.reshape(*scales.shape, 1)).clip(-32, 31) + 32).astype(np.uint8)
+ else:
+ scales, all_L = make_qx_quant(blocks, bits=6, rmse_type=1, qw=None)
+ imax = abs(scales).argmax(axis=-1, keepdims=True)
+ max_scales = np.take_along_axis(scales, imax, axis=-1)
+ iscales = np.where(max_scales != 0, -128 / max_scales, 0)
+ output_d = np.where(iscales != 0, 1 / iscales, -1)
+
+ output_scale = np.round(iscales * scales).clip(max=127).astype(np.int8)
+ d_tmp = output_d * output_scale.astype(np.float32)
+ replace_ids = d_tmp != 0
+ all_L[replace_ids] = np.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32) \
+ .clip(0, 63).astype(np.uint8)
+
+ tmp_L = all_L.reshape(nb, 4, 64) & 0xF
+ output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2)
+ output_qh = np.bitwise_or.reduce(
+ (all_L >> 4).reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), # pylint: disable=E1121
+ axis=2).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121
+
+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
+
+ # [ql, qh, scale, d]
+ output_scale = output_scale.view(np.uint8)
+ return np.concatenate([output_ql, output_qh, output_scale, output_d], axis=-1)
diff --git a/auto_round/export/export_to_gguf/quant_gpu.py b/auto_round/export/export_to_gguf/quant_gpu.py
index 7d669bfda..57407e021 100644
--- a/auto_round/export/export_to_gguf/quant_gpu.py
+++ b/auto_round/export/export_to_gguf/quant_gpu.py
@@ -15,26 +15,15 @@
import torch
import numpy as np
-QK_K = 256
-K_SCALE_SIZE = 12
-GGML_QUANT_SIZES = {
- "bf16": (1, 2),
- "q4_0": (32, 2 + 16),
- "q4_1": (32, 2 + 2 + 16),
- "q4_k": (256, 2 + 2 + QK_K//2 + 12),
- "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),
- "q8_0": (32, 2 + 32)
-}
+from .utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES
-GGML_QUANT_BLOCK = {}
+GGML_QUANT_TYPE = {}
-
-
-def register_block(name):
+def register_qtype(name):
def register(cls):
- GGML_QUANT_BLOCK[name] = cls
+ GGML_QUANT_TYPE[name] = cls
return cls
return register
@@ -54,7 +43,7 @@ def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=No
n_blocks = data.nelement() // block_size
blocks = data.reshape((n_blocks, block_size))
- quant_func = GGML_QUANT_BLOCK[ggml_type]
+ quant_func = GGML_QUANT_TYPE[ggml_type]
new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)
assert new_data.shape[-1] == type_size
@@ -69,7 +58,166 @@ def torch_roundf(n):
return torch.sign(n) * b
-@register_block("bf16")
+def make_qx_quant(data, bits, rmse_type=0, qw = None):
+ nmax = pow(2, bits - 1)
+ imax = abs(data).argmax(axis=-1, keepdims=True)
+ group_max = torch.take_along_dim(data, imax, axis=-1)
+ iscales = torch.where(group_max != 0, -nmax / group_max, 0)
+
+ if rmse_type == 0:
+ L = (torch.round(iscales * data).clip(-nmax, nmax-1) + nmax).to(torch.uint8)
+ scales = torch.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])
+ return scales, L
+
+ return_early = False
+ if rmse_type < 0:
+ rmse_type = -rmse_type
+ return_early = True
+
+ L = torch.round(iscales * data).clip(-nmax, nmax-1)
+ if qw is not None:
+ w = qw
+ elif rmse_type == 1:
+ w = data ** 2
+ elif rmse_type == 2:
+ w = 1
+ elif rmse_type == 3:
+ w = torch.abs(data)
+ else:
+ w = torch.sqrt(torch.abs(data))
+ sumlx = torch.sum(w * data * L, axis=-1)
+ suml2 = torch.sum(w * L * L, axis=-1)
+ scales = torch.where(suml2 !=0, sumlx / suml2, 0)
+ if return_early:
+ iscales_inv = torch.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])
+ scales = torch.where(suml2 > 0, 0.5 * (scales + iscales_inv), iscales_inv)
+ L = (L + nmax).to(torch.uint8)
+ return scales, L
+ best = scales * sumlx
+ for _is in range(-9, 10):
+ if _is == 0:
+ continue
+ iscales = torch.where(group_max != 0, -(nmax + -0.1 * _is) / nmax, 0)
+ tmp_L = torch.round(iscales * data).clip(-nmax, nmax - 1)
+ sumlx = torch.sum(w * data * L, axis=-1)
+ suml2 = torch.sum(w * L * L, axis=-1)
+
+ replace_id = (suml2 > 0) & (sumlx*sumlx > best*suml2)
+ L[replace_id] = tmp_L[replace_id]
+ scales[replace_id] = sumlx[replace_id] / suml2[replace_id]
+ best[replace_id] = scales[replace_id] * sumlx[replace_id]
+
+ L = (L + nmax).to(torch.uint8)
+ return scales, L
+
+
+def make_q3_quants(data, bits, do_rmse=False):
+ nmax = pow(2, bits - 1)
+ imax = abs(data).argmax(axis=-1, keepdims=True)
+ group_max = torch.take_along_dim(data, imax, axis=-1)
+ iscale = torch.where(group_max != 0, -nmax / group_max, 0)
+ if do_rmse:
+ L = torch.round(iscale * data).clip(-nmax, nmax - 1)
+ w = torch.pow(data, 2)
+ sumlx = torch.sum(w * data * L, axis=-1)[0]
+ suml2 = torch.sum(w * L * L, axis=-1)[0]
+
+ for itry in range(5):
+ # w = np.power(data, 2)
+ if len(sumlx.shape) == 2:
+ sumlx = sumlx.unsqueeze(-1)
+ if len(suml2.shape) == 2:
+ suml2 = suml2.unsqueeze(-1)
+ slx = sumlx - w*data*L
+ repalce_idx = slx > 0
+ sl2 = suml2 - w*L*L
+ new_L = torch.round(data * sl2 / slx).clip(-nmax, nmax - 1)
+ tmp_repalce_idx = repalce_idx & (new_L != L)
+ slx[tmp_repalce_idx] += w[tmp_repalce_idx] * data[tmp_repalce_idx] * new_L[tmp_repalce_idx]
+ sl2[tmp_repalce_idx] += w[tmp_repalce_idx] * new_L[tmp_repalce_idx] * new_L[tmp_repalce_idx]
+ repalce_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)
+
+ L[repalce_idx] = new_L[repalce_idx]
+ sumlx = slx
+ suml2 = sl2
+
+ L = torch.round(iscale * data).clip(-nmax, nmax - 1) + nmax
+ scales = torch.where(iscale != 0, 1 / iscale, 0).reshape(iscale.shape[:2])
+ return scales, L.to(torch.uint8)
+
+
+def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
+ nmax = pow(2, bits) - 1
+ # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K
+ if len(data.shape) == 2:
+ if bits in [4, 5]:
+ data_shape = (-1, 8, 32)
+ elif bits in [2, 3]:
+ data_shape = (-1, 16, 16)
+ else:
+ raise NotImplementedError(f"bits = {bits} is not supported")
+ data = data.reshape(data_shape)
+ sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)
+ av_x = torch.sqrt(sum_x2 / 32)
+ weight = torch.abs(data) + av_x
+
+ group_min = torch.min(data, axis=-1, keepdims=True)[0]
+ group_max = torch.max(data, axis=-1, keepdims=True)[0]
+
+ the_mins = -group_min
+
+ sum_w = torch.sum(weight, axis=-1, keepdims=True)
+ sum_x = torch.sum(weight * data, axis=-1, keepdims=True)
+
+ group_min[group_min > 0] = 0
+
+ scale = (group_max-group_min) / nmax
+ iscale = torch.where(scale == 0, 0, 1 / scale)
+
+ l_values = torch.round(iscale * (data-group_min))
+ L = torch.clip(l_values, 0, nmax).to(torch.uint8)
+
+ diffs = scale*L + group_min - data
+ diffs = torch.abs(diffs) if use_mad else diffs**2
+ best_mad = torch.sum(weight * diffs, axis=-1, keepdims=True)
+
+ if nstep < 1:
+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
+
+ for step in range(nstep):
+ new_scale = (group_max-group_min) / (rmin + rdelta*step + nmax)
+ new_iscale = torch.where(new_scale == 0, 0, 1 / new_scale)
+
+ l_values = torch.round(new_iscale * (data-group_min))
+ Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)
+
+ sum_l = torch.sum(weight * Laux, axis=-1, keepdims=True)
+ sum_l2 = torch.sum(weight * Laux**2, axis=-1, keepdims=True)
+ sum_xl = torch.sum(weight * Laux * data, axis=-1, keepdims=True)
+
+ D = sum_w*sum_l2 - sum_l*sum_l
+ replace_idx = D > 0
+
+ this_scale = (sum_w*sum_xl - sum_x*sum_l) / D
+ this_min = (sum_l2*sum_x - sum_l*sum_xl) / D
+ this_min[this_min > 0] = 0
+ this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]
+
+ diffs = this_scale*Laux + this_min - data
+ diffs = torch.abs(diffs) if use_mad else diffs**2
+ mad = torch.sum(weight * diffs, axis=-1, keepdims=True)
+
+ replace_idx &= mad < best_mad
+ best_mad[replace_idx] = mad[replace_idx]
+ L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]
+ scale[replace_idx] = this_scale[replace_idx]
+ group_min[replace_idx] = this_min[replace_idx]
+
+ the_mins = -group_min
+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
+
+
+@register_qtype("bf16")
def bf16_quant_block(blocks, scale=None, zp=None, **kwargs):
n = blocks.view(torch.uint32)
# force nan to quiet
@@ -79,12 +227,13 @@ def bf16_quant_block(blocks, scale=None, zp=None, **kwargs):
return n.cpu().numpy().astype(np.uint16).view(np.uint8)
-@register_block("q4_0")
-def q4_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
+@register_qtype("q4_0")
+def q4_0_quant_block(blocks, scale=None, zp=None, **kwargs):
if scale is not None:
d = scale.reshape((-1, 1))
else:
- max = torch.max(blocks, axis=-1, keepdims=True)[0]
+ imax = abs(blocks).argmax(axis=-1, keepdims=True)
+ max = np.take_along_axis(blocks, imax, axis=-1)
d = max / -8
id = torch.where(d == 0, 0, 1 / d)
@@ -100,8 +249,8 @@ def q4_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
return np.concatenate([d, qs], axis=-1)
-@register_block("q4_1")
-def q4_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
+@register_qtype("q4_1")
+def q4_1_quant_block(blocks, scale=None, zp=None, **kwargs):
if scale is not None:
d = scale.reshape((-1, 1))
min = zp.reshape((-1, 1)) * d * -1
@@ -123,107 +272,91 @@ def q4_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
return np.concatenate([d, m, qs], axis=-1)
-@register_block("q8_0")
-def q8_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs) -> np.ndarray:
+@register_qtype("q5_0")
+def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
if scale is not None:
d = scale.reshape((-1, 1))
else:
- d = torch.abs(blocks).max(axis=1, keepdims=True)[0] / 127
+ imax = abs(blocks).argmax(axis=-1, keepdims=True)
+ max = torch.take_along_dim(blocks, imax, axis=-1)[0]
+ d = max / -16
+
id = torch.where(d == 0, 0, 1 / d)
+ n_blocks = blocks.shape[0]
+ block_size = GGML_QUANT_SIZES["q5_0"][0]
- qs = torch_roundf(blocks * id)
+ # FIXME: Q5_0's reference rounding is cursed and depends on FMA
+ q = torch.trunc(blocks.to(torch.float64) * id.to(torch.float64) + 16.5).to(torch.uint8).clip(0, 31).cpu().numpy()
- # (n_blocks, 2)
- d = d.cpu().numpy().astype(np.float16).view(np.uint8)
- # (n_blocks, block_size)
- qs = qs.cpu().numpy().astype(np.int8).view(np.uint8)
-
- return np.concatenate([d, qs], axis=1)
+ qs = q.reshape((n_blocks, 2, block_size // 2))
+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))
+ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)
-def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
- assert bits in [2, 4], f"bits={bits} is not supported"
- nmax = bits ** 2 -1
- # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K
- if len(data.shape) == 2:
- data_shape = (-1, 8, 32) if bits == 4 else (-1, 16, 16)
- data = data.reshape(data_shape)
- sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)
- av_x = torch.sqrt(sum_x2 / 32)
- weight = torch.abs(data) + av_x
-
- group_min = torch.min(data, axis=-1, keepdims=True)[0]
- group_max = torch.max(data, axis=-1, keepdims=True)[0]
-
- the_mins = -group_min
+ d = d.cpu().numpy().astype(np.float16).view(np.uint8)
- sum_w = torch.sum(weight, axis=-1, keepdims=True)
- sum_x = torch.sum(weight * data, axis=-1, keepdims=True)
+ return np.concatenate([d, qh, qs], axis=-1)
- group_min[group_min > 0] = 0
- scale = (group_max - group_min) / nmax
- iscale = torch.where(scale == 0, 0, 1 /scale)
+@register_qtype("q5_1")
+def q5_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):
+ if scale is not None:
+ d = scale.reshape((-1, 1))
+ min = zp.reshape((-1, 1)) * d * -1
+ else:
+ max = blocks.max(axis=-1, keepdims=True)[0]
+ min = blocks.min(axis=-1, keepdims=True)[0]
+ d = (max-min) / 31
- l_values = torch.round(iscale * (data - group_min))
- L = torch.clip(l_values, 0, nmax).to(torch.uint8)
+ n_blocks = blocks.shape[0]
+ block_size = GGML_QUANT_SIZES["q5_1"][0]
- diffs = scale * L + group_min - data
- diffs = torch.abs(diffs) if use_mad else diffs**2
- best_mad = torch.sum(weight * diffs, axis=-1, keepdims=True)
+ id = torch.where(d == 0, 0, 1 / d)
+ q = torch.trunc((blocks-min) * id + 0.5).to(torch.uint8).clip(0, 31).cpu().numpy()
- if nstep < 1:
- return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
+ qs = q.reshape((n_blocks, 2, block_size // 2))
+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))
- for step in range(nstep):
- new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)
- new_iscale = torch.where(new_scale == 0, 0, 1 /new_scale)
+ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)
- l_values = torch.round(new_iscale * (data - group_min))
- Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)
+ d = d.cpu().numpy().astype(np.float16).view(np.uint8)
+ m = min.cpu().numpy().astype(np.float16).view(np.uint8)
- sum_l = torch.sum(weight * Laux, axis=-1, keepdims=True)
- sum_l2 = torch.sum(weight * Laux**2, axis=-1, keepdims=True)
- sum_xl = torch.sum(weight * Laux * data, axis=-1, keepdims=True)
+ return np.concatenate([d, m, qh, qs], axis=-1)
- D = sum_w * sum_l2 - sum_l * sum_l
- replace_idx = D > 0
- this_scale = (sum_w * sum_xl - sum_x * sum_l) / D
- this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D
- this_min[this_min > 0] = 0
- this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]
+@register_qtype("q8_0")
+def q8_0_quant_block(blocks, scale=None, zp=None, **kwargs) -> np.ndarray:
+ if scale is not None:
+ d = scale.reshape((-1, 1))
+ else:
+ d = torch.abs(blocks).max(axis=1, keepdims=True)[0] / 127
+ id = torch.where(d == 0, 0, 1 / d)
- diffs = this_scale * Laux + this_min - data
- diffs = torch.abs(diffs) if use_mad else diffs**2
- mad = torch.sum(weight * diffs, axis=-1, keepdims=True)
+ qs = torch_roundf(blocks * id)
- replace_idx &= mad < best_mad
- best_mad[replace_idx] = mad[replace_idx]
- L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]
- scale[replace_idx] = this_scale[replace_idx]
- group_min[replace_idx] = this_min[replace_idx]
+ # (n_blocks, 2)
+ d = d.cpu().numpy().astype(np.float16).view(np.uint8)
+ # (n_blocks, block_size)
+ qs = qs.cpu().numpy().astype(np.int8).view(np.uint8)
- the_mins = -group_min
- return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
+ return np.concatenate([d, qs], axis=1)
-@register_block("q2_k")
-def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+@register_qtype("q2_k")
+def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
nb = blocks.shape[0]
output_scale = np.empty((nb, 16), dtype=np.uint8)
output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)
- blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)
+ blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)
if scale is not None:
scales = scale.reshape((-1, QK_K // 16))
mins = wmin_m.reshape((-1, QK_K // 16))
- output_d = d_scale.to(torch.float32) if len(d_scale.shape) == 2 \
- else d_scale.reshape(-1, 1).to(torch.float32)
- output_dmin = d_wmin_m.to(torch.float32) if len(d_wmin_m.shape) == 2 \
- else d_wmin_m.reshape(-1, 1).to(torch.float32)
- inv_scales = torch.where(d_scale == 0, 0, 1 / output_d)
+ output_d = d_scale.reshape(-1, 1).to(torch.float32)
+ output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)
+ inv_scales = torch.where(output_d == 0, 0, 1 / output_d)
inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)
max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)
@@ -245,19 +378,18 @@ def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
if d_scale is None:
output_d = torch.where(max_scales > 0, max_scales / 15, 0)
if d_wmin_m is None:
- output_dmin = torch.where(max_mins > 0 ,max_mins / 15., 0)
+ output_dmin = torch.where(max_mins > 0, max_mins / 15., 0)
d_tmp = output_d * (output_scale & 0xF)
dm_tmp = output_dmin * (output_scale >> 4)
replace_ids = d_tmp != 0
all_L[replace_ids] = torch.round(
- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(
- torch.uint8)
+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)
all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 3)
output_scale = output_scale.cpu().numpy()
- output_qs = all_L[:,::4] | (all_L[:,1::4] << 2) | (all_L[:,2::4] << 4) | (all_L[:,3::4] << 6)
+ output_qs = all_L[:, ::4] | (all_L[:, 1::4] << 2) | (all_L[:, 2::4] << 4) | (all_L[:, 3::4] << 6)
output_d = output_d.cpu().numpy()
output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
output_dmin = output_dmin.cpu().numpy()
@@ -268,7 +400,54 @@ def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale
return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)
-@register_block("q4_k")
+@register_qtype("q3_k")
+def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ nb = blocks.shape[0]
+ blocks = blocks.reshape(nb, QK_K // 16, 16)
+
+ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
+
+ if scale is not None:
+ scales = scale.reshape(-1, QK_K // 16)
+ output_d = d_scale.reshape(-1, 1).to(torch.float32)
+ iscales = torch.where(output_d == 0, 0, 1 / output_d)
+ all_L = torch.round(blocks / scales.unsqueeze(-1)).clip(-4, 3) + 4
+ all_L = all_L.to(torch.uint8)
+ else:
+ scales, all_L = make_q3_quants(blocks, bits=3, do_rmse=True)
+ imax = abs(scales).argmax(axis=-1, keepdims=True)
+ max_scales = torch.take_along_dim(scales, imax, axis=-1)
+ iscales = torch.where(max_scales != 0, -32 / max_scales, 0)
+ output_d = torch.where(iscales != 0, 1 / iscales, 0)
+
+ q_scales = torch.round(iscales * scales).clip(-32, 31) + 32
+ q_scales = q_scales.cpu().numpy().astype(np.uint8)
+ output_scale[:, :8] = (q_scales[:, :8] & 0xF) | ((q_scales[:, 8:] & 0xF) << 4)
+ hmask = q_scales >> 4
+ output_scale[:, 8:] = hmask[:, :4] | hmask[:, 4:8] << 2 | hmask[:, 8:12] << 4 | hmask[:, 12:] << 6
+
+ sc = torch.round(iscales * scales).clip(-32, 31)
+ d = output_d.to(torch.float32) * sc
+ replace_idx = (d != 0)
+
+ all_L[replace_idx] = (torch.round(blocks[replace_idx] / d[replace_idx].unsqueeze(-1)).clip(-4, 3) + 4).to(
+ torch.uint8)
+
+ all_L = all_L.cpu().numpy()
+ output_hmask = np.bitwise_or.reduce(
+ all_L.reshape(nb, 8, 32) >> 2 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1) # pylint: disable=E1121
+ all_L = np.where(all_L > 3, all_L - 4, all_L)
+
+ output_qs = np.bitwise_or.reduce(
+ all_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121
+
+ output_qs = output_qs.reshape(nb, 64).astype(np.uint8)
+ output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)
+ # [hmask, qs, scale, d]
+ return np.concatenate([output_hmask, output_qs, output_scale, output_d], axis=-1)
+
+
+@register_qtype("q4_k")
def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
nb = blocks.shape[0]
blocks = blocks.reshape((nb, QK_K // 32, 32))
@@ -279,11 +458,9 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
if scale is not None:
scales = scale.reshape(-1, QK_K // 32)
mins = wmin_m.reshape(-1, QK_K // 32)
- output_d = d_scale.to(torch.float32) if len(d_scale.shape) == 2 \
- else d_scale.reshape(-1, 1).to(torch.float32)
- output_dmin = d_wmin_m.to(torch.float32) if len(d_wmin_m.shape) == 2 \
- else d_wmin_m.reshape(-1, 1).to(torch.float32)
- inv_scales = torch.where(d_scale == 0, 0, 1 / output_d)
+ output_d = d_scale.reshape(-1, 1).to(torch.float32)
+ output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)
+ inv_scales = torch.where(output_d == 0, 0, 1 / output_d)
inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)
max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)
@@ -308,18 +485,18 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
q_scales = q_scales.cpu().numpy().astype(np.uint8)
q_mins = q_mins.cpu().numpy().astype(np.uint8)
- output_scale[:,:4] = q_scales[:,:4]
- output_scale[:,4:8] = q_mins[:,:4]
+ output_scale[:, :4] = q_scales[:, :4]
+ output_scale[:, 4:8] = q_mins[:, :4]
- output_scale[:,8:] = (q_scales[:,4:] & 0xF) | ((q_mins[:,4:] & 0xF) << 4)
- output_scale[:,:4] |= ((q_scales[:,4:] >> 4) << 6)
- output_scale[:,4:8] |= ((q_mins[:,4:] >> 4) << 6)
+ output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)
+ output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)
+ output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)
replace_ids = d_tmp != 0
all_L[replace_ids] = torch.round(
(blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)
all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 15)
- output_qs = all_L[:,::2] | (all_L[:,1::2] << 4)
+ output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)
output_d = output_d.cpu().numpy()
output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
@@ -329,3 +506,105 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w
# [d, dmin, scale, qs]
return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)
+
+
+@register_qtype("q5_k")
+def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ nb = blocks.shape[0]
+ blocks = blocks.reshape((nb, QK_K // 32, 32))
+
+ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)
+ output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)
+ output_qh = np.empty((nb, QK_K // 8), dtype=np.uint8)
+
+ if scale is not None:
+ scales = scale.reshape(-1, QK_K // 32)
+ mins = wmin_m.reshape(-1, QK_K // 32)
+ output_d = d_scale.reshape(-1, 1).to(torch.float32)
+ output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)
+ inv_scales = torch.where(output_d == 0, 0, 1 / output_d)
+ inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)
+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)
+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)
+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).to(torch.uint8)
+ else:
+ scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)
+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0]
+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0]
+ inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)
+ inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)
+
+ q_scales = torch.round(inv_scales * scales).to(torch.uint8).clip(0, 63)
+ q_mins = torch.round(inv_mins * mins).to(torch.uint8).clip(0, 63)
+
+ if d_scale is None:
+ output_d = max_scales / 63
+ if d_wmin_m is None:
+ output_dmin = max_mins / 63
+
+ d_tmp = output_d * q_scales
+ dm_tmp = output_dmin * q_mins
+
+ q_scales = q_scales.cpu().numpy().astype(np.uint8)
+ q_mins = q_mins.cpu().numpy().astype(np.uint8)
+ output_scale[:, :4] = q_scales[:, :4]
+ output_scale[:, 4:8] = q_mins[:, :4]
+
+ output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)
+ output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)
+ output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)
+
+ replace_ids = d_tmp != 0
+ all_L[replace_ids] = torch.round(
+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)
+ all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 31) # (nb, 8, 32)
+
+ output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)
+ output_qh = np.bitwise_or.reduce(
+ all_L >> 4 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1).astype(np.uint8) # pylint: disable=E1121
+
+ output_d = output_d.cpu().numpy()
+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_dmin = output_dmin.cpu().numpy()
+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)
+ output_qs = output_qs.reshape(nb, QK_K // 2)
+ # [d, dmin, scale, qh, qs]
+ return np.concatenate([output_d, output_dmin, output_scale, output_qh, output_qs], axis=-1)
+
+
+@register_qtype("q6_k")
+def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):
+ nb = blocks.shape[0]
+ blocks = blocks.reshape((nb, QK_K // 16, 16))
+
+ scale = None
+ if scale is not None:
+ scales = scale.reshape(-1, QK_K // 16)
+ output_d = d_scale.reshape(-1, 1).to(torch.float32)
+ iscales = torch.where(output_d == 0, 0, 1 / output_d)
+ all_L = (torch.round(blocks / scales.reshape(*scales.shape, 1)).clip(-32, 31) + 32).to(torch.uint8)
+ else:
+ scales, all_L = make_qx_quant(blocks, bits=6, rmse_type=1, qw=None)
+ imax = abs(scales).argmax(axis=-1, keepdims=True)
+ max_scales = torch.take_along_dim(scales, imax, axis=-1)
+ iscales = torch.where(max_scales != 0, -128 / max_scales, 0)
+ output_d = torch.where(iscales != 0, 1 / iscales, -1)
+
+ output_scale = torch.round(iscales * scales).clip(max=127).to(torch.int8)
+ d_tmp = output_d * output_scale.to(torch.float32)
+ replace_ids = d_tmp != 0
+ all_L[replace_ids] = torch.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32) \
+ .clip(0, 63).to(torch.uint8)
+
+ all_L = all_L.cpu().numpy()
+ tmp_L = all_L.reshape(nb, 4, 64) & 0xF
+ output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2)
+ output_qh = np.bitwise_or.reduce(
+ (all_L >> 4).reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), # pylint: disable=E1121
+ axis=2).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121
+
+ output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)
+
+ # [ql, qh, scale, d]
+ output_scale = output_scale.cpu().numpy().view(np.uint8)
+ return np.concatenate([output_ql, output_qh, output_scale, output_d], axis=-1)
\ No newline at end of file
diff --git a/auto_round/export/export_to_gguf/utils.py b/auto_round/export/export_to_gguf/utils.py
new file mode 100644
index 000000000..99ab13c7d
--- /dev/null
+++ b/auto_round/export/export_to_gguf/utils.py
@@ -0,0 +1,30 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+QK_K = 256
+K_SCALE_SIZE = 12
+GGML_QUANT_SIZES = {
+ "bf16": (1, 2),
+ "q4_0": (32, 2 + 16),
+ "q4_1": (32, 2 + 2 + 16),
+ "q5_0": (32, 2 + 4 + 16),
+ "q5_1": (32, 2 + 2 + 4 + 16),
+ "q8_0": (32, 2 + 32),
+ "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),
+ "q3_k": (256, 2 + QK_K // 4 + QK_K // 8 + 12),
+ "q4_k": (256, 2 + 2 + QK_K//2 + 12),
+ "q5_k": (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),
+ "q6_k": (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),
+ "q8_k": (256, 4 + QK_K + QK_K // 8)
+}
\ No newline at end of file
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index b5c508677..435eed66d 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -225,13 +225,13 @@ def pack_model(
else:
scale = scale.to(dtype=convert_dtype)
zp = zp.to(dtype=torch.int32)
- if isinstance(m, transformers.modeling_utils.Conv1D):
+ if isinstance(m, transformers.pytorch_utils.Conv1D):
fp_weight = fp_weight.t_().contiguous()
int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size, fp_weight.device)
if isinstance(m, torch.nn.Linear):
in_features = m.in_features
out_features = m.out_features
- elif isinstance(m, transformers.modeling_utils.Conv1D):
+ elif isinstance(m, transformers.pytorch_utils.Conv1D):
in_features = m.weight.shape[0]
out_features = m.weight.shape[1]
int_weight = int_weight.type(torch.int32)
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 5a9594f61..f6046c64f 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -35,6 +35,7 @@ from auto_round.utils import (
clear_memory,
get_device_and_parallelism,
get_model_dtype,
+ str2bool,
set_cuda_visible_devices)
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
@@ -199,7 +200,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
type=str,
help="the torch_dytpe to load the model for evaluation.")
-
class EvalArgumentParser(argparse.ArgumentParser):
def __init__(self, *args, **kwargs):
@@ -338,19 +338,18 @@ def tune(args):
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel, AutoConfig
from auto_round.utils import detect_device, get_library_version
- from auto_round.utils import logger, _gguf_args_check
+ from auto_round.utils import logger
if args.format is None:
args.format = "auto_round"
+
formats = args.format.lower().replace(' ', '').split(",")
from auto_round.utils import supported_formats
for format in formats:
if format not in supported_formats:
raise ValueError(f"{format} is not supported, we only support {supported_formats}")
- args = _gguf_args_check(args)
-
if "auto_gptq" in args.format and args.asym is True:
logger.warning("the auto_gptq kernel has issues with asymmetric quantization. "
"It is recommended to use sym quantization or --format='auto_round'")
@@ -411,7 +410,7 @@ def tune(args):
layer_config = {}
for n, m in model.named_modules():
- if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
+ if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.pytorch_utils.Conv1D):
if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
layer_config[n] = {"bits": 16}
logger.info(
@@ -539,7 +538,7 @@ def tune(args):
for file in os.listdir(eval_folder):
gguf_file = file
- logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")
+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")
if eval_model_dtype == "float32" or eval_model_dtype == "auto":
logger.warning(
"set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
@@ -642,7 +641,7 @@ def eval(args):
from lm_eval.utils import make_table # pylint: disable=E0401
tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
- logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")
+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")
if eval_model_dtype == "float32" or eval_model_dtype == "auto":
logger.warning(
"set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
@@ -706,7 +705,7 @@ def eval_task_by_task(
eval_model_dtype = get_model_dtype(eval_model_dtype)
if is_gguf_file:
tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
- logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")
+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")
if eval_model_dtype == "float32" or eval_model_dtype == "auto":
logger.warning(
"set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 7fc42d1ae..212fb902a 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -23,7 +23,6 @@ from auto_round.utils import (
get_device_and_parallelism,
set_cuda_visible_devices,
logger,
- _gguf_args_check
)
@@ -299,9 +298,6 @@ def tune(args):
if format not in supported_formats:
raise ValueError(f"{format} is not supported, we only support {supported_formats}")
- args = _gguf_args_check(args)
-
-
##must set this before import torch
set_cuda_visible_devices(args.device)
device_str, use_auto_mapping = get_device_and_parallelism(args.device)
@@ -350,7 +346,7 @@ def tune(args):
if args.fp_layers != "":
fp_layers = args.fp_layers.replace(" ", "").split(",")
for n, m in model.named_modules():
- if not isinstance(m, (torch.nn.Linear, transformers.modeling_utils.Conv1D)):
+ if not isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):
continue
for fp_layer in fp_layers:
if fp_layer in n:
@@ -363,7 +359,7 @@ def tune(args):
logger.warning(f"mixed precision exporting does not support {format} currently")
for n, m in model.named_modules():
- if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
+ if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.pytorch_utils.Conv1D):
if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
layer_config[n] = {"bits": 32}
logger.info(
diff --git a/auto_round/utils.py b/auto_round/utils.py
index cbe7ae22d..e61ba480a 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -33,14 +33,26 @@ from auto_round.export.export_to_gguf.config import GGUF_CONFIG
shared_cache_keys = ("position_ids", "cache_position", "position_embeddings")
-supported_formats = (
- "auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:gptqmodel", "auto_round:auto_awq",
- "itrex", "itrex_xpu", "fake"
-)
+class SupportedFormats:
+ def __init__(self):
+ self._support_format = (
+ "auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:gptqmodel",
+ "auto_round:auto_awq", "itrex", "itrex_xpu", "fake")
+ self._gguf_format = tuple(GGUF_CONFIG.keys())
+ self._support_list = self._support_format + self._gguf_format
+
+ def __contains__(self, key):
+ return True if key in self._support_list else False
-supported_formats = supported_formats + tuple(GGUF_CONFIG.keys())
+ def __str__(self):
+ return "(%s)" % ', '.join(self._support_format + ("gguf:q*_0", "gguf:q*_1", "gguf:q*_k_s"))
+
+ def __getitem__(self, key):
+ return self._support_list[key]
-supported_layer_types = (torch.nn.Linear, transformers.modeling_utils.Conv1D)
+supported_formats = SupportedFormats()
+
+supported_layer_types = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)
supported_dtypes = ("int", "mx_fp", "fp", "nv_fp")
@@ -744,7 +756,7 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):
def get_layer_names_in_block(model, supported_types=(torch.nn.Linear,
- transformers.modeling_utils.Conv1D), quant_block_list=None):
+ transformers.pytorch_utils.Conv1D), quant_block_list=None):
"""Retrieves the names of layers within each block of the model.
Returns:
@@ -1038,7 +1050,7 @@ def get_fp_layer_names(model, fp_layers):
fp_layers = fp_layers.replace(" ", "").split(",")
all_layer_names = []
for n, m in model.named_modules():
- if isinstance(m, (torch.nn.Linear, transformers.modeling_utils.Conv1D)):
+ if isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):
all_layer_names.append(n)
not_to_quantized_layers = []
@@ -1076,7 +1088,7 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Non
if bits != 4:
return False, f"AutoAWQ GEMM kernel only supports 4 bits"
for n, m in model.named_modules():
- if isinstance(m, transformers.modeling_utils.Conv1D):
+ if isinstance(m, transformers.pytorch_utils.Conv1D):
return False, "AutoAWQ GEMM kernel does not support conv1d"
layer_names = get_layer_names_in_block(model)
@@ -1151,16 +1163,28 @@ def get_layer_features(layer):
return None, None # Unsupported layer type
-def _gguf_args_check(args):
+def _gguf_args_check(args_or_ar, format_str=None):
from auto_round.utils import logger
from auto_round.export.export_to_gguf.config import GGUF_CONFIG
+ import argparse
- formats = args.format.lower().replace(' ', '').split(",")
+ if format_str is None:
+ args_or_ar.format = args_or_ar.format.replace("q*_", f"q{args_or_ar.bits}_")
+ format_str = args_or_ar.format
+ else:
+ format_str = format_str.replace("q*_", f"q{args_or_ar.bits}_")
+ formats = format_str.lower().replace(' ', '').split(",")
formats = sorted(formats, key=lambda x: len(x))
+ for f in formats:
+ if f.startswith("gguf") and f not in GGUF_CONFIG:
+ logger.error(f"{f} is not supported, please check.")
pattern = re.compile("q\d_k")
pre_dq_format = ""
+ unsupport_list, reset_list = [], []
for format in GGUF_CONFIG:
if format in formats:
+ if format == "q6_k_s":
+ logger.warning("Please not that q6_k_s is q6_k.")
try:
from auto_round.export.export_to_gguf.convert import Model
except:
@@ -1175,10 +1199,10 @@ def _gguf_args_check(args):
else:
pre_dq_format = format
- if os.path.isdir(args.model):
+ if isinstance(args_or_ar.model, str) and os.path.isdir(args_or_ar.model):
from pathlib import Path
from auto_round.export.export_to_gguf.convert import Model
- hparams = Model.load_hparams(Path(args.model))
+ hparams = Model.load_hparams(Path(args_or_ar.model))
model_architecture = hparams["architectures"][0]
try:
model_class = Model.from_model_architecture(model_architecture)
@@ -1187,7 +1211,7 @@ def _gguf_args_check(args):
sys.exit(1)
if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % 256 != 0):
- model_name = args.model.split('/')
+ model_name = args_or_ar.model.split('/')
model_name = model_name[-1] if model_name[-1] else model_name[-2]
hidden_size = hparams["hidden_size"]
logger.error(
@@ -1198,17 +1222,28 @@ def _gguf_args_check(args):
unsupport_list, reset_list = [], []
gguf_config = GGUF_CONFIG[format]
for k, v in gguf_config.items():
- if getattr(args, k) != v:
- unsupport_list.append(f"{k}={getattr(args, k)}")
+ if not hasattr(args_or_ar, k):
+ continue
+ if k == "data_type":
+ if re.search("q\d_1", format) and len(formats) > 1:
+ v = "int"
+ if getattr(args_or_ar, k) != v:
+ unsupport_list.append(f"{k}={getattr(args_or_ar, k)}")
reset_list.append(f"{k}={v}")
- setattr(args, k, v)
+ setattr(args_or_ar, k, v)
if len(unsupport_list) > 0:
logger.error(
f"format {format} does not support for {', '.join(unsupport_list)},"
f" reset to {', '.join(reset_list)}.")
- logger.info(f"export format {format}, sym = {not args.asym}, group_size = {args.group_size}")
-
- return args
+ if not isinstance(args_or_ar, argparse.Namespace) and len(unsupport_list) > 0:
+ for layer_name in args_or_ar.layer_config:
+ if args_or_ar.layer_config[layer_name]['bits'] >= 16:
+ continue
+ for k in args_or_ar.layer_config[layer_name]:
+ if hasattr(args_or_ar, k):
+ args_or_ar.layer_config[layer_name][k] = getattr(args_or_ar, k)
+ args_or_ar.has_qlayer_outside_block = args_or_ar.set_layerwise_config(args_or_ar.layer_config)
+ return args_or_ar
def _to_model_dtype(model, model_dtype):
@@ -1438,6 +1473,16 @@ def get_model_dtype(model_dtype, default="auto"):
model_dtype = default
return model_dtype
+def str2bool(v):
+ import argparse
+ if isinstance(v, bool):
+ return v
+ if v.lower() in ('yes', 'true', 't', 'y', '1'):
+ return True
+ elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+ return False
+ else:
+ raise argparse.ArgumentTypeError('Boolean value expected.')
def filter_quantization_config(quantization_config):
default_dict = {"amp": True, "batch_size": 8, "data_type": int, "dataset": "NeelNanda/pile-10k",
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 3d787c128..703c2b1f5 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -93,7 +93,7 @@ class WrapperLinear(torch.nn.Module):
orig_layer = self.orig_layer
orig_weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()
- if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):
+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
orig_weight = orig_weight.t()
weight_reshape = reshape_and_pad_tensor(orig_weight.data, orig_layer.group_size)
self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)
@@ -106,7 +106,6 @@ class WrapperLinear(torch.nn.Module):
self.weight_quant_func, self.data_type = get_quant_func(orig_layer.data_type, orig_layer.bits,
orig_layer.sym)
-
if self.enable_act_quant:
self.act_quant_func, self.act_data_type = get_quant_func(orig_layer.act_data_type,
orig_layer.act_bits,
@@ -154,7 +153,7 @@ class WrapperLinear(torch.nn.Module):
weight = self.orig_layer.weight
if weight.device.type == 'meta':
weight = self.orig_layer.get_weight().to(self.device)
- if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):
+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
weight = weight.t()
quant_kwargs = {}
@@ -177,7 +176,7 @@ class WrapperLinear(torch.nn.Module):
**quant_kwargs
)
weight_q = weight_q.to(weight.dtype)
- if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):
+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
weight_q = weight_q.t()
return weight_q, scale, zp
@@ -237,7 +236,7 @@ class WrapperLinear(torch.nn.Module):
self.orig_layer.weight.grad = None
shape = qdq_weight.shape
- if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):
+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):
shape = qdq_weight.t().shape
def _set_dict_attr(attr_dict, attr_name):
@@ -507,14 +506,17 @@ def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device='
quantized_layers = []
unquantized_layers = []
for n, m in block.named_modules():
- if isinstance(m, (torch.nn.Linear, transformers.modeling_utils.Conv1D)):
+ if isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):
if not check_to_quantized(m):
unquantized_layers.append(n)
continue
- new_m = WrapperLinear(m, enable_minmax_tuning=enable_minmax_tuning,
- enable_norm_bias_tuning=enable_norm_bias_tuning, device=device,
- **kwargs,
- )
+ new_m = WrapperLinear(
+ m,
+ enable_minmax_tuning=enable_minmax_tuning,
+ enable_norm_bias_tuning=enable_norm_bias_tuning,
+ device=device,
+ **kwargs,
+ )
set_module(block, n, new_m)
quantized_layers.append(n)
diff --git a/test/requirements.txt b/test/requirements.txt
index 7bf7c3dd3..bc6615906 100644
--- a/test/requirements.txt
+++ b/test/requirements.txt
@@ -1,3 +1,4 @@
addict
modelscope
gguf
+torchvision
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 77ad0dacc..0fb7feed1 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -416,7 +416,10 @@ class TestAutoRound(unittest.TestCase):
model_name, quantization_config=quantization_config, device_map="cpu", torch_dtype=torch.float16)
self.assertTrue(isinstance(model.visual.blocks[0].attn.qkv, torch.nn.Linear))
self.assertFalse(isinstance(model.visual.merger.mlp[0], QuantLinear))
- self.assertTrue(isinstance(model.model.layers[0].self_attn.v_proj, QuantLinear))
+ if hasattr(model.model, "language_model"):
+ self.assertTrue(isinstance(model.model.language_model.layers[0].self_attn.v_proj, QuantLinear))
+ else:
+ self.assertTrue(isinstance(model.model.layers[0].self_attn.v_proj, QuantLinear))
processor = AutoProcessor.from_pretrained(model_name, size=None)
image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
diff --git a/test/test_gguf_format.py b/test/test_gguf_format.py
index c85eded58..d3ab8cb54 100644
--- a/test/test_gguf_format.py
+++ b/test/test_gguf_format.py
@@ -62,10 +62,9 @@ class TestGGUF(unittest.TestCase):
iters=1,
data_type="int"
)
- autoround.quantize()
quantized_model_path = "./saved"
- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")
gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_0.gguf"
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
text = "There is a girl who likes adventure,"
@@ -73,7 +72,7 @@ class TestGGUF(unittest.TestCase):
print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
from auto_round.eval.evaluation import simple_evaluate_user_model
- result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa")
+ result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16")
# 0.246
self.assertGreater(result['results']['openbookqa']['acc,none'], 0.23)
shutil.rmtree("./saved", ignore_errors=True)
@@ -89,10 +88,9 @@ class TestGGUF(unittest.TestCase):
iters=1,
data_type="int"
)
- autoround.quantize()
quantized_model_path = "./saved"
- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")
gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_1.gguf"
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
text = "There is a girl who likes adventure,"
@@ -100,10 +98,100 @@ class TestGGUF(unittest.TestCase):
print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
from auto_round.eval.evaluation import simple_evaluate_user_model
- result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa")
+ result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16")
# 0.23
self.assertGreater(result['results']['openbookqa']['acc,none'], 0.22)
shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_func(self):
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=0,
+ data_type="int"
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_1")
+ self.assertTrue(autoround.group_size == 32)
+ self.assertFalse(autoround.sym)
+ gguf_file = os.listdir("saved")[0]
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
+ text = "There is a girl who likes adventure,"
+ inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
+ print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ autoround = AutoRound(
+ model,
+ self.tokenizer,
+ bits=3,
+ group_size=16,
+ sym=True,
+ iters=0,
+ data_type="int_sym_dq",
+ super_group_size=16,
+ super_bits=6
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")
+ from auto_round.eval.evaluation import simple_evaluate_user_model
+ gguf_file = os.listdir("saved")[0]
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
+ result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="lambada_openai", eval_model_dtype="bf16")
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.5)
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_q5_k(self):
+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ autoround = AutoRound(
+ model,
+ self.tokenizer,
+ bits=5,
+ group_size=32,
+ sym=False,
+ iters=0,
+ data_type="int_asym_dq",
+ super_group_size=8,
+ super_bits=6
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")
+ gguf_file = os.listdir("saved")[0]
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
+ text = "There is a girl who likes adventure,"
+ inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
+ print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_q6_k(self):
+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ autoround = AutoRound(
+ model,
+ self.tokenizer,
+ bits=6,
+ group_size=16,
+ sym=True,
+ iters=0,
+ data_type="int_sym_dq",
+ super_group_size=16,
+ super_bits=8
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")
+ gguf_file = os.listdir("saved")[0]
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
+ text = "There is a girl who likes adventure,"
+ inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
+ print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
if __name__ == "__main__":
unittest.main()