Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix q2k bug (#599)

Heng Guof41bb3d5781d
diff --git a/auto_round/export/export_to_gguf/quant_cpu.py b/auto_round/export/export_to_gguf/quant_cpu.py
index 563284017..f4e097b35 100644
--- a/auto_round/export/export_to_gguf/quant_cpu.py
+++ b/auto_round/export/export_to_gguf/quant_cpu.py
@@ -150,28 +150,31 @@ def make_q3_quants(data, bits, do_rmse=False):
return scales, L
-def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
+def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False, weights=None):
# data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K
nmax = pow(2, bits) - 1
if len(data.shape) == 2:
if bits in [4, 5]:
data_shape = (-1, 8, 32)
- elif bits in [2, 3]:
+ elif bits in [2]:
data_shape = (-1, 16, 16)
else:
raise NotImplementedError(f"bits = {bits} is not supported")
data = data.reshape(data_shape)
- sum_x2 = np.sum(np.power(data, 2), axis=-1, keepdims=True)
- av_x = np.sqrt(sum_x2 / 32)
- weight = np.abs(data) + av_x
-
+ if weights is None:
+ sum_x2 = np.sum(np.pow(data, 2), axis=-1, keepdims=True)
+ if bits == 2:
+ av_x = 0
+ else:
+ av_x = np.sqrt(sum_x2 / data.shape[-1])
+ weights = np.abs(data) + av_x
group_min = np.min(data, axis=-1, keepdims=True)
group_max = np.max(data, axis=-1, keepdims=True)
the_mins = -group_min
- sum_w = np.sum(weight, axis=-1, keepdims=True)
- sum_x = np.sum(weight * data, axis=-1, keepdims=True)
+ sum_w = np.sum(weights, axis=-1, keepdims=True)
+ sum_x = np.sum(weights * data, axis=-1, keepdims=True)
group_min[group_min > 0] = 0
@@ -183,7 +186,7 @@ def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
diffs = scale*L + group_min - data
diffs = np.abs(diffs) if use_mad else diffs**2
- best_mad = np.sum(weight * diffs, axis=-1, keepdims=True)
+ best_mad = np.sum(weights * diffs, axis=-1, keepdims=True)
if nstep < 1:
return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
@@ -195,9 +198,9 @@ def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
l_values = np.round(new_iscale * (data-group_min))
Laux = np.clip(l_values, 0, nmax).astype(np.uint8)
- sum_l = np.sum(weight * Laux, axis=-1, keepdims=True)
- sum_l2 = np.sum(weight * Laux**2, axis=-1, keepdims=True)
- sum_xl = np.sum(weight * Laux * data, axis=-1, keepdims=True)
+ sum_l = np.sum(weights * Laux, axis=-1, keepdims=True)
+ sum_l2 = np.sum(weights * Laux**2, axis=-1, keepdims=True)
+ sum_xl = np.sum(weights * Laux * data, axis=-1, keepdims=True)
D = sum_w*sum_l2 - sum_l*sum_l
replace_idx = D > 0
@@ -209,7 +212,7 @@ def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
diffs = this_scale*Laux + this_min - data
diffs = np.abs(diffs) if use_mad else diffs**2
- mad = np.sum(weight * diffs, axis=-1, keepdims=True)
+ mad = np.sum(weights * diffs, axis=-1, keepdims=True)
replace_idx &= mad < best_mad
best_mad[replace_idx] = mad[replace_idx]
diff --git a/auto_round/export/export_to_gguf/quant_gpu.py b/auto_round/export/export_to_gguf/quant_gpu.py
index d1e773c65..99da6eb4e 100644
--- a/auto_round/export/export_to_gguf/quant_gpu.py
+++ b/auto_round/export/export_to_gguf/quant_gpu.py
@@ -146,28 +146,32 @@ def make_q3_quants(data, bits, do_rmse=False):
return scales, L.to(torch.uint8)
-def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
+def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False, weights=None):
nmax = pow(2, bits) - 1
# data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K
if len(data.shape) == 2:
if bits in [4, 5]:
data_shape = (-1, 8, 32)
- elif bits in [2, 3]:
+ elif bits in [2]:
data_shape = (-1, 16, 16)
else:
raise NotImplementedError(f"bits = {bits} is not supported")
data = data.reshape(data_shape)
- sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)
- av_x = torch.sqrt(sum_x2 / data.shape[-1])
- weight = torch.abs(data) + av_x
+ if weights is None:
+ sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)
+ if bits == 2:
+ av_x = 0
+ else:
+ av_x = torch.sqrt(sum_x2 / data.shape[-1])
+ weights = torch.abs(data) + av_x
group_min = torch.min(data, axis=-1, keepdims=True)[0]
group_max = torch.max(data, axis=-1, keepdims=True)[0]
the_mins = -group_min
- sum_w = torch.sum(weight, axis=-1, keepdims=True)
- sum_x = torch.sum(weight * data, axis=-1, keepdims=True)
+ sum_w = torch.sum(weights, axis=-1, keepdims=True)
+ sum_x = torch.sum(weights * data, axis=-1, keepdims=True)
group_min[group_min > 0] = 0
@@ -179,7 +183,7 @@ def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
diffs = scale*L + group_min - data
diffs = torch.abs(diffs) if use_mad else diffs**2
- best_mad = torch.sum(weight * diffs, axis=-1, keepdims=True)
+ best_mad = torch.sum(weights * diffs, axis=-1, keepdims=True)
if nstep < 1:
return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])
@@ -191,9 +195,9 @@ def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
l_values = torch.round(new_iscale * (data-group_min))
Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)
- sum_l = torch.sum(weight * Laux, axis=-1, keepdims=True)
- sum_l2 = torch.sum(weight * Laux**2, axis=-1, keepdims=True)
- sum_xl = torch.sum(weight * Laux * data, axis=-1, keepdims=True)
+ sum_l = torch.sum(weights * Laux, axis=-1, keepdims=True)
+ sum_l2 = torch.sum(weights * Laux**2, axis=-1, keepdims=True)
+ sum_xl = torch.sum(weights * Laux * data, axis=-1, keepdims=True)
D = sum_w*sum_l2 - sum_l*sum_l
replace_idx = D > 0
@@ -205,7 +209,7 @@ def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):
diffs = this_scale*Laux + this_min - data
diffs = torch.abs(diffs) if use_mad else diffs**2
- mad = torch.sum(weight * diffs, axis=-1, keepdims=True)
+ mad = torch.sum(weights * diffs, axis=-1, keepdims=True)
replace_idx &= mad < best_mad
best_mad[replace_idx] = mad[replace_idx]