Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix SDPA bug (#1862)

Luo Yuc51d9fe703c0
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.py
index bd844b656..167cd0c08 100644
--- a/auto_round_extension/ark/auto_round_kernel/__init__.py
+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py
@@ -530,7 +530,8 @@ class ARK:
- key: [B, Hkv, Skv, D] int8
- value: [B, Hkv, Skv, D] int8
- qscale: [B, Hq, ceil(Sq / quant_block_size), 1] float32
- - kscale/vscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32
+ - kscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32
+ - vscale: [B, Hkv, ceil(Skv / quant_block_size), D] float32
Returns:
- O: [B, Hq, Sq, D] float16
@@ -563,6 +564,21 @@ class ARK:
if D not in (64, 128):
raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
+ q_blocks = (Sq + quant_block_size - 1) // quant_block_size
+ kv_blocks = (Skv + quant_block_size - 1) // quant_block_size
+ if qscale.numel() != B * Hq * q_blocks:
+ raise ValueError(
+ f"qscale must have {B * Hq * q_blocks} elements for shape [B, Hq, ceil(Sq/block), 1], got {qscale.numel()}"
+ )
+ if kscale.numel() != B * Hkv * kv_blocks:
+ raise ValueError(
+ f"kscale must have {B * Hkv * kv_blocks} elements for shape [B, Hkv, ceil(Skv/block), 1], got {kscale.numel()}"
+ )
+ if vscale.numel() != B * Hkv * kv_blocks * D:
+ raise ValueError(
+ f"vscale must have {B * Hkv * kv_blocks * D} elements for shape [B, Hkv, ceil(Skv/block), D], got {vscale.numel()}"
+ )
+
lib = self.get_lib(query)
stream = get_stream(query)
O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=out_dtype)
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp
index f80f08cc9..ddde49c6c 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp
@@ -270,8 +270,9 @@ class XeSageFwdKernel {
? (float*)params.mainloop.kscale + (idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad)
: nullptr; // per head kscale
auto scaleV = params.mainloop.scale_block_size && params.mainloop.vscale
- ? (float*)params.mainloop.vscale + (idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad)
- : nullptr; // per head vscale
+ ? (float*)params.mainloop.vscale +
+ ((idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad) * s.head_size_vo)
+ : nullptr; // per head vscale, laid out as [seq_block, d]
auto ptrO = p.O + offset_o;
auto stride_q = is_var_len ? cutlass::make_cute_packed_stride(StrideQ{}, shape_Q) : p.dQ;
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp
index 0762423ff..60b42618c 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp
@@ -146,6 +146,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
decltype(FragS{}.tv_layout()){}));
using FragPRow = decltype(reduce<1>(FragP{}, sycl::plus<void>{}));
+ using FragPCol = decltype(reduce<0>(FragP{}, sycl::plus<void>{}));
using SingleFragPV = FragC<TiledMMAPV>; // (atom val,q',v')
using SingleFragAFloat = std::remove_reference_t<decltype(make_subgroup_tensor(
make_fragment_like<float>(typename SingleFragPV::Base{}), decltype(SingleFragPV{}.tv_layout()){}))>;
@@ -279,6 +280,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
Tensor cK_cache = make_identity_tensor(K_cache_2D.shape()); // (k,d)
Tensor cV_cache = make_identity_tensor(V_cache_2D.shape()); // (v,k)
Tensor cP = make_identity_tensor(take<0, 2>(TileShapeQK{})); // (q,k)
+ Tensor cPV = make_identity_tensor(select<0, 1>(TileShapePV{}));
/* Partition global tensors into workgroup tiles */
Tensor gQ = local_tile(cQ, TileShapeQK{}, append(blk_qv, _), Step<_1, X, _1>{}); // (q,d,D)
@@ -335,6 +337,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
auto tVrV = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));
auto tArV = thr_mma_pv.partition_sg_fragment_B(gV_split(_, _, 0, 0));
+ auto tCrA = thr_mma_pv_i8.partition_C(cPV);
/* Create TiledCopy objects for prefetches */
auto prefetch_q = make_block_2d_prefetch(copy_q);
@@ -501,7 +504,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
/* k masking for remainder tiles */
if constexpr (!is_cache) {
if (check_remainder_k && K == total_blk - 1) {
- FragPRow k_rem_mask;
+ FragPCol k_rem_mask;
int k_val = get<0>(tKgK_cur(0, 0, 0, k_idx, 0)) + kblocks_cache * get<1>(TileShapeQK{});
int k = k_val + get_sub_group().get_local_id()[0];
CUTLASS_PRAGMA_UNROLL
@@ -538,10 +541,10 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
cute::gemm(mma_pv, tArP, tArV, tArA(_, _, _, VV));
}
} else {
- ElementA pv_dequant_scale = ElementA(0.0f);
- if constexpr (WriteBackInt8PV) {
- pv_dequant_scale = ElementA(scaleV[scalek_idx]) * (ElementA(1.0f) / ElementA(127.0f));
- }
+ constexpr ElementA kInvVQuantScale = ElementA(1.0f / 127.0f);
+ int scalev_head_dim = int(size<0>(V_2D));
+ int v_block_base = int(get<1>(blk_qv)) * int(get<1>(TileShapePV{})) * VTiles;
+ int scalev_block_base = scalek_idx * scalev_head_dim;
CUTLASS_PRAGMA_UNROLL
for (int VV = 0; VV < VTiles; VV++) {
auto tArA_v = tArA(_, _, _, VV);
@@ -550,7 +553,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
for (int i = 0; i < tArA.size() / VTiles; i++) tArA_v(i) *= broadcast<0>(rescale, tArA, i);
}
- auto tArAcc = thr_mma_pv_i8.partition_sg_fragment_C(make_identity_tensor(select<0, 1>(TileShapePV{})));
+ auto tArAcc = thr_mma_pv_i8.partition_sg_fragment_C(cPV);
auto tVrV_i8 = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));
auto tArV_i8 = thr_mma_pv_i8.partition_sg_fragment_B(gV_split(_, _, 0, 0));
copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV_i8);
@@ -564,7 +567,9 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
if constexpr (WriteBackInt8PV) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < tArAcc.size(); ++i) {
- tArA_v(i) += ElementA(tArAcc(i)) * pv_dequant_scale;
+ int local_v = int(get<1>(tCrA(i)));
+ int scalev_idx = scalev_block_base + v_block_base + VV * int(get<1>(TileShapePV{})) + local_v;
+ tArA_v(i) += ElementA(tArAcc(i)) * ElementA(scaleV[scalev_idx]) * kInvVQuantScale;
}
}
}
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hpp
index c8826cbff..f060ae77b 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hpp
@@ -130,6 +130,7 @@ struct SDPAFwdMainloop<sdpa::XeDefault<Stages>, CausalMask_, FullMask_, CachedKV
using FragS = FragC<TiledMMAQK>;
using FragSRow = decltype(reduce<1>(FragS{}, sycl::plus<void>{}));
+ using FragSCol = decltype(reduce<0>(FragS{}, sycl::plus<void>{}));
using ElementS = typename TiledMMAQK::ValTypeD;
using ElementM = typename TiledMMAQK::ValTypeA;
@@ -400,7 +401,7 @@ struct SDPAFwdMainloop<sdpa::XeDefault<Stages>, CausalMask_, FullMask_, CachedKV
/* k masking for remainder tiles */
if constexpr (!is_cache) {
if (check_remainder_k && K == total_blk - 1) {
- FragSRow k_rem_mask;
+ FragSCol k_rem_mask;
int k_val = get<0>(tKgK_cur(0, 0, 0, k_idx, 0)) + kblocks_cache * get<1>(TileShapeQK{});
int k = k_val + get_sub_group().get_local_id()[0];
CUTLASS_PRAGMA_UNROLL
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
index 13a3ef875..51a346e2b 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
@@ -1041,6 +1041,40 @@ class XpuWrapper {
}
}
+ template <typename T>
+ static void sage_dynamic_quant_v(sycl::queue* q, const T* in_ptr, int8_t* out_ptr, float* scale_ptr, int num_rows,
+ int seq, int n_seq_blk, int head_dim, int block_size) {
+ size_t num_scales = size_t(num_rows) * size_t(n_seq_blk) * size_t(head_dim);
+ q->parallel_for(sycl::range<1>(num_scales), [=](sycl::id<1> item) {
+ size_t linear_idx = item[0];
+ int dim = int(linear_idx % size_t(head_dim));
+ size_t block_idx = linear_idx / size_t(head_dim);
+ int seq_id = int(block_idx % size_t(n_seq_blk));
+ int row_id = int(block_idx / size_t(n_seq_blk));
+ int seq_begin = seq_id * block_size;
+ int seq_end = std::min(seq_begin + block_size, seq);
+
+ float absmax = 0.0f;
+ for (int token = seq_begin; token < seq_end; ++token) {
+ size_t offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) + size_t(dim);
+ float value = static_cast<float>(in_ptr[offset]);
+ absmax = sycl::fmax(absmax, sycl::fabs(value));
+ }
+
+ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;
+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;
+ scale_ptr[linear_idx] = scale;
+
+ for (int token = seq_begin; token < seq_end; ++token) {
+ size_t offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) + size_t(dim);
+ float value = static_cast<float>(in_ptr[offset]) * inv_scale;
+ value = sycl::round(value);
+ value = sycl::clamp(value, -127.0f, 127.0f);
+ out_ptr[offset] = static_cast<int8_t>(value);
+ }
+ });
+ }
+
static void sagev1_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, bool use_int8_pv) {
@@ -1052,7 +1086,7 @@ class XpuWrapper {
size_t k_size = num_heads_kv * seq_len_kv * head_dim * batch;
size_t k_scale_size = num_heads_kv * seq_kv_blk * batch;
size_t v_tmp_size = use_int8_pv ? k_size : 0;
- size_t v_scale_size = use_int8_pv ? k_scale_size * sizeof(float) : 0;
+ size_t v_scale_size = use_int8_pv ? k_scale_size * head_dim * sizeof(float) : 0;
size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(sycl::half) : 0;
size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size +
v_scale_size + k_bias_size;
@@ -1082,8 +1116,8 @@ class XpuWrapper {
sage_dynamic_quant<sycl::half>(q, (sycl::half*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,
seq_len_kv, seq_kv_blk, head_dim, scale_block_size, kbias);
if (use_int8_pv) {
- sage_dynamic_quant<sycl::half>(q, (sycl::half*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale, batch * num_heads_kv,
- seq_len_kv, seq_kv_blk, head_dim, scale_block_size);
+ sage_dynamic_quant_v<sycl::half>(q, (sycl::half*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale,
+ batch * num_heads_kv, seq_len_kv, seq_kv_blk, head_dim, scale_block_size);
ark::sdpa_impl_qks8_pvi8(q, q_out_ptr, k_out_ptr, v_out_ptr, O_ptr, mask, scale_block_size, qscale, kscale,
vscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
softmax_scale, is_causal);
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
index a42dfc3b9..ac7d75eb7 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
@@ -195,7 +195,7 @@ struct TestSDPA {
size_t o_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;
size_t q_scale_count = size_t(batch) * num_heads_q * ((seq_len_q + scale_block_size - 1) / scale_block_size);
size_t k_scale_count = size_t(batch) * num_heads_kv * ((seq_len_kv + scale_block_size - 1) / scale_block_size);
- size_t v_scale_count = k_scale_count;
+ size_t v_scale_count = k_scale_count * head_dim;
float softmax_scale = 1.0f / std::sqrt(float(head_dim));
auto host_q = to_fp16_vector(make_random_vector(q_count, -1.0f, 1.0f, 901u + uint32_t(seq_len_q)));
@@ -224,9 +224,10 @@ struct TestSDPA {
ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_k, dev_ki8, dev_kscale, batch * num_heads_kv, seq_len_kv,
(seq_len_kv + scale_block_size - 1) / scale_block_size, head_dim,
scale_block_size);
- ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_v, dev_vi8, dev_vscale, batch * num_heads_kv, seq_len_kv,
- (seq_len_kv + scale_block_size - 1) / scale_block_size, head_dim,
- scale_block_size);
+ ark::XpuWrapper::sage_dynamic_quant_v<sycl::half>(q, dev_v, dev_vi8, dev_vscale, batch * num_heads_kv,
+ seq_len_kv,
+ (seq_len_kv + scale_block_size - 1) / scale_block_size,
+ head_dim, scale_block_size);
double sage_ms = run_bench(
[&]() {
diff --git a/auto_round_extension/ark/test/test_sage_dynquant.py b/auto_round_extension/ark/test/test_sage_dynquant.py
index 37a90a0e9..72beb2fb4 100644
--- a/auto_round_extension/ark/test/test_sage_dynquant.py
+++ b/auto_round_extension/ark/test/test_sage_dynquant.py
@@ -35,7 +35,7 @@ def bench(fn, n_warmup=50, n_runs=100):
return (time.time() - st) / n_runs * 1e3
-def run_benchmark(batch=2, seq=17776, h_q=30, h_kv=30, H=128, H_v=128, dt=torch.float16, is_causal=False, dev="xpu"):
+def run_benchmark(batch=1, seq=4096, h_q=30, h_kv=30, H=128, H_v=128, dt=torch.float16, is_causal=False, dev="xpu"):
# Align seq to largest block_size for divisibility
block_sizes = [1, 32, 64, 128, 256]
max_bs = max(block_sizes)
diff --git a/auto_round_extension/ark/test/test_sdpa_parity.py b/auto_round_extension/ark/test/test_sdpa_parity.py
new file mode 100644
index 000000000..77efc6f79
--- /dev/null
+++ b/auto_round_extension/ark/test/test_sdpa_parity.py
@@ -0,0 +1,70 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import math
+
+import auto_round_kernel
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(
+ not (hasattr(torch, "xpu") and torch.xpu.is_available()),
+ reason="XPU not available",
+)
+
+
[email protected]("seq_q,seq_kv", [(64, 64), (80, 64), (64, 80), (226, 226)])
+def test_ark_sdpa_matches_torch_for_kv_remainders(seq_q, seq_kv):
+ torch.manual_seed(2026)
+ batch, heads, head_dim = 1, 2, 64
+ scale = 1 / math.sqrt(head_dim)
+ q = torch.randn(batch, heads, seq_q, head_dim, device="xpu", dtype=torch.float16)
+ k = torch.randn(batch, heads, seq_kv, head_dim, device="xpu", dtype=torch.float16)
+ v = torch.randn(batch, heads, seq_kv, head_dim, device="xpu", dtype=torch.float16)
+
+ expected = torch.nn.functional.scaled_dot_product_attention(
+ q,
+ k,
+ v,
+ scale=scale,
+ is_causal=False,
+ )
+ actual = auto_round_kernel.ARK().sdpa(q, k, v, scale=scale, is_causal=False)
+ torch.xpu.synchronize()
+
+ torch.testing.assert_close(actual, expected, atol=1e-2, rtol=1e-2)
+
+
+def test_ark_sagev1_matches_torch_for_kv_remainder_tile():
+ """SAGEV1 uses the same final partial-K tile mask pattern after Q/K quantization.
+
+ If the SAGEV1 remainder mask is built from the row-reduced fragment, this
+ case produces a large error because the mask is broadcast along the K/column
+ dimension. The column-reduced mask keeps the error within the expected
+ SAGEV1 quantization tolerance.
+ """
+ torch.manual_seed(2027)
+ batch, heads, seq_q, seq_kv, head_dim = 1, 2, 80, 80, 64
+ scale = 1 / math.sqrt(head_dim)
+ q = torch.randn(batch, heads, seq_q, head_dim, device="xpu", dtype=torch.float16)
+ k = torch.randn(batch, heads, seq_kv, head_dim, device="xpu", dtype=torch.float16)
+ v = torch.randn(batch, heads, seq_kv, head_dim, device="xpu", dtype=torch.float16)
+
+ expected = torch.nn.functional.scaled_dot_product_attention(
+ q,
+ k,
+ v,
+ scale=scale,
+ is_causal=False,
+ )
+ actual = auto_round_kernel.ARK().sagev1(
+ q,
+ k,
+ v,
+ scale=scale,
+ is_causal=False,
+ quant_block_size=64,
+ )
+ torch.xpu.synchronize()
+
+ torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)