PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix SDPA bug (#1862)
Luo Yuc51d9fe703c0
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex bd844b656..167cd0c08 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -530,7 +530,8 @@ class ARK:- key: [B, Hkv, Skv, D] int8- value: [B, Hkv, Skv, D] int8- qscale: [B, Hq, ceil(Sq / quant_block_size), 1] float32- - kscale/vscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32+ - kscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32+ - vscale: [B, Hkv, ceil(Skv / quant_block_size), D] float32Returns:- O: [B, Hq, Sq, D] float16@@ -563,6 +564,21 @@ class ARK:if D not in (64, 128):raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")+ q_blocks = (Sq + quant_block_size - 1) // quant_block_size+ kv_blocks = (Skv + quant_block_size - 1) // quant_block_size+ if qscale.numel() != B * Hq * q_blocks:+ raise ValueError(+ f"qscale must have {B * Hq * q_blocks} elements for shape [B, Hq, ceil(Sq/block), 1], got {qscale.numel()}"+ )+ if kscale.numel() != B * Hkv * kv_blocks:+ raise ValueError(+ f"kscale must have {B * Hkv * kv_blocks} elements for shape [B, Hkv, ceil(Skv/block), 1], got {kscale.numel()}"+ )+ if vscale.numel() != B * Hkv * kv_blocks * D:+ raise ValueError(+ f"vscale must have {B * Hkv * kv_blocks * D} elements for shape [B, Hkv, ceil(Skv/block), D], got {vscale.numel()}"+ )+lib = self.get_lib(query)stream = get_stream(query)O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=out_dtype)diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hppindex f80f08cc9..ddde49c6c 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp@@ -270,8 +270,9 @@ class XeSageFwdKernel {? (float*)params.mainloop.kscale + (idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad): nullptr; // per head kscaleauto scaleV = params.mainloop.scale_block_size && params.mainloop.vscale- ? (float*)params.mainloop.vscale + (idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad)- : nullptr; // per head vscale+ ? (float*)params.mainloop.vscale ++ ((idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad) * s.head_size_vo)+ : nullptr; // per head vscale, laid out as [seq_block, d]auto ptrO = p.O + offset_o;auto stride_q = is_var_len ? cutlass::make_cute_packed_stride(StrideQ{}, shape_Q) : p.dQ;diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hppindex 0762423ff..60b42618c 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp@@ -146,6 +146,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cacheddecltype(FragS{}.tv_layout()){}));using FragPRow = decltype(reduce<1>(FragP{}, sycl::plus<void>{}));+ using FragPCol = decltype(reduce<0>(FragP{}, sycl::plus<void>{}));using SingleFragPV = FragC<TiledMMAPV>; // (atom val,q',v')using SingleFragAFloat = std::remove_reference_t<decltype(make_subgroup_tensor(make_fragment_like<float>(typename SingleFragPV::Base{}), decltype(SingleFragPV{}.tv_layout()){}))>;@@ -279,6 +280,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, CachedTensor cK_cache = make_identity_tensor(K_cache_2D.shape()); // (k,d)Tensor cV_cache = make_identity_tensor(V_cache_2D.shape()); // (v,k)Tensor cP = make_identity_tensor(take<0, 2>(TileShapeQK{})); // (q,k)+ Tensor cPV = make_identity_tensor(select<0, 1>(TileShapePV{}));/* Partition global tensors into workgroup tiles */Tensor gQ = local_tile(cQ, TileShapeQK{}, append(blk_qv, _), Step<_1, X, _1>{}); // (q,d,D)@@ -335,6 +337,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cachedauto tVrV = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));auto tArV = thr_mma_pv.partition_sg_fragment_B(gV_split(_, _, 0, 0));+ auto tCrA = thr_mma_pv_i8.partition_C(cPV);/* Create TiledCopy objects for prefetches */auto prefetch_q = make_block_2d_prefetch(copy_q);@@ -501,7 +504,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached/* k masking for remainder tiles */if constexpr (!is_cache) {if (check_remainder_k && K == total_blk - 1) {- FragPRow k_rem_mask;+ FragPCol k_rem_mask;int k_val = get<0>(tKgK_cur(0, 0, 0, k_idx, 0)) + kblocks_cache * get<1>(TileShapeQK{});int k = k_val + get_sub_group().get_local_id()[0];CUTLASS_PRAGMA_UNROLL@@ -538,10 +541,10 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cachedcute::gemm(mma_pv, tArP, tArV, tArA(_, _, _, VV));}} else {- ElementA pv_dequant_scale = ElementA(0.0f);- if constexpr (WriteBackInt8PV) {- pv_dequant_scale = ElementA(scaleV[scalek_idx]) * (ElementA(1.0f) / ElementA(127.0f));- }+ constexpr ElementA kInvVQuantScale = ElementA(1.0f / 127.0f);+ int scalev_head_dim = int(size<0>(V_2D));+ int v_block_base = int(get<1>(blk_qv)) * int(get<1>(TileShapePV{})) * VTiles;+ int scalev_block_base = scalek_idx * scalev_head_dim;CUTLASS_PRAGMA_UNROLLfor (int VV = 0; VV < VTiles; VV++) {auto tArA_v = tArA(_, _, _, VV);@@ -550,7 +553,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cachedfor (int i = 0; i < tArA.size() / VTiles; i++) tArA_v(i) *= broadcast<0>(rescale, tArA, i);}- auto tArAcc = thr_mma_pv_i8.partition_sg_fragment_C(make_identity_tensor(select<0, 1>(TileShapePV{})));+ auto tArAcc = thr_mma_pv_i8.partition_sg_fragment_C(cPV);auto tVrV_i8 = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));auto tArV_i8 = thr_mma_pv_i8.partition_sg_fragment_B(gV_split(_, _, 0, 0));copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV_i8);@@ -564,7 +567,9 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cachedif constexpr (WriteBackInt8PV) {CUTLASS_PRAGMA_UNROLLfor (int i = 0; i < tArAcc.size(); ++i) {- tArA_v(i) += ElementA(tArAcc(i)) * pv_dequant_scale;+ int local_v = int(get<1>(tCrA(i)));+ int scalev_idx = scalev_block_base + v_block_base + VV * int(get<1>(TileShapePV{})) + local_v;+ tArA_v(i) += ElementA(tArAcc(i)) * ElementA(scaleV[scalev_idx]) * kInvVQuantScale;}}}diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hppindex c8826cbff..f060ae77b 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hpp@@ -130,6 +130,7 @@ struct SDPAFwdMainloop<sdpa::XeDefault<Stages>, CausalMask_, FullMask_, CachedKVusing FragS = FragC<TiledMMAQK>;using FragSRow = decltype(reduce<1>(FragS{}, sycl::plus<void>{}));+ using FragSCol = decltype(reduce<0>(FragS{}, sycl::plus<void>{}));using ElementS = typename TiledMMAQK::ValTypeD;using ElementM = typename TiledMMAQK::ValTypeA;@@ -400,7 +401,7 @@ struct SDPAFwdMainloop<sdpa::XeDefault<Stages>, CausalMask_, FullMask_, CachedKV/* k masking for remainder tiles */if constexpr (!is_cache) {if (check_remainder_k && K == total_blk - 1) {- FragSRow k_rem_mask;+ FragSCol k_rem_mask;int k_val = get<0>(tKgK_cur(0, 0, 0, k_idx, 0)) + kblocks_cache * get<1>(TileShapeQK{});int k = k_val + get_sub_group().get_local_id()[0];CUTLASS_PRAGMA_UNROLLdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hppindex 13a3ef875..51a346e2b 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp@@ -1041,6 +1041,40 @@ class XpuWrapper {}}+ template <typename T>+ static void sage_dynamic_quant_v(sycl::queue* q, const T* in_ptr, int8_t* out_ptr, float* scale_ptr, int num_rows,+ int seq, int n_seq_blk, int head_dim, int block_size) {+ size_t num_scales = size_t(num_rows) * size_t(n_seq_blk) * size_t(head_dim);+ q->parallel_for(sycl::range<1>(num_scales), [=](sycl::id<1> item) {+ size_t linear_idx = item[0];+ int dim = int(linear_idx % size_t(head_dim));+ size_t block_idx = linear_idx / size_t(head_dim);+ int seq_id = int(block_idx % size_t(n_seq_blk));+ int row_id = int(block_idx / size_t(n_seq_blk));+ int seq_begin = seq_id * block_size;+ int seq_end = std::min(seq_begin + block_size, seq);++ float absmax = 0.0f;+ for (int token = seq_begin; token < seq_end; ++token) {+ size_t offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) + size_t(dim);+ float value = static_cast<float>(in_ptr[offset]);+ absmax = sycl::fmax(absmax, sycl::fabs(value));+ }++ float scale = absmax > 0.0f ? absmax / 127.0f : 0.0f;+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;+ scale_ptr[linear_idx] = scale;++ for (int token = seq_begin; token < seq_end; ++token) {+ size_t offset = (size_t(row_id) * size_t(seq) + size_t(token)) * size_t(head_dim) + size_t(dim);+ float value = static_cast<float>(in_ptr[offset]) * inv_scale;+ value = sycl::round(value);+ value = sycl::clamp(value, -127.0f, 127.0f);+ out_ptr[offset] = static_cast<int8_t>(value);+ }+ });+ }+static void sagev1_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, bool use_int8_pv) {@@ -1052,7 +1086,7 @@ class XpuWrapper {size_t k_size = num_heads_kv * seq_len_kv * head_dim * batch;size_t k_scale_size = num_heads_kv * seq_kv_blk * batch;size_t v_tmp_size = use_int8_pv ? k_size : 0;- size_t v_scale_size = use_int8_pv ? k_scale_size * sizeof(float) : 0;+ size_t v_scale_size = use_int8_pv ? k_scale_size * head_dim * sizeof(float) : 0;size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(sycl::half) : 0;size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size +v_scale_size + k_bias_size;@@ -1082,8 +1116,8 @@ class XpuWrapper {sage_dynamic_quant<sycl::half>(q, (sycl::half*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,seq_len_kv, seq_kv_blk, head_dim, scale_block_size, kbias);if (use_int8_pv) {- sage_dynamic_quant<sycl::half>(q, (sycl::half*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale, batch * num_heads_kv,- seq_len_kv, seq_kv_blk, head_dim, scale_block_size);+ sage_dynamic_quant_v<sycl::half>(q, (sycl::half*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale,+ batch * num_heads_kv, seq_len_kv, seq_kv_blk, head_dim, scale_block_size);ark::sdpa_impl_qks8_pvi8(q, q_out_ptr, k_out_ptr, v_out_ptr, O_ptr, mask, scale_block_size, qscale, kscale,vscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,softmax_scale, is_causal);diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hppindex a42dfc3b9..ac7d75eb7 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp@@ -195,7 +195,7 @@ struct TestSDPA {size_t o_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;size_t q_scale_count = size_t(batch) * num_heads_q * ((seq_len_q + scale_block_size - 1) / scale_block_size);size_t k_scale_count = size_t(batch) * num_heads_kv * ((seq_len_kv + scale_block_size - 1) / scale_block_size);- size_t v_scale_count = k_scale_count;+ size_t v_scale_count = k_scale_count * head_dim;float softmax_scale = 1.0f / std::sqrt(float(head_dim));auto host_q = to_fp16_vector(make_random_vector(q_count, -1.0f, 1.0f, 901u + uint32_t(seq_len_q)));@@ -224,9 +224,10 @@ struct TestSDPA {ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_k, dev_ki8, dev_kscale, batch * num_heads_kv, seq_len_kv,(seq_len_kv + scale_block_size - 1) / scale_block_size, head_dim,scale_block_size);- ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_v, dev_vi8, dev_vscale, batch * num_heads_kv, seq_len_kv,- (seq_len_kv + scale_block_size - 1) / scale_block_size, head_dim,- scale_block_size);+ ark::XpuWrapper::sage_dynamic_quant_v<sycl::half>(q, dev_v, dev_vi8, dev_vscale, batch * num_heads_kv,+ seq_len_kv,+ (seq_len_kv + scale_block_size - 1) / scale_block_size,+ head_dim, scale_block_size);double sage_ms = run_bench([&]() {diff --git a/auto_round_extension/ark/test/test_sage_dynquant.py b/auto_round_extension/ark/test/test_sage_dynquant.pyindex 37a90a0e9..72beb2fb4 100644--- a/auto_round_extension/ark/test/test_sage_dynquant.py+++ b/auto_round_extension/ark/test/test_sage_dynquant.py@@ -35,7 +35,7 @@ def bench(fn, n_warmup=50, n_runs=100):return (time.time() - st) / n_runs * 1e3-def run_benchmark(batch=2, seq=17776, h_q=30, h_kv=30, H=128, H_v=128, dt=torch.float16, is_causal=False, dev="xpu"):+def run_benchmark(batch=1, seq=4096, h_q=30, h_kv=30, H=128, H_v=128, dt=torch.float16, is_causal=False, dev="xpu"):# Align seq to largest block_size for divisibilityblock_sizes = [1, 32, 64, 128, 256]max_bs = max(block_sizes)diff --git a/auto_round_extension/ark/test/test_sdpa_parity.py b/auto_round_extension/ark/test/test_sdpa_parity.pynew file mode 100644index 000000000..77efc6f79--- /dev/null+++ b/auto_round_extension/ark/test/test_sdpa_parity.py@@ -0,0 +1,70 @@+# Copyright (C) 2026 Intel Corporation+# SPDX-License-Identifier: Apache-2.0++import math++import auto_round_kernel+import pytest+import torch++pytestmark = pytest.mark.skipif(+ not (hasattr(torch, "xpu") and torch.xpu.is_available()),+ reason="XPU not available",+)++[email protected]("seq_q,seq_kv", [(64, 64), (80, 64), (64, 80), (226, 226)])+def test_ark_sdpa_matches_torch_for_kv_remainders(seq_q, seq_kv):+ torch.manual_seed(2026)+ batch, heads, head_dim = 1, 2, 64+ scale = 1 / math.sqrt(head_dim)+ q = torch.randn(batch, heads, seq_q, head_dim, device="xpu", dtype=torch.float16)+ k = torch.randn(batch, heads, seq_kv, head_dim, device="xpu", dtype=torch.float16)+ v = torch.randn(batch, heads, seq_kv, head_dim, device="xpu", dtype=torch.float16)++ expected = torch.nn.functional.scaled_dot_product_attention(+ q,+ k,+ v,+ scale=scale,+ is_causal=False,+ )+ actual = auto_round_kernel.ARK().sdpa(q, k, v, scale=scale, is_causal=False)+ torch.xpu.synchronize()++ torch.testing.assert_close(actual, expected, atol=1e-2, rtol=1e-2)+++def test_ark_sagev1_matches_torch_for_kv_remainder_tile():+ """SAGEV1 uses the same final partial-K tile mask pattern after Q/K quantization.++ If the SAGEV1 remainder mask is built from the row-reduced fragment, this+ case produces a large error because the mask is broadcast along the K/column+ dimension. The column-reduced mask keeps the error within the expected+ SAGEV1 quantization tolerance.+ """+ torch.manual_seed(2027)+ batch, heads, seq_q, seq_kv, head_dim = 1, 2, 80, 80, 64+ scale = 1 / math.sqrt(head_dim)+ q = torch.randn(batch, heads, seq_q, head_dim, device="xpu", dtype=torch.float16)+ k = torch.randn(batch, heads, seq_kv, head_dim, device="xpu", dtype=torch.float16)+ v = torch.randn(batch, heads, seq_kv, head_dim, device="xpu", dtype=torch.float16)++ expected = torch.nn.functional.scaled_dot_product_attention(+ q,+ k,+ v,+ scale=scale,+ is_causal=False,+ )+ actual = auto_round_kernel.ARK().sagev1(+ q,+ k,+ v,+ scale=scale,+ is_causal=False,+ quant_block_size=64,+ )+ torch.xpu.synchronize()++ torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)