PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[ARK] feat: expose smooth_k (use_mean_bias) parameter from Python to C++ in SAGE attention kernels (#1995)
Luo Yud55ba0042b5d
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex 0f075d4b5..545dcb53f 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -976,6 +976,7 @@ def sagev1(quant_block_size: int = 64,tensor_layout: str = "HND",return_lse: bool = False,+ smooth_k: bool = True,) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:"""SAGE v1 attention prefill+decode.@@ -988,6 +989,8 @@ def sagev1(- quant_block_size: Quantization block size used by the kernel.- tensor_layout: Layout of Q/K/V/O tensors.- return_lse: If True, returns (O, LSE) where LSE[b, h, q] = log(sum_j exp(score_{b,h,q,j})).+ - smooth_k: Whether to smooth the key by subtracting the sequence mean+ before INT8 quantization (handled by the C++ kernel). Default: True.Returns:- O: same layout as the input tensors.@@ -1066,6 +1069,7 @@ def sagev1(float(scale) if scale is not None else 1.0 / (D**0.5),bool(is_causal),layout_code,+ bool(smooth_k),LSE.data_ptr() if LSE is not None else 0,)@@ -1086,6 +1090,7 @@ def sagev1_pvi8(quant_block_size: int = 64,tensor_layout: str = "HND",return_lse: bool = False,+ smooth_k: bool = True,) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:"""SAGE v1 attention with PV int8 path.@@ -1165,6 +1170,7 @@ def sagev1_pvi8(float(scale) if scale is not None else 1.0 / (D**0.5),bool(is_causal),layout_code,+ bool(smooth_k),LSE.data_ptr() if LSE is not None else 0,)@@ -1236,6 +1242,7 @@ def sageattn_varlen(kernel: str = "v1_pvhalf",quant_block_size: int = 64,return_lse: bool = False,+ smooth_k: bool = True,**kwargs,) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:"""SAGE attention with variable-length sequences (no padding).@@ -1258,6 +1265,8 @@ def sageattn_varlen(sm_scale: Softmax scale. Uses 1/sqrt(D) when None.kernel: "v1_pvhalf" (PV half) or "v1_pvi8" (PV int8).quant_block_size: Block size for INT8 quantization (default 64).+ smooth_k: Whether to smooth the key by subtracting the sequence mean+ before INT8 quantization (handled by the C++ kernel). Default: True.**kwargs: Forwarded (attn_mask, dropout_p etc. not yet supported).Returns:@@ -1362,6 +1371,7 @@ def sageattn_varlen(cu_seqlens_q_i32.data_ptr(),cu_seqlens_k_i32.data_ptr(),use_int8_pv,+ bool(smooth_k),LSE.data_ptr() if LSE is not None else 0,)diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppindex 841b0eb4a..3716d79ef 100755--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -200,7 +200,7 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptrint total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,int head_dim, float softmax_scale, bool is_causal,torch_ptr cu_seqlens_q, torch_ptr cu_seqlens_k,- int use_int8_pv, torch_ptr lse = 0) {+ int use_int8_pv, bool use_mean_bias, torch_ptr lse = 0) {if (mask && is_causal) {throw std::invalid_argument("ark::sagev1_varlen: mask and is_causal cannot both be set");}@@ -233,6 +233,7 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptrbatch, num_heads_q, num_heads_kv,total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,head_dim, softmax_scale, is_causal, bool(use_int8_pv),+ bool(use_mean_bias),(const int*)cu_seqlens_q, (const int*)cu_seqlens_k,(float*)lse);} else {@@ -245,6 +246,7 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptrbatch, num_heads_q, num_heads_kv,total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,head_dim, softmax_scale, is_causal, bool(use_int8_pv),+ bool(use_mean_bias),(const int*)cu_seqlens_q, (const int*)cu_seqlens_k,(float*)lse);}@@ -254,7 +256,7 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype,int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,float softmax_scale, bool is_causal, bool use_int8_pv,- int tensor_layout, torch_ptr lse = 0) {+ int tensor_layout, bool use_mean_bias, torch_ptr lse = 0) {if (mask && is_causal) {throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");}@@ -296,7 +298,7 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h,k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,- softmax_scale, is_causal, (BTLA_DTYPE)q_dtype, (float*)lse);+ softmax_scale, is_causal, (BTLA_DTYPE)q_dtype, (float*)lse, use_mean_bias);}#elsethrow std::runtime_error("ark::sagev1 is only supported on XPU");@@ -307,20 +309,20 @@ static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torcint scale_block_size,int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv,int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- int tensor_layout, torch_ptr lse = 0) {+ int tensor_layout, bool use_mean_bias, torch_ptr lse = 0) {sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch,num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, false,- tensor_layout, lse);+ tensor_layout, use_mean_bias, lse);}static void sagev1_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,int scale_block_size,int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv,int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- int tensor_layout, torch_ptr lse = 0) {+ int tensor_layout, bool use_mean_bias, torch_ptr lse = 0) {sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch,num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, true,- tensor_layout, lse);+ tensor_layout, use_mean_bias, lse);}static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,@@ -625,10 +627,24 @@ PYBIND11_MODULE(PY_NAME, m) {pybind11::arg("max_seqlen_q"), pybind11::arg("max_seqlen_kv"),pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),pybind11::arg("cu_seqlens_q"), pybind11::arg("cu_seqlens_k"),- pybind11::arg("use_int8_pv"), pybind11::arg("lse") = 0);- m.def("sagev1", &ark::sagev1);+ pybind11::arg("use_int8_pv"), pybind11::arg("use_mean_bias"), pybind11::arg("lse") = 0);+ m.def("sagev1", &ark::sagev1, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),+ pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),+ pybind11::arg("scale_block_size"),+ pybind11::arg("q_dtype"), pybind11::arg("k_dtype"), pybind11::arg("v_dtype"), pybind11::arg("o_dtype"),+ pybind11::arg("batch"), pybind11::arg("num_heads_q"), pybind11::arg("num_heads_kv"),+ pybind11::arg("seq_len_q"), pybind11::arg("seq_len_kv"),+ pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),+ pybind11::arg("tensor_layout"), pybind11::arg("use_mean_bias"), pybind11::arg("lse") = 0);// High-level SAGEV1 PVi8 API: input Q/K/V are FP16 and quantized internally.- m.def("sagev1_pvi8", &ark::sagev1_pvi8);+ m.def("sagev1_pvi8", &ark::sagev1_pvi8, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),+ pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),+ pybind11::arg("scale_block_size"),+ pybind11::arg("q_dtype"), pybind11::arg("k_dtype"), pybind11::arg("v_dtype"), pybind11::arg("o_dtype"),+ pybind11::arg("batch"), pybind11::arg("num_heads_q"), pybind11::arg("num_heads_kv"),+ pybind11::arg("seq_len_q"), pybind11::arg("seq_len_kv"),+ pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),+ pybind11::arg("tensor_layout"), pybind11::arg("use_mean_bias"), pybind11::arg("lse") = 0);m.def("sage", &ark::sage, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),pybind11::arg("scale_block_size"),diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hppindex b67b356af..09fd1ccac 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp@@ -1478,8 +1478,7 @@ class XpuWrapper {int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- bool use_int8_pv, float* lse = nullptr) {- bool use_mean_bias = env_params::Instance()->sage_use_mean_bias != 0;+ bool use_int8_pv, bool use_mean_bias, float* lse = nullptr) {bool q_packed = is_packed_hnd(q_stride_s, q_stride_d, q_stride_h, q_stride_b, num_heads_q, seq_len_q, head_dim);bool k_packed =is_packed_hnd(k_stride_s, k_stride_d, k_stride_h, k_stride_b, num_heads_kv, seq_len_kv, head_dim);@@ -1586,6 +1585,7 @@ class XpuWrapper {int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,int head_dim, float softmax_scale, bool is_causal, bool use_int8_pv,+ bool use_mean_bias,const int* cu_seqlens_q, const int* cu_seqlens_k, float* lse = nullptr) {size_t q_size = size_t(num_heads_q) * total_seqlen_q * head_dim;size_t q_seq_blk = (size_t(total_seqlen_q) + scale_block_size - 1) / scale_block_size;@@ -1595,7 +1595,9 @@ class XpuWrapper {size_t k_scale_size = size_t(num_heads_kv) * k_seq_blk;size_t v_tmp_size = use_int8_pv ? k_size : 0;size_t v_scale_size = use_int8_pv ? k_scale_size * head_dim * sizeof(float) : 0;- size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size + v_scale_size;+ size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * sizeof(T) : 0;+ size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size ++ v_scale_size + k_bias_size;auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);auto q_out_ptr = (int8_t*)ptr;auto k_out_ptr = (int8_t*)ptr + q_size;@@ -1607,6 +1609,17 @@ class XpuWrapper {auto vscale = use_int8_pv? (float*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) + k_scale_size * sizeof(float) + k_size): nullptr;+ auto kbias = use_mean_bias+ ? (T*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) ++ k_scale_size * sizeof(float) + v_tmp_size + v_scale_size)+ : nullptr;++ if (use_mean_bias) {+ // Per-head sequence mean bias for the flat 3-D [total_kv, Hkv, D] tensor.+ // batch=1 because the varlen tensor is flat (all sequences concatenated).+ compute_seq_mean_bias_strided<T>(q, (T*)K_ptr, kbias, 1, num_heads_kv, total_seqlen_kv,+ head_dim, k_stride_s, k_stride_d, k_stride_h, k_stride_b);+ }// Flat [total, H, D] quantize via strided path (batch=1 — the tensor is flat,// not a batched 4-D tensor, so batch must be 1 to avoid reading past the end@@ -1616,7 +1629,7 @@ class XpuWrapper {q_stride_s, q_stride_d, q_stride_h, q_stride_b);sage_dynamic_quant_strided<T>(q, (T*)K_ptr, k_out_ptr, kscale, 1, num_heads_kv,total_seqlen_kv, k_seq_blk, head_dim, scale_block_size,- k_stride_s, k_stride_d, k_stride_h, k_stride_b);+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, kbias);// Quantized output stride: packed HND using total_seqlen.int qo_s = head_dim, qo_d = 1, qo_h = total_seqlen_q * head_dim, qo_b = num_heads_q * total_seqlen_q * head_dim;@@ -1658,19 +1671,20 @@ class XpuWrapper {int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr) {+ BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr,+ bool use_mean_bias = true) {if (dtype == BTLA_DTYPE::BF16) {sagev1_impl<sycl::ext::oneapi::bfloat16>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,- head_dim, softmax_scale, is_causal, false, lse);+ head_dim, softmax_scale, is_causal, false, use_mean_bias, lse);} else {sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,- is_causal, false, lse);+ is_causal, false, use_mean_bias, lse);}}@@ -1680,19 +1694,20 @@ class XpuWrapper {int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr) {+ BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr,+ bool use_mean_bias = true) {if (dtype == BTLA_DTYPE::BF16) {sagev1_impl<sycl::ext::oneapi::bfloat16>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,- head_dim, softmax_scale, is_causal, true, lse);+ head_dim, softmax_scale, is_causal, true, use_mean_bias, lse);} else {sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,- is_causal, true, lse);+ is_causal, true, use_mean_bias, lse);}}};