Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[ARK] feat: expose smooth_k (use_mean_bias) parameter from Python to C++ in SAGE attention kernels (#1995)

Luo Yud55ba0042b5d
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.py
index 0f075d4b5..545dcb53f 100644
--- a/auto_round_extension/ark/auto_round_kernel/__init__.py
+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py
@@ -976,6 +976,7 @@ def sagev1(
quant_block_size: int = 64,
tensor_layout: str = "HND",
return_lse: bool = False,
+ smooth_k: bool = True,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""SAGE v1 attention prefill+decode.
@@ -988,6 +989,8 @@ def sagev1(
- quant_block_size: Quantization block size used by the kernel.
- tensor_layout: Layout of Q/K/V/O tensors.
- return_lse: If True, returns (O, LSE) where LSE[b, h, q] = log(sum_j exp(score_{b,h,q,j})).
+ - smooth_k: Whether to smooth the key by subtracting the sequence mean
+ before INT8 quantization (handled by the C++ kernel). Default: True.
Returns:
- O: same layout as the input tensors.
@@ -1066,6 +1069,7 @@ def sagev1(
float(scale) if scale is not None else 1.0 / (D**0.5),
bool(is_causal),
layout_code,
+ bool(smooth_k),
LSE.data_ptr() if LSE is not None else 0,
)
@@ -1086,6 +1090,7 @@ def sagev1_pvi8(
quant_block_size: int = 64,
tensor_layout: str = "HND",
return_lse: bool = False,
+ smooth_k: bool = True,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""SAGE v1 attention with PV int8 path.
@@ -1165,6 +1170,7 @@ def sagev1_pvi8(
float(scale) if scale is not None else 1.0 / (D**0.5),
bool(is_causal),
layout_code,
+ bool(smooth_k),
LSE.data_ptr() if LSE is not None else 0,
)
@@ -1236,6 +1242,7 @@ def sageattn_varlen(
kernel: str = "v1_pvhalf",
quant_block_size: int = 64,
return_lse: bool = False,
+ smooth_k: bool = True,
**kwargs,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""SAGE attention with variable-length sequences (no padding).
@@ -1258,6 +1265,8 @@ def sageattn_varlen(
sm_scale: Softmax scale. Uses 1/sqrt(D) when None.
kernel: "v1_pvhalf" (PV half) or "v1_pvi8" (PV int8).
quant_block_size: Block size for INT8 quantization (default 64).
+ smooth_k: Whether to smooth the key by subtracting the sequence mean
+ before INT8 quantization (handled by the C++ kernel). Default: True.
**kwargs: Forwarded (attn_mask, dropout_p etc. not yet supported).
Returns:
@@ -1362,6 +1371,7 @@ def sageattn_varlen(
cu_seqlens_q_i32.data_ptr(),
cu_seqlens_k_i32.data_ptr(),
use_int8_pv,
+ bool(smooth_k),
LSE.data_ptr() if LSE is not None else 0,
)
diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cpp
index 841b0eb4a..3716d79ef 100755
--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp
@@ -200,7 +200,7 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr
int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,
int head_dim, float softmax_scale, bool is_causal,
torch_ptr cu_seqlens_q, torch_ptr cu_seqlens_k,
- int use_int8_pv, torch_ptr lse = 0) {
+ int use_int8_pv, bool use_mean_bias, torch_ptr lse = 0) {
if (mask && is_causal) {
throw std::invalid_argument("ark::sagev1_varlen: mask and is_causal cannot both be set");
}
@@ -233,6 +233,7 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr
batch, num_heads_q, num_heads_kv,
total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,
head_dim, softmax_scale, is_causal, bool(use_int8_pv),
+ bool(use_mean_bias),
(const int*)cu_seqlens_q, (const int*)cu_seqlens_k,
(float*)lse);
} else {
@@ -245,6 +246,7 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr
batch, num_heads_q, num_heads_kv,
total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,
head_dim, softmax_scale, is_causal, bool(use_int8_pv),
+ bool(use_mean_bias),
(const int*)cu_seqlens_q, (const int*)cu_seqlens_k,
(float*)lse);
}
@@ -254,7 +256,7 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,
int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype,
int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
float softmax_scale, bool is_causal, bool use_int8_pv,
- int tensor_layout, torch_ptr lse = 0) {
+ int tensor_layout, bool use_mean_bias, torch_ptr lse = 0) {
if (mask && is_causal) {
throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");
}
@@ -296,7 +298,7 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,
q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h,
k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,
o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
- softmax_scale, is_causal, (BTLA_DTYPE)q_dtype, (float*)lse);
+ softmax_scale, is_causal, (BTLA_DTYPE)q_dtype, (float*)lse, use_mean_bias);
}
#else
throw std::runtime_error("ark::sagev1 is only supported on XPU");
@@ -307,20 +309,20 @@ static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torc
int scale_block_size,
int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv,
int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- int tensor_layout, torch_ptr lse = 0) {
+ int tensor_layout, bool use_mean_bias, torch_ptr lse = 0) {
sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch,
num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, false,
- tensor_layout, lse);
+ tensor_layout, use_mean_bias, lse);
}
static void sagev1_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
int scale_block_size,
int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv,
int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- int tensor_layout, torch_ptr lse = 0) {
+ int tensor_layout, bool use_mean_bias, torch_ptr lse = 0) {
sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch,
num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, true,
- tensor_layout, lse);
+ tensor_layout, use_mean_bias, lse);
}
static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
@@ -625,10 +627,24 @@ PYBIND11_MODULE(PY_NAME, m) {
pybind11::arg("max_seqlen_q"), pybind11::arg("max_seqlen_kv"),
pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),
pybind11::arg("cu_seqlens_q"), pybind11::arg("cu_seqlens_k"),
- pybind11::arg("use_int8_pv"), pybind11::arg("lse") = 0);
- m.def("sagev1", &ark::sagev1);
+ pybind11::arg("use_int8_pv"), pybind11::arg("use_mean_bias"), pybind11::arg("lse") = 0);
+ m.def("sagev1", &ark::sagev1, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),
+ pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),
+ pybind11::arg("scale_block_size"),
+ pybind11::arg("q_dtype"), pybind11::arg("k_dtype"), pybind11::arg("v_dtype"), pybind11::arg("o_dtype"),
+ pybind11::arg("batch"), pybind11::arg("num_heads_q"), pybind11::arg("num_heads_kv"),
+ pybind11::arg("seq_len_q"), pybind11::arg("seq_len_kv"),
+ pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),
+ pybind11::arg("tensor_layout"), pybind11::arg("use_mean_bias"), pybind11::arg("lse") = 0);
// High-level SAGEV1 PVi8 API: input Q/K/V are FP16 and quantized internally.
- m.def("sagev1_pvi8", &ark::sagev1_pvi8);
+ m.def("sagev1_pvi8", &ark::sagev1_pvi8, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),
+ pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),
+ pybind11::arg("scale_block_size"),
+ pybind11::arg("q_dtype"), pybind11::arg("k_dtype"), pybind11::arg("v_dtype"), pybind11::arg("o_dtype"),
+ pybind11::arg("batch"), pybind11::arg("num_heads_q"), pybind11::arg("num_heads_kv"),
+ pybind11::arg("seq_len_q"), pybind11::arg("seq_len_kv"),
+ pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),
+ pybind11::arg("tensor_layout"), pybind11::arg("use_mean_bias"), pybind11::arg("lse") = 0);
m.def("sage", &ark::sage, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),
pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),
pybind11::arg("scale_block_size"),
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
index b67b356af..09fd1ccac 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
@@ -1478,8 +1478,7 @@ class XpuWrapper {
int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- bool use_int8_pv, float* lse = nullptr) {
- bool use_mean_bias = env_params::Instance()->sage_use_mean_bias != 0;
+ bool use_int8_pv, bool use_mean_bias, float* lse = nullptr) {
bool q_packed = is_packed_hnd(q_stride_s, q_stride_d, q_stride_h, q_stride_b, num_heads_q, seq_len_q, head_dim);
bool k_packed =
is_packed_hnd(k_stride_s, k_stride_d, k_stride_h, k_stride_b, num_heads_kv, seq_len_kv, head_dim);
@@ -1586,6 +1585,7 @@ class XpuWrapper {
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,
int head_dim, float softmax_scale, bool is_causal, bool use_int8_pv,
+ bool use_mean_bias,
const int* cu_seqlens_q, const int* cu_seqlens_k, float* lse = nullptr) {
size_t q_size = size_t(num_heads_q) * total_seqlen_q * head_dim;
size_t q_seq_blk = (size_t(total_seqlen_q) + scale_block_size - 1) / scale_block_size;
@@ -1595,7 +1595,9 @@ class XpuWrapper {
size_t k_scale_size = size_t(num_heads_kv) * k_seq_blk;
size_t v_tmp_size = use_int8_pv ? k_size : 0;
size_t v_scale_size = use_int8_pv ? k_scale_size * head_dim * sizeof(float) : 0;
- size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size + v_scale_size;
+ size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * sizeof(T) : 0;
+ size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size +
+ v_scale_size + k_bias_size;
auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);
auto q_out_ptr = (int8_t*)ptr;
auto k_out_ptr = (int8_t*)ptr + q_size;
@@ -1607,6 +1609,17 @@ class XpuWrapper {
auto vscale = use_int8_pv
? (float*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) + k_scale_size * sizeof(float) + k_size)
: nullptr;
+ auto kbias = use_mean_bias
+ ? (T*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) +
+ k_scale_size * sizeof(float) + v_tmp_size + v_scale_size)
+ : nullptr;
+
+ if (use_mean_bias) {
+ // Per-head sequence mean bias for the flat 3-D [total_kv, Hkv, D] tensor.
+ // batch=1 because the varlen tensor is flat (all sequences concatenated).
+ compute_seq_mean_bias_strided<T>(q, (T*)K_ptr, kbias, 1, num_heads_kv, total_seqlen_kv,
+ head_dim, k_stride_s, k_stride_d, k_stride_h, k_stride_b);
+ }
// Flat [total, H, D] quantize via strided path (batch=1 — the tensor is flat,
// not a batched 4-D tensor, so batch must be 1 to avoid reading past the end
@@ -1616,7 +1629,7 @@ class XpuWrapper {
q_stride_s, q_stride_d, q_stride_h, q_stride_b);
sage_dynamic_quant_strided<T>(q, (T*)K_ptr, k_out_ptr, kscale, 1, num_heads_kv,
total_seqlen_kv, k_seq_blk, head_dim, scale_block_size,
- k_stride_s, k_stride_d, k_stride_h, k_stride_b);
+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, kbias);
// Quantized output stride: packed HND using total_seqlen.
int qo_s = head_dim, qo_d = 1, qo_h = total_seqlen_q * head_dim, qo_b = num_heads_q * total_seqlen_q * head_dim;
@@ -1658,19 +1671,20 @@ class XpuWrapper {
int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr) {
+ BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr,
+ bool use_mean_bias = true) {
if (dtype == BTLA_DTYPE::BF16) {
sagev1_impl<sycl::ext::oneapi::bfloat16>(
q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,
o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,
- head_dim, softmax_scale, is_causal, false, lse);
+ head_dim, softmax_scale, is_causal, false, use_mean_bias, lse);
} else {
sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,
q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,
v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,
batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
- is_causal, false, lse);
+ is_causal, false, use_mean_bias, lse);
}
}
@@ -1680,19 +1694,20 @@ class XpuWrapper {
int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr) {
+ BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr,
+ bool use_mean_bias = true) {
if (dtype == BTLA_DTYPE::BF16) {
sagev1_impl<sycl::ext::oneapi::bfloat16>(
q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,
o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,
- head_dim, softmax_scale, is_causal, true, lse);
+ head_dim, softmax_scale, is_causal, true, use_mean_bias, lse);
} else {
sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,
q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,
v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,
batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
- is_causal, true, lse);
+ is_causal, true, use_mean_bias, lse);
}
}
};