Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: add return_lse support for SDPA and SAGEV1 kernel — enable sequence-parallel (Ring Attention) merge (#1991)

Luo Yub481cebbc4af
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.py
index 8eb831f40..0f075d4b5 100644
--- a/auto_round_extension/ark/auto_round_kernel/__init__.py
+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py
@@ -12,10 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
from typing import Optional
import torch
import sys
+# Intel GPU compiler (IGC) environment variable: ensure implicit local IDs are
+# not removed by the compiler even when unused. This guarantees consistent
+# behavior across driver upgrades and prevents performance regressions.
+# Reference: https://github.com/intel/intel-graphics-compiler/issues/412
+os.environ.setdefault("IGC_RemoveUnusedIdImplicitLocalIDs", "0")
+
# Tensor layout constants passed to native C++ backends.
# 0 = HND ([B, H, S, D]), 1 = NHD ([B, S, H, D]).
LAYOUT_HND = 0
@@ -509,7 +516,8 @@ def sdpa(
is_causal: bool = False,
scale: float | None = None,
tensor_layout: str = "HND",
-) -> torch.Tensor:
+ return_lse: bool = False,
+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Scaled dot-product attention (SDPA) prefill+decode.
Supported tensor layouts:
@@ -519,9 +527,11 @@ def sdpa(
Args:
- scale: Softmax scale. Uses 1 / sqrt(D) when None.
- tensor_layout: Layout of Q/K/V/O tensors.
+ - return_lse: If True, returns (O, LSE) where LSE[b, h, q] = log(sum_j exp(score_{b,h,q,j})).
Returns:
- O: same layout as the input tensors.
+ - (O, LSE): if return_lse is True.
"""
if query.device.type != "xpu":
raise NotImplementedError("sdpa is only supported on XPU")
@@ -574,6 +584,9 @@ def sdpa(
device=query.device,
tensor_layout=tensor_layout,
)
+
+ LSE = torch.empty(B, Hq, Sq, dtype=torch.float32, device=query.device) if return_lse else None
+
layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHD
lib.sdpa(
stream,
@@ -594,7 +607,11 @@ def sdpa(
float(scale) if scale is not None else 1.0 / (D**0.5),
bool(is_causal),
layout_code,
+ LSE.data_ptr() if LSE is not None else 0,
)
+
+ if return_lse:
+ return O, LSE
return O
@@ -610,7 +627,8 @@ def sdpa_varlen(
dropout_p: float = 0.0,
is_causal: bool = False,
scale: float | None = None,
-) -> torch.Tensor:
+ return_lse: bool = False,
+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Scaled dot-product attention (SDPA) prefill+decode with variable-length sequences.
Q, K, V use a flat 3-D layout where tokens from all sequences in the batch
@@ -701,6 +719,18 @@ def sdpa_varlen(
O = torch.empty(total_q, Hq, D, dtype=value.dtype, device=query.device)
+ if return_lse:
+ max_q = int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())
+ if max_seqlen_q < max_q:
+ raise ValueError(f"max_seqlen_q ({max_seqlen_q}) < max sequence length in cu_seqlens_q ({max_q})")
+ LSE = torch.full(
+ (batch, Hq, max_seqlen_q),
+ float("-inf"),
+ dtype=torch.float32,
+ device=query.device,
+ )
+ else:
+ LSE = None
lib.sdpa_varlen(
stream,
query.data_ptr(),
@@ -724,7 +754,11 @@ def sdpa_varlen(
cu_seqlens_q_i32.data_ptr(),
cu_seqlens_k_i32.data_ptr(),
LAYOUT_HND, # unused by the native varlen path (always flat 3-D)
+ LSE.data_ptr() if LSE is not None else 0,
)
+
+ if return_lse:
+ return O, LSE
return O
@@ -941,7 +975,8 @@ def sagev1(
enable_gqa: bool = False,
quant_block_size: int = 64,
tensor_layout: str = "HND",
-) -> torch.Tensor:
+ return_lse: bool = False,
+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""SAGE v1 attention prefill+decode.
Supported tensor layouts:
@@ -952,9 +987,11 @@ def sagev1(
- scale: Attention scale. Uses 1 / sqrt(D) when None.
- quant_block_size: Quantization block size used by the kernel.
- tensor_layout: Layout of Q/K/V/O tensors.
+ - return_lse: If True, returns (O, LSE) where LSE[b, h, q] = log(sum_j exp(score_{b,h,q,j})).
Returns:
- O: same layout as the input tensors.
+ - (O, LSE): if return_lse is True.
"""
if quant_block_size <= 0:
return sdpa(
@@ -966,6 +1003,7 @@ def sagev1(
is_causal=is_causal,
scale=scale,
tensor_layout=tensor_layout,
+ return_lse=return_lse,
)
if query.device.type != "xpu":
raise NotImplementedError("sdpa is only supported on XPU")
@@ -1003,6 +1041,9 @@ def sagev1(
device=query.device,
tensor_layout=tensor_layout,
)
+
+ LSE = torch.empty(B, Hq, Sq, dtype=torch.float32, device=query.device) if return_lse else None
+
layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHD
lib.sagev1(
stream,
@@ -1025,7 +1066,11 @@ def sagev1(
float(scale) if scale is not None else 1.0 / (D**0.5),
bool(is_causal),
layout_code,
+ LSE.data_ptr() if LSE is not None else 0,
)
+
+ if return_lse:
+ return O, LSE
return O
@@ -1040,7 +1085,8 @@ def sagev1_pvi8(
enable_gqa: bool = False,
quant_block_size: int = 64,
tensor_layout: str = "HND",
-) -> torch.Tensor:
+ return_lse: bool = False,
+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""SAGE v1 attention with PV int8 path.
Expects FP16 Q/K/V input and quantizes Q/K/V internally before calling
@@ -1056,6 +1102,7 @@ def sagev1_pvi8(
is_causal=is_causal,
scale=scale,
tensor_layout=tensor_layout,
+ return_lse=return_lse,
)
if query.device.type != "xpu":
raise NotImplementedError("sagev1_pvi8 is only supported on XPU")
@@ -1093,6 +1140,9 @@ def sagev1_pvi8(
device=query.device,
tensor_layout=tensor_layout,
)
+
+ LSE = torch.empty(B, Hq, Sq, dtype=torch.float32, device=query.device) if return_lse else None
+
layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHD
lib.sagev1_pvi8(
stream,
@@ -1115,7 +1165,11 @@ def sagev1_pvi8(
float(scale) if scale is not None else 1.0 / (D**0.5),
bool(is_causal),
layout_code,
+ LSE.data_ptr() if LSE is not None else 0,
)
+
+ if return_lse:
+ return O, LSE
return O
@@ -1129,7 +1183,7 @@ def sageattn(
return_lse: bool = False,
kernel: str = "v1_pvhalf",
**kwargs,
-) -> torch.Tensor:
+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""SAGE attention dispatcher.
Signature mirrors ``sageattention.sageattn``.
@@ -1139,7 +1193,7 @@ def sageattn(
- tensor_layout: "HND" or "NHD".
- is_causal: Whether to apply causal mask.
- sm_scale: Softmax scale. Uses ``1 / sqrt(head_dim)`` when None.
- - return_lse: Not supported; must be False.
+ - return_lse: If True, returns (O, LSE) tuple.
- kernel: Which SAGE variant to dispatch to.
- "v1_pvhalf" (default): PV in half precision (calls ``sagev1``).
- "v1_pvi8": PV in INT8 precision (calls ``sagev1_pvi8``).
@@ -1148,10 +1202,8 @@ def sageattn(
Returns:
- O: same layout as the input tensors.
+ - (O, LSE): if return_lse is True.
"""
- if return_lse:
- raise NotImplementedError("return_lse is not supported in ARK sageattn")
-
if kernel == "v1_pvhalf":
impl = sagev1
elif kernel == "v1_pvi8":
@@ -1166,6 +1218,7 @@ def sageattn(
is_causal=is_causal,
scale=sm_scale,
tensor_layout=tensor_layout,
+ return_lse=return_lse,
**kwargs,
)
@@ -1182,8 +1235,9 @@ def sageattn_varlen(
sm_scale: float | None = None,
kernel: str = "v1_pvhalf",
quant_block_size: int = 64,
+ return_lse: bool = False,
**kwargs,
-) -> torch.Tensor:
+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""SAGE attention with variable-length sequences (no padding).
Q/K/V are flat 3-D tensors: [total_tokens, num_heads, head_dim].
@@ -1271,6 +1325,18 @@ def sageattn_varlen(
O = torch.empty(total_q, Hq, D, dtype=v.dtype, device=q.device)
+ if return_lse:
+ max_q = int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())
+ if max_seqlen_q < max_q:
+ raise ValueError(f"max_seqlen_q ({max_seqlen_q}) < max sequence length in cu_seqlens_q ({max_q})")
+ LSE = torch.full(
+ (batch, Hq, max_seqlen_q),
+ float("-inf"),
+ dtype=torch.float32,
+ device=q.device,
+ )
+ else:
+ LSE = None
lib.sagev1_varlen(
stream,
q.data_ptr(),
@@ -1296,7 +1362,11 @@ def sageattn_varlen(
cu_seqlens_q_i32.data_ptr(),
cu_seqlens_k_i32.data_ptr(),
use_int8_pv,
+ LSE.data_ptr() if LSE is not None else 0,
)
+
+ if return_lse:
+ return O, LSE
return O
diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cpp
index 708e3100b..841b0eb4a 100755
--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp
@@ -109,7 +109,7 @@ static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_
int q_dtype, int k_dtype, int o_dtype,
int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
float softmax_scale, bool is_causal,
- int tensor_layout) {
+ int tensor_layout, torch_ptr lse = 0) {
if (k_dtype != q_dtype || o_dtype != q_dtype) {
throw std::invalid_argument("ark::sdpa: k_dtype and o_dtype must match q_dtype");
}
@@ -141,7 +141,8 @@ static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_
ark::sdpa_impl((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (BTLA_DTYPE)(q_dtype),
q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b,
v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,
- batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal,
+ (float*)lse);
}
static void sdpa_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
@@ -151,7 +152,7 @@ static void sdpa_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,
int max_seqlen_q, int max_seqlen_kv,
int head_dim, float softmax_scale, bool is_causal,
torch_ptr cu_seqlens_q, torch_ptr cu_seqlens_k,
- int tensor_layout) {
+ int tensor_layout, torch_ptr lse = 0) {
if (k_dtype != q_dtype || o_dtype != q_dtype) {
throw std::invalid_argument("ark::sdpa_varlen: k_dtype and o_dtype must match q_dtype");
}
@@ -188,7 +189,8 @@ static void sdpa_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,
total_seqlen_q, total_seqlen_kv,
max_seqlen_q, max_seqlen_kv,
head_dim, softmax_scale, is_causal,
- (const int*)cu_seqlens_q, (const int*)cu_seqlens_k);
+ (const int*)cu_seqlens_q, (const int*)cu_seqlens_k,
+ (float*)lse);
}
// Varlen SageV1 bridge: quantizes Q/K to INT8, then dispatches with varlen=true.
@@ -198,7 +200,7 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr
int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,
int head_dim, float softmax_scale, bool is_causal,
torch_ptr cu_seqlens_q, torch_ptr cu_seqlens_k,
- int use_int8_pv) {
+ int use_int8_pv, torch_ptr lse = 0) {
if (mask && is_causal) {
throw std::invalid_argument("ark::sagev1_varlen: mask and is_causal cannot both be set");
}
@@ -231,7 +233,8 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr
batch, num_heads_q, num_heads_kv,
total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,
head_dim, softmax_scale, is_causal, bool(use_int8_pv),
- (const int*)cu_seqlens_q, (const int*)cu_seqlens_k);
+ (const int*)cu_seqlens_q, (const int*)cu_seqlens_k,
+ (float*)lse);
} else {
XpuWrapper::sagev1_varlen_impl<sycl::half>(
(sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,
@@ -242,7 +245,8 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr
batch, num_heads_q, num_heads_kv,
total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,
head_dim, softmax_scale, is_causal, bool(use_int8_pv),
- (const int*)cu_seqlens_q, (const int*)cu_seqlens_k);
+ (const int*)cu_seqlens_q, (const int*)cu_seqlens_k,
+ (float*)lse);
}
}
@@ -250,7 +254,7 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,
int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype,
int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
float softmax_scale, bool is_causal, bool use_int8_pv,
- int tensor_layout) {
+ int tensor_layout, torch_ptr lse = 0) {
if (mask && is_causal) {
throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");
}
@@ -285,13 +289,14 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,
scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s,
k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,
o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,
- seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, (BTLA_DTYPE)q_dtype);
+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, (BTLA_DTYPE)q_dtype,
+ (float*)lse);
} else {
XpuWrapper::sagev1((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, scale_block_size,
q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h,
k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,
o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
- softmax_scale, is_causal, (BTLA_DTYPE)q_dtype);
+ softmax_scale, is_causal, (BTLA_DTYPE)q_dtype, (float*)lse);
}
#else
throw std::runtime_error("ark::sagev1 is only supported on XPU");
@@ -302,27 +307,27 @@ static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torc
int scale_block_size,
int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv,
int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- int tensor_layout) {
+ int tensor_layout, torch_ptr lse = 0) {
sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch,
num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, false,
- tensor_layout);
+ tensor_layout, lse);
}
static void sagev1_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
int scale_block_size,
int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv,
int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- int tensor_layout) {
+ int tensor_layout, torch_ptr lse = 0) {
sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch,
num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, true,
- tensor_layout);
+ tensor_layout, lse);
}
static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
int scale_block_size, torch_ptr qscale, torch_ptr kscale,
int q_dtype, int k_dtype, int o_dtype, int batch, int num_heads_q,
int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,
- bool is_causal, int tensor_layout) {
+ bool is_causal, int tensor_layout, torch_ptr lse = 0) {
if (mask && is_causal) {
throw std::invalid_argument("ark::sage: mask and is_causal cannot both be set");
}
@@ -350,14 +355,14 @@ static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_
q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,
v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch,
num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal,
- (BTLA_DTYPE)o_dtype);
+ (BTLA_DTYPE)o_dtype, (float*)lse);
}
static void sage_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
int scale_block_size, torch_ptr qscale, torch_ptr kscale, torch_ptr vscale,
int q_dtype, int k_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- int tensor_layout) {
+ int tensor_layout, torch_ptr lse = 0) {
if (mask && is_causal) {
throw std::invalid_argument("ark::sage_pvi8: mask and is_causal cannot both be set");
}
@@ -385,7 +390,7 @@ static void sage_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, t
q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,
v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,
batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
- is_causal, (BTLA_DTYPE)o_dtype);
+ is_causal, (BTLA_DTYPE)o_dtype, (float*)lse);
}
static void moe_gemm_wrapper(torch_ptr stream, torch_ptr activations, torch_ptr weights, torch_ptr scales,
@@ -609,7 +614,7 @@ PYBIND11_MODULE(PY_NAME, m) {
pybind11::arg("max_seqlen_q"), pybind11::arg("max_seqlen_kv"),
pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),
pybind11::arg("cu_seqlens_q"), pybind11::arg("cu_seqlens_k"),
- pybind11::arg("tensor_layout"));
+ pybind11::arg("tensor_layout"), pybind11::arg("lse") = 0);
// Varlen SAGEV1: flat 3-D Q/K/V + cu_seqlens (use_int8_pv=0) or pvi8 (use_int8_pv=1).
m.def("sagev1_varlen", &ark::sagev1_varlen, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),
pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),
@@ -620,13 +625,29 @@ PYBIND11_MODULE(PY_NAME, m) {
pybind11::arg("max_seqlen_q"), pybind11::arg("max_seqlen_kv"),
pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),
pybind11::arg("cu_seqlens_q"), pybind11::arg("cu_seqlens_k"),
- pybind11::arg("use_int8_pv"));
+ pybind11::arg("use_int8_pv"), pybind11::arg("lse") = 0);
m.def("sagev1", &ark::sagev1);
// High-level SAGEV1 PVi8 API: input Q/K/V are FP16 and quantized internally.
m.def("sagev1_pvi8", &ark::sagev1_pvi8);
- m.def("sage", &ark::sage);
+ m.def("sage", &ark::sage, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),
+ pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),
+ pybind11::arg("scale_block_size"),
+ pybind11::arg("qscale"), pybind11::arg("kscale"),
+ pybind11::arg("q_dtype"), pybind11::arg("k_dtype"), pybind11::arg("o_dtype"),
+ pybind11::arg("batch"), pybind11::arg("num_heads_q"), pybind11::arg("num_heads_kv"),
+ pybind11::arg("seq_len_q"), pybind11::arg("seq_len_kv"),
+ pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),
+ pybind11::arg("tensor_layout"), pybind11::arg("lse") = 0);
// Low-level SAGE PVi8 API: input Q/K/V are pre-quantized int8 with qscale/kscale/vscale.
- m.def("sage_pvi8", &ark::sage_pvi8);
+ m.def("sage_pvi8", &ark::sage_pvi8, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),
+ pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),
+ pybind11::arg("scale_block_size"),
+ pybind11::arg("qscale"), pybind11::arg("kscale"), pybind11::arg("vscale"),
+ pybind11::arg("q_dtype"), pybind11::arg("k_dtype"), pybind11::arg("o_dtype"),
+ pybind11::arg("batch"), pybind11::arg("num_heads_q"), pybind11::arg("num_heads_kv"),
+ pybind11::arg("seq_len_q"), pybind11::arg("seq_len_kv"),
+ pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),
+ pybind11::arg("tensor_layout"), pybind11::arg("lse") = 0);
m.def("sage_dynamic_quant", &ark::sage_dynamic_quant);
m.def("sage_compute_seq_mean_bias_layout", &ark::sage_compute_seq_mean_bias_layout);
m.def("sage_dynamic_quant_layout", &ark::sage_dynamic_quant_layout);
diff --git a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
index 3b03c4647..10afcf646 100644
--- a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
@@ -232,7 +232,7 @@ void sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O
int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,
int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,
int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
- float softmax_scale, bool is_causal) {
+ float softmax_scale, bool is_causal, float* lse = nullptr) {
detail::Options options =
make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,
@@ -242,6 +242,7 @@ void sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O
options.qscale = qscale;
options.kscale = kscale;
options.vscale = vscale;
+ options.lse = lse;
compat::set_default_queue(*q);
KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, pv_dtype, head_dim, use_int8_pv);
@@ -258,12 +259,14 @@ void flash_attn_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, v
int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
- int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
+ int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
+ float* lse = nullptr) {
detail::Options options =
make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,
v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ options.lse = lse;
compat::set_default_queue(*q);
KernelLauncher launcher = select_prefill_launcher(q_dtype, head_dim);
@@ -280,12 +283,14 @@ void flash_attn_decode(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, vo
int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
+ float* lse = nullptr) {
detail::Options options =
make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,
v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
num_heads_kv, 1, seq_len_kv, head_dim, softmax_scale, is_causal);
+ options.lse = lse;
compat::set_default_queue(*q);
KernelLauncher launcher = select_decode_launcher(q_dtype, head_dim);
@@ -302,7 +307,7 @@ void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_pt
int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b,
int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch, int num_heads_q,
int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,
- bool is_causal) {
+ bool is_causal, float* lse = nullptr) {
// if (q_dtype != BTLA_DTYPE::F16 && q_dtype != BTLA_DTYPE::BF16) {
// throw std::invalid_argument("sdpa_impl: only FP16 and BF16 are supported");
// }
@@ -317,14 +322,14 @@ void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_pt
flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,
v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,
- seq_len_kv, head_dim, softmax_scale, is_causal);
+ seq_len_kv, head_dim, softmax_scale, is_causal, lse);
return;
}
flash_attn_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,
v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,
- seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, lse);
}
void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
@@ -332,7 +337,8 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr
int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,
int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h,
int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE pv_dtype) {
+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE pv_dtype,
+ float* lse = nullptr) {
if (mask && is_causal) {
throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");
}
@@ -347,7 +353,7 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr
flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, pv_dtype, q_stride_s, q_stride_d, q_stride_h,
q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,
v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
- num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal);
+ num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal, lse);
return;
}
@@ -355,7 +361,7 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr
BTLA_DTYPE::S8, pv_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,
k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s,
o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q,
- seq_len_kv, head_dim, softmax_scale, is_causal);
+ seq_len_kv, head_dim, softmax_scale, is_causal, lse);
}
void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
@@ -364,7 +370,8 @@ void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,
int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,
int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,
int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
- int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype) {
+ int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype,
+ float* lse = nullptr) {
if (mask && is_causal) {
throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");
}
@@ -382,7 +389,7 @@ void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,
flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, o_dtype, q_stride_s, q_stride_d, q_stride_h,
q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,
v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
- num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal);
+ num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal, lse);
return;
}
@@ -390,7 +397,7 @@ void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,
BTLA_DTYPE::S8, o_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,
k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,
o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
- softmax_scale, is_causal);
+ softmax_scale, is_causal, lse);
}
void sage_prefill_varlen(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
@@ -403,7 +410,8 @@ void sage_prefill_varlen(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,
int batch, int num_heads_q, int num_heads_kv,
int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,
int head_dim, float softmax_scale, bool is_causal,
- const int* cu_seqlens_q, const int* cu_seqlens_k) {
+ const int* cu_seqlens_q, const int* cu_seqlens_k,
+ float* lse = nullptr) {
if (mask && is_causal) {
throw std::invalid_argument("sage_prefill_varlen: mask and is_causal cannot both be set");
}
@@ -437,6 +445,7 @@ void sage_prefill_varlen(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,
options.qscale = qscale;
options.kscale = kscale;
options.vscale = vscale;
+ options.lse = lse;
compat::set_default_queue(*q);
@@ -463,7 +472,8 @@ void sdpa_varlen_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, voi
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,
int head_dim, float softmax_scale, bool is_causal,
- const int* cu_seqlens_q, const int* cu_seqlens_k) {
+ const int* cu_seqlens_q, const int* cu_seqlens_k,
+ float* lse = nullptr) {
if (mask && is_causal) {
throw std::invalid_argument("sdpa_varlen_impl: mask and is_causal cannot both be set");
}
@@ -496,6 +506,7 @@ void sdpa_varlen_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, voi
options.seq_len_kv_cache = 0;
options.total_seqlen_kv_cache = 0;
options.max_seqlen_kv_cache = 0;
+ options.lse = lse;
compat::set_default_queue(*q);
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp
index a5a64638d..4e773f49f 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp
@@ -131,6 +131,7 @@ class XeSageFwdKernel {
StrideK dK_cache{};
const ElementV* V_cache;
StrideV dV_cache{};
+ float* Lse = nullptr; // LSE output buffer (null = skip)
};
using KernelParams = KernelArguments;
@@ -154,8 +155,15 @@ class XeSageFwdKernel {
//
static Params to_underlying_arguments(Arguments const& args, void* workspace) {
+ // Forward LSE info to epilogue if requested
+ auto epi_args = args.epilogue;
+ if (args.kernel.Lse) {
+ epi_args.lse_ptr = args.kernel.Lse;
+ epi_args.seq_len_qo = args.kernel.shape.seq_len_qo;
+ epi_args.num_heads_q = args.kernel.shape.num_heads_q;
+ }
return {args.kernel, CollectiveMainloop::to_underlying_arguments(args.mainloop, workspace),
- CollectiveEpilogue::to_underlying_arguments(args.epilogue, workspace),
+ CollectiveEpilogue::to_underlying_arguments(epi_args, workspace),
TileScheduler::to_underlying_arguments(args.kernel.shape, args.hw_info, TileShapeO{})};
}
@@ -340,7 +348,7 @@ class XeSageFwdKernel {
// Epilogue
CollectiveEpilogue epilogue{params.epilogue, shared_storage.epilogue};
- epilogue(O(_, _, head_q, l_coord), tArA, tA_max, tA_sum, blk_qv, thr_id);
+ epilogue(O(_, _, head_q, l_coord), tArA, tA_max, tA_sum, blk_qv, thr_id, head_q, idx_b);
}
}
};
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp
index 8df165821..acd34137a 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp
@@ -62,7 +62,8 @@ void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_pt
int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b,
int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch, int num_heads_q,
int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
- float softmax_scale, bool is_causal);
+ float softmax_scale, bool is_causal,
+ float* lse = nullptr);
void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
int scale_block_size, void* qscale, void* kscale, int q_stride_s, int q_stride_d, int q_stride_h,
@@ -70,7 +71,8 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr
int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h,
int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- BTLA_DTYPE pv_dtype = BTLA_DTYPE::F16);
+ BTLA_DTYPE pv_dtype = BTLA_DTYPE::F16,
+ float* lse = nullptr);
void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
int scale_block_size, void* qscale, void* kscale, void* vscale, int q_stride_s,
@@ -78,7 +80,8 @@ void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,
int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,
int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,
int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
- float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype = BTLA_DTYPE::F16);
+ float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype = BTLA_DTYPE::F16,
+ float* lse = nullptr);
/**
* @brief Flash Attention Prefill with variable-length sequences (no padding).
@@ -97,7 +100,8 @@ void sdpa_varlen_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, voi
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,
int head_dim, float softmax_scale, bool is_causal,
- const int* cu_seqlens_q, const int* cu_seqlens_k);
+ const int* cu_seqlens_q, const int* cu_seqlens_k,
+ float* lse = nullptr);
/**
* @brief SAGE (INT8 Q/K) attention prefill with variable-length sequences.
@@ -118,7 +122,8 @@ void sage_prefill_varlen(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,
int batch, int num_heads_q, int num_heads_kv,
int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,
int head_dim, float softmax_scale, bool is_causal,
- const int* cu_seqlens_q, const int* cu_seqlens_k);
+ const int* cu_seqlens_q, const int* cu_seqlens_k,
+ float* lse = nullptr);
#endif // ARK_SYCL_TLA
} // namespace ark
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp
index 378a21511..aabe8bff0 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp
@@ -87,6 +87,7 @@ struct Options {
int v_stride_d = 1, v_stride_s = 0, v_stride_h = 0, v_stride_b = 0;
int o_stride_s = 0, o_stride_d = 1, o_stride_h = 0, o_stride_b = 0;
float softmax_scale = 0.0f;
+ float* lse = nullptr; // LSE output buffer (null = skip)
bool persistent = false;
void print(std::ostream& os = std::cout) const {
@@ -313,6 +314,7 @@ struct KernelRunner {
stride_K_cache,
static_cast<const FMHAKernel::ElementV*>(options.block_V),
stride_V_cache,
+ static_cast<float*>(options.lse),
},
{options.softmax_scale, static_cast<FMHAKernel::ElementQ*>(options.mask),
options.use_paged_kv ? options.page_table : nullptr, options.use_paged_kv ? options.page_size : 0,
@@ -669,6 +671,7 @@ struct SageKernelRunner {
stride_K_cache,
static_cast<const FMHAKernel::ElementV*>(options.block_V),
stride_V_cache,
+ static_cast<float*>(options.lse),
},
{options.softmax_scale, static_cast<float*>(options.mask), options.scale_block_size,
static_cast<const float*>(options.qscale), static_cast<const float*>(options.kscale),
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
index 759042e28..b67b356af 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
@@ -1478,7 +1478,7 @@ class XpuWrapper {
int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- bool use_int8_pv) {
+ bool use_int8_pv, float* lse = nullptr) {
bool use_mean_bias = env_params::Instance()->sage_use_mean_bias != 0;
bool q_packed = is_packed_hnd(q_stride_s, q_stride_d, q_stride_h, q_stride_b, num_heads_q, seq_len_q, head_dim);
bool k_packed =
@@ -1565,13 +1565,14 @@ class XpuWrapper {
k_out_stride_s, k_out_stride_d, k_out_stride_h, k_out_stride_b, v_out_stride_d,
v_out_stride_s, v_out_stride_h, v_out_stride_b, o_stride_s, o_stride_d, o_stride_h,
o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
- softmax_scale, is_causal, pv_dtype);
+ softmax_scale, is_causal, pv_dtype, lse);
} else {
ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale,
q_out_stride_s, q_out_stride_d, q_out_stride_h, q_out_stride_b, k_out_stride_s,
k_out_stride_d, k_out_stride_h, k_out_stride_b, v_stride_d, v_stride_s, v_stride_h,
v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,
- num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, pv_dtype);
+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, pv_dtype,
+ lse);
}
}
@@ -1585,7 +1586,7 @@ class XpuWrapper {
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,
int head_dim, float softmax_scale, bool is_causal, bool use_int8_pv,
- const int* cu_seqlens_q, const int* cu_seqlens_k) {
+ const int* cu_seqlens_q, const int* cu_seqlens_k, float* lse = nullptr) {
size_t q_size = size_t(num_heads_q) * total_seqlen_q * head_dim;
size_t q_seq_blk = (size_t(total_seqlen_q) + scale_block_size - 1) / scale_block_size;
size_t q_scale_size = size_t(num_heads_q) * q_seq_blk;
@@ -1636,7 +1637,7 @@ class XpuWrapper {
batch, num_heads_q, num_heads_kv,
total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,
head_dim, softmax_scale, is_causal,
- cu_seqlens_q, cu_seqlens_k);
+ cu_seqlens_q, cu_seqlens_k, lse);
} else {
ark::sage_prefill_varlen(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale,
nullptr, false, BTLA_DTYPE::S8, pv_dtype,
@@ -1647,7 +1648,7 @@ class XpuWrapper {
batch, num_heads_q, num_heads_kv,
total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,
head_dim, softmax_scale, is_causal,
- cu_seqlens_q, cu_seqlens_k);
+ cu_seqlens_q, cu_seqlens_k, lse);
}
}
@@ -1657,19 +1658,19 @@ class XpuWrapper {
int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- BTLA_DTYPE dtype = BTLA_DTYPE::F16) {
+ BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr) {
if (dtype == BTLA_DTYPE::BF16) {
sagev1_impl<sycl::ext::oneapi::bfloat16>(
q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,
o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,
- head_dim, softmax_scale, is_causal, false);
+ head_dim, softmax_scale, is_causal, false, lse);
} else {
sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,
q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,
v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,
batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
- is_causal, false);
+ is_causal, false, lse);
}
}
@@ -1679,19 +1680,19 @@ class XpuWrapper {
int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,
int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,
int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,
- BTLA_DTYPE dtype = BTLA_DTYPE::F16) {
+ BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr) {
if (dtype == BTLA_DTYPE::BF16) {
sagev1_impl<sycl::ext::oneapi::bfloat16>(
q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,
k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,
o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,
- head_dim, softmax_scale, is_causal, true);
+ head_dim, softmax_scale, is_causal, true, lse);
} else {
sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,
q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,
v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,
batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
- is_causal, true);
+ is_causal, true, lse);
}
}
};
diff --git a/auto_round_extension/ark/test/test_lse.py b/auto_round_extension/ark/test/test_lse.py
new file mode 100644
index 000000000..65fe2d78b
--- /dev/null
+++ b/auto_round_extension/ark/test/test_lse.py
@@ -0,0 +1,203 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""UT for return_lse support across SDPA and SAGEV1 kernels.
+
+Verifies:
+1. return_lse=False → only O returned (backward compatible).
+2. return_lse=True → (O, LSE) tuple with correct shapes.
+3. LSE correctness → LSE-merged split-KV matches full attention.
+4. Zero regressions → return_lse=False output identical to True's O.
+"""
+
+import math
+import sys
+from pathlib import Path
+
+import pytest
+import torch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import auto_round_kernel as ark
+
+pytestmark = pytest.mark.skipif(
+ not (hasattr(torch, "xpu") and torch.xpu.is_available()),
+ reason="XPU not available",
+)
+
+# ── helpers ──────────────────────────────────────────────────────────────
+
+
+def _lse_merge(O_chunks, lse_chunks):
+ """Merge N split-KV attention outputs via LSE rescaling (log2 domain)."""
+ O_all = torch.stack(O_chunks, dim=0) # [N, B, H, S, D]
+ lse_all = torch.stack(lse_chunks, dim=0) # [N, B, H, S]
+ lse_max = torch.max(lse_all, dim=0).values
+ weights = torch.exp2(lse_all - lse_max) # log2 → linear via exp2
+ O_weighted = (O_all * weights.unsqueeze(-1)).sum(dim=0)
+ weight_sum = weights.sum(dim=0)
+ O_merged = O_weighted / weight_sum.unsqueeze(-1)
+ lse_merged = lse_max + torch.log2(weight_sum)
+ return O_merged.to(O_chunks[0].dtype), lse_merged
+
+
+def _split_kv(K, V, n_chunks):
+ """Split K/V along sequence dim into n_chunks contiguous pieces."""
+ B, Hkv, Skv, D = K.shape
+ chunk_size = (Skv + n_chunks - 1) // n_chunks
+ Kc, Vc = [], []
+ for i in range(n_chunks):
+ start = i * chunk_size
+ end = min(start + chunk_size, Skv)
+ Kc.append(K[:, :, start:end, :].contiguous())
+ Vc.append(V[:, :, start:end, :].contiguous())
+ return Kc, Vc
+
+
+def _run_full_and_split(cfg, kernel_fn, kernel_kwargs, n_chunks=2):
+ """Run full-attention (return_lse=True), split-attention merge, compare."""
+ B, Hq, Hkv, Sq, Skv, D, causal = cfg
+ torch.manual_seed(3030 + Skv)
+ scale = 1.0 / math.sqrt(D)
+ q = torch.randn(B, Hq, Sq, D, dtype=torch.float16, device="xpu")
+ k = torch.randn(B, Hkv, Skv, D, dtype=torch.float16, device="xpu")
+ v = torch.randn(B, Hkv, Skv, D, dtype=torch.float16, device="xpu")
+
+ # Full attention with LSE
+ O_full, lse_full = kernel_fn(q, k, v, scale=scale, is_causal=causal, return_lse=True, **kernel_kwargs)
+
+ # Split KV, compute each chunk, merge via LSE
+ K_chunks, V_chunks = _split_kv(k, v, n_chunks)
+ O_chunks, lse_chunks = [], []
+ for i in range(n_chunks):
+ O_c, lse_c = kernel_fn(
+ q, K_chunks[i], V_chunks[i], scale=scale, is_causal=causal, return_lse=True, **kernel_kwargs
+ )
+ O_chunks.append(O_c)
+ lse_chunks.append(lse_c)
+
+ O_merged, lse_merged = _lse_merge(O_chunks, lse_chunks)
+
+ return (O_full, lse_full), (O_merged, lse_merged)
+
+
+# ── test data ────────────────────────────────────────────────────────────
+
+SDPA_ONLY_KWARGS = [{}]
+SAGEV1_KWARGS = [{"quant_block_size": bs} for bs in (32, 64, 128)]
+
+NONCAUSAL_CFGS = [
+ (1, 96, 8, 4096, 8192, 128, False),
+ (1, 96, 8, 4096, 16384, 128, False),
+ (1, 128, 8, 4096, 8192, 64, False),
+ (1, 64, 8, 4096, 8192, 64, False),
+ (1, 32, 8, 2048, 4096, 64, False),
+ (1, 16, 4, 2048, 6144, 64, False), # non-power-of-2 KV
+]
+
+CAUSAL_CFGS = [
+ (1, 64, 8, 4096, 8192, 64, True),
+ (1, 32, 8, 2048, 2048, 64, True),
+]
+
+# ── tests ────────────────────────────────────────────────────────────────
+
+
+class TestReturnLseAPI:
+ """Verify API contract: return_lse=False vs True."""
+
+ @pytest.mark.parametrize(
+ "kernel_name,kwargs",
+ [
+ ("sdpa", {}),
+ ("sagev1", {"quant_block_size": 64}),
+ ("sagev1_pvi8", {"quant_block_size": 64}),
+ ],
+ )
+ def test_return_lse_false_returns_tensor(self, kernel_name, kwargs):
+ """return_lse=False (default) returns a single tensor."""
+ q = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")
+ k = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")
+ v = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")
+ fn = getattr(ark, kernel_name)
+ out = fn(q, k, v, scale=1 / 8.0, **kwargs)
+ assert isinstance(out, torch.Tensor), f"{kernel_name} return_lse=False should return Tensor, got {type(out)}"
+
+ @pytest.mark.parametrize(
+ "kernel_name,kwargs",
+ [
+ ("sdpa", {}),
+ ("sagev1", {"quant_block_size": 64}),
+ ("sagev1_pvi8", {"quant_block_size": 64}),
+ ],
+ )
+ def test_return_lse_true_returns_tuple(self, kernel_name, kwargs):
+ """return_lse=True returns (O, LSE) tuple."""
+ q = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")
+ k = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")
+ v = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")
+ fn = getattr(ark, kernel_name)
+ out = fn(q, k, v, scale=1 / 8.0, return_lse=True, **kwargs)
+ assert (
+ isinstance(out, tuple) and len(out) == 2
+ ), f"{kernel_name} return_lse=True should return (O, LSE), got {type(out)}"
+ O, LSE = out
+ assert isinstance(O, torch.Tensor) and isinstance(LSE, torch.Tensor)
+ assert LSE.shape == (1, 4, 128), f"LSE shape mismatch: {LSE.shape} vs (1,4,128)"
+
+ @pytest.mark.parametrize(
+ "kernel_name,kwargs",
+ [
+ ("sdpa", {}),
+ ("sagev1", {"quant_block_size": 64}),
+ ],
+ )
+ def test_return_lse_false_matches_true_O(self, kernel_name, kwargs):
+ """O from return_lse=False must match O from return_lse=True."""
+ q = torch.randn(1, 8, 256, 64, dtype=torch.float16, device="xpu")
+ k = torch.randn(1, 4, 512, 64, dtype=torch.float16, device="xpu")
+ v = torch.randn(1, 4, 512, 64, dtype=torch.float16, device="xpu")
+ fn = getattr(ark, kernel_name)
+ O_false = fn(q, k, v, scale=1 / 8.0, **kwargs)
+ O_true, _ = fn(q, k, v, scale=1 / 8.0, return_lse=True, **kwargs)
+ torch.testing.assert_close(O_false, O_true, atol=0, rtol=0, msg="return_lse=False O differs from True O")
+
+
+class TestLseSequenceParallelMerge:
+ """LSE-based merging of split-KV attention = full attention."""
+
+ @pytest.mark.parametrize("cfg", NONCAUSAL_CFGS)
+ @pytest.mark.parametrize("kwargs", SDPA_ONLY_KWARGS)
+ def test_sdpa_2way(self, cfg, kwargs):
+ self._check(cfg, ark.sdpa, kwargs, n_chunks=2)
+
+ @pytest.mark.parametrize("cfg", NONCAUSAL_CFGS[:2]) # subset for speed
+ @pytest.mark.parametrize("kwargs", SAGEV1_KWARGS)
+ def test_sagev1_2way(self, cfg, kwargs):
+ self._check(cfg, ark.sagev1, kwargs, n_chunks=2)
+
+ @pytest.mark.parametrize("cfg", [(1, 32, 8, 2048, 4096, 64, False)])
+ @pytest.mark.parametrize("kwargs", SDPA_ONLY_KWARGS)
+ def test_sdpa_3way(self, cfg, kwargs):
+ self._check(cfg, ark.sdpa, kwargs, n_chunks=3)
+
+ def _check(self, cfg, fn, kwargs, n_chunks):
+ """Run full + split, then assert both O and LSE match."""
+ (O_full, lse_full), (O_merged, lse_merged) = _run_full_and_split(cfg, fn, kwargs, n_chunks)
+
+ bs = kwargs.get("quant_block_size", 0)
+ # SAGEV1 with split-then-merge compounds quantization noise across chunks.
+ # The larger the error comes from per-chunk INT8 quantization, not from the
+ # LSE merge itself. LSE is validated separately with its own tolerance.
+ atol_o = 5.0e-2 if not bs else 1.6e-1
+ atol_lse = 1e-1
+
+ O_close = torch.allclose(O_merged, O_full, atol=atol_o, rtol=1e-2)
+ LSE_close = torch.allclose(lse_merged, lse_full, atol=atol_lse, rtol=1e-1)
+
+ if not O_close:
+ err = (O_merged - O_full).abs().max().item()
+ pytest.fail(f"O merged vs full max_err={err:.2e} atol={atol_o:.2e}")
+ if not LSE_close:
+ err = (lse_merged - lse_full).abs().max().item()
+ pytest.fail(f"LSE merged vs full max_err={err:.2e} atol={atol_lse:.2e}")