PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: add return_lse support for SDPA and SAGEV1 kernel — enable sequence-parallel (Ring Attention) merge (#1991)
Luo Yub481cebbc4af
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex 8eb831f40..0f075d4b5 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -12,10 +12,17 @@# See the License for the specific language governing permissions and# limitations under the License.+import osfrom typing import Optionalimport torchimport sys+# Intel GPU compiler (IGC) environment variable: ensure implicit local IDs are+# not removed by the compiler even when unused. This guarantees consistent+# behavior across driver upgrades and prevents performance regressions.+# Reference: https://github.com/intel/intel-graphics-compiler/issues/412+os.environ.setdefault("IGC_RemoveUnusedIdImplicitLocalIDs", "0")+# Tensor layout constants passed to native C++ backends.# 0 = HND ([B, H, S, D]), 1 = NHD ([B, S, H, D]).LAYOUT_HND = 0@@ -509,7 +516,8 @@ def sdpa(is_causal: bool = False,scale: float | None = None,tensor_layout: str = "HND",-) -> torch.Tensor:+ return_lse: bool = False,+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:"""Scaled dot-product attention (SDPA) prefill+decode.Supported tensor layouts:@@ -519,9 +527,11 @@ def sdpa(Args:- scale: Softmax scale. Uses 1 / sqrt(D) when None.- tensor_layout: Layout of Q/K/V/O tensors.+ - return_lse: If True, returns (O, LSE) where LSE[b, h, q] = log(sum_j exp(score_{b,h,q,j})).Returns:- O: same layout as the input tensors.+ - (O, LSE): if return_lse is True."""if query.device.type != "xpu":raise NotImplementedError("sdpa is only supported on XPU")@@ -574,6 +584,9 @@ def sdpa(device=query.device,tensor_layout=tensor_layout,)++ LSE = torch.empty(B, Hq, Sq, dtype=torch.float32, device=query.device) if return_lse else None+layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHDlib.sdpa(stream,@@ -594,7 +607,11 @@ def sdpa(float(scale) if scale is not None else 1.0 / (D**0.5),bool(is_causal),layout_code,+ LSE.data_ptr() if LSE is not None else 0,)++ if return_lse:+ return O, LSEreturn O@@ -610,7 +627,8 @@ def sdpa_varlen(dropout_p: float = 0.0,is_causal: bool = False,scale: float | None = None,-) -> torch.Tensor:+ return_lse: bool = False,+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:"""Scaled dot-product attention (SDPA) prefill+decode with variable-length sequences.Q, K, V use a flat 3-D layout where tokens from all sequences in the batch@@ -701,6 +719,18 @@ def sdpa_varlen(O = torch.empty(total_q, Hq, D, dtype=value.dtype, device=query.device)+ if return_lse:+ max_q = int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())+ if max_seqlen_q < max_q:+ raise ValueError(f"max_seqlen_q ({max_seqlen_q}) < max sequence length in cu_seqlens_q ({max_q})")+ LSE = torch.full(+ (batch, Hq, max_seqlen_q),+ float("-inf"),+ dtype=torch.float32,+ device=query.device,+ )+ else:+ LSE = Nonelib.sdpa_varlen(stream,query.data_ptr(),@@ -724,7 +754,11 @@ def sdpa_varlen(cu_seqlens_q_i32.data_ptr(),cu_seqlens_k_i32.data_ptr(),LAYOUT_HND, # unused by the native varlen path (always flat 3-D)+ LSE.data_ptr() if LSE is not None else 0,)++ if return_lse:+ return O, LSEreturn O@@ -941,7 +975,8 @@ def sagev1(enable_gqa: bool = False,quant_block_size: int = 64,tensor_layout: str = "HND",-) -> torch.Tensor:+ return_lse: bool = False,+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:"""SAGE v1 attention prefill+decode.Supported tensor layouts:@@ -952,9 +987,11 @@ def sagev1(- scale: Attention scale. Uses 1 / sqrt(D) when None.- quant_block_size: Quantization block size used by the kernel.- tensor_layout: Layout of Q/K/V/O tensors.+ - return_lse: If True, returns (O, LSE) where LSE[b, h, q] = log(sum_j exp(score_{b,h,q,j})).Returns:- O: same layout as the input tensors.+ - (O, LSE): if return_lse is True."""if quant_block_size <= 0:return sdpa(@@ -966,6 +1003,7 @@ def sagev1(is_causal=is_causal,scale=scale,tensor_layout=tensor_layout,+ return_lse=return_lse,)if query.device.type != "xpu":raise NotImplementedError("sdpa is only supported on XPU")@@ -1003,6 +1041,9 @@ def sagev1(device=query.device,tensor_layout=tensor_layout,)++ LSE = torch.empty(B, Hq, Sq, dtype=torch.float32, device=query.device) if return_lse else None+layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHDlib.sagev1(stream,@@ -1025,7 +1066,11 @@ def sagev1(float(scale) if scale is not None else 1.0 / (D**0.5),bool(is_causal),layout_code,+ LSE.data_ptr() if LSE is not None else 0,)++ if return_lse:+ return O, LSEreturn O@@ -1040,7 +1085,8 @@ def sagev1_pvi8(enable_gqa: bool = False,quant_block_size: int = 64,tensor_layout: str = "HND",-) -> torch.Tensor:+ return_lse: bool = False,+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:"""SAGE v1 attention with PV int8 path.Expects FP16 Q/K/V input and quantizes Q/K/V internally before calling@@ -1056,6 +1102,7 @@ def sagev1_pvi8(is_causal=is_causal,scale=scale,tensor_layout=tensor_layout,+ return_lse=return_lse,)if query.device.type != "xpu":raise NotImplementedError("sagev1_pvi8 is only supported on XPU")@@ -1093,6 +1140,9 @@ def sagev1_pvi8(device=query.device,tensor_layout=tensor_layout,)++ LSE = torch.empty(B, Hq, Sq, dtype=torch.float32, device=query.device) if return_lse else None+layout_code = LAYOUT_HND if _normalize_tensor_layout(tensor_layout) == "HND" else LAYOUT_NHDlib.sagev1_pvi8(stream,@@ -1115,7 +1165,11 @@ def sagev1_pvi8(float(scale) if scale is not None else 1.0 / (D**0.5),bool(is_causal),layout_code,+ LSE.data_ptr() if LSE is not None else 0,)++ if return_lse:+ return O, LSEreturn O@@ -1129,7 +1183,7 @@ def sageattn(return_lse: bool = False,kernel: str = "v1_pvhalf",**kwargs,-) -> torch.Tensor:+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:"""SAGE attention dispatcher.Signature mirrors ``sageattention.sageattn``.@@ -1139,7 +1193,7 @@ def sageattn(- tensor_layout: "HND" or "NHD".- is_causal: Whether to apply causal mask.- sm_scale: Softmax scale. Uses ``1 / sqrt(head_dim)`` when None.- - return_lse: Not supported; must be False.+ - return_lse: If True, returns (O, LSE) tuple.- kernel: Which SAGE variant to dispatch to.- "v1_pvhalf" (default): PV in half precision (calls ``sagev1``).- "v1_pvi8": PV in INT8 precision (calls ``sagev1_pvi8``).@@ -1148,10 +1202,8 @@ def sageattn(Returns:- O: same layout as the input tensors.+ - (O, LSE): if return_lse is True."""- if return_lse:- raise NotImplementedError("return_lse is not supported in ARK sageattn")-if kernel == "v1_pvhalf":impl = sagev1elif kernel == "v1_pvi8":@@ -1166,6 +1218,7 @@ def sageattn(is_causal=is_causal,scale=sm_scale,tensor_layout=tensor_layout,+ return_lse=return_lse,**kwargs,)@@ -1182,8 +1235,9 @@ def sageattn_varlen(sm_scale: float | None = None,kernel: str = "v1_pvhalf",quant_block_size: int = 64,+ return_lse: bool = False,**kwargs,-) -> torch.Tensor:+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:"""SAGE attention with variable-length sequences (no padding).Q/K/V are flat 3-D tensors: [total_tokens, num_heads, head_dim].@@ -1271,6 +1325,18 @@ def sageattn_varlen(O = torch.empty(total_q, Hq, D, dtype=v.dtype, device=q.device)+ if return_lse:+ max_q = int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())+ if max_seqlen_q < max_q:+ raise ValueError(f"max_seqlen_q ({max_seqlen_q}) < max sequence length in cu_seqlens_q ({max_q})")+ LSE = torch.full(+ (batch, Hq, max_seqlen_q),+ float("-inf"),+ dtype=torch.float32,+ device=q.device,+ )+ else:+ LSE = Nonelib.sagev1_varlen(stream,q.data_ptr(),@@ -1296,7 +1362,11 @@ def sageattn_varlen(cu_seqlens_q_i32.data_ptr(),cu_seqlens_k_i32.data_ptr(),use_int8_pv,+ LSE.data_ptr() if LSE is not None else 0,)++ if return_lse:+ return O, LSEreturn Odiff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppindex 708e3100b..841b0eb4a 100755--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -109,7 +109,7 @@ static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_int q_dtype, int k_dtype, int o_dtype,int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,float softmax_scale, bool is_causal,- int tensor_layout) {+ int tensor_layout, torch_ptr lse = 0) {if (k_dtype != q_dtype || o_dtype != q_dtype) {throw std::invalid_argument("ark::sdpa: k_dtype and o_dtype must match q_dtype");}@@ -141,7 +141,8 @@ static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ark::sdpa_impl((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (BTLA_DTYPE)(q_dtype),q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b,v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,- batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal,+ (float*)lse);}static void sdpa_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,@@ -151,7 +152,7 @@ static void sdpa_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,int max_seqlen_q, int max_seqlen_kv,int head_dim, float softmax_scale, bool is_causal,torch_ptr cu_seqlens_q, torch_ptr cu_seqlens_k,- int tensor_layout) {+ int tensor_layout, torch_ptr lse = 0) {if (k_dtype != q_dtype || o_dtype != q_dtype) {throw std::invalid_argument("ark::sdpa_varlen: k_dtype and o_dtype must match q_dtype");}@@ -188,7 +189,8 @@ static void sdpa_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,total_seqlen_q, total_seqlen_kv,max_seqlen_q, max_seqlen_kv,head_dim, softmax_scale, is_causal,- (const int*)cu_seqlens_q, (const int*)cu_seqlens_k);+ (const int*)cu_seqlens_q, (const int*)cu_seqlens_k,+ (float*)lse);}// Varlen SageV1 bridge: quantizes Q/K to INT8, then dispatches with varlen=true.@@ -198,7 +200,7 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptrint total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,int head_dim, float softmax_scale, bool is_causal,torch_ptr cu_seqlens_q, torch_ptr cu_seqlens_k,- int use_int8_pv) {+ int use_int8_pv, torch_ptr lse = 0) {if (mask && is_causal) {throw std::invalid_argument("ark::sagev1_varlen: mask and is_causal cannot both be set");}@@ -231,7 +233,8 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptrbatch, num_heads_q, num_heads_kv,total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,head_dim, softmax_scale, is_causal, bool(use_int8_pv),- (const int*)cu_seqlens_q, (const int*)cu_seqlens_k);+ (const int*)cu_seqlens_q, (const int*)cu_seqlens_k,+ (float*)lse);} else {XpuWrapper::sagev1_varlen_impl<sycl::half>((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,@@ -242,7 +245,8 @@ static void sagev1_varlen(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptrbatch, num_heads_q, num_heads_kv,total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,head_dim, softmax_scale, is_causal, bool(use_int8_pv),- (const int*)cu_seqlens_q, (const int*)cu_seqlens_k);+ (const int*)cu_seqlens_q, (const int*)cu_seqlens_k,+ (float*)lse);}}@@ -250,7 +254,7 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype,int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,float softmax_scale, bool is_causal, bool use_int8_pv,- int tensor_layout) {+ int tensor_layout, torch_ptr lse = 0) {if (mask && is_causal) {throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");}@@ -285,13 +289,14 @@ static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s,k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,- seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, (BTLA_DTYPE)q_dtype);+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, (BTLA_DTYPE)q_dtype,+ (float*)lse);} else {XpuWrapper::sagev1((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, scale_block_size,q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h,k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,- softmax_scale, is_causal, (BTLA_DTYPE)q_dtype);+ softmax_scale, is_causal, (BTLA_DTYPE)q_dtype, (float*)lse);}#elsethrow std::runtime_error("ark::sagev1 is only supported on XPU");@@ -302,27 +307,27 @@ static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torcint scale_block_size,int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv,int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- int tensor_layout) {+ int tensor_layout, torch_ptr lse = 0) {sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch,num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, false,- tensor_layout);+ tensor_layout, lse);}static void sagev1_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,int scale_block_size,int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv,int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- int tensor_layout) {+ int tensor_layout, torch_ptr lse = 0) {sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch,num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, true,- tensor_layout);+ tensor_layout, lse);}static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,int scale_block_size, torch_ptr qscale, torch_ptr kscale,int q_dtype, int k_dtype, int o_dtype, int batch, int num_heads_q,int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,- bool is_causal, int tensor_layout) {+ bool is_causal, int tensor_layout, torch_ptr lse = 0) {if (mask && is_causal) {throw std::invalid_argument("ark::sage: mask and is_causal cannot both be set");}@@ -350,14 +355,14 @@ static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch,num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal,- (BTLA_DTYPE)o_dtype);+ (BTLA_DTYPE)o_dtype, (float*)lse);}static void sage_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,int scale_block_size, torch_ptr qscale, torch_ptr kscale, torch_ptr vscale,int q_dtype, int k_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- int tensor_layout) {+ int tensor_layout, torch_ptr lse = 0) {if (mask && is_causal) {throw std::invalid_argument("ark::sage_pvi8: mask and is_causal cannot both be set");}@@ -385,7 +390,7 @@ static void sage_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, tq_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,- is_causal, (BTLA_DTYPE)o_dtype);+ is_causal, (BTLA_DTYPE)o_dtype, (float*)lse);}static void moe_gemm_wrapper(torch_ptr stream, torch_ptr activations, torch_ptr weights, torch_ptr scales,@@ -609,7 +614,7 @@ PYBIND11_MODULE(PY_NAME, m) {pybind11::arg("max_seqlen_q"), pybind11::arg("max_seqlen_kv"),pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),pybind11::arg("cu_seqlens_q"), pybind11::arg("cu_seqlens_k"),- pybind11::arg("tensor_layout"));+ pybind11::arg("tensor_layout"), pybind11::arg("lse") = 0);// Varlen SAGEV1: flat 3-D Q/K/V + cu_seqlens (use_int8_pv=0) or pvi8 (use_int8_pv=1).m.def("sagev1_varlen", &ark::sagev1_varlen, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),@@ -620,13 +625,29 @@ PYBIND11_MODULE(PY_NAME, m) {pybind11::arg("max_seqlen_q"), pybind11::arg("max_seqlen_kv"),pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),pybind11::arg("cu_seqlens_q"), pybind11::arg("cu_seqlens_k"),- pybind11::arg("use_int8_pv"));+ pybind11::arg("use_int8_pv"), pybind11::arg("lse") = 0);m.def("sagev1", &ark::sagev1);// High-level SAGEV1 PVi8 API: input Q/K/V are FP16 and quantized internally.m.def("sagev1_pvi8", &ark::sagev1_pvi8);- m.def("sage", &ark::sage);+ m.def("sage", &ark::sage, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),+ pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),+ pybind11::arg("scale_block_size"),+ pybind11::arg("qscale"), pybind11::arg("kscale"),+ pybind11::arg("q_dtype"), pybind11::arg("k_dtype"), pybind11::arg("o_dtype"),+ pybind11::arg("batch"), pybind11::arg("num_heads_q"), pybind11::arg("num_heads_kv"),+ pybind11::arg("seq_len_q"), pybind11::arg("seq_len_kv"),+ pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),+ pybind11::arg("tensor_layout"), pybind11::arg("lse") = 0);// Low-level SAGE PVi8 API: input Q/K/V are pre-quantized int8 with qscale/kscale/vscale.- m.def("sage_pvi8", &ark::sage_pvi8);+ m.def("sage_pvi8", &ark::sage_pvi8, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),+ pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),+ pybind11::arg("scale_block_size"),+ pybind11::arg("qscale"), pybind11::arg("kscale"), pybind11::arg("vscale"),+ pybind11::arg("q_dtype"), pybind11::arg("k_dtype"), pybind11::arg("o_dtype"),+ pybind11::arg("batch"), pybind11::arg("num_heads_q"), pybind11::arg("num_heads_kv"),+ pybind11::arg("seq_len_q"), pybind11::arg("seq_len_kv"),+ pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),+ pybind11::arg("tensor_layout"), pybind11::arg("lse") = 0);m.def("sage_dynamic_quant", &ark::sage_dynamic_quant);m.def("sage_compute_seq_mean_bias_layout", &ark::sage_compute_seq_mean_bias_layout);m.def("sage_dynamic_quant_layout", &ark::sage_dynamic_quant_layout);diff --git a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp b/auto_round_extension/ark/auto_round_kernel/sdpa.cppindex 3b03c4647..10afcf646 100644--- a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp+++ b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp@@ -232,7 +232,7 @@ void sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* Oint k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,- float softmax_scale, bool is_causal) {+ float softmax_scale, bool is_causal, float* lse = nullptr) {detail::Options options =make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,@@ -242,6 +242,7 @@ void sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* Ooptions.qscale = qscale;options.kscale = kscale;options.vscale = vscale;+ options.lse = lse;compat::set_default_queue(*q);KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, pv_dtype, head_dim, use_int8_pv);@@ -258,12 +259,14 @@ void flash_attn_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, vint k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,- int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,+ float* lse = nullptr) {detail::Options options =make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ options.lse = lse;compat::set_default_queue(*q);KernelLauncher launcher = select_prefill_launcher(q_dtype, head_dim);@@ -280,12 +283,14 @@ void flash_attn_decode(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, voint k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,+ float* lse = nullptr) {detail::Options options =make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,num_heads_kv, 1, seq_len_kv, head_dim, softmax_scale, is_causal);+ options.lse = lse;compat::set_default_queue(*q);KernelLauncher launcher = select_decode_launcher(q_dtype, head_dim);@@ -302,7 +307,7 @@ void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptint k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b,int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch, int num_heads_q,int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,- bool is_causal) {+ bool is_causal, float* lse = nullptr) {// if (q_dtype != BTLA_DTYPE::F16 && q_dtype != BTLA_DTYPE::BF16) {// throw std::invalid_argument("sdpa_impl: only FP16 and BF16 are supported");// }@@ -317,14 +322,14 @@ void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptflash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b,k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,- seq_len_kv, head_dim, softmax_scale, is_causal);+ seq_len_kv, head_dim, softmax_scale, is_causal, lse);return;}flash_attn_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b,k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv,- seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, lse);}void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,@@ -332,7 +337,8 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptrint q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h,int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,- int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE pv_dtype) {+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE pv_dtype,+ float* lse = nullptr) {if (mask && is_causal) {throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");}@@ -347,7 +353,7 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptrflash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, pv_dtype, q_stride_s, q_stride_d, q_stride_h,q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,- num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal);+ num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal, lse);return;}@@ -355,7 +361,7 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptrBTLA_DTYPE::S8, pv_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s,o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q,- seq_len_kv, head_dim, softmax_scale, is_causal);+ seq_len_kv, head_dim, softmax_scale, is_causal, lse);}void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,@@ -364,7 +370,8 @@ void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,- int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype) {+ int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype,+ float* lse = nullptr) {if (mask && is_causal) {throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");}@@ -382,7 +389,7 @@ void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, o_dtype, q_stride_s, q_stride_d, q_stride_h,q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s,v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,- num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal);+ num_heads_kv, seq_len_kv, head_dim, softmax_scale, is_causal, lse);return;}@@ -390,7 +397,7 @@ void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,BTLA_DTYPE::S8, o_dtype, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,- softmax_scale, is_causal);+ softmax_scale, is_causal, lse);}void sage_prefill_varlen(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,@@ -403,7 +410,8 @@ void sage_prefill_varlen(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,int batch, int num_heads_q, int num_heads_kv,int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,int head_dim, float softmax_scale, bool is_causal,- const int* cu_seqlens_q, const int* cu_seqlens_k) {+ const int* cu_seqlens_q, const int* cu_seqlens_k,+ float* lse = nullptr) {if (mask && is_causal) {throw std::invalid_argument("sage_prefill_varlen: mask and is_causal cannot both be set");}@@ -437,6 +445,7 @@ void sage_prefill_varlen(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,options.qscale = qscale;options.kscale = kscale;options.vscale = vscale;+ options.lse = lse;compat::set_default_queue(*q);@@ -463,7 +472,8 @@ void sdpa_varlen_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, voiint o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,int head_dim, float softmax_scale, bool is_causal,- const int* cu_seqlens_q, const int* cu_seqlens_k) {+ const int* cu_seqlens_q, const int* cu_seqlens_k,+ float* lse = nullptr) {if (mask && is_causal) {throw std::invalid_argument("sdpa_varlen_impl: mask and is_causal cannot both be set");}@@ -496,6 +506,7 @@ void sdpa_varlen_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, voioptions.seq_len_kv_cache = 0;options.total_seqlen_kv_cache = 0;options.max_seqlen_kv_cache = 0;+ options.lse = lse;compat::set_default_queue(*q);diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hppindex a5a64638d..4e773f49f 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp@@ -131,6 +131,7 @@ class XeSageFwdKernel {StrideK dK_cache{};const ElementV* V_cache;StrideV dV_cache{};+ float* Lse = nullptr; // LSE output buffer (null = skip)};using KernelParams = KernelArguments;@@ -154,8 +155,15 @@ class XeSageFwdKernel {//static Params to_underlying_arguments(Arguments const& args, void* workspace) {+ // Forward LSE info to epilogue if requested+ auto epi_args = args.epilogue;+ if (args.kernel.Lse) {+ epi_args.lse_ptr = args.kernel.Lse;+ epi_args.seq_len_qo = args.kernel.shape.seq_len_qo;+ epi_args.num_heads_q = args.kernel.shape.num_heads_q;+ }return {args.kernel, CollectiveMainloop::to_underlying_arguments(args.mainloop, workspace),- CollectiveEpilogue::to_underlying_arguments(args.epilogue, workspace),+ CollectiveEpilogue::to_underlying_arguments(epi_args, workspace),TileScheduler::to_underlying_arguments(args.kernel.shape, args.hw_info, TileShapeO{})};}@@ -340,7 +348,7 @@ class XeSageFwdKernel {// EpilogueCollectiveEpilogue epilogue{params.epilogue, shared_storage.epilogue};- epilogue(O(_, _, head_q, l_coord), tArA, tA_max, tA_sum, blk_qv, thr_id);+ epilogue(O(_, _, head_q, l_coord), tArA, tA_max, tA_sum, blk_qv, thr_id, head_q, idx_b);}}};diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hppindex 8df165821..acd34137a 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp@@ -62,7 +62,8 @@ void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptint k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h, int v_stride_b,int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch, int num_heads_q,int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,- float softmax_scale, bool is_causal);+ float softmax_scale, bool is_causal,+ float* lse = nullptr);void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,int scale_block_size, void* qscale, void* kscale, int q_stride_s, int q_stride_d, int q_stride_h,@@ -70,7 +71,8 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptrint v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h,int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- BTLA_DTYPE pv_dtype = BTLA_DTYPE::F16);+ BTLA_DTYPE pv_dtype = BTLA_DTYPE::F16,+ float* lse = nullptr);void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,int scale_block_size, void* qscale, void* kscale, void* vscale, int q_stride_s,@@ -78,7 +80,8 @@ void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,- float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype = BTLA_DTYPE::F16);+ float softmax_scale, bool is_causal, BTLA_DTYPE o_dtype = BTLA_DTYPE::F16,+ float* lse = nullptr);/*** @brief Flash Attention Prefill with variable-length sequences (no padding).@@ -97,7 +100,8 @@ void sdpa_varlen_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, voiint o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,int head_dim, float softmax_scale, bool is_causal,- const int* cu_seqlens_q, const int* cu_seqlens_k);+ const int* cu_seqlens_q, const int* cu_seqlens_k,+ float* lse = nullptr);/*** @brief SAGE (INT8 Q/K) attention prefill with variable-length sequences.@@ -118,7 +122,8 @@ void sage_prefill_varlen(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,int batch, int num_heads_q, int num_heads_kv,int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,int head_dim, float softmax_scale, bool is_causal,- const int* cu_seqlens_q, const int* cu_seqlens_k);+ const int* cu_seqlens_q, const int* cu_seqlens_k,+ float* lse = nullptr);#endif // ARK_SYCL_TLA} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hppindex 378a21511..aabe8bff0 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp@@ -87,6 +87,7 @@ struct Options {int v_stride_d = 1, v_stride_s = 0, v_stride_h = 0, v_stride_b = 0;int o_stride_s = 0, o_stride_d = 1, o_stride_h = 0, o_stride_b = 0;float softmax_scale = 0.0f;+ float* lse = nullptr; // LSE output buffer (null = skip)bool persistent = false;void print(std::ostream& os = std::cout) const {@@ -313,6 +314,7 @@ struct KernelRunner {stride_K_cache,static_cast<const FMHAKernel::ElementV*>(options.block_V),stride_V_cache,+ static_cast<float*>(options.lse),},{options.softmax_scale, static_cast<FMHAKernel::ElementQ*>(options.mask),options.use_paged_kv ? options.page_table : nullptr, options.use_paged_kv ? options.page_size : 0,@@ -669,6 +671,7 @@ struct SageKernelRunner {stride_K_cache,static_cast<const FMHAKernel::ElementV*>(options.block_V),stride_V_cache,+ static_cast<float*>(options.lse),},{options.softmax_scale, static_cast<float*>(options.mask), options.scale_block_size,static_cast<const float*>(options.qscale), static_cast<const float*>(options.kscale),diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hppindex 759042e28..b67b356af 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp@@ -1478,7 +1478,7 @@ class XpuWrapper {int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- bool use_int8_pv) {+ bool use_int8_pv, float* lse = nullptr) {bool use_mean_bias = env_params::Instance()->sage_use_mean_bias != 0;bool q_packed = is_packed_hnd(q_stride_s, q_stride_d, q_stride_h, q_stride_b, num_heads_q, seq_len_q, head_dim);bool k_packed =@@ -1565,13 +1565,14 @@ class XpuWrapper {k_out_stride_s, k_out_stride_d, k_out_stride_h, k_out_stride_b, v_out_stride_d,v_out_stride_s, v_out_stride_h, v_out_stride_b, o_stride_s, o_stride_d, o_stride_h,o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,- softmax_scale, is_causal, pv_dtype);+ softmax_scale, is_causal, pv_dtype, lse);} else {ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale,q_out_stride_s, q_out_stride_d, q_out_stride_h, q_out_stride_b, k_out_stride_s,k_out_stride_d, k_out_stride_h, k_out_stride_b, v_stride_d, v_stride_s, v_stride_h,v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,- num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, pv_dtype);+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, pv_dtype,+ lse);}}@@ -1585,7 +1586,7 @@ class XpuWrapper {int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,int total_seqlen_q, int total_seqlen_kv, int max_seqlen_q, int max_seqlen_kv,int head_dim, float softmax_scale, bool is_causal, bool use_int8_pv,- const int* cu_seqlens_q, const int* cu_seqlens_k) {+ const int* cu_seqlens_q, const int* cu_seqlens_k, float* lse = nullptr) {size_t q_size = size_t(num_heads_q) * total_seqlen_q * head_dim;size_t q_seq_blk = (size_t(total_seqlen_q) + scale_block_size - 1) / scale_block_size;size_t q_scale_size = size_t(num_heads_q) * q_seq_blk;@@ -1636,7 +1637,7 @@ class XpuWrapper {batch, num_heads_q, num_heads_kv,total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,head_dim, softmax_scale, is_causal,- cu_seqlens_q, cu_seqlens_k);+ cu_seqlens_q, cu_seqlens_k, lse);} else {ark::sage_prefill_varlen(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale,nullptr, false, BTLA_DTYPE::S8, pv_dtype,@@ -1647,7 +1648,7 @@ class XpuWrapper {batch, num_heads_q, num_heads_kv,total_seqlen_q, total_seqlen_kv, max_seqlen_q, max_seqlen_kv,head_dim, softmax_scale, is_causal,- cu_seqlens_q, cu_seqlens_k);+ cu_seqlens_q, cu_seqlens_k, lse);}}@@ -1657,19 +1658,19 @@ class XpuWrapper {int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- BTLA_DTYPE dtype = BTLA_DTYPE::F16) {+ BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr) {if (dtype == BTLA_DTYPE::BF16) {sagev1_impl<sycl::ext::oneapi::bfloat16>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,- head_dim, softmax_scale, is_causal, false);+ head_dim, softmax_scale, is_causal, false, lse);} else {sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,- is_causal, false);+ is_causal, false, lse);}}@@ -1679,19 +1680,19 @@ class XpuWrapper {int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,- BTLA_DTYPE dtype = BTLA_DTYPE::F16) {+ BTLA_DTYPE dtype = BTLA_DTYPE::F16, float* lse = nullptr) {if (dtype == BTLA_DTYPE::BF16) {sagev1_impl<sycl::ext::oneapi::bfloat16>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d, q_stride_h, q_stride_b,k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b,o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,- head_dim, softmax_scale, is_causal, true);+ head_dim, softmax_scale, is_causal, true, lse);} else {sagev1_impl<sycl::half>(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, q_stride_s, q_stride_d,q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d,v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b,batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,- is_causal, true);+ is_causal, true, lse);}}};diff --git a/auto_round_extension/ark/test/test_lse.py b/auto_round_extension/ark/test/test_lse.pynew file mode 100644index 000000000..65fe2d78b--- /dev/null+++ b/auto_round_extension/ark/test/test_lse.py@@ -0,0 +1,203 @@+# Copyright (C) 2026 Intel Corporation+# SPDX-License-Identifier: Apache-2.0++"""UT for return_lse support across SDPA and SAGEV1 kernels.++Verifies:+1. return_lse=False → only O returned (backward compatible).+2. return_lse=True → (O, LSE) tuple with correct shapes.+3. LSE correctness → LSE-merged split-KV matches full attention.+4. Zero regressions → return_lse=False output identical to True's O.+"""++import math+import sys+from pathlib import Path++import pytest+import torch++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))+import auto_round_kernel as ark++pytestmark = pytest.mark.skipif(+ not (hasattr(torch, "xpu") and torch.xpu.is_available()),+ reason="XPU not available",+)++# ── helpers ──────────────────────────────────────────────────────────────+++def _lse_merge(O_chunks, lse_chunks):+ """Merge N split-KV attention outputs via LSE rescaling (log2 domain)."""+ O_all = torch.stack(O_chunks, dim=0) # [N, B, H, S, D]+ lse_all = torch.stack(lse_chunks, dim=0) # [N, B, H, S]+ lse_max = torch.max(lse_all, dim=0).values+ weights = torch.exp2(lse_all - lse_max) # log2 → linear via exp2+ O_weighted = (O_all * weights.unsqueeze(-1)).sum(dim=0)+ weight_sum = weights.sum(dim=0)+ O_merged = O_weighted / weight_sum.unsqueeze(-1)+ lse_merged = lse_max + torch.log2(weight_sum)+ return O_merged.to(O_chunks[0].dtype), lse_merged+++def _split_kv(K, V, n_chunks):+ """Split K/V along sequence dim into n_chunks contiguous pieces."""+ B, Hkv, Skv, D = K.shape+ chunk_size = (Skv + n_chunks - 1) // n_chunks+ Kc, Vc = [], []+ for i in range(n_chunks):+ start = i * chunk_size+ end = min(start + chunk_size, Skv)+ Kc.append(K[:, :, start:end, :].contiguous())+ Vc.append(V[:, :, start:end, :].contiguous())+ return Kc, Vc+++def _run_full_and_split(cfg, kernel_fn, kernel_kwargs, n_chunks=2):+ """Run full-attention (return_lse=True), split-attention merge, compare."""+ B, Hq, Hkv, Sq, Skv, D, causal = cfg+ torch.manual_seed(3030 + Skv)+ scale = 1.0 / math.sqrt(D)+ q = torch.randn(B, Hq, Sq, D, dtype=torch.float16, device="xpu")+ k = torch.randn(B, Hkv, Skv, D, dtype=torch.float16, device="xpu")+ v = torch.randn(B, Hkv, Skv, D, dtype=torch.float16, device="xpu")++ # Full attention with LSE+ O_full, lse_full = kernel_fn(q, k, v, scale=scale, is_causal=causal, return_lse=True, **kernel_kwargs)++ # Split KV, compute each chunk, merge via LSE+ K_chunks, V_chunks = _split_kv(k, v, n_chunks)+ O_chunks, lse_chunks = [], []+ for i in range(n_chunks):+ O_c, lse_c = kernel_fn(+ q, K_chunks[i], V_chunks[i], scale=scale, is_causal=causal, return_lse=True, **kernel_kwargs+ )+ O_chunks.append(O_c)+ lse_chunks.append(lse_c)++ O_merged, lse_merged = _lse_merge(O_chunks, lse_chunks)++ return (O_full, lse_full), (O_merged, lse_merged)+++# ── test data ────────────────────────────────────────────────────────────++SDPA_ONLY_KWARGS = [{}]+SAGEV1_KWARGS = [{"quant_block_size": bs} for bs in (32, 64, 128)]++NONCAUSAL_CFGS = [+ (1, 96, 8, 4096, 8192, 128, False),+ (1, 96, 8, 4096, 16384, 128, False),+ (1, 128, 8, 4096, 8192, 64, False),+ (1, 64, 8, 4096, 8192, 64, False),+ (1, 32, 8, 2048, 4096, 64, False),+ (1, 16, 4, 2048, 6144, 64, False), # non-power-of-2 KV+]++CAUSAL_CFGS = [+ (1, 64, 8, 4096, 8192, 64, True),+ (1, 32, 8, 2048, 2048, 64, True),+]++# ── tests ────────────────────────────────────────────────────────────────+++class TestReturnLseAPI:+ """Verify API contract: return_lse=False vs True."""++ @pytest.mark.parametrize(+ "kernel_name,kwargs",+ [+ ("sdpa", {}),+ ("sagev1", {"quant_block_size": 64}),+ ("sagev1_pvi8", {"quant_block_size": 64}),+ ],+ )+ def test_return_lse_false_returns_tensor(self, kernel_name, kwargs):+ """return_lse=False (default) returns a single tensor."""+ q = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")+ k = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")+ v = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")+ fn = getattr(ark, kernel_name)+ out = fn(q, k, v, scale=1 / 8.0, **kwargs)+ assert isinstance(out, torch.Tensor), f"{kernel_name} return_lse=False should return Tensor, got {type(out)}"++ @pytest.mark.parametrize(+ "kernel_name,kwargs",+ [+ ("sdpa", {}),+ ("sagev1", {"quant_block_size": 64}),+ ("sagev1_pvi8", {"quant_block_size": 64}),+ ],+ )+ def test_return_lse_true_returns_tuple(self, kernel_name, kwargs):+ """return_lse=True returns (O, LSE) tuple."""+ q = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")+ k = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")+ v = torch.randn(1, 4, 128, 64, dtype=torch.float16, device="xpu")+ fn = getattr(ark, kernel_name)+ out = fn(q, k, v, scale=1 / 8.0, return_lse=True, **kwargs)+ assert (+ isinstance(out, tuple) and len(out) == 2+ ), f"{kernel_name} return_lse=True should return (O, LSE), got {type(out)}"+ O, LSE = out+ assert isinstance(O, torch.Tensor) and isinstance(LSE, torch.Tensor)+ assert LSE.shape == (1, 4, 128), f"LSE shape mismatch: {LSE.shape} vs (1,4,128)"++ @pytest.mark.parametrize(+ "kernel_name,kwargs",+ [+ ("sdpa", {}),+ ("sagev1", {"quant_block_size": 64}),+ ],+ )+ def test_return_lse_false_matches_true_O(self, kernel_name, kwargs):+ """O from return_lse=False must match O from return_lse=True."""+ q = torch.randn(1, 8, 256, 64, dtype=torch.float16, device="xpu")+ k = torch.randn(1, 4, 512, 64, dtype=torch.float16, device="xpu")+ v = torch.randn(1, 4, 512, 64, dtype=torch.float16, device="xpu")+ fn = getattr(ark, kernel_name)+ O_false = fn(q, k, v, scale=1 / 8.0, **kwargs)+ O_true, _ = fn(q, k, v, scale=1 / 8.0, return_lse=True, **kwargs)+ torch.testing.assert_close(O_false, O_true, atol=0, rtol=0, msg="return_lse=False O differs from True O")+++class TestLseSequenceParallelMerge:+ """LSE-based merging of split-KV attention = full attention."""++ @pytest.mark.parametrize("cfg", NONCAUSAL_CFGS)+ @pytest.mark.parametrize("kwargs", SDPA_ONLY_KWARGS)+ def test_sdpa_2way(self, cfg, kwargs):+ self._check(cfg, ark.sdpa, kwargs, n_chunks=2)++ @pytest.mark.parametrize("cfg", NONCAUSAL_CFGS[:2]) # subset for speed+ @pytest.mark.parametrize("kwargs", SAGEV1_KWARGS)+ def test_sagev1_2way(self, cfg, kwargs):+ self._check(cfg, ark.sagev1, kwargs, n_chunks=2)++ @pytest.mark.parametrize("cfg", [(1, 32, 8, 2048, 4096, 64, False)])+ @pytest.mark.parametrize("kwargs", SDPA_ONLY_KWARGS)+ def test_sdpa_3way(self, cfg, kwargs):+ self._check(cfg, ark.sdpa, kwargs, n_chunks=3)++ def _check(self, cfg, fn, kwargs, n_chunks):+ """Run full + split, then assert both O and LSE match."""+ (O_full, lse_full), (O_merged, lse_merged) = _run_full_and_split(cfg, fn, kwargs, n_chunks)++ bs = kwargs.get("quant_block_size", 0)+ # SAGEV1 with split-then-merge compounds quantization noise across chunks.+ # The larger the error comes from per-chunk INT8 quantization, not from the+ # LSE merge itself. LSE is validated separately with its own tolerance.+ atol_o = 5.0e-2 if not bs else 1.6e-1+ atol_lse = 1e-1++ O_close = torch.allclose(O_merged, O_full, atol=atol_o, rtol=1e-2)+ LSE_close = torch.allclose(lse_merged, lse_full, atol=atol_lse, rtol=1e-1)++ if not O_close:+ err = (O_merged - O_full).abs().max().item()+ pytest.fail(f"O merged vs full max_err={err:.2e} atol={atol_o:.2e}")+ if not LSE_close:+ err = (lse_merged - lse_full).abs().max().item()+ pytest.fail(f"LSE merged vs full max_err={err:.2e} atol={atol_lse:.2e}")