PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
ark: add BF16/FP16 Sparge variants and reduce compilation RAM (#2145)
Yi Liu402b0edd9329
diff --git a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txtindex 718ea46dd..a6fffb969 100755--- a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt+++ b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt@@ -101,6 +101,7 @@ if(NOT ARK_XPU)endif()set(SDPA_GENERATED_SRCS)set(SDPA_KERNEL_DECLARATIONS)+set(SDPA_SPARSE_KERNEL_DECLARATIONS)set(SYCL_TLA_GENERATED_SRCS)include_directories(wrapper/include)@@ -184,6 +185,18 @@ if(ARK_XPU AND ARK_SYCL_TLA)target_compile_definitions(${PY_NAME} PRIVATE ARK_SYCL_TLA=1 CUTLASS_ENABLE_SYCL=1 SYCL_INTEL_TARGET=1)target_include_directories(${PY_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/wrapper/include)target_include_directories(${PY_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/generated/sdpa)+ # sycl-tla may provision oneMKL through an ExternalProject. Wire its staged+ # headers into this module directly so parallel builds do not compile before+ # oneMKL has been downloaded and installed.+ if(TARGET onemkl_project)+ add_dependencies(${PY_NAME} onemkl_project)+ target_include_directories(${PY_NAME} SYSTEM PRIVATE ${CMAKE_BINARY_DIR}/deps/oneMKL/include)+ elseif(TARGET MKL::MKL)+ target_link_libraries(${PY_NAME} PRIVATE MKL::MKL)+ if(DEFINED ENV{MKLROOT} AND EXISTS "$ENV{MKLROOT}/include")+ target_include_directories(${PY_NAME} SYSTEM PRIVATE "$ENV{MKLROOT}/include")+ endif()+ endif()# Use SYSTEM include directories for sycl-tla to suppress warnings/errors from third-party headers# (e.g., std::common_type specialization issues in traits.hpp)foreach(_inc_dir IN LISTS _sycl_tla_include_dirs)@@ -210,6 +223,15 @@ if(ARK_UT)if(ARK_XPU AND ARK_SYCL_TLA)target_compile_definitions(${TEST_NAME} PRIVATE ARK_SYCL_TLA=1 CUTLASS_ENABLE_SYCL=1 SYCL_INTEL_TARGET=1)target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/generated/sdpa)+ if(TARGET onemkl_project)+ add_dependencies(${TEST_NAME} onemkl_project)+ target_include_directories(${TEST_NAME} SYSTEM PRIVATE ${CMAKE_BINARY_DIR}/deps/oneMKL/include)+ elseif(TARGET MKL::MKL)+ target_link_libraries(${TEST_NAME} PRIVATE MKL::MKL)+ if(DEFINED ENV{MKLROOT} AND EXISTS "$ENV{MKLROOT}/include")+ target_include_directories(${TEST_NAME} SYSTEM PRIVATE "$ENV{MKLROOT}/include")+ endif()+ endif()foreach(_inc_dir IN LISTS _sycl_tla_include_dirs)if(EXISTS "${_inc_dir}")target_include_directories(${TEST_NAME} SYSTEM PRIVATE "${_inc_dir}")diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex a98543c7f..33bb1f38d 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -2158,10 +2158,12 @@ from .sparse_attention import (_sequence_mean_native_layout,_slice_sequence_native_layout,_to_hnd,+ block_sparse_sdpa,sage_sparse,sparge_block_map_to_mask,sparge_preprocess_topk,sparge_sage2_attn_meansim_topk_xpu,+ sparge_sage2_attn_meansim_topk_xpu_sdpa,)diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppindex 870cc924a..9a76d238b 100755--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -532,6 +532,98 @@ static void sage_sparse(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V,throw std::invalid_argument("ark::sage_sparse: unsupported head_dim; supported values are 64 and 128");}+static void block_sparse_sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,+ torch_ptr lut, torch_ptr valid_block_num, int num_q_blocks, int num_k_blocks,+ int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b,+ int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d,+ int v_stride_s, int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d,+ int o_stride_h, int o_stride_b, int q_dtype, int batch, int num_heads_q,+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal) {+ if (q_dtype != (int)FlashAttnDtype::FP16 && q_dtype != (int)FlashAttnDtype::BF16) {+ throw std::invalid_argument("ark::block_sparse_sdpa: q_dtype must be FP16 or BF16");+ }+ if (mask && is_causal) {+ throw std::invalid_argument("ark::block_sparse_sdpa: mask and is_causal cannot both be set");+ }+ if (!lut || !valid_block_num) {+ throw std::invalid_argument("ark::block_sparse_sdpa: lut and valid_block_num must be provided");+ }+ auto matches_block_size = [](int seq_len, int num_blocks, int block_size) {+ return block_size > 0 && num_blocks == ((seq_len + block_size - 1) / block_size);+ };+ const bool key_block_is_64 = matches_block_size(seq_len_kv, num_k_blocks, 64);+ if (!key_block_is_64) {+ throw std::invalid_argument("ark::block_sparse_sdpa: only key block size 64 is supported");+ }+ const bool is_bf16 = (q_dtype == (int)FlashAttnDtype::BF16);+ if (head_dim == 64) {+ if (!matches_block_size(seq_len_q, num_q_blocks, 64)) {+ throw std::invalid_argument("ark::block_sparse_sdpa: head_dim=64 requires query block size 64");+ }+ if (is_bf16) {+ ark::sdpa_impl_bf16_sparse_sdpa_d64(+ (sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (void*)lut,+ (void*)valid_block_num, num_q_blocks, num_k_blocks, q_tile_override, q_stride_s, q_stride_d, q_stride_h,+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q,+ seq_len_kv, head_dim, softmax_scale, is_causal);+ } else {+ ark::sdpa_impl_fp16_sparse_sdpa_d64(+ (sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (void*)lut,+ (void*)valid_block_num, num_q_blocks, num_k_blocks, q_tile_override, q_stride_s, q_stride_d, q_stride_h,+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q,+ seq_len_kv, head_dim, softmax_scale, is_causal);+ }+ return;+ }+ if (head_dim == 128) {+ if (matches_block_size(seq_len_q, num_q_blocks, 256)) {+ if (q_tile_override != 256) {+ throw std::invalid_argument(+ "ark::block_sparse_sdpa: head_dim=128 query block size 256 requires q_tile_override=256");+ }+ if (is_bf16) {+ ark::sdpa_impl_bf16_sparse_sdpa_qtile256_row64k(+ (sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (void*)lut,+ (void*)valid_block_num, num_q_blocks, num_k_blocks, q_tile_override, q_stride_s, q_stride_d, q_stride_h,+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q,+ seq_len_kv, head_dim, softmax_scale, is_causal);+ } else {+ ark::sdpa_impl_fp16_sparse_sdpa_qtile256_row64k(+ (sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (void*)lut,+ (void*)valid_block_num, num_q_blocks, num_k_blocks, q_tile_override, q_stride_s, q_stride_d, q_stride_h,+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q,+ seq_len_kv, head_dim, softmax_scale, is_causal);+ }+ return;+ }+ if (matches_block_size(seq_len_q, num_q_blocks, 64)) {+ if (is_bf16) {+ ark::sdpa_impl_bf16_sparse_sdpa_row_linear(+ (sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (void*)lut,+ (void*)valid_block_num, num_q_blocks, num_k_blocks, q_tile_override, q_stride_s, q_stride_d, q_stride_h,+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q,+ seq_len_kv, head_dim, softmax_scale, is_causal);+ } else {+ ark::sdpa_impl_fp16_sparse_sdpa_row_linear(+ (sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (void*)lut,+ (void*)valid_block_num, num_q_blocks, num_k_blocks, q_tile_override, q_stride_s, q_stride_d, q_stride_h,+ q_stride_b, k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q,+ seq_len_kv, head_dim, softmax_scale, is_causal);+ }+ return;+ }+ throw std::invalid_argument("ark::block_sparse_sdpa: head_dim=128 supports query block sizes 64 and 256 only");+ }+ throw std::invalid_argument("ark::block_sparse_sdpa: unsupported head_dim; supported values are 64 and 128");+}+static void moe_gemm_wrapper(torch_ptr stream, torch_ptr activations, torch_ptr weights, torch_ptr scales,torch_ptr outputs, int dtype, int N, int K, torch_ptr num_tokens_per_expert,int num_experts) {@@ -1550,6 +1642,7 @@ PYBIND11_MODULE(PY_NAME, m) {pybind11::arg("head_dim"), pybind11::arg("softmax_scale"), pybind11::arg("is_causal"),pybind11::arg("tensor_layout"), pybind11::arg("lse") = 0);m.def("sage_sparse", &ark::sage_sparse);+ m.def("block_sparse_sdpa", &ark::block_sparse_sdpa);// Low-level SAGE PVi8 API: input Q/K/V are pre-quantized int8 with qscale/kscale/vscale.m.def("sage_pvi8", &ark::sage_pvi8, pybind11::arg("stream"), pybind11::arg("Q"), pybind11::arg("K"),pybind11::arg("V"), pybind11::arg("O"), pybind11::arg("mask"),diff --git a/auto_round_extension/ark/auto_round_kernel/sdpa_generation.cmake b/auto_round_extension/ark/auto_round_kernel/sdpa_generation.cmakeindex aebbbf8f8..aaea93766 100644--- a/auto_round_extension/ark/auto_round_kernel/sdpa_generation.cmake+++ b/auto_round_extension/ark/auto_round_kernel/sdpa_generation.cmake@@ -70,4 +70,116 @@ foreach(mode IN ITEMS prefill decode)endforeach()endforeach()-configure_file(${CMAKE_CURRENT_LIST_DIR}/sdpa_kernel_declarations.hpp.in ${SDPA_DECLARATIONS_FILE} @ONLY)\ No newline at end of file+configure_file(${CMAKE_CURRENT_LIST_DIR}/sdpa_kernel_declarations.hpp.in ${SDPA_DECLARATIONS_FILE} @ONLY)++# ========================================================================+# Sparse attention instantiations+#+# The sparse families (INT8 qks8 "sage" and native BF16/FP16 "sdpa") used to+# instantiate every causal/mask combination of every launcher inside a single+# translation unit, which peaked around 18 GiB of compiler RSS per TU. Emit one+# translation unit per (family, shape, dtype, causal, mask) combination instead,+# mirroring the dense SDPA split above.+# ========================================================================+set(SDPA_SPARSE_DECLARATIONS_FILE+ ${CMAKE_CURRENT_BINARY_DIR}/generated/sdpa/sdpa_sparse_kernel_declarations.hpp)++function(generate_sdpa_sparse_instantiation family shape launcher el_q el_k el_v dtype_name causal_value+ has_mask_value)+ if(causal_value)+ set(variant_name causal)+ elseif(has_mask_value)+ set(variant_name noncausal_masked)+ else()+ set(variant_name noncausal_unmasked)+ endif()+ set(function_name launch_${family}_sparse_${shape}_${dtype_name}_${variant_name})+ set(SDPA_SPARSE_FUNCTION_NAME ${function_name})+ set(SDPA_SPARSE_TEMPLATE_NAME ${launcher})+ set(SDPA_SPARSE_CAUSAL ${causal_value})+ set(SDPA_SPARSE_HAS_MASK ${has_mask_value})+ set(SDPA_SPARSE_EL_Q ${el_q})+ set(SDPA_SPARSE_EL_K ${el_k})+ set(SDPA_SPARSE_EL_V ${el_v})+ string(APPEND SDPA_SPARSE_KERNEL_DECLARATIONS "int ${function_name}(Options const& options);\n")+ set(SDPA_SPARSE_KERNEL_DECLARATIONS ${SDPA_SPARSE_KERNEL_DECLARATIONS} PARENT_SCOPE)+ set(_generated_src+ ${CMAKE_CURRENT_BINARY_DIR}/generated/sdpa/sparse_${family}_${shape}_${dtype_name}_${variant_name}.cpp)+ configure_file(${CMAKE_CURRENT_LIST_DIR}/sdpa_sparse_kernel_instantiation.cpp.in ${_generated_src} @ONLY)+ list(APPEND SDPA_GENERATED_SRCS ${_generated_src})+ set(SDPA_GENERATED_SRCS ${SDPA_GENERATED_SRCS} PARENT_SCOPE)+endfunction()++# Expands the causal / noncausal-masked / noncausal-unmasked variants for one+# (family, shape, dtype) triple. Causal never needs a mask, matching the+# runtime behaviour of the launchers this replaces.+function(generate_sdpa_sparse_family family shape launcher el_q el_k el_v dtype_name)+ foreach(causal_name IN ITEMS causal noncausal)+ if(causal_name STREQUAL "causal")+ set(causal_value true)+ set(mask_names unmasked)+ else()+ set(causal_value false)+ set(mask_names masked unmasked)+ endif()++ foreach(mask_name IN LISTS mask_names)+ if(mask_name STREQUAL "masked")+ set(has_mask_value true)+ else()+ set(has_mask_value false)+ endif()++ generate_sdpa_sparse_instantiation(${family} ${shape} ${launcher} "${el_q}" "${el_k}" "${el_v}"+ ${dtype_name} ${causal_value} ${has_mask_value})+ endforeach()+ endforeach()++ # generate_sdpa_sparse_instantiation() writes through PARENT_SCOPE, which here+ # is this function's scope. Re-export the accumulators so the caller sees them.+ set(SDPA_GENERATED_SRCS ${SDPA_GENERATED_SRCS} PARENT_SCOPE)+ set(SDPA_SPARSE_KERNEL_DECLARATIONS ${SDPA_SPARSE_KERNEL_DECLARATIONS} PARENT_SCOPE)+endfunction()++# Native BF16/FP16 sparse SDPA (sdpa_sparse_sdpa.cpp).+foreach(shape IN ITEMS 128 128_qtile64 64)+ if(shape STREQUAL "128")+ set(launcher launch_sparse_sdpa_prefill_kernel_128)+ elseif(shape STREQUAL "128_qtile64")+ set(launcher launch_sparse_sdpa_prefill_kernel_128_qtile64)+ else()+ set(launcher launch_sparse_sdpa_prefill_kernel_64)+ endif()++ foreach(dtype_name IN ITEMS bf16 f16)+ if(dtype_name STREQUAL "f16")+ set(element_type "cute::half_t")+ else()+ set(element_type "cute::bfloat16_t")+ endif()+ generate_sdpa_sparse_family(sdpa ${shape} ${launcher} "${element_type}" "${element_type}" "${element_type}"+ ${dtype_name})+ endforeach()+endforeach()++# INT8 qks8 sparse SAGE (sdpa_sparse.cpp): int8 Q/K with half or bf16 P*V.+foreach(shape IN ITEMS 128 128_qtile64 64)+ if(shape STREQUAL "128")+ set(launcher launch_sparse_sage_prefill_kernel_128)+ elseif(shape STREQUAL "128_qtile64")+ set(launcher launch_sparse_sage_prefill_kernel_128_qtile64)+ else()+ set(launcher launch_sparse_sage_prefill_kernel_64)+ endif()++ foreach(dtype_name IN ITEMS i8half i8bf16)+ if(dtype_name STREQUAL "i8half")+ set(pv_type "cute::half_t")+ else()+ set(pv_type "cute::bfloat16_t")+ endif()+ generate_sdpa_sparse_family(sage ${shape} ${launcher} "cute::int8_t" "cute::int8_t" "${pv_type}" ${dtype_name})+ endforeach()+endforeach()++configure_file(${CMAKE_CURRENT_LIST_DIR}/sdpa_sparse_kernel_declarations.hpp.in ${SDPA_SPARSE_DECLARATIONS_FILE} @ONLY)\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/sdpa_sparse.cpp b/auto_round_extension/ark/auto_round_kernel/sdpa_sparse.cppindex 2f4dfe598..79e7c10f7 100644--- a/auto_round_extension/ark/auto_round_kernel/sdpa_sparse.cpp+++ b/auto_round_extension/ark/auto_round_kernel/sdpa_sparse.cpp@@ -21,6 +21,7 @@#include <sycl/sycl.hpp>#include "sycl_tla_sdpa_sparse.hpp"+#include "sdpa_sparse_kernel_declarations.hpp"namespace ark {@@ -30,49 +31,42 @@ namespace {using KernelLauncher = int (*)(detail::Options const& options);-int launch_prefill_kernel_f16_128_sparse_sage(detail::Options const& options) {- return detail::launch_sparse_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::half_t>(options);-}--int launch_prefill_kernel_f16_128_sparse_sage_qtile64(detail::Options const& options) {- return detail::launch_sparse_sage_prefill_kernel_128_qtile64<cute::int8_t, cute::int8_t, cute::half_t>(options);-}--int launch_prefill_kernel_f16_64_sparse_sage(detail::Options const& options) {- return detail::launch_sparse_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::half_t>(options);-}--int launch_prefill_kernel_bf16_128_sparse_sage(detail::Options const& options) {- return detail::launch_sparse_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::bfloat16_t>(options);-}+// The launcher bodies live one variant per translation unit; see the sparse+// section of sdpa_generation.cmake. Causal never carries a mask, and the sparse+// path never uses cached KV, which matches the behaviour of the runtime+// selection these replaced.+#define ARK_SPARSE_SAGE_LAUNCHER(shape, dtype) \+ do { \+ if (is_causal) return detail::launch_sage_sparse_##shape##_##dtype##_causal; \+ if (has_mask) return detail::launch_sage_sparse_##shape##_##dtype##_noncausal_masked; \+ return detail::launch_sage_sparse_##shape##_##dtype##_noncausal_unmasked; \+ } while (0)-int launch_prefill_kernel_bf16_128_sparse_sage_qtile64(detail::Options const& options) {- return detail::launch_sparse_sage_prefill_kernel_128_qtile64<cute::int8_t, cute::int8_t, cute::bfloat16_t>(options);-}--int launch_prefill_kernel_bf16_64_sparse_sage(detail::Options const& options) {- return detail::launch_sparse_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::bfloat16_t>(options);-}--KernelLauncher select_sparse_sage_prefill_launcher(BTLA_DTYPE pv_dtype, int head_dim, int q_tile_override) {+KernelLauncher select_sparse_sage_prefill_launcher(BTLA_DTYPE pv_dtype, int head_dim, int q_tile_override,+ bool is_causal, bool has_mask) {+ if (is_causal && has_mask) {+ return nullptr;+ }switch (head_dim) {case 128:if (q_tile_override == 64) {- return pv_dtype == BTLA_DTYPE::BF16 ? launch_prefill_kernel_bf16_128_sparse_sage_qtile64- : launch_prefill_kernel_f16_128_sparse_sage_qtile64;+ if (pv_dtype == BTLA_DTYPE::BF16) ARK_SPARSE_SAGE_LAUNCHER(128_qtile64, i8bf16);+ ARK_SPARSE_SAGE_LAUNCHER(128_qtile64, i8half);}if (q_tile_override != 0 && q_tile_override != 256) return nullptr;- return pv_dtype == BTLA_DTYPE::BF16 ? launch_prefill_kernel_bf16_128_sparse_sage- : launch_prefill_kernel_f16_128_sparse_sage;+ if (pv_dtype == BTLA_DTYPE::BF16) ARK_SPARSE_SAGE_LAUNCHER(128, i8bf16);+ ARK_SPARSE_SAGE_LAUNCHER(128, i8half);case 64:if (q_tile_override != 0 && q_tile_override != 128) return nullptr;- return pv_dtype == BTLA_DTYPE::BF16 ? launch_prefill_kernel_bf16_64_sparse_sage- : launch_prefill_kernel_f16_64_sparse_sage;+ if (pv_dtype == BTLA_DTYPE::BF16) ARK_SPARSE_SAGE_LAUNCHER(64, i8bf16);+ ARK_SPARSE_SAGE_LAUNCHER(64, i8half);default:return nullptr;}}+#undef ARK_SPARSE_SAGE_LAUNCHER+detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int q_stride_s,int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,@@ -144,7 +138,8 @@ void sparse_sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,options.num_k_blocks = num_k_blocks;compat::set_default_queue(*q);- KernelLauncher launcher = select_sparse_sage_prefill_launcher(pv_dtype, head_dim, effective_q_tile_override);+ KernelLauncher launcher =+ select_sparse_sage_prefill_launcher(pv_dtype, head_dim, effective_q_tile_override, is_causal, mask != nullptr);if (launcher == nullptr) {throw std::runtime_error("Unsupported sparse_sage_prefill config");}diff --git a/auto_round_extension/ark/auto_round_kernel/sdpa_sparse_kernel_declarations.hpp.in b/auto_round_extension/ark/auto_round_kernel/sdpa_sparse_kernel_declarations.hpp.innew file mode 100644index 000000000..d0f67abdd--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/sdpa_sparse_kernel_declarations.hpp.in@@ -0,0 +1,15 @@+// Generated by CMake. Do not edit directly.++#pragma once++#if defined(ARK_SYCL_TLA)++namespace ark {+namespace sparse_detail {++@SDPA_SPARSE_KERNEL_DECLARATIONS@++} // namespace sparse_detail+} // namespace ark++#endif // ARK_SYCL_TLAdiff --git a/auto_round_extension/ark/auto_round_kernel/sdpa_sparse_kernel_instantiation.cpp.in b/auto_round_extension/ark/auto_round_kernel/sdpa_sparse_kernel_instantiation.cpp.innew file mode 100644index 000000000..89f23cb18--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/sdpa_sparse_kernel_instantiation.cpp.in@@ -0,0 +1,19 @@+// Generated by CMake. Do not edit directly.++#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)++#include <sycl/sycl.hpp>+#include "sycl_tla_sdpa_sparse.hpp"++namespace ark {+namespace sparse_detail {++int @SDPA_SPARSE_FUNCTION_NAME@(Options const& options) {+ return @SDPA_SPARSE_TEMPLATE_NAME@<@SDPA_SPARSE_CAUSAL@, @SDPA_SPARSE_HAS_MASK@, @SDPA_SPARSE_EL_Q@,+ @SDPA_SPARSE_EL_K@, @SDPA_SPARSE_EL_V@>(options);+}++} // namespace sparse_detail+} // namespace ark++#endif // ARK_XPU && ARK_SYCL_TLAdiff --git a/auto_round_extension/ark/auto_round_kernel/sdpa_sparse_sdpa.cpp b/auto_round_extension/ark/auto_round_kernel/sdpa_sparse_sdpa.cppnew file mode 100644index 000000000..77bfc99ec--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/sdpa_sparse_sdpa.cpp@@ -0,0 +1,324 @@+// Copyright (c) 2026 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++// Independent native-precision sparse SDPA path (BF16 + FP16). This dispatches to+// the sparse SDPA mainloop (SPARSESDPAFwdMainloop via SparseSDPAConfig), separate+// from the INT8-centric sparse SAGE path in sdpa_sparse.cpp.++#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)++#include <cute/numeric/numeric_types.hpp>+#include <sycl/aliases.hpp>+#include "bestla/bestla.h"++#include <sycl/sycl.hpp>+#include "sycl_tla_sdpa_sparse.hpp"+#include "sdpa_sparse_kernel_declarations.hpp"++namespace ark {++namespace detail = sparse_detail;++namespace {++using KernelLauncher = int (*)(detail::Options const& options);++// The launcher bodies live one variant per translation unit; see the sparse+// section of sdpa_generation.cmake. Causal never carries a mask, and the sparse+// path never uses cached KV, which matches the behaviour of the runtime+// selection these replaced.+#define ARK_SPARSE_SDPA_LAUNCHER(shape, dtype) \+ do { \+ if (is_causal) return detail::launch_sdpa_sparse_##shape##_##dtype##_causal; \+ if (has_mask) return detail::launch_sdpa_sparse_##shape##_##dtype##_noncausal_masked; \+ return detail::launch_sdpa_sparse_##shape##_##dtype##_noncausal_unmasked; \+ } while (0)++KernelLauncher select_sparse_sdpa_prefill_launcher(BTLA_DTYPE q_dtype, int head_dim, int q_tile_override,+ bool is_causal, bool has_mask) {+ if (is_causal && has_mask) {+ return nullptr;+ }+ switch (head_dim) {+ case 128:+ if (q_tile_override == 64) {+ if (q_dtype == BTLA_DTYPE::BF16) ARK_SPARSE_SDPA_LAUNCHER(128_qtile64, bf16);+ ARK_SPARSE_SDPA_LAUNCHER(128_qtile64, f16);+ }+ if (q_tile_override != 0 && q_tile_override != 256) return nullptr;+ if (q_dtype == BTLA_DTYPE::BF16) ARK_SPARSE_SDPA_LAUNCHER(128, bf16);+ ARK_SPARSE_SDPA_LAUNCHER(128, f16);+ case 64:+ if (q_tile_override != 0 && q_tile_override != 64 && q_tile_override != 128) return nullptr;+ if (q_dtype == BTLA_DTYPE::BF16) ARK_SPARSE_SDPA_LAUNCHER(64, bf16);+ ARK_SPARSE_SDPA_LAUNCHER(64, f16);+ default:+ return nullptr;+ }+}++#undef ARK_SPARSE_SDPA_LAUNCHER++detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int q_stride_s,+ int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s, int k_stride_d,+ int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s, int v_stride_h,+ int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,+ int head_dim, float softmax_scale, bool is_causal) {+ if (q_stride_d != 1 || k_stride_d != 1 || v_stride_d != 1 || o_stride_d != 1) {+ throw std::invalid_argument("make_common_options: head-dim stride must be 1 for Q/K/V/O");+ }+ detail::Options options;+ options.q = Q_ptr;+ options.k = K_ptr;+ options.v = V_ptr;+ options.mask = mask;+ options.o = O_ptr;+ options.use_tensor_strides = true;+ options.q_stride_s = q_stride_s;+ options.q_stride_d = q_stride_d;+ options.q_stride_h = q_stride_h;+ options.q_stride_b = q_stride_b;+ options.k_stride_s = k_stride_s;+ options.k_stride_d = k_stride_d;+ options.k_stride_h = k_stride_h;+ options.k_stride_b = k_stride_b;+ options.v_stride_d = v_stride_d;+ options.v_stride_s = v_stride_s;+ options.v_stride_h = v_stride_h;+ options.v_stride_b = v_stride_b;+ options.o_stride_s = o_stride_s;+ options.o_stride_d = o_stride_d;+ options.o_stride_h = o_stride_h;+ options.o_stride_b = o_stride_b;+ options.batch = batch;+ options.num_heads_q = num_heads_q;+ options.num_heads_kv = num_heads_kv;+ options.seq_len_qo = seq_len_q;+ options.seq_len_kv = seq_len_kv;+ options.head_size_qk = head_dim;+ options.head_size_vo = head_dim;+ options.softmax_scale = softmax_scale;+ options.is_causal = is_causal;+ return options;+}++void sparse_sdpa_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut,+ void* valid_block_num, int num_q_blocks, int num_k_blocks, int q_tile_override,+ int q_stride_s, int q_stride_d, int q_stride_h, int q_stride_b, int k_stride_s,+ int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h,+ int o_stride_b, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,+ int head_dim, float softmax_scale, bool is_causal, BTLA_DTYPE q_dtype,+ int sparse_q_block_size = 0) {+ const int effective_q_tile_override =+ (head_dim == 128 && q_tile_override == 0) ? 64 : ((head_dim == 64 && q_tile_override == 0) ? 64 : q_tile_override);+ detail::Options options =+ make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_stride_s, q_stride_d, q_stride_h, q_stride_b,+ k_stride_s, k_stride_d, k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h,+ v_stride_b, o_stride_s, o_stride_d, o_stride_h, o_stride_b, batch, num_heads_q,+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ // Native-precision path: no INT8 dequant scales, softmax scale applied directly.+ options.scale_block_size = 0;+ options.sparse_q_block_size = sparse_q_block_size;+ options.q_tile_override = effective_q_tile_override;+ options.qscale = nullptr;+ options.kscale = nullptr;+ options.vscale = nullptr;+ options.lut = static_cast<int const*>(lut);+ options.valid_block_num = static_cast<int const*>(valid_block_num);+ options.num_q_blocks = num_q_blocks;+ options.num_k_blocks = num_k_blocks;+ compat::set_default_queue(*q);++ KernelLauncher launcher =+ select_sparse_sdpa_prefill_launcher(q_dtype, head_dim, effective_q_tile_override, is_causal, mask != nullptr);+ if (launcher == nullptr) {+ throw std::runtime_error("Unsupported sparse_sdpa_prefill config");+ }++ launcher(options);+}++} // namespace++void sdpa_impl_bf16_sparse_sdpa_d64(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal) {+ if (mask && is_causal) {+ throw std::invalid_argument("sdpa_impl_bf16_sparse_sdpa_d64: mask and is_causal cannot both be set");+ }+ if (seq_len_q <= 0 || seq_len_kv <= 0) {+ throw std::invalid_argument("sdpa_impl_bf16_sparse_sdpa_d64: seq_len_q and seq_len_kv must be greater than 0");+ }+ if (lut == nullptr || valid_block_num == nullptr) {+ throw std::invalid_argument("sdpa_impl_bf16_sparse_sdpa_d64: lut and valid_block_num must be provided");+ }+ if (num_q_blocks <= 0 || num_k_blocks <= 0) {+ throw std::invalid_argument("sdpa_impl_bf16_sparse_sdpa_d64: num_q_blocks and num_k_blocks must be greater than 0");+ }+ if (head_dim != 64) {+ throw std::invalid_argument("sdpa_impl_bf16_sparse_sdpa_d64: head_dim must be 64");+ }++ sparse_sdpa_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, lut, valid_block_num, num_q_blocks, num_k_blocks,+ q_tile_override, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,+ k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,+ o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,+ softmax_scale, is_causal, BTLA_DTYPE::BF16, /*sparse_q_block_size=*/64);+}++void sdpa_impl_fp16_sparse_sdpa_d64(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal) {+ if (mask && is_causal) {+ throw std::invalid_argument("sdpa_impl_fp16_sparse_sdpa_d64: mask and is_causal cannot both be set");+ }+ if (seq_len_q <= 0 || seq_len_kv <= 0) {+ throw std::invalid_argument("sdpa_impl_fp16_sparse_sdpa_d64: seq_len_q and seq_len_kv must be greater than 0");+ }+ if (lut == nullptr || valid_block_num == nullptr) {+ throw std::invalid_argument("sdpa_impl_fp16_sparse_sdpa_d64: lut and valid_block_num must be provided");+ }+ if (num_q_blocks <= 0 || num_k_blocks <= 0) {+ throw std::invalid_argument("sdpa_impl_fp16_sparse_sdpa_d64: num_q_blocks and num_k_blocks must be greater than 0");+ }+ if (head_dim != 64) {+ throw std::invalid_argument("sdpa_impl_fp16_sparse_sdpa_d64: head_dim must be 64");+ }++ sparse_sdpa_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, lut, valid_block_num, num_q_blocks, num_k_blocks,+ q_tile_override, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,+ k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,+ o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,+ softmax_scale, is_causal, BTLA_DTYPE::F16, /*sparse_q_block_size=*/64);+}++void sdpa_impl_bf16_sparse_sdpa_row_linear(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal) {+ if (q_tile_override != 0 && q_tile_override != 64) {+ throw std::invalid_argument("sdpa_impl_bf16_sparse_sdpa_row_linear: q_tile_override must be 0 or 64");+ }+ if (mask && is_causal) {+ throw std::invalid_argument("sdpa_impl_bf16_sparse_sdpa_row_linear: mask and is_causal cannot both be set");+ }+ if (seq_len_q <= 0 || seq_len_kv <= 0) {+ throw std::invalid_argument(+ "sdpa_impl_bf16_sparse_sdpa_row_linear: seq_len_q and seq_len_kv must be greater than 0");+ }+ if (lut == nullptr || valid_block_num == nullptr) {+ throw std::invalid_argument("sdpa_impl_bf16_sparse_sdpa_row_linear: lut and valid_block_num must be provided");+ }+ if (num_q_blocks <= 0 || num_k_blocks <= 0) {+ throw std::invalid_argument(+ "sdpa_impl_bf16_sparse_sdpa_row_linear: num_q_blocks and num_k_blocks must be greater than 0");+ }++ sparse_sdpa_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, lut, valid_block_num, num_q_blocks, num_k_blocks, 64,+ q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h,+ k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h,+ o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,+ is_causal, BTLA_DTYPE::BF16, /*sparse_q_block_size=*/64);+}++void sdpa_impl_fp16_sparse_sdpa_row_linear(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal) {+ if (q_tile_override != 0 && q_tile_override != 64) {+ throw std::invalid_argument("sdpa_impl_fp16_sparse_sdpa_row_linear: q_tile_override must be 0 or 64");+ }+ if (mask && is_causal) {+ throw std::invalid_argument("sdpa_impl_fp16_sparse_sdpa_row_linear: mask and is_causal cannot both be set");+ }+ if (seq_len_q <= 0 || seq_len_kv <= 0) {+ throw std::invalid_argument(+ "sdpa_impl_fp16_sparse_sdpa_row_linear: seq_len_q and seq_len_kv must be greater than 0");+ }+ if (lut == nullptr || valid_block_num == nullptr) {+ throw std::invalid_argument("sdpa_impl_fp16_sparse_sdpa_row_linear: lut and valid_block_num must be provided");+ }+ if (num_q_blocks <= 0 || num_k_blocks <= 0) {+ throw std::invalid_argument(+ "sdpa_impl_fp16_sparse_sdpa_row_linear: num_q_blocks and num_k_blocks must be greater than 0");+ }++ sparse_sdpa_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, lut, valid_block_num, num_q_blocks, num_k_blocks, 64,+ q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d, k_stride_h,+ k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d, o_stride_h,+ o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,+ is_causal, BTLA_DTYPE::F16, /*sparse_q_block_size=*/64);+}++void sdpa_impl_bf16_sparse_sdpa_qtile256_row64k(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal) {+ if (q_tile_override != 256) {+ throw std::invalid_argument("sdpa_impl_bf16_sparse_sdpa_qtile256_row64k: q_tile_override must be 256");+ }+ if (head_dim != 128) {+ throw std::invalid_argument("sdpa_impl_bf16_sparse_sdpa_qtile256_row64k: head_dim must be 128");+ }++ sparse_sdpa_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, lut, valid_block_num, num_q_blocks, num_k_blocks,+ q_tile_override, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,+ k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,+ o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,+ softmax_scale, is_causal, BTLA_DTYPE::BF16, /*sparse_q_block_size=*/256);+}++void sdpa_impl_fp16_sparse_sdpa_qtile256_row64k(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal) {+ if (q_tile_override != 256) {+ throw std::invalid_argument("sdpa_impl_fp16_sparse_sdpa_qtile256_row64k: q_tile_override must be 256");+ }+ if (head_dim != 128) {+ throw std::invalid_argument("sdpa_impl_fp16_sparse_sdpa_qtile256_row64k: head_dim must be 128");+ }++ sparse_sdpa_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, lut, valid_block_num, num_q_blocks, num_k_blocks,+ q_tile_override, q_stride_s, q_stride_d, q_stride_h, q_stride_b, k_stride_s, k_stride_d,+ k_stride_h, k_stride_b, v_stride_d, v_stride_s, v_stride_h, v_stride_b, o_stride_s, o_stride_d,+ o_stride_h, o_stride_b, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,+ softmax_scale, is_causal, BTLA_DTYPE::F16, /*sparse_q_block_size=*/256);+}++} // namespace ark++#endif // ARK_XPU && ARK_SYCL_TLAdiff --git a/auto_round_extension/ark/auto_round_kernel/sparge_preprocess_triton.py b/auto_round_extension/ark/auto_round_kernel/sparge_preprocess_triton.pyindex d94263c47..2fe1091f8 100644--- a/auto_round_extension/ark/auto_round_kernel/sparge_preprocess_triton.py+++ b/auto_round_extension/ark/auto_round_kernel/sparge_preprocess_triton.py@@ -16,7 +16,11 @@ _APPLIED_TRITON_PREDICATED_CHECK = Falsedef _apply_xpu_triton_workarounds() -> None:- """Patch triton's Intel compiler so JIT kernels avoid the rejected SPIR-V extension."""+ """Patch triton's Intel compiler so JIT kernels avoid the rejected SPIR-V extension.++ Triton 3.7.x can emit SPV_INTEL_predicated_io for non-LTS Intel GPU drivers,+ but some level-zero loaders reject that extension at kernel-load time.+ """global _APPLIED_TRITON_PREDICATED_CHECKif _APPLIED_TRITON_PREDICATED_CHECK:return@@ -416,8 +420,9 @@ def _run_triton_xpu_preprocess(ctx: Any) -> dict[str, Any]:.contiguous())final_tile_map = torch.zeros_like(pooled_prob, dtype=torch.bool)- final_tile_map[~sim_k_expand] = True- final_tile_map[~sim_q_expand] = True+ # Boolean-mask assignment (`t[mask] = val`) is unreliable on XPU; use |= instead.+ final_tile_map |= ~sim_k_expand+ final_tile_map |= ~sim_q_expandfinal_tile_map = _fill_block_map_triton(final_tile_map, num_to_select, sorted_prob.indices)if causal_mask is not None:final_tile_map &= causal_mask.view(1, 1, ctx.num_q_tiles, num_k_route_blocks)diff --git a/auto_round_extension/ark/auto_round_kernel/sparse_attention.py b/auto_round_extension/ark/auto_round_kernel/sparse_attention.pyindex 0d5932747..e45febf88 100644--- a/auto_round_extension/ark/auto_round_kernel/sparse_attention.py+++ b/auto_round_extension/ark/auto_round_kernel/sparse_attention.py@@ -199,6 +199,122 @@ def sage_sparse(return O+def block_sparse_sdpa(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ lut: torch.Tensor,+ valid_block_num: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ q_tile_override: int = 0,+ sparse_q_block_tokens: int | None = None,+ sparse_k_block_tokens: int | None = None,+ tensor_layout: str = "HND",+) -> torch.Tensor:+ """Low-level native-precision sparse attention (BF16/FP16) routed by sparse LUT metadata.++ Independent path built on the dense-SDPA sparse mainloop (SparseSDPAConfig).+ Unlike the INT8 ``sage_sparse`` path it does not reuse the INT8-centric SAGE+ mainloop; Q/K/V are native BF16/FP16 and the softmax scale is applied directly.+ """+ del dropout_p, enable_gqa+ if query.device.type != "xpu":+ raise NotImplementedError("block_sparse_sdpa is only supported on XPU")+ if query.dtype not in (torch.bfloat16, torch.float16):+ raise ValueError(f"Q/K/V must be bfloat16 or float16, got {query.dtype}")+ if key.dtype != query.dtype or value.dtype != query.dtype:+ raise ValueError(f"Q/K/V dtypes must match, got Q={query.dtype}, K={key.dtype}, V={value.dtype}")+ if lut.dtype != torch.int32 or valid_block_num.dtype != torch.int32:+ raise ValueError("lut and valid_block_num must be int32 tensors")+ if lut.device != query.device or valid_block_num.device != query.device:+ raise ValueError("lut and valid_block_num must be on the same XPU device as Q/K/V")++ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout)+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout)+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout)++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ _validate_gqa_head_config(Hq, Hkv, op_name="block_sparse_sdpa")+ if D not in (64, 128):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")+ effective_sparse_q_block_tokens = 64 if sparse_q_block_tokens is None else int(sparse_q_block_tokens)+ effective_sparse_k_block_tokens = 64 if sparse_k_block_tokens is None else int(sparse_k_block_tokens)+ if effective_sparse_q_block_tokens <= 0 or effective_sparse_k_block_tokens <= 0:+ raise ValueError(+ "sparse_q_block_tokens and sparse_k_block_tokens must be positive when provided; "+ f"got {effective_sparse_q_block_tokens} and {effective_sparse_k_block_tokens}"+ )++ q_sparse_blocks = (Sq + effective_sparse_q_block_tokens - 1) // effective_sparse_q_block_tokens+ kv_sparse_blocks = (Skv + effective_sparse_k_block_tokens - 1) // effective_sparse_k_block_tokens+ if tuple(lut.shape) != (B, Hq, q_sparse_blocks, kv_sparse_blocks):+ raise ValueError(f"lut must have shape {(B, Hq, q_sparse_blocks, kv_sparse_blocks)}, got {tuple(lut.shape)}")+ if tuple(valid_block_num.shape) != (B, Hq, q_sparse_blocks):+ raise ValueError(+ f"valid_block_num must have shape {(B, Hq, q_sparse_blocks)}, got {tuple(valid_block_num.shape)}"+ )+ if torch.any(valid_block_num < 0).item():+ raise ValueError("valid_block_num entries must be non-negative")+ if torch.any(valid_block_num > kv_sparse_blocks).item():+ raise ValueError(f"valid_block_num entries must be <= {kv_sparse_blocks}")++ if D == 64 and q_tile_override not in (0, 64, 128):+ raise ValueError(+ f"q_tile_override must be one of {{0, 64, 128}} for block_sparse_sdpa with head_dim=64, got {q_tile_override}"+ )++ lib = get_lib(query)+ if not hasattr(lib, "block_sparse_sdpa"):+ raise RuntimeError("Loaded XPU extension does not expose block_sparse_sdpa")+ effective_q_tile_override = _resolve_sparse_prefill_q_tile_override(+ head_dim=D,+ quant_block_size=64,+ q_tile_override=q_tile_override,+ sparse_q_block_tokens=effective_sparse_q_block_tokens,+ sparse_k_block_tokens=effective_sparse_k_block_tokens,+ tensor_layout=tensor_layout,+ )+ stream = get_stream(query)+ O = _empty_attention_output(B, Hq, Sq, D, dtype=value.dtype, device=query.device, tensor_layout=tensor_layout)+ q_dtype = 1 if query.dtype == torch.bfloat16 else 0 # FlashAttnDtype: 0=FP16, 1=BF16+ lib.block_sparse_sdpa(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ lut.data_ptr(),+ valid_block_num.data_ptr(),+ q_sparse_blocks,+ kv_sparse_blocks,+ effective_q_tile_override,+ *_attention_strides_qko(query, tensor_layout),+ *_attention_strides_qko(key, tensor_layout),+ *_attention_strides_v(value, tensor_layout),+ *_attention_strides_qko(O, tensor_layout),+ q_dtype,+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0 / (D**0.5),+ bool(is_causal),+ )+ return O++def sage_sparse_row_linear(query: torch.Tensor,key: torch.Tensor,@@ -399,8 +515,10 @@ def _normalize_per_head_hparam(def _query_tile_tokens_for_head_dim(head_dim: int) -> int:+ # Keep the preprocess default aligned with the sparse e2e wrappers: for head_dim 64+ # the wrappers default to 64-token query tiles, for head_dim 128 to 64 as well.if head_dim == 64:- return 128+ return 64if head_dim == 128:return 64raise ValueError(f"Unsupported head_dim={head_dim}; supported: 64, 128")@@ -616,19 +734,25 @@ def _build_block_causal_mask(def _fill_block_map_torch(final_map: torch.Tensor, num_to_select: torch.Tensor, sorted_indices: torch.Tensor) -> torch.Tensor:+ """Fill ``final_map`` with the top ``num_to_select`` NEW blocks in sorted order.++ The naive implementation looped over every K block (``for rank in range(k_blocks)``),+ launching a handful of elementwise kernels per iteration. At long sequences that is+ thousands of sequential kernel launches and dominates the torch preprocess. This+ version is fully vectorized: for a block at sorted ``rank r`` it is the+ ``(r - #already_filled_before_r)``-th *new* block, and it is selected iff that index+ is below the per-row quota.+ """k_blocks = final_map.shape[-1]- filled = final_map.clone()- column_ids = torch.arange(k_blocks, device=final_map.device).view(1, 1, 1, k_blocks)target_new = torch.maximum(num_to_select, torch.ones_like(num_to_select))- added = torch.zeros_like(num_to_select)- for rank in range(k_blocks):- idx_match = column_ids == sorted_indices[..., rank : rank + 1]- is_new = idx_match & ~filled- should_add = (added < target_new).unsqueeze(-1)- newly_selected = should_add & is_new- filled |= newly_selected- added = added + newly_selected.any(dim=-1).to(added.dtype)- return filled+ already = torch.gather(final_map, -1, sorted_indices) # is the block at this rank already filled?+ already_cum = torch.cumsum(already.to(torch.int64), dim=-1) - already.to(torch.int64)+ rank = torch.arange(k_blocks, device=final_map.device).view(1, 1, 1, k_blocks)+ new_idx = rank - already_cum # 0-based index among new blocks at this rank+ add_new = (new_idx < target_new.unsqueeze(-1)) & ~already+ chosen = torch.zeros_like(final_map)+ chosen.scatter_(-1, sorted_indices, add_new)+ return final_map | chosendef _block_map_lut_torch(block_map: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:@@ -637,8 +761,11 @@ def _block_map_lut_torch(block_map: torch.Tensor) -> tuple[torch.Tensor, torch.Tone_matrix = torch.ones(block_map.shape, dtype=torch.int32, device=block_map.device)cum_matrix = torch.cumsum(one_matrix, dim=-1)masked_cum_matrix = cum_matrix * block_map.to(torch.int32)- filled_matrix = masked_cum_matrix.clone()- filled_matrix[~block_map] = 10_000_000+ # NOTE: do not use `filled_matrix[~block_map] = 10_000_000` here — boolean-mask+ # in-place assignment silently misses entries on the XPU backend (torch-xpu-ops),+ # leaving 0s in the mask that become -1 in the LUT after sort and drive the+ # ScatterGatherKernels out-of-bounds assert downstream. Use torch.where instead.+ filled_matrix = torch.where(block_map, masked_cum_matrix, torch.full_like(masked_cum_matrix, 10_000_000))lut = torch.sort(filled_matrix, dim=-1)[0] - 1lut[..., 1:] = lut[..., 1:] - lut[..., :-1]invalid_mask = torch.arange(num_k_blocks, device=block_map.device).view(@@ -648,6 +775,27 @@ def _block_map_lut_torch(block_map: torch.Tensor) -> tuple[torch.Tensor, torch.Treturn lut.to(torch.int32).contiguous(), valid_block_num.to(torch.int32).contiguous()+def _lut_to_block_map(lut: torch.Tensor, valid_block_num: torch.Tensor) -> torch.Tensor:+ if lut.dtype != torch.int32 or valid_block_num.dtype != torch.int32:+ raise ValueError("lut and valid_block_num must be int32 tensors")+ if lut.shape[:-1] != valid_block_num.shape:+ raise ValueError(+ f"lut and valid_block_num shape mismatch: expected lut.shape[:-1] == valid_block_num.shape, "+ f"got {tuple(lut.shape)} and {tuple(valid_block_num.shape)}"+ )+ # Run-length encoded LUT: cumsum recovers the 0-indexed selected block positions.+ logical_blocks = torch.cumsum(lut.to(torch.int64), dim=-1)+ # Scatter True at every recovered position. The invalid tail of `logical_blocks`+ # plateaus at the last selected position, so those duplicate indices do not add+ # spurious blocks. Do NOT scatter the tail with False (e.g. `scatter_(-1, idx, mask)`):+ # masked-out positions all index slot 0, and their trailing False writes clobber a+ # selected block at position 0, silently dropping it. Empty rows are cleared below.+ block_map = torch.zeros_like(lut, dtype=torch.bool)+ block_map.scatter_(-1, logical_blocks, True)+ block_map &= valid_block_num.unsqueeze(-1) > 0+ return block_map.contiguous()++def _prefix_keep_cross_attn_enabled() -> bool:return os.getenv("SPARGE_KEEP_PREFIX_ON_CROSS_ATTN", "0").strip().lower() in {"1", "true", "yes"}@@ -1088,8 +1236,8 @@ def _sparge_preprocess_topk_torch_impl(ctx: _SpargePreprocessContext) -> dict[st.contiguous())tail_map = torch.zeros_like(tail_prob, dtype=torch.bool)- tail_map[~tail_sim_k_expand] = True- tail_map[~tail_sim_q_expand] = True+ tail_map |= ~tail_sim_k_expand+ tail_map |= ~tail_sim_q_expandtail_map = _fill_block_map_torch(tail_map, num_to_select_tail, tail_sorted_prob.indices)if ctx.cdfthreshd is not None:prefix_mass = pooled_prob[..., :prefix_route_blocks].sum(dim=-1, keepdim=True)@@ -1111,8 +1259,9 @@ def _sparge_preprocess_topk_torch_impl(ctx: _SpargePreprocessContext) -> dict[st.contiguous())final_tile_map = torch.zeros_like(pooled_prob, dtype=torch.bool)- final_tile_map[~sim_k_expand] = True- final_tile_map[~sim_q_expand] = True+ # Boolean-mask assignment (`t[mask] = val`) is unreliable on XPU; use |= instead.+ final_tile_map |= ~sim_k_expand+ final_tile_map |= ~sim_q_expandfinal_tile_map = _fill_block_map_torch(final_tile_map, num_to_select, sorted_prob.indices)if ctx.cdfthreshd is not None:final_tile_map |= _select_blocks_for_cdf(@@ -1383,17 +1532,31 @@ def sparge_sage2_attn_meansim_topk_xpu(effective_query_tile_tokens = query_tile_tokenseffective_q_tile_override = q_tile_override+ if D == 64 and effective_query_tile_tokens is None and q_tile_override == 0:+ effective_query_tile_tokens = 64if effective_query_tile_tokens is None and q_tile_override in (64, 128, 256):effective_query_tile_tokens = q_tile_overrideelif effective_query_tile_tokens is not None:if q_tile_override == 0:- effective_q_tile_override = int(effective_query_tile_tokens)+ effective_q_tile_override = (+ 0 if D == 64 and int(effective_query_tile_tokens) == 64 else int(effective_query_tile_tokens)+ )elif q_tile_override != int(effective_query_tile_tokens):raise ValueError("query_tile_tokens and q_tile_override must match when both are set; "f"got query_tile_tokens={effective_query_tile_tokens}, q_tile_override={q_tile_override}")- _validate_sparse_q_tile_override_for_head_dim(D, effective_q_tile_override)+ if D == 64:+ supported_q_tiles = (0, 64, 128)+ elif D == 128:+ supported_q_tiles = (0, 64, 256)+ else:+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")+ if effective_q_tile_override not in supported_q_tiles:+ raise ValueError(+ f"q_tile_override={effective_q_tile_override} is not supported for BF16 sparse head_dim={D}; "+ f"supported values: {', '.join(str(v) for v in supported_q_tiles)}"+ )normalized_mask = _normalize_sparse_mask(attn_mask, B, Sq, Skv, query.device)metadata = sparge_preprocess_topk(@@ -1438,3 +1601,112 @@ def sparge_sage2_attn_meansim_topk_xpu(if return_sparsity:return out, sparsity_ratioreturn out+++def sparge_sage2_attn_meansim_topk_xpu_sdpa(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ smooth_k: bool = True,+ simthreshd1: float | torch.Tensor = -0.1,+ cdfthreshd: float | torch.Tensor | None = None,+ topk: float | torch.Tensor = 0.5,+ pvthreshd: float | torch.Tensor = 50,+ attention_sink: bool = False,+ tensor_layout: str = "HND",+ output_dtype: torch.dtype | None = None,+ return_sparsity: bool = False,+ return_metadata: bool = False,+ k_quant_granularity: int = 64,+ query_tile_tokens: int | None = None,+ q_tile_override: int = 0,+ sparse_q_block_tokens: int | None = None,+ sparse_k_block_tokens: int | None = None,+) -> torch.Tensor | tuple[Any, ...]:+ """E2E sparse attention on the independent native-precision (BF16/FP16) SDPA path."""+ if query.device.type != "xpu":+ raise NotImplementedError("sparge_sage2_attn_meansim_topk_xpu_sdpa is only supported on XPU")+ if dropout_p != 0.0:+ raise NotImplementedError("dropout_p must be 0.0 for sparge_sage2_attn_meansim_topk_xpu_sdpa")+ if attn_mask is not None and is_causal:+ raise ValueError("attn_mask and is_causal cannot both be set")+ if query.dtype not in (torch.bfloat16, torch.float16):+ raise ValueError(+ "sparge_sage2_attn_meansim_topk_xpu_sdpa requires bfloat16 or float16 Q/K/V, "+ f"got Q={query.dtype}, K={key.dtype}, V={value.dtype}"+ )+ if key.dtype != query.dtype or value.dtype != query.dtype:+ raise ValueError(f"Q/K/V dtypes must match, got Q={query.dtype}, K={key.dtype}, V={value.dtype}")+ if output_dtype is not None and output_dtype != query.dtype:+ raise ValueError(f"output_dtype must match Q dtype in the current implementation, got {output_dtype}")+ if pvthreshd not in (None, 50):+ warnings.warn("pvthreshd is not supported by the current ARK sparse kernel and is ignored", stacklevel=2)++ B, Hq, Sq, D = _validate_attention_tensor(query, "Q", tensor_layout, expected_dtype=query.dtype)+ Bk, Hkv, Skv, Dk = _validate_attention_tensor(key, "K", tensor_layout, expected_dtype=key.dtype)+ Bv, Hkv2, Skv2, Dv = _validate_attention_tensor(value, "V", tensor_layout, expected_dtype=value.dtype)+ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")++ effective_query_tile_tokens = query_tile_tokens+ effective_q_tile_override = q_tile_override+ if effective_query_tile_tokens is None and q_tile_override in (64, 128, 256):+ effective_query_tile_tokens = q_tile_override+ elif effective_query_tile_tokens is not None:+ if q_tile_override == 0:+ effective_q_tile_override = int(effective_query_tile_tokens)+ elif q_tile_override != int(effective_query_tile_tokens):+ raise ValueError(+ "query_tile_tokens and q_tile_override must match when both are set; "+ f"got query_tile_tokens={effective_query_tile_tokens}, q_tile_override={q_tile_override}"+ )+ _validate_sparse_q_tile_override_for_head_dim(D, effective_q_tile_override)++ normalized_mask = _normalize_sparse_mask(attn_mask, B, Sq, Skv, query.device)+ metadata = sparge_preprocess_topk(+ query,+ key,+ is_causal=is_causal,+ smooth_k=smooth_k,+ simthreshd1=simthreshd1,+ topk=topk,+ cdfthreshd=cdfthreshd,+ attention_sink=attention_sink,+ quant_block_size=64,+ tensor_layout=tensor_layout,+ k_quant_granularity=k_quant_granularity,+ query_tile_tokens=effective_query_tile_tokens,+ sparse_q_block_tokens=sparse_q_block_tokens,+ sparse_k_block_tokens=sparse_k_block_tokens,+ )+ _get_xpu_sparse_kernel_backend()+ out = block_sparse_sdpa(+ query,+ key,+ value,+ metadata["lut"],+ metadata["valid_block_num"],+ attn_mask=normalized_mask,+ is_causal=is_causal,+ scale=scale,+ q_tile_override=effective_q_tile_override,+ sparse_q_block_tokens=metadata["sparse_q_block_tokens"],+ sparse_k_block_tokens=metadata["sparse_k_block_tokens"],+ tensor_layout=tensor_layout,+ )+ sparsity_ratio = metadata["stats"]["sparsity_ratio"]+ if return_metadata and return_sparsity:+ return out, sparsity_ratio, metadata+ if return_metadata:+ return out, metadata+ if return_sparsity:+ return out, sparsity_ratio+ return outdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/flash_attn_dtype.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/flash_attn_dtype.hppnew file mode 100644index 000000000..193870700--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/flash_attn_dtype.hpp@@ -0,0 +1,25 @@+// Flash Attention data type codes shared by the SYCL-TLA wrappers.+//+// Copyright (C) 2026 Intel Corporation+// SPDX-License-Identifier: MIT++// This enum lives in its own dependency-free header so that translation units+// which only dispatch on dtype (for example ark.cpp, which must include only+// declaration-only headers to keep compiler memory down) can use it without+// pulling in the heavy SYCL-TLA kernel headers or the public API declarations+// in sycl_tla_common.hpp.++#pragma once++namespace ark {++/// Flash Attention data type codes (matches Python side)+enum class FlashAttnDtype : int {+ FP16 = 0,+ BF16 = 1,+ FP32 = 2,+ FP8_E4M3 = 3,+ FP8_E5M2 = 4,+};++} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sparse_sdpa_fwd_mainloop.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sparse_sdpa_fwd_mainloop.hppnew file mode 100644index 000000000..c1b2d1fae--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sparse_sdpa_fwd_mainloop.hpp@@ -0,0 +1,636 @@+/***************************************************************************************************+ * Copyright (C) 2026 Intel Corporation, All rights reserved.+ * SPDX-License-Identifier: BSD-3-Clause+ **************************************************************************************************/++#pragma once++#include "cute/util/print_tensor.hpp"+#include "cutlass/cutlass.h"+#include "cutlass/gemm/dispatch_policy.hpp"++#include "cute/algorithm/functional.hpp"+#include "cute/algorithm/gemm.hpp"+#include "cute/algorithm/subgroup_algorithms.hpp"+#include "cute/atom/mma_atom.hpp"+#include "flash_attention_v2/collective/fmha_fusion.hpp"+#include <cute/util/xe_split_barrier.hpp>++namespace cutlass::sdpa {++template <int Stages>+class XeDefault {};++} // namespace cutlass::sdpa++namespace cutlass::fmha::collective {++using namespace cute;++template <class DispatchPolicy_, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_,+ class TiledMMAQK_, class TiledMMAPV_, int VTiles_, class TensorQ_, class TensorK_, class TensorV_,+ class TensorK_cache_, class TensorV_cache_, class TiledCopyQ_ = void, class TiledCopyK_ = void,+ class TiledCopyV_ = void, class TiledCopyK_cache_ = void, class TiledCopyV_cache_ = void>+struct SPARSESDPAFwdMainloop {+ static_assert(cutlass::detail::dependent_false<DispatchPolicy_>, "Could not find a mainloop specialization.");+};++template <int Stages, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_, class TiledMMAQK_,+ class TiledMMAPV_, int VTiles_, class TensorQ_, class TensorK_, class TensorV_, class TensorK_cache_,+ class TensorV_cache_, class TiledCopyQ_, class TiledCopyK_, class TiledCopyV_, class TiledCopyK_cache_,+ class TiledCopyV_cache_>+struct SPARSESDPAFwdMainloop<cutlass::sdpa::XeDefault<Stages>, CausalMask_, FullMask_, CachedKV_, PagedKV_,+ TiledMMAQK_, TiledMMAPV_, VTiles_, TensorQ_, TensorK_, TensorV_, TensorK_cache_,+ TensorV_cache_, TiledCopyQ_, TiledCopyK_, TiledCopyV_, TiledCopyK_cache_,+ TiledCopyV_cache_> {+ using TiledMMAQK = TiledMMAQK_;+ using TiledMMAPV = TiledMMAPV_;+ using TileShapeQK = decltype(TiledMMAQK{}.tile_mnk());+ using TileShapePV = decltype(TiledMMAPV{}.tile_mnk());+ static constexpr int VTiles = VTiles_;+ using SubgroupLayoutQK = decltype(TiledMMAQK{}.get_atom_layout_mnk());+ using SGPerWG = decltype(product(take<1, 4>(shape(typename TiledMMAQK::ThrLayoutVMNK{}))));++ using TensorQ = TensorQ_;+ using TensorK = TensorK_;+ using TensorV = TensorV_;++ using TensorQ2D = decltype(TensorQ_{}(append<rank_v<TensorQ_>>(make_coord(_, _), 0)));+ using TensorK2D = decltype(TensorK_{}(append<rank_v<TensorK_>>(make_coord(_, _), 0)));+ using TensorV2D = decltype(TensorV_{}(append<rank_v<TensorV_>>(make_coord(_, _), 0)));++ using TiledCopyQ =+ conditional_t<is_void_v<TiledCopyQ_>, decltype(make_block_2d_copy_A(TiledMMAQK{}, TensorQ2D{})), TiledCopyQ_>;+ using TiledCopyK =+ conditional_t<is_void_v<TiledCopyK_>, decltype(make_block_2d_copy_B(TiledMMAQK{}, TensorK2D{})), TiledCopyK_>;+ using TiledCopyV =+ conditional_t<is_void_v<TiledCopyV_>, decltype(make_block_2d_copy_B(TiledMMAPV{}, TensorV2D{})), TiledCopyV_>;+ using TensorK_cache = TensorK_cache_;+ using TensorV_cache = TensorV_cache_;+ using TensorK_cache2D = decltype(TensorK_cache_{}(append<rank_v<TensorK_cache_>>(make_coord(_, _), 0)));+ using TensorV_cache2D = decltype(TensorV_cache_{}(append<rank_v<TensorV_cache_>>(make_coord(_, _), 0)));+ using TiledCopyK_cache =+ conditional_t<is_void_v<TiledCopyK_cache_>, decltype(make_block_2d_copy_B(TiledMMAQK{}, TensorK_cache2D{})),+ TiledCopyK_cache_>;+ using TiledCopyV_cache =+ conditional_t<is_void_v<TiledCopyV_cache_>, decltype(make_block_2d_copy_B(TiledMMAPV{}, TensorV_cache2D{})),+ TiledCopyV_cache_>;++ template <typename TiledMMA>+ using FragC = decltype(TiledMMA{}.get_slice(0).partition_sg_fragment_C(+ make_identity_tensor(select<0, 1>(TiledMMA{}.tile_mnk()))));++ using FragS = FragC<TiledMMAQK>;+ using FragSRow = decltype(reduce<1>(FragS{}, sycl::plus<void>{}));+ using FragSCol = decltype(reduce<0>(FragS{}, sycl::plus<void>{}));+ using ElementS = typename TiledMMAQK::ValTypeD;+ using ElementM = typename TiledMMAQK::ValTypeA;++ using SingleFragA = FragC<TiledMMAPV>;+ using FragA = expand_sg_fragment_t<SingleFragA, 1, VTiles>;+ using FragARow = decltype(reduce<1>(FragA{}, sycl::plus<void>{}));+ using ElementA = typename TiledMMAPV::ValTypeD;++ static constexpr bool CausalMask = CausalMask_;+ static constexpr bool CachedKV = CachedKV_;+ static constexpr bool PagedKV = PagedKV_;++ struct Arguments {+ float const scale;+ float const* mask = nullptr;+ int scale_block_size = 0;+ float const* qscale = nullptr;+ float const* kscale = nullptr;+ float const* vscale = nullptr;+ int const* lut = nullptr;+ int const* valid_block_num = nullptr;+ int num_q_blocks = 0;+ int num_k_blocks = 0;+ int sparse_q_block_size = 0;+ bool canonical_nhd_k = false;+ int const* ptr_page_table = nullptr;+ int page_size = 0;+ int const* num_pages_per_seq = nullptr;+ };++ using Params = Arguments;+ struct EmptySharedStorage {};+ using SharedStorage = EmptySharedStorage;++ Params params;+ SharedStorage& shared_storage;++ SPARSESDPAFwdMainloop(Params const& params_, SharedStorage& shared_storage_)+ : params(params_), shared_storage(shared_storage_) {}++ static constexpr Params to_underlying_arguments(Arguments const& args, void* /* workspace */) {+ constexpr double kLog2e = 1.4426950408889634074;+ float val = args.scale * static_cast<float>(kLog2e);+ return Params{val, args.mask, args.scale_block_size, args.qscale, args.kscale, args.vscale, args.lut,+ args.valid_block_num, args.num_q_blocks, args.num_k_blocks, args.sparse_q_block_size,+ args.canonical_nhd_k, args.ptr_page_table, args.page_size, args.num_pages_per_seq};+ }++ CUTLASS_HOST_DEVICE static bool can_implement(Arguments const&) { return true; }++ CUTLASS_DEVICE+ int get_physical_k_tile(int K, int l_coord, int seq_len_kv_cache) {+ int next_page_logical_idx = K * get<1>(TileShapeQK{}) / params.page_size;+ int tiles_per_page = params.page_size / get<1>(TileShapeQK{});+ int batch_offset =+ params.num_pages_per_seq ? params.num_pages_per_seq[l_coord] : l_coord * (seq_len_kv_cache / params.page_size);++ return params.ptr_page_table[batch_offset + next_page_logical_idx] * tiles_per_page + K % tiles_per_page;+ }++ CUTLASS_DEVICE+ static int logical_block_from_delta_row(int const* row, int valid_blocks, int idx) {+ int logical_block = 0;+ for (int i = 0; i <= idx && i < valid_blocks; ++i) {+ logical_block += row[i];+ }+ return logical_block;+ }++ template <typename QVCoord>+ CUTLASS_DEVICE void operator()(TensorQ2D const& Q_2D, TensorK2D const& K_2D, TensorV2D const& V_2D, FragA& tArA,+ FragARow& tA_max, FragARow& tA_sum, QVCoord blk_qv, int blk_k0, int blk_k1,+ int total_blk, int thr_id, int seq_len, int seq_len_kv_cache, int l_coord,+ [[maybe_unused]] float* scaleQ, [[maybe_unused]] float* scaleK,+ [[maybe_unused]] float* scaleV, int full_tile_offset, int discard_seq_coord,+ int const* lut_rows_base = nullptr, int const* valid_blocks_base = nullptr,+ int sparse_q_rows_in_tile = 1,+ TensorK_cache2D const& K_cache_2D = TensorK_cache2D{},+ TensorV_cache2D const& V_cache_2D = TensorV_cache2D{}) {+ using namespace sycl::ext::oneapi::this_work_item;++ auto tile_shape_v = make_shape(get<1>(TileShapePV{}) * C<VTiles>{}, get<2>(TileShapePV{}));++ Tensor cQ = make_identity_tensor(Q_2D.shape());+ Tensor cK = make_identity_tensor(K_2D.shape());+ Tensor cV = make_identity_tensor(V_2D.shape());+ Tensor cK_cache = make_identity_tensor(K_cache_2D.shape());+ Tensor cV_cache = make_identity_tensor(V_cache_2D.shape());+ Tensor cP = make_identity_tensor(take<0, 2>(TileShapeQK{}));++ Tensor gQ = local_tile(cQ, TileShapeQK{}, append(blk_qv, _), Step<_1, X, _1>{});+ Tensor gK = local_tile(cK, TileShapeQK{}, make_coord(_, _, _), Step<X, _1, _1>{});+ Tensor gV = local_tile(cV, tile_shape_v, make_coord(get<1>(blk_qv), _));+ Tensor gV_split = local_tile(gV, TileShapePV{}, make_coord(_, _, 0), Step<X, _1, _1>{});++ Tensor gK_cache = local_tile(cK_cache, TileShapeQK{}, make_coord(_, _, _), Step<X, _1, _1>{});+ Tensor gV_cache = local_tile(cV_cache, tile_shape_v, make_coord(get<1>(blk_qv), _));+ Tensor gV_cache_split = local_tile(gV_cache, TileShapePV{}, make_coord(_, _, 0), Step<X, _1, _1>{});++ TiledCopyQ copy_q{Q_2D};+ TiledCopyK copy_k{K_2D};+ TiledCopyV copy_v{V_2D};+ TiledCopyK_cache copy_k_cache{K_cache_2D};+ TiledCopyV_cache copy_v_cache{V_cache_2D};++ TiledMMAQK mma_qk{};+ TiledMMAPV mma_pv{};++ auto thr_copy_q = copy_q.get_slice(thr_id);+ auto thr_copy_k = copy_k.get_slice(thr_id);+ auto thr_copy_v = copy_v.get_slice(thr_id);+ auto thr_copy_k_cache = copy_k_cache.get_slice(thr_id);+ auto thr_copy_v_cache = copy_v_cache.get_slice(thr_id);+ auto thr_mma_qk = mma_qk.get_slice(thr_id);+ auto thr_mma_pv = mma_pv.get_slice(thr_id);++ auto tQgQ = thr_copy_q.partition_S(gQ);+ auto tKgK = thr_copy_k.partition_S(gK);+ auto tVgV = thr_copy_v.partition_S(gV_split);+ auto tKgK_cache = thr_copy_k_cache.partition_S(gK_cache);+ auto tVgV_cache = thr_copy_v_cache.partition_S(gV_cache_split);++ auto tQrQ = thr_copy_q.partition_sg_fragment_D(gQ(_, _, 0));+ auto tSrQ = thr_mma_qk.partition_sg_fragment_A(gQ(_, _, 0));+ auto tKrK = thr_copy_k.partition_sg_fragment_D(gK(_, _, 0, 0));+ auto tSrK = thr_mma_qk.partition_sg_fragment_B(gK(_, _, 0, 0));+ auto tSrS = thr_mma_qk.partition_sg_fragment_C(cP);+ auto tArP = thr_mma_pv.partition_sg_fragment_A(cP);+ auto tVrV = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));+ auto tArV = thr_mma_pv.partition_sg_fragment_B(gV_split(_, _, 0, 0));++ auto prefetch_q = make_block_2d_prefetch(copy_q);+ auto prefetch_k = make_block_2d_prefetch(copy_k);+ auto prefetch_v = make_block_2d_prefetch(copy_v);+ auto prefetch_k_cache = make_block_2d_prefetch(copy_k_cache);+ auto prefetch_v_cache = make_block_2d_prefetch(copy_v_cache);++ auto pQgQ = prefetch_q.get_slice(thr_id).partition_S(gQ);+ auto pKgK = prefetch_k.get_slice(thr_id).partition_S(gK);+ auto pVgV = prefetch_v.get_slice(thr_id).partition_S(gV_split);+ auto pKgK_cache = prefetch_k_cache.get_slice(thr_id).partition_S(gK_cache);+ auto pVgV_cache = prefetch_v_cache.get_slice(thr_id).partition_S(gV_cache_split);++ int kblocks_cache = ceil_div(seq_len_kv_cache, get<1>(TileShapeQK{}));+ for (int D = 0; D < size<3>(pQgQ); D++) {+ prefetch(prefetch_q, pQgQ(_, _, _, D));+ }+ if (lut_rows_base == nullptr) {+ for (int D = 0; D < size<4>(pKgK); D++) {+ CUTLASS_PRAGMA_UNROLL+ for (int K = 0; K < Stages; K++) {+ if (K < kblocks_cache) {+ if constexpr (PagedKV) {+ int physical_K_tile = get_physical_k_tile(K, l_coord, seq_len_kv_cache);+ prefetch(prefetch_k_cache, pKgK_cache(_, _, _, physical_K_tile, D));+ } else {+ prefetch(prefetch_k_cache, pKgK_cache(_, _, _, K, D));+ }+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, K - kblocks_cache, D));+ }+ }+ }+ }+ if (blk_k0 == 0) {+ clear(tArA);+ fill(tA_max, cutlass::platform::numeric_limits<ElementA>::lowest());+ clear(tA_sum);+ }++ bool check_remainder_k = (seq_len % get<1>(TileShapeQK{}) != 0);+ int q_sg_tile = get<0>(shape_div(TileShapeQK{}, shape(SubgroupLayoutQK{})));+ int sparse_q_block_size = params.sparse_q_block_size > 0 ? params.sparse_q_block_size : params.scale_block_size;+ int q_blocks_per_wg_tile =+ sparse_q_block_size > 0 ? cute::max(1, int(get<0>(TileShapeQK{})) / sparse_q_block_size) : 1;+ int sg_rows_per_sparse_q_block =+ sparse_q_block_size > 0 ? cute::max(1, sparse_q_block_size / q_sg_tile) : 1;+ int subgroup_q_row_in_tile = get_sub_group_id() / sg_rows_per_sparse_q_block;+ subgroup_q_row_in_tile = cute::min(subgroup_q_row_in_tile, q_blocks_per_wg_tile - 1);++ int sparse_route_blocks = params.num_k_blocks > 0 ? params.num_k_blocks : total_blk;+ // The causal scheduler shortens total_blk for each Q subgroup. Deriving+ // this ratio from total_blk would then collapse a 64-token route block to+ // one K32 tile near the beginning of the sequence and drop half its keys.+ // Keep the route-to-physical-tile mapping tied to the full K shape instead.+ int k_route_block_tokens = cute::ceil_div(int(size<0>(K_2D)), sparse_route_blocks);+ int k_tiles_per_sparse_route_block =+ cute::max(1, ceil_div(k_route_block_tokens, get<1>(TileShapeQK{})));+ auto route_block_to_k_tile = [&](int route_block, int micro_tile) {+ return route_block * k_tiles_per_sparse_route_block + micro_tile;+ };++ auto prefetch_sparse_k_block = [&](int route_block) {+ if (route_block < 0 || route_block >= sparse_route_blocks) return;+ for (int micro_tile = 0; micro_tile < k_tiles_per_sparse_route_block; ++micro_tile) {+ int logical_block = route_block_to_k_tile(route_block, micro_tile);+ if (logical_block >= total_blk) break;+ for (int D = 0; D < size<4>(pKgK); D++) {+ if constexpr (CachedKV) {+ if (logical_block < kblocks_cache) {+ int physical_block = logical_block;+ if constexpr (PagedKV) {+ physical_block = get_physical_k_tile(logical_block, l_coord, seq_len_kv_cache);+ }+ prefetch(prefetch_k_cache, pKgK_cache(_, _, _, physical_block, D));+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, logical_block - kblocks_cache, D));+ }+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, logical_block - kblocks_cache, D));+ }+ }+ }+ };++ auto mainloop_body = [&](auto cached_k, int K, bool first_block, bool subgroup_selected, int sparse_prefetch_block,+ auto& copy_k_cur, auto& copy_v_cur, auto& prefetch_v_cur, auto& tKgK_cur,+ auto& tVgV_cur, auto& pVgV_cur) {+ barrier_arrive(ScopeWorkgroup);+ constexpr bool is_cache = decltype(cached_k)::value;++ int k_idx;+ if constexpr (is_cache) {+ k_idx = K;+ if constexpr (PagedKV) {+ k_idx = get_physical_k_tile(K, l_coord, seq_len_kv_cache);+ }+ } else {+ k_idx = K - kblocks_cache;+ }++ clear(tSrS);+ CUTLASS_PRAGMA_UNROLL+ for (int D = 0; D < size<4>(tKgK); D++) {+ copy(copy_q, tQgQ(_, _, _, D), tQrQ);+ copy(copy_k_cur, tKgK_cur(_, _, _, k_idx, D), tKrK);+ reorder(tQrQ, tSrQ);+ reorder(tKrK, tSrK);+ cute::gemm(mma_qk, tSrQ, tSrK, tSrS);+ }++ CUTLASS_PRAGMA_UNROLL+ for (int VV = 0; VV < VTiles; VV++) {+ prefetch(prefetch_v_cur, pVgV_cur(_, _, _, VV, k_idx));+ }++ if (subgroup_selected) {+ if constexpr (!is_cache && CausalMask) {+ // K32 splits each logical 64-token sparse route block into two+ // physical K tiles. The diagonal can therefore occur in any+ // physical tile, rather than only in the final K tile.+ Tensor cPgP = make_identity_tensor(make_shape(seq_len, seq_len));+ Tensor gP = local_tile(cPgP, take<0, 2>(TileShapeQK{}), make_coord(get<0>(blk_qv), K));+ auto cS_thread = thr_mma_qk.partition_C(gP);+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tSrS.size(); ++i) {+ int row_idx = get<0>(cS_thread(i));+ int col_idx = get<1>(cS_thread(i));+ if (col_idx - seq_len_kv_cache - full_tile_offset > row_idx - discard_seq_coord) {+ tSrS(i) = ElementS(-INFINITY);+ }+ }+ } else if constexpr (FullMask_) {+ Tensor cPgP = make_identity_tensor(make_shape(seq_len, seq_len));+ Tensor gP = local_tile(cPgP, take<0, 2>(TileShapeQK{}), make_coord(get<0>(blk_qv), K));+ auto cS_thread = thr_mma_qk.partition_C(gP);+ int row_idx_begin = get<0>(cS_thread(0));+ int row_idx_end = row_idx_begin + q_sg_tile;+ int col_idx_begin = get<1>(cS_thread(0));+ int col_idx_end = col_idx_begin + get<1>(TileShapeQK{});+ if (row_idx_end <= seq_len && col_idx_end <= seq_len) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tSrS.size(); ++i) {+ int row_idx = get<0>(cS_thread(i));+ int col_idx = get<1>(cS_thread(i));+ tSrS(i) += ElementS(params.mask[col_idx + row_idx * seq_len]);+ }+ } else {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tSrS.size(); ++i) {+ int row_idx = get<0>(cS_thread(i));+ int col_idx = get<1>(cS_thread(i));+ tSrS(i) +=+ (row_idx < seq_len && col_idx < seq_len) ? ElementS(params.mask[col_idx + row_idx * seq_len])+ : ElementS(-INFINITY);+ }+ }+ }++ if constexpr (!is_cache) {+ if (check_remainder_k && K == total_blk - 1) {+ FragSCol k_rem_mask;+ int k_val = get<0>(tKgK_cur(0, 0, 0, k_idx, 0)) + kblocks_cache * get<1>(TileShapeQK{});+ int k = k_val + get_sub_group().get_local_id()[0];+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < k_rem_mask.size(); i++, k += intel::sg_size) {+ k_rem_mask(i) = (k < seq_len) ? ElementS(sycl::nan(0u)) : ElementS(-INFINITY);+ }+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tSrS.size(); i++) {+ tSrS(i) = sycl::fmin(tSrS(i), broadcast<1>(k_rem_mask, tSrS, i));+ }+ }+ }++ auto rescale = softmax(first_block, tSrS, tA_max, tA_sum);+ reorder(tSrS, tArP);++ CUTLASS_PRAGMA_UNROLL+ for (int VV = 0; VV < VTiles; VV++) {+ copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV);+ reorder(tVrV, tArV);+ if (!first_block) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tArA.size() / VTiles; i++) {+ tArA(_, _, _, VV)(i) *= broadcast<0>(rescale, tArA, i);+ }+ }+ cute::gemm(mma_pv, tArP, tArV, tArA(_, _, _, VV));+ }+ }++ if (lut_rows_base == nullptr) {+ int K_next = K + Stages;+ for (int D = 0; D < size<4>(pKgK); D++) {+ if constexpr (is_cache) {+ bool is_cache_next = K_next < kblocks_cache;+ int physical_K_next = K_next;+ if constexpr (PagedKV) {+ if (is_cache_next) {+ physical_K_next = get_physical_k_tile(K_next, l_coord, seq_len_kv_cache);+ }+ }+ if (is_cache_next) {+ prefetch(prefetch_k_cache, pKgK_cache(_, _, _, physical_K_next, D));+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, K_next - kblocks_cache, D));+ }+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, K_next - kblocks_cache, D));+ }+ }+ } else if (sparse_prefetch_block < sparse_route_blocks) {+ prefetch_sparse_k_block(sparse_prefetch_block);+ }++ barrier_wait(ScopeWorkgroup);+ };++ auto run_sparse_route_block = [&](int route_block, bool subgroup_selected, auto& subgroup_started,+ int sparse_prefetch_block) {+ if (route_block < 0 || route_block >= sparse_route_blocks) return;+ for (int micro_tile = 0; micro_tile < k_tiles_per_sparse_route_block; ++micro_tile) {+ int K = route_block_to_k_tile(route_block, micro_tile);+ if (K >= total_blk) break;+ bool first_selected_block = subgroup_selected ? !subgroup_started : false;+ if constexpr (CachedKV) {+ if (K < kblocks_cache) {+ if (K >= blk_k0 && K < blk_k1) {+ mainloop_body(std::bool_constant<true>{}, K, first_selected_block, subgroup_selected,+ sparse_prefetch_block, copy_k_cache, copy_v_cache, prefetch_v_cache, tKgK_cache,+ tVgV_cache, pVgV_cache);+ if (subgroup_selected) subgroup_started = true;+ }+ } else if (K >= (blk_k0 > kblocks_cache ? blk_k0 : kblocks_cache) && K < blk_k1) {+ mainloop_body(std::bool_constant<false>{}, K, first_selected_block, subgroup_selected,+ sparse_prefetch_block, copy_k, copy_v, prefetch_v, tKgK, tVgV, pVgV);+ if (subgroup_selected) subgroup_started = true;+ }+ } else if (K >= blk_k0 && K < blk_k1) {+ mainloop_body(std::bool_constant<false>{}, K, first_selected_block, subgroup_selected, sparse_prefetch_block,+ copy_k, copy_v, prefetch_v, tKgK, tVgV, pVgV);+ if (subgroup_selected) subgroup_started = true;+ }+ }+ };++ if (lut_rows_base != nullptr && valid_blocks_base != nullptr) {+ bool subgroup_started = false;+ if (sparse_q_rows_in_tile == 1) {+ int const* row_ptr = lut_rows_base;+ int row_valid = valid_blocks_base[0];+ int row_pos = 0;+ int row_cur_block = row_valid > 0 ? logical_block_from_delta_row(row_ptr, row_valid, 0) : sparse_route_blocks;+ int prefetch_pos = row_pos;+ int prefetch_cur_block = row_cur_block;++ auto advance_single_sparse_block = [&](int& frontier_pos, int& frontier_cur_block) {+ if (frontier_cur_block >= sparse_route_blocks) return;+ frontier_pos += 1;+ if (frontier_pos < row_valid) {+ frontier_cur_block += row_ptr[frontier_pos];+ } else {+ frontier_cur_block = sparse_route_blocks;+ }+ };++ auto pop_single_sparse_block = [&](int& frontier_pos, int& frontier_cur_block) {+ int block = frontier_cur_block;+ advance_single_sparse_block(frontier_pos, frontier_cur_block);+ return block;+ };++ for (int stage = 0; stage < Stages; ++stage) {+ int sparse_prefetch_block = pop_single_sparse_block(prefetch_pos, prefetch_cur_block);+ if (sparse_prefetch_block >= sparse_route_blocks) break;+ prefetch_sparse_k_block(sparse_prefetch_block);+ }++ while (row_cur_block < sparse_route_blocks) {+ int next_block = row_cur_block;+ bool subgroup_selected = (subgroup_q_row_in_tile == 0);+ int sparse_prefetch_block = pop_single_sparse_block(prefetch_pos, prefetch_cur_block);+ run_sparse_route_block(next_block, subgroup_selected, subgroup_started, sparse_prefetch_block);+ advance_single_sparse_block(row_pos, row_cur_block);+ }+ } else {+ static constexpr int kMaxSparseRowsPerTile = cute::max(1, int(get<0>(TileShapeQK{})) / 64);+ int const* row_ptrs[kMaxSparseRowsPerTile];+ int active_rows[kMaxSparseRowsPerTile];+ int active_row_count = 0;+ bool subgroup_started_rows[kMaxSparseRowsPerTile];+ int row_valid[kMaxSparseRowsPerTile];+ int row_pos[kMaxSparseRowsPerTile];+ int row_cur_block[kMaxSparseRowsPerTile];++ for (int row = 0; row < kMaxSparseRowsPerTile; ++row) {+ row_ptrs[row] = lut_rows_base + row * params.num_k_blocks;+ subgroup_started_rows[row] = false;+ if (row < sparse_q_rows_in_tile) {+ row_valid[row] = valid_blocks_base[row];+ row_pos[row] = 0;+ row_cur_block[row] =+ row_valid[row] > 0 ? logical_block_from_delta_row(row_ptrs[row], row_valid[row], 0) : sparse_route_blocks;+ if (row_valid[row] > 0) {+ active_rows[active_row_count++] = row;+ }+ } else {+ row_valid[row] = 0;+ row_pos[row] = 0;+ row_cur_block[row] = sparse_route_blocks;+ }+ }++ auto find_sparse_block = [&](int* frontier_pos, int* frontier_cur_block) {+ int block = sparse_route_blocks;+ for (int active = 0; active < active_row_count; ++active) {+ int row = active_rows[active];+ if (frontier_pos[row] < row_valid[row]) {+ block = cute::min(block, frontier_cur_block[row]);+ }+ }+ return block;+ };++ auto advance_sparse_block = [&](int block, int* frontier_pos, int* frontier_cur_block) {+ if (block >= sparse_route_blocks) return;+ for (int active = 0; active < active_row_count; ++active) {+ int row = active_rows[active];+ if (frontier_pos[row] < row_valid[row] && frontier_cur_block[row] == block) {+ frontier_pos[row] += 1;+ if (frontier_pos[row] < row_valid[row]) {+ frontier_cur_block[row] += row_ptrs[row][frontier_pos[row]];+ } else {+ frontier_cur_block[row] = sparse_route_blocks;+ }+ }+ }+ };++ auto pop_sparse_block = [&](int* frontier_pos, int* frontier_cur_block) {+ int block = find_sparse_block(frontier_pos, frontier_cur_block);+ advance_sparse_block(block, frontier_pos, frontier_cur_block);+ return block;+ };++ int prefetch_pos[kMaxSparseRowsPerTile];+ int prefetch_cur_block[kMaxSparseRowsPerTile];+ for (int row = 0; row < kMaxSparseRowsPerTile; ++row) {+ prefetch_pos[row] = row_pos[row];+ prefetch_cur_block[row] = row_cur_block[row];+ }++ for (int stage = 0; stage < Stages; ++stage) {+ int sparse_prefetch_block = pop_sparse_block(prefetch_pos, prefetch_cur_block);+ if (sparse_prefetch_block >= sparse_route_blocks) break;+ prefetch_sparse_k_block(sparse_prefetch_block);+ }++ int next_block = find_sparse_block(row_pos, row_cur_block);+ while (next_block < sparse_route_blocks) {+ int selected_row = subgroup_q_row_in_tile;+ bool subgroup_selected = subgroup_q_row_in_tile < sparse_q_rows_in_tile &&+ row_pos[selected_row] < row_valid[selected_row] &&+ row_cur_block[selected_row] == next_block;+ int sparse_prefetch_block = pop_sparse_block(prefetch_pos, prefetch_cur_block);+ run_sparse_route_block(next_block, subgroup_selected, subgroup_started_rows[selected_row],+ sparse_prefetch_block);++ advance_sparse_block(next_block, row_pos, row_cur_block);+ next_block = find_sparse_block(row_pos, row_cur_block);+ }+ }+ } else {+ if constexpr (CachedKV) {+ for (int K = blk_k0; K < kblocks_cache; K++) {+ mainloop_body(std::bool_constant<true>{}, K, K == blk_k0, true, total_blk, copy_k_cache, copy_v_cache,+ prefetch_v_cache, tKgK_cache, tVgV_cache, pVgV_cache);+ }+ }+ for (int K = (blk_k0 > kblocks_cache ? blk_k0 : kblocks_cache); K < blk_k1; K++) {+ mainloop_body(std::bool_constant<false>{}, K,+ K == (blk_k0 > kblocks_cache ? blk_k0 : kblocks_cache), true, total_blk, copy_k, copy_v,+ prefetch_v, tKgK, tVgV, pVgV);+ }+ }+ }++ CUTLASS_DEVICE+ FragSRow softmax(bool first_block, FragS& tS, FragSRow& tS_max, FragSRow& tS_sum) {+ auto tS_bmax = reduce<1>(tS, sycl::maximum{});+ FragSRow rescale;+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_max.size(); i++) {+ ElementS new_max = sycl::max(tS_max(i), params.scale * tS_bmax(i));+ rescale(i) = sycl::native::exp2(tS_max(i) - new_max);+ tS_max(i) = new_max;+ }++ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS.size(); i++) {+ tS(i) = sycl::native::exp2(params.scale * tS(i) - broadcast<0>(tS_max, tS, i));+ }++ if (!first_block) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_sum.size(); i++) {+ tS_sum(i) *= rescale(i);+ }+ }++ auto tS_bsum = reduce<1>(tS, sycl::plus<void>{});+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_sum.size(); i++) {+ tS_sum(i) += tS_bsum(i);+ }+ return rescale;+ }+};++} // namespace cutlass::fmha::collectivediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hppindex fcfe62f0e..cea890b5d 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp@@ -9,6 +9,7 @@#include <stdexcept>#include <string>#include "bestla/bestla.h"+#include "flash_attn_dtype.hpp"#ifdef ARK_XPU#include <sycl/sycl.hpp>#endif@@ -335,6 +336,54 @@ void sdpa_impl_qks8_sparse_qtile256_row64k_pvhalf(int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch, int num_heads_q, int num_heads_kv,int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal,BTLA_DTYPE pv_dtype = BTLA_DTYPE::F16);++void sdpa_impl_bf16_sparse_sdpa_d64(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal);++void sdpa_impl_fp16_sparse_sdpa_d64(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal);++void sdpa_impl_bf16_sparse_sdpa_row_linear(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal);++void sdpa_impl_fp16_sparse_sdpa_row_linear(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal);++void sdpa_impl_bf16_sparse_sdpa_qtile256_row64k(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal);++void sdpa_impl_fp16_sparse_sdpa_qtile256_row64k(+ sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, void* lut, void* valid_block_num,+ int num_q_blocks, int num_k_blocks, int q_tile_override, int q_stride_s, int q_stride_d, int q_stride_h,+ int q_stride_b, int k_stride_s, int k_stride_d, int k_stride_h, int k_stride_b, int v_stride_d, int v_stride_s,+ int v_stride_h, int v_stride_b, int o_stride_s, int o_stride_d, int o_stride_h, int o_stride_b, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal);#endif // ARK_SYCL_TLA} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hppindex b339ccd23..5134e9f51 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp@@ -465,4 +465,4 @@ void run_igemm_kblock_fixed_tile(sycl::queue* q, int m, int n, int k, const int8#endif // ARK_XPU && ARK_SYCL_TLA-} // namespace ark\ No newline at end of file+} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hppindex ef91b238b..c2aca12f4 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp@@ -14,6 +14,7 @@#include <string>#include <sycl/aliases.hpp>#include "bestla/bestla.h"+#include "flash_attn_dtype.hpp"#ifdef ARK_XPU#include <sycl/sycl.hpp>#endif@@ -47,14 +48,8 @@namespace ark {-/// Flash Attention data type codes (matches Python side)-enum class FlashAttnDtype : int {- FP16 = 0,- BF16 = 1,- FP32 = 2,- FP8_E4M3 = 3,- FP8_E5M2 = 4,-};+// FlashAttnDtype is defined in flash_attn_dtype.hpp so that dispatch-only+// translation units such as ark.cpp do not pull in this heavy header.#if defined(ARK_SYCL_TLA)diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa_sparse.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa_sparse.hppindex a8b4d2d10..95d13bec0 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa_sparse.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa_sparse.hpp@@ -14,6 +14,7 @@#include <string>#include <sycl/aliases.hpp>#include "bestla/bestla.h"+#include "flash_attn_dtype.hpp"#ifdef ARK_XPU#include <sycl/sycl.hpp>#endif@@ -35,6 +36,7 @@#if defined(ARK_SDPA_ENABLE_SPARSE)#include "stla/xe_sparse_fmha_fwd_epilogue.hpp"#include "stla/xe_sparse_sagev1_fwd_mainloop.hpp"+#include "stla/xe_sparse_sdpa_fwd_mainloop.hpp"#include "stla/xe_sparse_sage_fwd_kernel.hpp"#endif#include "flash_attention_v2/collective/fmha_fusion.hpp"@@ -61,14 +63,8 @@namespace ark {-/// Flash Attention data type codes (matches Python side)-enum class FlashAttnDtype : int {- FP16 = 0,- BF16 = 1,- FP32 = 2,- FP8_E4M3 = 3,- FP8_E5M2 = 4,-};+// FlashAttnDtype is defined in flash_attn_dtype.hpp so that dispatch-only+// translation units such as ark.cpp do not pull in this heavy header.#if defined(ARK_SYCL_TLA)@@ -945,11 +941,11 @@ struct SparseSageConfig {cute::conditional_t<is_void_v<SubgroupLayoutPV_>,decltype(cutlass::fmha::collective::get_sg_layout_pv(SubgroupLayoutQK{})), SubgroupLayoutPV_>;- template <bool isVarLen, bool CachedKV, bool PagedKV, class Scheduler>+ template <bool HasMask, bool CachedKV, class Scheduler>static int run(const Options& options) {cutlass::KernelHardwareInfo hw_info;hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id);- using ProblemShapeType = cutlass::fmha::kernel::SparseSageProblemShape<isVarLen>;+ using ProblemShapeType = cutlass::fmha::kernel::SparseSageProblemShape<false>;using TiledMMAQK = typename TiledMMAHelper<MMA_Atom<MMAOperation>, Layout<TileShapeQK>, SubgroupLayoutQK>::TiledMMA;using TiledMMAPV =typename TiledMMAHelper<MMA_Atom<MMAOperationPV>, Layout<TileShapePV>, SubgroupLayoutPV>::TiledMMA;@@ -970,7 +966,7 @@ struct SparseSageConfig {using MainloopDispatchPolicy = cutlass::sage::XeDefault<PipelineStages>;if constexpr (Causal) {using CollectiveMainloop =- cutlass::fmha::collective::SPARSESAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, PagedKV,+ cutlass::fmha::collective::SPARSESAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, false,UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,@@ -980,44 +976,41 @@ struct SparseSageConfig {GmemTiledCopyO>;using FMHAKernel = cutlass::fmha::kernel::XeSparseSageFwdKernel<ProblemShapeType, CollectiveMainloop,CollectiveEpilogue, Scheduler>;- SageKernelRunner<FMHAKernel, isVarLen> runner;+ SageKernelRunner<FMHAKernel, false> runner;+ CUTLASS_CHECK(runner.run(options, hw_info));+ } else if constexpr (HasMask) {+ using CollectiveMainloop =+ cutlass::fmha::collective::SPARSESAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, true, CachedKV, false,+ UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,+ TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;+ using CollectiveEpilogue =+ cutlass::fmha::collective::SparseFMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO,+ GmemTiledCopyO>;+ using FMHAKernel = cutlass::fmha::kernel::XeSparseSageFwdKernel<ProblemShapeType, CollectiveMainloop,+ CollectiveEpilogue, Scheduler>;+ SageKernelRunner<FMHAKernel, false> runner;CUTLASS_CHECK(runner.run(options, hw_info));} else {- if (options.mask) {using CollectiveMainloop =- cutlass::fmha::collective::SPARSESAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, true, CachedKV, PagedKV,- UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,- TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,- TensorK_cache, TensorV_cache, GmemTiledCopyQ,- GmemTiledCopyK, GmemTiledCopyV, GmemTiledCopyK_cache,- GmemTiledCopyV_cache>;- using CollectiveEpilogue =- cutlass::fmha::collective::SparseFMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO,- GmemTiledCopyO>;- using FMHAKernel = cutlass::fmha::kernel::XeSparseSageFwdKernel<ProblemShapeType, CollectiveMainloop,- CollectiveEpilogue, Scheduler>;- SageKernelRunner<FMHAKernel, isVarLen> runner;- CUTLASS_CHECK(runner.run(options, hw_info));- } else {- using CollectiveMainloop =- cutlass::fmha::collective::SPARSESAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, PagedKV,- UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,- TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,- TensorK_cache, TensorV_cache, GmemTiledCopyQ,- GmemTiledCopyK, GmemTiledCopyV, GmemTiledCopyK_cache,- GmemTiledCopyV_cache>;- using CollectiveEpilogue =- cutlass::fmha::collective::SparseFMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO,- GmemTiledCopyO>;- using FMHAKernel = cutlass::fmha::kernel::XeSparseSageFwdKernel<ProblemShapeType, CollectiveMainloop,- CollectiveEpilogue, Scheduler>;- SageKernelRunner<FMHAKernel, isVarLen> runner;- CUTLASS_CHECK(runner.run(options, hw_info));- }+ cutlass::fmha::collective::SPARSESAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, false,+ UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,+ TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;+ using CollectiveEpilogue =+ cutlass::fmha::collective::SparseFMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO,+ GmemTiledCopyO>;+ using FMHAKernel = cutlass::fmha::kernel::XeSparseSageFwdKernel<ProblemShapeType, CollectiveMainloop,+ CollectiveEpilogue, Scheduler>;+ SageKernelRunner<FMHAKernel, false> runner;+ CUTLASS_CHECK(runner.run(options, hw_info));}return 0;}+ template <bool HasMask, bool CachedKV = false>static int run(const Options& options) {if (options.use_paged_kv || options.varlen) {throw std::runtime_error("Sparse Sage does not support paged KV or varlen in v1");@@ -1033,9 +1026,124 @@ struct SparseSageConfig {throw std::runtime_error("Sparse Sage requires lut and valid_block_num");}if (options.seq_len_kv_cache > 0) {- return run<false, true, false, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ throw std::runtime_error("Sparse Sage cached decode is not supported by the split sparse kernel path in v1");}- return run<false, false, false, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ return run<HasMask, CachedKV, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ }+};++// Independent native-precision sparse path built on the dense SDPA mainloop+// (SPARSESDPAFwdMainloop). Unlike SparseSageConfig it does not dequantize INT8+// Q/K: Q/K/V are bf16/fp16 and the softmax applies `params.scale` directly.+template <bool Causal, typename TileShapeQK, typename TileShapePV, typename TileShapeOutput,+ typename SubgroupLayoutQK, typename SubgroupLayoutPV_, int PipelineStages, bool persistent,+ typename ElementQ = bfloat16_t, typename ElementK = bfloat16_t, typename ElementV = bfloat16_t,+ typename ElementO = ElementQ, typename MMAOperation_ = void,+ typename StrideQ = Stride<int, _1, int, int>, typename StrideK = Stride<int, _1, int, int>,+ typename StrideV = Stride<_1, int, int, int>, typename StrideO = Stride<int, _1, int, int>,+ typename GmemTiledCopyQ = void, typename GmemTiledCopyK = void, typename GmemTiledCopyV = void,+ typename GmemTiledCopyO = void>+struct SparseSDPAConfig {+ static constexpr int SGTileQ = get<0>(shape_div(TileShapeQK{}, shape(SubgroupLayoutQK{})))();+ // Native-precision Q*K MMA: bf16 -> float/bf16 DPAS, fp16 -> float/half DPAS.+ using MMAOperation = cute::conditional_t<+ is_void_v<MMAOperation_>,+ typename cute::conditional_t<+ cute::is_same_v<ElementQ, cutlass::float_e5m2_t> || cute::is_same_v<ElementQ, cutlass::float_e4m3_t>,+ XE_DPAS_TT<cute::gcd(SGTileQ, 8), float, half_t>,+ XE_DPAS_TT<cute::gcd(SGTileQ, 8), float, ElementQ>>,+ MMAOperation_>;+ using MMAOperationPV = cute::conditional_t<is_void_v<MMAOperation_>,+ XE_DPAS_TT<cute::gcd(SGTileQ, 8), float, ElementO>, MMAOperation_>;+ using SubgroupLayoutPV =+ cute::conditional_t<is_void_v<SubgroupLayoutPV_>,+ decltype(cutlass::fmha::collective::get_sg_layout_pv(SubgroupLayoutQK{})), SubgroupLayoutPV_>;++ template <bool HasMask, bool CachedKV, class Scheduler>+ static int run(const Options& options) {+ cutlass::KernelHardwareInfo hw_info;+ hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id);+ using ProblemShapeType = cutlass::fmha::kernel::SparseSageProblemShape<false>;+ using TiledMMAQK = typename TiledMMAHelper<MMA_Atom<MMAOperation>, Layout<TileShapeQK>, SubgroupLayoutQK>::TiledMMA;+ using TiledMMAPV =+ typename TiledMMAHelper<MMA_Atom<MMAOperationPV>, Layout<TileShapePV>, SubgroupLayoutPV>::TiledMMA;+ static_assert(get<0>(TileShapeOutput{}) == get<0>(TileShapePV{}),+ "Output tile and P*V tile have different sizes in Q dimension");+ constexpr int VTiles = get<1>(TileShapeOutput{}) / get<1>(TileShapePV{});+ auto make_dummy_tensor = [&](auto val, auto stride) {+ return make_tensor(make_gmem_ptr(&val), make_layout(repeat<rank_v<decltype(stride)>>(1), stride));+ };+ using TensorQ = decltype(make_dummy_tensor(ElementQ{}, StrideQ{}));+ using TensorK = decltype(make_dummy_tensor(ElementK{}, StrideK{}));+ using TensorV = decltype(make_dummy_tensor(ElementV{}, StrideV{}));+ using TensorO = decltype(make_dummy_tensor(ElementO{}, StrideO{}));+ using TensorK_cache = TensorK;+ using TensorV_cache = TensorV;+ using GmemTiledCopyK_cache = GmemTiledCopyK;+ using GmemTiledCopyV_cache = GmemTiledCopyV;+ using MainloopDispatchPolicy = cutlass::sdpa::XeDefault<PipelineStages>;+ if constexpr (Causal) {+ using CollectiveMainloop =+ cutlass::fmha::collective::SPARSESDPAFwdMainloop<MainloopDispatchPolicy, true, false, CachedKV, false,+ TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;+ using CollectiveEpilogue =+ cutlass::fmha::collective::SparseFMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO,+ GmemTiledCopyO>;+ using FMHAKernel = cutlass::fmha::kernel::XeSparseSageFwdKernel<ProblemShapeType, CollectiveMainloop,+ CollectiveEpilogue, Scheduler>;+ SageKernelRunner<FMHAKernel, false> runner;+ CUTLASS_CHECK(runner.run(options, hw_info));+ } else if constexpr (HasMask) {+ using CollectiveMainloop =+ cutlass::fmha::collective::SPARSESDPAFwdMainloop<MainloopDispatchPolicy, false, true, CachedKV, false,+ TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;+ using CollectiveEpilogue =+ cutlass::fmha::collective::SparseFMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO,+ GmemTiledCopyO>;+ using FMHAKernel = cutlass::fmha::kernel::XeSparseSageFwdKernel<ProblemShapeType, CollectiveMainloop,+ CollectiveEpilogue, Scheduler>;+ SageKernelRunner<FMHAKernel, false> runner;+ CUTLASS_CHECK(runner.run(options, hw_info));+ } else {+ using CollectiveMainloop =+ cutlass::fmha::collective::SPARSESDPAFwdMainloop<MainloopDispatchPolicy, false, false, CachedKV, false,+ TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;+ using CollectiveEpilogue =+ cutlass::fmha::collective::SparseFMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO,+ GmemTiledCopyO>;+ using FMHAKernel = cutlass::fmha::kernel::XeSparseSageFwdKernel<ProblemShapeType, CollectiveMainloop,+ CollectiveEpilogue, Scheduler>;+ SageKernelRunner<FMHAKernel, false> runner;+ CUTLASS_CHECK(runner.run(options, hw_info));+ }+ return 0;+ }++ template <bool HasMask, bool CachedKV = false>+ static int run(const Options& options) {+ if (options.use_paged_kv || options.varlen) {+ throw std::runtime_error("Sparse SDPA does not support paged KV or varlen in v1");+ }+ if (options.block_K != nullptr || options.block_V != nullptr) {+ if (options.seq_len_kv_cache <= 0 || options.seq_len_qo != 1) {+ throw std::runtime_error("Sparse SDPA only supports block_K/block_V for seq_len_q == 1 cached decode in v1");+ }+ } else if (options.seq_len_kv_cache > 0) {+ throw std::runtime_error("Sparse SDPA cached decode requires block_K/block_V cache tensors");+ }+ if (options.lut == nullptr || options.valid_block_num == nullptr) {+ throw std::runtime_error("Sparse SDPA requires lut and valid_block_num");+ }+ if (options.seq_len_kv_cache > 0) {+ throw std::runtime_error("Sparse SDPA cached decode is not supported by the split sparse kernel path in v1");+ }+ return run<HasMask, CachedKV, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);}};#endif // ARK_SDPA_ENABLE_SPARSE@@ -1105,7 +1213,8 @@ inline int launch_sage_prefill_kernel_64(Options const& options) {#endif // ARK_SDPA_ENABLE_DENSE#if defined(ARK_SDPA_ENABLE_SPARSE)-template <typename ElementQ, typename ElementK, typename ElementV, typename ElementO = ElementV>+template <bool Causal, bool HasMask, typename ElementQ, typename ElementK, typename ElementV,+ typename ElementO = ElementV>inline int launch_sparse_sage_prefill_kernel_128(Options const& options) {constexpr int PipelineStages = 2;using ShapeQK = Shape<_256, _64, _32>;@@ -1116,15 +1225,19 @@ inline int launch_sparse_sage_prefill_kernel_128(Options const& options) {using ShapePV1 = Shape<_256, _32, _64>;using ShapeOut1 = Shape<_256, _128>;using SubgroupLayoutQK1 = Layout<Shape<_16, _1, _1>>;- return options.is_causal ? SparseSageConfig<true, false, true, true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK,- void, PipelineStages, false, ElementQ, ElementK, ElementV,- ElementO>::run(options)- : SparseSageConfig<false, false, true, true, ShapeQK1, ShapePV1, ShapeOut1,- SubgroupLayoutQK1, void, PipelineStages, false, ElementQ, ElementK,- ElementV, ElementO>::run(options);+ if constexpr (Causal) {+ return SparseSageConfig<true, false, true, true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void,+ PipelineStages, false, ElementQ, ElementK, ElementV, ElementO>::template run<HasMask>(+ options);+ } else {+ return SparseSageConfig<false, false, true, true, ShapeQK1, ShapePV1, ShapeOut1, SubgroupLayoutQK1, void,+ PipelineStages, false, ElementQ, ElementK, ElementV, ElementO>::template run<HasMask>(+ options);+ }}-template <typename ElementQ, typename ElementK, typename ElementV, typename ElementO = ElementV>+template <bool Causal, bool HasMask, typename ElementQ, typename ElementK, typename ElementV,+ typename ElementO = ElementV>inline int launch_sparse_sage_prefill_kernel_128_qtile128(Options const& options) {constexpr int PipelineStages = 2;using ShapeQK = Shape<_128, _64, _32>;@@ -1135,15 +1248,19 @@ inline int launch_sparse_sage_prefill_kernel_128_qtile128(Options const& optionsusing ShapePV1 = Shape<_128, _32, _64>;using ShapeOut1 = Shape<_128, _128>;using SubgroupLayoutQK1 = Layout<Shape<_8, _1, _1>>;- return options.is_causal ? SparseSageConfig<true, false, true, true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK,- void, PipelineStages, false, ElementQ, ElementK, ElementV,- ElementO>::run(options)- : SparseSageConfig<false, false, true, true, ShapeQK1, ShapePV1, ShapeOut1,- SubgroupLayoutQK1, void, PipelineStages, false, ElementQ, ElementK,- ElementV, ElementO>::run(options);+ if constexpr (Causal) {+ return SparseSageConfig<true, false, true, true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void,+ PipelineStages, false, ElementQ, ElementK, ElementV, ElementO>::template run<HasMask>(+ options);+ } else {+ return SparseSageConfig<false, false, true, true, ShapeQK1, ShapePV1, ShapeOut1, SubgroupLayoutQK1, void,+ PipelineStages, false, ElementQ, ElementK, ElementV, ElementO>::template run<HasMask>(+ options);+ }}-template <typename ElementQ, typename ElementK, typename ElementV, typename ElementO = ElementV>+template <bool Causal, bool HasMask, typename ElementQ, typename ElementK, typename ElementV,+ typename ElementO = ElementV>inline int launch_sparse_sage_prefill_kernel_128_qtile64(Options const& options) {constexpr int PipelineStages = 2;using ShapeQK = Shape<_64, _64, _32>;@@ -1154,15 +1271,19 @@ inline int launch_sparse_sage_prefill_kernel_128_qtile64(Options const& options)using ShapePV1 = Shape<_64, _32, _64>;using ShapeOut1 = Shape<_64, _128>;using SubgroupLayoutQK1 = Layout<Shape<_4, _1, _1>>;- return options.is_causal ? SparseSageConfig<true, false, true, true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK,- void, PipelineStages, false, ElementQ, ElementK, ElementV,- ElementO>::run(options)- : SparseSageConfig<false, false, true, true, ShapeQK1, ShapePV1, ShapeOut1,- SubgroupLayoutQK1, void, PipelineStages, false, ElementQ, ElementK,- ElementV, ElementO>::run(options);+ if constexpr (Causal) {+ return SparseSageConfig<true, false, true, true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void,+ PipelineStages, false, ElementQ, ElementK, ElementV, ElementO>::template run<HasMask>(+ options);+ } else {+ return SparseSageConfig<false, false, true, true, ShapeQK1, ShapePV1, ShapeOut1, SubgroupLayoutQK1, void,+ PipelineStages, false, ElementQ, ElementK, ElementV, ElementO>::template run<HasMask>(+ options);+ }}-template <typename ElementQ, typename ElementK, typename ElementV, typename ElementO = ElementV>+template <bool Causal, bool HasMask, typename ElementQ, typename ElementK, typename ElementV,+ typename ElementO = ElementV>inline int launch_sparse_sage_prefill_kernel_64(Options const& options) {constexpr int PipelineStages = 2;using ShapeQK = Shape<_128, _64, _32>;@@ -1170,13 +1291,75 @@ inline int launch_sparse_sage_prefill_kernel_64(Options const& options) {using ShapeOut = Shape<_128, _64>;using SubgroupLayoutQK = Layout<Shape<_8, _1, _1>>;using SubgroupLayoutPV = void;- return options.is_causal- ? SparseSageConfig<true, false, true, true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK,- SubgroupLayoutPV, PipelineStages, false, ElementQ, ElementK, ElementV,- ElementO>::run(options)- : SparseSageConfig<false, false, true, true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK,- SubgroupLayoutPV, PipelineStages, false, ElementQ, ElementK, ElementV,- ElementO>::run(options);+ if constexpr (Causal) {+ return SparseSageConfig<true, false, true, true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK,+ SubgroupLayoutPV, PipelineStages, false, ElementQ, ElementK, ElementV,+ ElementO>::template run<HasMask>(options);+ } else {+ return SparseSageConfig<false, false, true, true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK,+ SubgroupLayoutPV, PipelineStages, false, ElementQ, ElementK, ElementV,+ ElementO>::template run<HasMask>(options);+ }+}++template <bool Causal, bool HasMask, typename ElementQ, typename ElementK, typename ElementV,+ typename ElementO = ElementV>+inline int launch_sparse_sdpa_prefill_kernel_128(Options const& options) {+ constexpr int PipelineStages = 2;+ using ShapeQK = Shape<_256, _32, _32>;+ using ShapePV = Shape<_256, _32, _32>;+ using ShapeOut = Shape<_256, _128>;+ using SubgroupLayoutQK = Layout<Shape<_16, _1, _1>>;+ using ShapeQK1 = Shape<_256, _32, _32>;+ using ShapePV1 = Shape<_256, _32, _32>;+ using ShapeOut1 = Shape<_256, _128>;+ using SubgroupLayoutQK1 = Layout<Shape<_16, _1, _1>>;+ if constexpr (Causal) {+ return SparseSDPAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages, false, ElementQ,+ ElementK, ElementV, ElementO>::template run<HasMask>(options);+ } else {+ return SparseSDPAConfig<false, ShapeQK1, ShapePV1, ShapeOut1, SubgroupLayoutQK1, void, PipelineStages, false,+ ElementQ, ElementK, ElementV, ElementO>::template run<HasMask>(options);+ }+}++template <bool Causal, bool HasMask, typename ElementQ, typename ElementK, typename ElementV,+ typename ElementO = ElementV>+inline int launch_sparse_sdpa_prefill_kernel_128_qtile64(Options const& options) {+ constexpr int PipelineStages = 2;+ using ShapeQK = Shape<_64, _64, _32>;+ using ShapePV = Shape<_64, _32, _64>;+ using ShapeOut = Shape<_64, _128>;+ using SubgroupLayoutQK = Layout<Shape<_4, _1, _1>>;+ using ShapeQK1 = Shape<_64, _64, _32>;+ using ShapePV1 = Shape<_64, _32, _64>;+ using ShapeOut1 = Shape<_64, _128>;+ using SubgroupLayoutQK1 = Layout<Shape<_4, _1, _1>>;+ if constexpr (Causal) {+ return SparseSDPAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages, false, ElementQ,+ ElementK, ElementV, ElementO>::template run<HasMask>(options);+ } else {+ return SparseSDPAConfig<false, ShapeQK1, ShapePV1, ShapeOut1, SubgroupLayoutQK1, void, PipelineStages, false,+ ElementQ, ElementK, ElementV, ElementO>::template run<HasMask>(options);+ }+}++template <bool Causal, bool HasMask, typename ElementQ, typename ElementK, typename ElementV,+ typename ElementO = ElementV>+inline int launch_sparse_sdpa_prefill_kernel_64(Options const& options) {+ constexpr int PipelineStages = 2;+ using ShapeQK = Shape<_128, _64, _32>;+ using ShapePV = Shape<_128, _32, _64>;+ using ShapeOut = Shape<_128, _64>;+ using SubgroupLayoutQK = Layout<Shape<_8, _1, _1>>;+ using SubgroupLayoutPV = void;+ if constexpr (Causal) {+ return SparseSDPAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages,+ false, ElementQ, ElementK, ElementV, ElementO>::template run<HasMask>(options);+ } else {+ return SparseSDPAConfig<false, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages,+ false, ElementQ, ElementK, ElementV, ElementO>::template run<HasMask>(options);+ }}#endif // ARK_SDPA_ENABLE_SPARSEdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hppindex 10b309328..5ecf3b7d5 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp@@ -15,6 +15,7 @@#ifdef ARK_XPU#include <sycl/sycl.hpp>#endif+#include "flash_attn_dtype.hpp"#if defined(ARK_SYCL_TLA)@@ -55,14 +56,8 @@namespace ark {-/// Flash Attention data type codes (matches Python side)-enum class FlashAttnDtype : int {- FP16 = 0,- BF16 = 1,- FP32 = 2,- FP8_E4M3 = 3,- FP8_E5M2 = 4,-};+// FlashAttnDtype is defined in flash_attn_dtype.hpp so that dispatch-only+// translation units such as ark.cpp do not pull in this heavy header.#if defined(ARK_SYCL_TLA)diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sage_sparse_prefill_e2e.py b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sage_sparse_prefill_e2e.pyindex 27ab86385..f258a5a05 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sage_sparse_prefill_e2e.py+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sage_sparse_prefill_e2e.py@@ -15,8 +15,18 @@ import auto_round_kernel as arkfrom auto_round_kernel.xpu_loader import ensure_xpu_lib-def ensure_sparse_binding() -> None:- ensure_xpu_lib(required_symbols=("sage_sparse",))+def ensure_sparse_binding(*, required_symbols: tuple[str, ...] = ("sage_sparse",)) -> None:+ search_roots = (+ REPO_PARENT / "auto_round_kernel",+ REPO_PARENT / "auto_round_kernel" / "xbuild_diffuser",+ REPO_PARENT / "auto_round_kernel" / "xbuild",+ REPO_PARENT / "auto_round_kernel" / "xbuild_bf16_v2",+ REPO_PARENT / "auto_round_kernel" / "ark-xbuild",+ )+ ensure_xpu_lib(+ required_symbols=required_symbols,+ search_roots=search_roots,+ )def quantize_qk(tensor: torch.Tensor, block_size: int) -> tuple[torch.Tensor, torch.Tensor]:@@ -43,7 +53,7 @@ def quantize_qk(tensor: torch.Tensor, block_size: int) -> tuple[torch.Tensor, todef build_sparse_metadata_and_mask(batch: int,heads: int,- seq_len: int,+ seq_len_q: int,quant_block_size: int,query_tile_tokens: int,per_query_tile_selection: list[list[int]],@@ -51,17 +61,19 @@ def build_sparse_metadata_and_mask(is_causal: bool = False,sparse_q_block_tokens: int | None = None,sparse_k_block_tokens: int | None = None,+ seq_len_kv: int | None = None,) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ seq_len_kv = seq_len_q if seq_len_kv is None else seq_len_kvq_block_tokens = quant_block_size if sparse_q_block_tokens is None else sparse_q_block_tokensk_block_tokens = quant_block_size if sparse_k_block_tokens is None else sparse_k_block_tokens- q_blocks = (seq_len + q_block_tokens - 1) // q_block_tokens- kv_blocks = (seq_len + k_block_tokens - 1) // k_block_tokens- active_query_tiles = (seq_len + query_tile_tokens - 1) // query_tile_tokens+ q_blocks = (seq_len_q + q_block_tokens - 1) // q_block_tokens+ kv_blocks = (seq_len_kv + k_block_tokens - 1) // k_block_tokens+ active_query_tiles = (seq_len_q + query_tile_tokens - 1) // query_tile_tokensassert len(per_query_tile_selection) == active_query_tileslut = torch.zeros((batch, heads, q_blocks, kv_blocks), dtype=torch.int32, device=device)valid = torch.zeros((batch, heads, q_blocks), dtype=torch.int32, device=device)- mask = torch.full((batch, 1, seq_len, seq_len), -1.0e9, dtype=torch.float32, device=device)+ mask = torch.full((batch, 1, seq_len_q, seq_len_kv), -1.0e9, dtype=torch.float32, device=device)q_blocks_per_query_tile = max(1, query_tile_tokens // q_block_tokens)for qblk in range(q_blocks):@@ -75,11 +87,11 @@ def build_sparse_metadata_and_mask(for qtile, selected_blocks in enumerate(per_query_tile_selection):q_start = qtile * query_tile_tokens- q_end = min(q_start + query_tile_tokens, seq_len)+ q_end = min(q_start + query_tile_tokens, seq_len_q)for qt in range(q_start, q_end):for selected in selected_blocks:k_start = selected * k_block_tokens- k_end = min(k_start + k_block_tokens, seq_len)+ k_end = min(k_start + k_block_tokens, seq_len_kv)if not is_causal:mask[:, :, qt : qt + 1, k_start:k_end] = 0.0else:@@ -98,6 +110,33 @@ def _clamp_selected_blocks(kv_blocks: int, per_query_tile_selection: list[list[ireturn clamped+def bf16_sparse_reference(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None,+ *,+ scale: float,+ enable_gqa: bool,+) -> torch.Tensor:+ # Run the reference on CPU: torch-xpu's softmax is numerically broken on large+ # masked tensors (sums to <1, drops visible entries), which corrupts the reference.+ q = query.float().cpu()+ k = key.float().cpu()+ v = value.float().cpu()+ mask = None if attn_mask is None else attn_mask.float().cpu()+ if enable_gqa and q.shape[1] != k.shape[1]:+ repeat = q.shape[1] // k.shape[1]+ k = k.repeat_interleave(repeat, dim=1)+ v = v.repeat_interleave(repeat, dim=1)++ scores = torch.matmul(q, k.transpose(-1, -2)) * scale+ if mask is not None:+ scores = scores + mask+ probs = torch.softmax(scores, dim=-1)+ return torch.matmul(probs, v).to(query.dtype).to(query.device)++def run_case(head_dim: int,block_size: int = 64,@@ -201,6 +240,163 @@ def run_case()+def run_case_sdpa(+ dtype: torch.dtype,+ head_dim: int,+ *,+ seq_len_q: int = 256,+ seq_len_kv: int = 256,+ num_heads_q: int = 4,+ num_heads_kv: int | None = None,+ is_causal: bool = False,+ q_tile_override: int = 0,+ sparse_q_block_tokens: int | None = None,+ sparse_k_block_tokens: int | None = None,+) -> None:+ """Validate the independent native-precision sparse-SDPA path (bf16/fp16)+ against a dense reference built from the same block selection."""+ ensure_sparse_binding(required_symbols=("sage_sparse", "block_sparse_sdpa"))+ device = torch.device("xpu")+ batch = 1+ num_heads_kv = num_heads_q if num_heads_kv is None else num_heads_kv+ scale = 1.0 / math.sqrt(head_dim)+ query_tile_tokens = q_tile_override or (64 if head_dim == 64 else 256)+ k_block_tokens = 64 if sparse_k_block_tokens is None else sparse_k_block_tokens+ kv_blocks = (seq_len_kv + k_block_tokens - 1) // k_block_tokens+ active_query_tiles = (seq_len_q + query_tile_tokens - 1) // query_tile_tokens+ if not is_causal:+ per_query_tile_selection = [list(range(max(1, min(kv_blocks, 3)))) for _ in range(active_query_tiles)]+ if active_query_tiles > 1 and kv_blocks > 3:+ per_query_tile_selection[-1] = list(range(kv_blocks - 3, kv_blocks))+ case_name = f"{dtype}_prefill"+ else:+ per_query_tile_selection = [list(range(min(kv_blocks, idx + 1))) for idx in range(active_query_tiles)]+ case_name = f"{dtype}_prefill_causal"+ per_query_tile_selection = _clamp_selected_blocks(kv_blocks, per_query_tile_selection)++ torch.manual_seed(9026 + head_dim + num_heads_q + (num_heads_kv * 13) + seq_len_q + seq_len_kv)+ query = torch.randn((batch, num_heads_q, seq_len_q, head_dim), dtype=dtype, device=device)+ key = torch.randn((batch, num_heads_kv, seq_len_kv, head_dim), dtype=dtype, device=device)+ value = torch.randn((batch, num_heads_kv, seq_len_kv, head_dim), dtype=dtype, device=device)++ effective_sparse_q_block_tokens = 64 if sparse_q_block_tokens is None else sparse_q_block_tokens+ effective_sparse_k_block_tokens = 64 if sparse_k_block_tokens is None else sparse_k_block_tokens+ lut, valid, dense_mask = build_sparse_metadata_and_mask(+ batch,+ num_heads_q,+ seq_len_q,+ 64,+ query_tile_tokens,+ per_query_tile_selection,+ device,+ is_causal=is_causal,+ sparse_q_block_tokens=effective_sparse_q_block_tokens,+ sparse_k_block_tokens=effective_sparse_k_block_tokens,+ seq_len_kv=seq_len_kv,+ )++ dense_out = bf16_sparse_reference(+ query,+ key,+ value,+ dense_mask,+ scale=scale,+ enable_gqa=num_heads_q != num_heads_kv,+ )+ sparse_out = ark.block_sparse_sdpa(+ query,+ key,+ value,+ lut,+ valid,+ is_causal=is_causal,+ scale=scale,+ q_tile_override=q_tile_override,+ sparse_q_block_tokens=sparse_q_block_tokens,+ sparse_k_block_tokens=sparse_k_block_tokens,+ tensor_layout="HND",+ )+ torch.xpu.synchronize()++ diff = (dense_out.float() - sparse_out.float()).abs()+ max_diff = float(diff.max().cpu())+ mean_diff = float(diff.mean().cpu())+ print(+ f"[block_sparse_sdpa][{case_name}] D={head_dim} Hq={num_heads_q} Hkv={num_heads_kv} "+ f"Sq={seq_len_q} Skv={seq_len_kv} max_diff={max_diff:.6f} mean_diff={mean_diff:.6f}"+ )+ if max_diff > 2e-2 or mean_diff > 2e-3:+ raise RuntimeError(+ f"block_sparse_sdpa mismatch for dtype={dtype}, D={head_dim}, Hq={num_heads_q}, Hkv={num_heads_kv}, "+ f"Sq={seq_len_q}, Skv={seq_len_kv}, causal={is_causal}"+ )+++def run_case_sdpa_full(+ dtype: torch.dtype,+ head_dim: int,+ *,+ seq_len_q: int = 256,+ seq_len_kv: int = 256,+ num_heads_q: int = 4,+ q_tile_override: int = 0,+ sparse_q_block_tokens: int | None = None,+ sparse_k_block_tokens: int | None = None,+) -> None:+ """Dense gate: selecting every KV block must make sparse-SDPA match the dense reference."""+ ensure_sparse_binding(required_symbols=("sage_sparse", "block_sparse_sdpa"))+ device = torch.device("xpu")+ batch = 1+ scale = 1.0 / math.sqrt(head_dim)+ query_tile_tokens = q_tile_override or (64 if head_dim == 64 else 256)+ k_block_tokens = 64 if sparse_k_block_tokens is None else sparse_k_block_tokens+ kv_blocks = (seq_len_kv + k_block_tokens - 1) // k_block_tokens+ active_query_tiles = (seq_len_q + query_tile_tokens - 1) // query_tile_tokens+ per_query_tile_selection = [list(range(kv_blocks)) for _ in range(active_query_tiles)]++ torch.manual_seed(77 + head_dim + num_heads_q)+ query = torch.randn((batch, num_heads_q, seq_len_q, head_dim), dtype=dtype, device=device)+ key = torch.randn((batch, num_heads_q, seq_len_kv, head_dim), dtype=dtype, device=device)+ value = torch.randn((batch, num_heads_q, seq_len_kv, head_dim), dtype=dtype, device=device)++ lut, valid, dense_mask = build_sparse_metadata_and_mask(+ batch,+ num_heads_q,+ seq_len_q,+ 64,+ query_tile_tokens,+ per_query_tile_selection,+ device,+ is_causal=False,+ sparse_q_block_tokens=64 if sparse_q_block_tokens is None else sparse_q_block_tokens,+ sparse_k_block_tokens=k_block_tokens,+ seq_len_kv=seq_len_kv,+ )++ dense_out = bf16_sparse_reference(query, key, value, dense_mask, scale=scale, enable_gqa=False)+ sparse_out = ark.block_sparse_sdpa(+ query,+ key,+ value,+ lut,+ valid,+ is_causal=False,+ scale=scale,+ q_tile_override=q_tile_override,+ sparse_q_block_tokens=sparse_q_block_tokens,+ sparse_k_block_tokens=sparse_k_block_tokens,+ tensor_layout="HND",+ )+ torch.xpu.synchronize()++ diff = (dense_out.float() - sparse_out.float()).abs()+ max_diff = float(diff.max().cpu())+ mean_diff = float(diff.mean().cpu())+ print(f"[block_sparse_sdpa][{dtype}_all_selected] D={head_dim} max_diff={max_diff:.6f} mean_diff={mean_diff:.6f}")+ if max_diff > 2e-2 or mean_diff > 2e-3:+ raise RuntimeError(f"block_sparse_sdpa all-selected mismatch for dtype={dtype}, D={head_dim}")++def run_multi_row_tile_case() -> None:device = torch.device("xpu")batch = 1@@ -341,6 +537,37 @@ def main() -> None:)run_case(64, is_causal=True)run_case(128, is_causal=True)+ # Independent sparse-SDPA path (bf16 + fp16)+ for dtype in (torch.bfloat16, torch.float16):+ run_case_sdpa(dtype, 64)+ run_case_sdpa(dtype, 128, q_tile_override=64, num_heads_q=32, num_heads_kv=8)+ run_case_sdpa(+ dtype,+ 128,+ seq_len_q=512,+ seq_len_kv=768,+ num_heads_q=32,+ num_heads_kv=8,+ q_tile_override=256,+ sparse_q_block_tokens=256,+ sparse_k_block_tokens=64,+ )+ run_case_sdpa(dtype, 128, is_causal=True, q_tile_override=64)+ # Regression coverage for causal masking across K32 qtile256 microtiles.+ run_case_sdpa(+ dtype,+ 128,+ seq_len_q=512,+ seq_len_kv=512,+ num_heads_q=8,+ num_heads_kv=8,+ is_causal=True,+ q_tile_override=256,+ sparse_q_block_tokens=256,+ sparse_k_block_tokens=64,+ )+ run_case_sdpa_full(dtype, 64)+ run_case_sdpa_full(dtype, 128, q_tile_override=64)if __name__ == "__main__":diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sparge_preprocess_topk_e2e.py b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sparge_preprocess_topk_e2e.pyindex 02426f391..547620728 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sparge_preprocess_topk_e2e.py+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sparge_preprocess_topk_e2e.py@@ -16,7 +16,16 @@ from auto_round_kernel.xpu_loader import ensure_xpu_libdef ensure_sparse_binding() -> None:- ensure_xpu_lib(required_symbols=("sage_sparse",))+ ensure_xpu_lib(+ required_symbols=("sage_sparse",),+ search_roots=(+ REPO_PARENT / "auto_round_kernel",+ REPO_PARENT / "auto_round_kernel" / "xbuild_diffuser",+ REPO_PARENT / "auto_round_kernel" / "xbuild",+ REPO_PARENT / "auto_round_kernel" / "xbuild_bf16_v2",+ REPO_PARENT / "auto_round_kernel" / "ark-xbuild",+ ),+ )def _to_layout(tensor: torch.Tensor, tensor_layout: str) -> torch.Tensor:@@ -264,6 +273,74 @@ def run_case(assert torch.equal(crafted_meta["block_map"], crafted_meta["raw_block_map"])+def run_lut_roundtrip_case(*, num_heads_q: int = 40, q_blocks: int = 8, k_blocks: int = 32) -> None:+ """Regression test for the XPU boolean-mask assignment bug.++ torch-xpu-ops' boolean-mask in-place assignment (`tensor[mask] = value`) silently+ misses entries on 4-D int64/bool tensors of these sizes, which used to leave 0s in+ ``_block_map_lut_torch``'s ``filled_matrix`` and produce **-1** LUT entries. Those+ -1s then drove the ``scatter_`` in ``_lut_to_block_map`` out of bounds+ (``ScatterGatherKernels.cpp:233``). The LUT must be non-negative and the+ block_map -> lut -> block_map round-trip must be lossless.+ """+ from auto_round_kernel.sparse_attention import _block_map_lut_torch, _lut_to_block_map++ device = torch.device("xpu")+ torch.manual_seed(1234)+ block_map = torch.rand(1, num_heads_q, q_blocks, k_blocks, device=device) < 0.5+ lut, valid_block_num = _block_map_lut_torch(block_map)+ torch.xpu.synchronize()++ assert lut.dtype == torch.int32+ assert valid_block_num.dtype == torch.int32+ assert tuple(lut.shape) == tuple(block_map.shape)+ # Regression: on XPU this was -1 because the masked assignment silently failed.+ assert int(lut.min().item()) >= 0, f"lut must be non-negative, got min={int(lut.min().item())}"+ assert int(valid_block_num.max().item()) <= k_blocks++ reconstructed = _lut_to_block_map(lut, valid_block_num)+ torch.xpu.synchronize()+ assert torch.equal(reconstructed, block_map), "block_map -> lut -> block_map round-trip must be lossless"+++def run_fill_block_map_case() -> None:+ """Regression test for the vectorized ``_fill_block_map_torch``.++ The naive implementation looped ``for rank in range(k_blocks)`` (thousands of+ sequential kernel launches at long sequences); the vectorized rewrite must be+ exactly equivalent. Runs on CPU against a reference loop implementation.+ """+ from auto_round_kernel.sparse_attention import _fill_block_map_torch++ def reference_loop(final_map, num_to_select, sorted_indices):+ k_blocks = final_map.shape[-1]+ filled = final_map.clone()+ column_ids = torch.arange(k_blocks, device=final_map.device).view(1, 1, 1, k_blocks)+ target_new = torch.maximum(num_to_select, torch.ones_like(num_to_select))+ added = torch.zeros_like(num_to_select)+ for rank in range(k_blocks):+ idx_match = column_ids == sorted_indices[..., rank : rank + 1]+ is_new = idx_match & ~filled+ should_add = (added < target_new).unsqueeze(-1)+ newly_selected = should_add & is_new+ filled |= newly_selected+ added = added + newly_selected.any(dim=-1).to(added.dtype)+ return filled++ for shape in [(1, 4, 8, 32), (1, 40, 8, 32), (1, 40, 128, 512), (2, 16, 64, 128)]:+ b, h, q, k = shape+ for seed in range(4):+ torch.manual_seed(seed)+ probs = torch.rand(shape)+ _, sorted_indices = torch.sort(probs, dim=-1, descending=True)+ num_to_select = torch.randint(0, k + 1, (b, h, q), dtype=torch.int64)+ final_map = torch.rand(shape) < 0.1 # some blocks already forced+ assert torch.equal(+ _fill_block_map_torch(final_map, num_to_select, sorted_indices),+ reference_loop(final_map, num_to_select, sorted_indices),+ ), f"vectorized _fill_block_map_torch differs from reference at shape={shape} seed={seed}"++def run_causal_decoupled_wrapper_case(*, tensor_layout: str) -> None:device = torch.device("xpu")batch = 1@@ -410,6 +487,8 @@ def main() -> None:num_heads_kv=1,)run_causal_decoupled_wrapper_case(tensor_layout=tensor_layout)+ run_lut_roundtrip_case()+ run_fill_block_map_case()if __name__ == "__main__":diff --git a/auto_round_extension/ark/benchmarks/bench_sparse_topk.py b/auto_round_extension/ark/benchmarks/bench_sparse_topk.pyindex 2813ed48b..9efe91e07 100644--- a/auto_round_extension/ark/benchmarks/bench_sparse_topk.py+++ b/auto_round_extension/ark/benchmarks/bench_sparse_topk.py@@ -23,13 +23,13 @@ import auto_round_kernel as arkfrom auto_round_kernel.xpu_loader import load_xpu_lib-def _load_sparse_binding(ext_path: Path) -> None:+def _load_sparse_binding(ext_path: Path, *, required_symbols: tuple[str, ...]) -> None:module_name = "auto_round_kernel._bench.auto_round_kernel_xpu"print(f"Loading XPU extension from {ext_path.resolve()} as {module_name}")- load_xpu_lib(ext_path, required_symbols=("sage_sparse",), module_name=module_name)+ load_xpu_lib(ext_path, required_symbols=required_symbols, module_name=module_name)-def _resolve_sparse_binding(args: argparse.Namespace) -> Path | None:+def _resolve_sparse_binding(args: argparse.Namespace, *, required_symbols: tuple[str, ...]) -> Path | None:local_kernel_dir = REPO_ROOT / "auto_round_extension" / "ark" / "auto_round_kernel"current_file = getattr(getattr(ark, "xpu_lib", None), "__file__", None)if args.xpu_so is not None:@@ -37,9 +37,11 @@ def _resolve_sparse_binding(args: argparse.Namespace) -> Path | None:if args.xbuild_dir is not None:candidates = sorted(args.xbuild_dir.resolve().glob("auto_round_kernel_xpu*.so"))if not candidates:- raise RuntimeError(f"Unable to locate built XPU extension with sage_sparse in {args.xbuild_dir}")+ raise RuntimeError(+ f"Unable to locate built XPU extension with required symbols {required_symbols} in {args.xbuild_dir}"+ )return candidates[-1]- if current_file is not None and hasattr(ark.xpu_lib, "sage_sparse"):+ if current_file is not None and all(hasattr(ark.xpu_lib, symbol) for symbol in required_symbols):try:current_path = Path(current_file).resolve()if current_path.is_relative_to(local_kernel_dir.resolve()):@@ -47,22 +49,38 @@ def _resolve_sparse_binding(args: argparse.Namespace) -> Path | None:return Noneexcept Exception:pass- candidates = sorted((local_kernel_dir / "xbuild").glob("auto_round_kernel_xpu*.so"))+ build_roots = [root for root in local_kernel_dir.iterdir() if root.is_dir() and "build" in root.name]+ candidates: list[Path] = []+ for root in build_roots:+ candidates.extend(sorted(root.glob("auto_round_kernel_xpu*.so")))if not candidates:- raise RuntimeError("Unable to locate built XPU extension with sage_sparse in auto_round_kernel/xbuild")+ raise RuntimeError(+ f"Unable to locate built XPU extension with required symbols {required_symbols} under {local_kernel_dir}"+ )+ candidates.sort(key=lambda path: path.stat().st_mtime)return candidates[-1]def ensure_sparse_binding(args: argparse.Namespace) -> None:- ext_path = _resolve_sparse_binding(args)+ required_symbols = ("sage_sparse", "block_sparse_sdpa")+ ext_path = _resolve_sparse_binding(args, required_symbols=required_symbols)if ext_path is not None:- _load_sparse_binding(ext_path)+ _load_sparse_binding(ext_path, required_symbols=required_symbols)def is_xpu_available() -> bool:return hasattr(torch, "xpu") and torch.xpu.is_available()+def resolve_dtype(name: str) -> torch.dtype:+ normalized = name.strip().lower()+ if normalized == "fp16":+ return torch.float16+ if normalized == "bf16":+ return torch.bfloat16+ raise ValueError(f"Unsupported dtype={name!r}; supported values: fp16, bf16")++def bench(fn, warmup: int, iters: int) -> float:for _ in range(warmup):out = fn()@@ -476,18 +494,22 @@ def hnd_to_nhd(x: torch.Tensor) -> torch.Tensor:def summarize_speedups(rows: list[dict[str, object]]) -> list[dict[str, object]]:torch_row = next((row for row in rows if row["mode"] == "dense_torch_sdpa" and row["status"] == "ok"), None)sage_row = next((row for row in rows if row["mode"] == "dense_sagev1" and row["status"] == "ok"), None)+ ark_row = next((row for row in rows if row["mode"] == "dense_ark_sdpa" and row["status"] == "ok"), None)torch_ms = None if torch_row is None else float(torch_row["latency_ms"])sage_ms = None if sage_row is None else float(sage_row["latency_ms"])+ ark_ms = None if ark_row is None else float(ark_row["latency_ms"])for row in rows:latency_ms = row["latency_ms"]if latency_ms is None:row["speedup_vs_torch"] = Nonerow["speedup_vs_sagev1"] = None+ row["speedup_vs_ark"] = Nonerow["baseline_tflops"] = Nonerow["effective_tflops"] = Nonecontinuerow["speedup_vs_torch"] = (torch_ms / latency_ms) if torch_ms is not None else Nonerow["speedup_vs_sagev1"] = (sage_ms / latency_ms) if sage_ms is not None else None+ row["speedup_vs_ark"] = (ark_ms / latency_ms) if ark_ms is not None else Nonebatch = int(row["batch"])num_heads_q = int(row["num_heads_q"])seq_len = int(row["seq_len"])@@ -499,10 +521,15 @@ def summarize_speedups(rows: list[dict[str, object]]) -> list[dict[str, object]]if mode in {"dense_torch_sdpa","dense_sagev1",+ "dense_ark_sdpa","sparse_kernel_only","sparse_e2e","sparse_qtile256_row64k_kernel_only","sparse_qtile256_row64k_e2e",+ "sparse_sdpa_bf16_kernel_only",+ "sparse_sdpa_bf16_e2e",+ "sparse_sdpa_fp16_kernel_only",+ "sparse_sdpa_fp16_e2e",}:row["baseline_tflops"] = flops_to_tflops(dense_flops, float(latency_ms))work_ratio = 1.0@@ -514,9 +541,9 @@ def summarize_speedups(rows: list[dict[str, object]]) -> list[dict[str, object]]def print_summary(rows: list[dict[str, object]]) -> None:print(- "| layout | seq_len | mode | pattern | gqa_group | topk | selected_ratio | blocks/row | latency (ms) | baseline_tflops | effective_tflops | status | speedup_vs_torch | speedup_vs_sagev1 |"+ "| layout | seq_len | mode | pattern | gqa_group | topk | selected_ratio | blocks/row | latency (ms) | baseline_tflops | effective_tflops | status | speedup_vs_torch | speedup_vs_sagev1 | speedup_vs_ark |")- print("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|")+ print("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|")for row in rows:topk = "-" if row["requested_topk"] is None else f"{float(row['requested_topk']):.3f}"ratio = "-" if row["selected_ratio"] is None else f"{float(row['selected_ratio']):.6f}"@@ -526,10 +553,11 @@ def print_summary(rows: list[dict[str, object]]) -> None:effective_tflops = "-" if row.get("effective_tflops") is None else f"{float(row['effective_tflops']):.3f}"sp_torch = "-" if row.get("speedup_vs_torch") is None else f"{float(row['speedup_vs_torch']):.3f}"sp_sage = "-" if row.get("speedup_vs_sagev1") is None else f"{float(row['speedup_vs_sagev1']):.3f}"+ sp_ark = "-" if row.get("speedup_vs_ark") is None else f"{float(row['speedup_vs_ark']):.3f}"print(f"| {row['tensor_layout']} | {row['seq_len']} | {row['mode']} | {row['attention_pattern']} | {row['gqa_group_size']} | {topk} | {ratio} | "f"{blocks} | {latency} | {baseline_tflops} | {effective_tflops} | {row['status']} | {sp_torch} | "- f"{sp_sage} |"+ f"{sp_sage} | {sp_ark} |")if row["note"]:print(f"note[{row['mode']}]: {row['note']}")@@ -548,7 +576,7 @@ def write_csv(rows: list[dict[str, object]], output_csv: Path) -> None:def run_single_benchmark(args: argparse.Namespace, *, seq_len: int, tensor_layout: str) -> list[dict[str, object]]:device = torch.device("xpu")- dtype = torch.float16+ dtype = resolve_dtype(args.dtype)scale = 1.0 / math.sqrt(args.head_dim)enable_gqa = args.num_heads_q // args.num_heads_kv > 1q_hnd_src, k_hnd_src, v_hnd_src = build_inputs(@@ -641,6 +669,29 @@ def run_single_benchmark(args: argparse.Namespace, *, seq_len: int, tensor_layouiters=args.iters,))+ rows.append(+ try_benchmark(+ "dense_ark_sdpa",+ lambda: ark.sdpa(+ q,+ k,+ v,+ is_causal=args.causal,+ scale=scale,+ tensor_layout=tensor_layout,+ ),+ batch=args.batch,+ num_heads_q=args.num_heads_q,+ num_heads_kv=args.num_heads_kv,+ seq_len=seq_len,+ tensor_layout=tensor_layout,+ head_dim=args.head_dim,+ dtype=dtype,+ is_causal=args.causal,+ warmup=args.warmup,+ iters=args.iters,+ )+ )for topk in args.topk:preprocess = None@@ -891,6 +942,108 @@ def run_single_benchmark(args: argparse.Namespace, *, seq_len: int, tensor_layouselected_blocks_per_row=selected_blocks_per_row,))+ rows.append(+ try_benchmark(+ "sparse_sdpa_fp16_kernel_only" if dtype == torch.float16 else "sparse_sdpa_bf16_kernel_only",+ lambda preprocess=preprocess: (+ hnd_to_nhd(+ ark.block_sparse_sdpa(+ nhd_to_hnd(q_nhd_src),+ nhd_to_hnd(k_nhd_src),+ nhd_to_hnd(v_nhd_src),+ preprocess["lut"],+ preprocess["valid_block_num"],+ is_causal=args.causal,+ scale=scale,+ q_tile_override=args.q_tile_override,+ sparse_q_block_tokens=preprocess["sparse_q_block_tokens"],+ sparse_k_block_tokens=preprocess["sparse_k_block_tokens"],+ tensor_layout="HND",+ )+ )+ if tensor_layout == "HND"+ else ark.block_sparse_sdpa(+ q,+ k,+ v,+ preprocess["lut"],+ preprocess["valid_block_num"],+ is_causal=args.causal,+ scale=scale,+ q_tile_override=args.q_tile_override,+ sparse_q_block_tokens=preprocess["sparse_q_block_tokens"],+ sparse_k_block_tokens=preprocess["sparse_k_block_tokens"],+ tensor_layout=tensor_layout,+ )+ ),+ batch=args.batch,+ num_heads_q=args.num_heads_q,+ num_heads_kv=args.num_heads_kv,+ seq_len=seq_len,+ tensor_layout=tensor_layout,+ head_dim=args.head_dim,+ dtype=dtype,+ is_causal=args.causal,+ warmup=args.warmup,+ iters=args.iters,+ requested_topk=topk,+ selected_ratio=selected_ratio,+ selected_blocks_per_row=selected_blocks_per_row,+ )+ )+ rows.append(+ try_benchmark(+ "sparse_sdpa_fp16_e2e" if dtype == torch.float16 else "sparse_sdpa_bf16_e2e",+ lambda topk=topk: (+ hnd_to_nhd(+ ark.sparge_sage2_attn_meansim_topk_xpu_sdpa(+ nhd_to_hnd(q_nhd_src),+ nhd_to_hnd(k_nhd_src),+ nhd_to_hnd(v_nhd_src),+ is_causal=args.causal,+ scale=scale,+ smooth_k=True,+ simthreshd1=-1.0,+ topk=topk,+ attention_sink=False,+ tensor_layout="HND",+ q_tile_override=args.q_tile_override,+ sparse_q_block_tokens=args.sparse_q_block_tokens,+ sparse_k_block_tokens=args.sparse_k_block_tokens,+ )+ )+ if tensor_layout == "HND"+ else ark.sparge_sage2_attn_meansim_topk_xpu_sdpa(+ q,+ k,+ v,+ is_causal=args.causal,+ scale=scale,+ smooth_k=True,+ simthreshd1=-1.0,+ topk=topk,+ attention_sink=False,+ tensor_layout=tensor_layout,+ q_tile_override=args.q_tile_override,+ sparse_q_block_tokens=args.sparse_q_block_tokens,+ sparse_k_block_tokens=args.sparse_k_block_tokens,+ )+ ),+ batch=args.batch,+ num_heads_q=args.num_heads_q,+ num_heads_kv=args.num_heads_kv,+ seq_len=seq_len,+ tensor_layout=tensor_layout,+ head_dim=args.head_dim,+ dtype=dtype,+ is_causal=args.causal,+ warmup=args.warmup,+ iters=args.iters,+ requested_topk=topk,+ selected_ratio=selected_ratio,+ selected_blocks_per_row=selected_blocks_per_row,+ )+ )del preprocessempty_xpu_cache()@@ -921,6 +1074,12 @@ def parse_args() -> argparse.Namespace:parser.add_argument("--num-heads-kv", type=int, default=40)parser.add_argument("--seq-len", type=int, nargs="+", default=[32768, 75600])parser.add_argument("--head-dim", type=int, default=128)+ parser.add_argument(+ "--dtype",+ choices=("fp16", "bf16"),+ default="fp16",+ help="Input/output dtype used for dense SDPA, dense SAGE, and sparse attention benchmarks.",+ )parser.add_argument("--topk", type=float, nargs="+", default=[0.5, 0.25, 0.125])parser.add_argument("--quant-block-size", type=int, default=64)parser.add_argument(diff --git a/auto_round_extension/ark/examples/flux_gen_bf16_sweep.py b/auto_round_extension/ark/examples/flux_gen_bf16_sweep.pynew file mode 100644index 000000000..904c8e7e7--- /dev/null+++ b/auto_round_extension/ark/examples/flux_gen_bf16_sweep.py@@ -0,0 +1,182 @@+# Copyright (C) 2026 Intel Corporation+# SPDX-License-Identifier: Apache-2.0++import contextlib+import csv+import os+import sys+import time+from pathlib import Path++import torch+from diffusers import FluxPipeline+from flux_sparse_patch import patch_flux_sparse_attention_from_env++dtype = torch.bfloat16++MODEL_DEFAULT = "/mnt/disk3/models/black-forest-labs/FLUX.1-dev"+PROMPT_DEFAULT = "A cat holding a sign that says hello world"+++def env_flag(name, default="1"):+ return os.getenv(name, default).lower() not in {"0", "false", "no", "off"}+++def sync_xpu():+ if hasattr(torch, "xpu") and hasattr(torch.xpu, "synchronize"):+ torch.xpu.synchronize()+++def main():+ model_id = os.getenv("FLUX_MODEL", MODEL_DEFAULT)+ topks = os.getenv("FLUX_SPARSE_TOPKS", "0.5 0.25 0.125").split()+ run_dense = env_flag("FLUX_RUN_DENSE", "1")+ prompt = os.getenv("FLUX_PROMPT", PROMPT_DEFAULT)+ height = int(os.getenv("FLUX_HEIGHT", "512"))+ width = int(os.getenv("FLUX_WIDTH", "512"))+ steps = int(os.getenv("FLUX_STEPS", "50"))+ seed = int(os.getenv("FLUX_SEED", "0"))+ guidance_scale = float(os.getenv("FLUX_GUIDANCE_SCALE", "3.5"))+ max_sequence_length = int(os.getenv("FLUX_MAX_SEQUENCE_LENGTH", "512"))++ default_out = Path("benchmarks/results") / f"flux_bf16_{time.strftime('%Y%m%d_%H%M%S')}"+ out_dir = Path(os.getenv("FLUX_OUTPUT_DIR", str(default_out)))+ out_dir.mkdir(parents=True, exist_ok=True)++ device_id = os.getenv("ZE_AFFINITY_MASK_VALUE", "")+ dev_suffix = f"_dev{device_id}" if device_id else ""++ print(f"[flux_sweep] model={model_id}", flush=True)+ print(+ f"[flux_sweep] topks={topks} run_dense={run_dense} size={height}x{width} steps={steps} seed={seed}", flush=True+ )+ print(f"[flux_sweep] out_dir={out_dir}", flush=True)+ print(+ f"[flux_sweep] kernel={os.getenv('FLUX_SPARSE_KERNEL', '?')} "+ f"q_tile={os.getenv('FLUX_SPARSE_Q_TILE_OVERRIDE', '0')} "+ f"q_block={os.getenv('FLUX_SPARSE_Q_BLOCK_TOKENS', 'default')} "+ f"k_block={os.getenv('FLUX_SPARSE_K_BLOCK_TOKENS', 'default')}",+ flush=True,+ )++ pipe = FluxPipeline.from_pretrained(model_id, torch_dtype=dtype)+ # Offload-only: pipe.to(device) would load the whole ~54 GB model onto the+ # 24.4 GB device and OOM.+ #+ # Use sequential (block-level) offload, NOT enable_model_cpu_offload():+ # model-level offload pulls the whole ~46 GB transformer onto the device,+ # peaking at ~24 GB (the full 24.4 GB device) during every transformer+ # forward. The sparse BF16 preprocess (triton_xpu) then has <0.4 GB of+ # headroom, and its kernel launch flakily OOMs / resets the GPU+ # (UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY / OUT_OF_RESOURCES / DEVICE_LOST).+ # Sequential offload keeps only a few blocks resident (peak ~0.16 GB), so+ # the triton_xpu sparse path always has room.+ pipe.enable_sequential_cpu_offload()++ common_kwargs = dict(+ height=height,+ width=width,+ guidance_scale=guidance_scale,+ num_inference_steps=steps,+ max_sequence_length=max_sequence_length,+ generator=torch.Generator("cpu").manual_seed(seed),+ )++ rows = []++ def do_run(tag, topk, png_name):+ if topk is None:+ ctx = contextlib.nullcontext(None)+ os.environ.pop("FLUX_SPARSE_TOPK", None)+ else:+ os.environ["FLUX_SPARSE_TOPK"] = str(topk)+ ctx = patch_flux_sparse_attention_from_env(pipe.transformer)++ sync_xpu()+ t0 = time.perf_counter()+ with ctx as stats:+ image = pipe(prompt, output_type="pil", **common_kwargs).images[0]+ sync_xpu()+ wall_s = time.perf_counter() - t0++ png_path = out_dir / png_name+ image.save(png_path)++ row = {+ "tag": tag,+ "topk": "" if topk is None else topk,+ "wall_s": round(wall_s, 3),+ "sparsity": "",+ "calls": "",+ "sparse_calls": "",+ "runtime_fallbacks": "",+ "unsupported_fallbacks": "",+ "png": str(png_path),+ }+ if topk is not None:+ if stats is None or stats.sparse_calls == 0:+ raise RuntimeError(+ f"topk={topk}: sparse path did not run (sparse_calls=0) — check FLUX_SPARSE_KERNEL / qtile256 config"+ )+ if stats.runtime_fallbacks or stats.unsupported_fallbacks:+ raise RuntimeError(+ f"topk={topk}: silent fallback detected — runtime={stats.runtime_fallbacks}, "+ f"unsupported={stats.unsupported_fallbacks}"+ )+ row["sparsity"] = round(float(stats.avg_sparsity), 4)+ row["calls"] = stats.total_calls+ row["sparse_calls"] = stats.sparse_calls+ row["runtime_fallbacks"] = stats.runtime_fallbacks+ row["unsupported_fallbacks"] = stats.unsupported_fallbacks+ print(+ f"[flux_sweep] DONE tag={tag} topk={topk} wall={wall_s:.3f}s "+ f"sparsity={stats.avg_sparsity:.4f} sparse_calls={stats.sparse_calls} "+ f"runtime_fallbacks={stats.runtime_fallbacks}",+ flush=True,+ )+ else:+ print(f"[flux_sweep] DONE tag={tag} wall={wall_s:.3f}s", flush=True)+ rows.append(row)+ return row++ if run_dense:+ do_run("dense", None, f"flux_dense_512{dev_suffix}.png")++ for topk in topks:+ do_run("sparse_bf16_qtile256", topk, f"flux_bf16_qtile256_topk{topk}_512{dev_suffix}.png")++ # CSV + markdown summary (device-suffixed so parallel instances sharing+ # FLUX_OUTPUT_DIR do not clobber each other).+ csv_path = out_dir / f"sweep_summary{dev_suffix}.csv"+ fieldnames = [+ "tag",+ "topk",+ "wall_s",+ "sparsity",+ "calls",+ "sparse_calls",+ "runtime_fallbacks",+ "unsupported_fallbacks",+ "png",+ ]+ with open(csv_path, "w", newline="", encoding="utf-8") as fh:+ writer = csv.DictWriter(fh, fieldnames=fieldnames)+ writer.writeheader()+ writer.writerows(rows)++ md_path = out_dir / f"SWEEP_SUMMARY{dev_suffix}.md"+ with open(md_path, "w", encoding="utf-8") as fh:+ fh.write("| tag | topk | wall_s | sparsity | calls | sparse_calls | runtime_fallbacks | png |\n")+ fh.write("|---|---|---|---|---|---|---|---|\n")+ fh.writelines(+ f"| {r['tag']} | {r['topk']} | {r['wall_s']} | {r['sparsity']} | {r['calls']} "+ f"| {r['sparse_calls']} | {r['runtime_fallbacks']} | `{r['png']}` |\n"+ for r in rows+ )++ print(f"[flux_sweep] summary: {csv_path}", flush=True)+ print(f"[flux_sweep] summary: {md_path}", flush=True)+++if __name__ == "__main__":+ main()diff --git a/auto_round_extension/ark/examples/flux_sparse_patch.py b/auto_round_extension/ark/examples/flux_sparse_patch.pyindex 6695bcbb9..ab512c9aa 100644--- a/auto_round_extension/ark/examples/flux_sparse_patch.py+++ b/auto_round_extension/ark/examples/flux_sparse_patch.py@@ -20,6 +20,19 @@ from diffusers.models.transformers.transformer_flux import FluxAttention, _get_qfrom wan_sparse_patch import _parse_bool_env, _parse_optional_int_env, ensure_ark_sparse_binding+def _parse_sparse_kernel_env(name: str = "FLUX_SPARSE_KERNEL", default: str = "int8") -> str:+ value = os.getenv(name, default).strip().lower()+ aliases = {+ "int8": "int8",+ "qks8": "int8",+ "bf16": "bf16",+ }+ normalized = aliases.get(value)+ if normalized is None:+ raise ValueError(f"{name} must be one of: int8, bf16")+ return normalized++def _normalize_attention_mask(attn_mask: torch.Tensor | None,batch: int,@@ -90,6 +103,7 @@ class FluxSparseAttnProcessor:q_tile_override: int,sparse_q_block_tokens: int | None,sparse_k_block_tokens: int | None,+ sparse_kernel: str,):self.original_processor = original_processorself.stats = stats@@ -100,6 +114,7 @@ class FluxSparseAttnProcessor:self.q_tile_override = q_tile_overrideself.sparse_q_block_tokens = sparse_q_block_tokensself.sparse_k_block_tokens = sparse_k_block_tokens+ self.sparse_kernel = sparse_kernelself._warned_runtime_fallback = False@staticmethod@@ -192,22 +207,40 @@ class FluxSparseAttnProcessor:seq_kv=key.shape[1],device=query.device,)- hidden_states, sparsity = ark.sparge_sage2_attn_meansim_topk_xpu(- query,- key,- value,- attn_mask=mask,- is_causal=False,- smooth_k=self.smooth_k,- simthreshd1=-1.0,- topk=self.topk,- attention_sink=self.attention_sink,- tensor_layout="NHD",- q_tile_override=self.q_tile_override,- sparse_q_block_tokens=self.sparse_q_block_tokens,- sparse_k_block_tokens=self.sparse_k_block_tokens,- return_sparsity=True,- )+ if self.sparse_kernel == "bf16":+ hidden_states, sparsity = ark.sparge_sage2_attn_meansim_topk_xpu_sdpa(+ query,+ key,+ value,+ attn_mask=mask,+ is_causal=False,+ smooth_k=self.smooth_k,+ simthreshd1=-1.0,+ topk=self.topk,+ attention_sink=self.attention_sink,+ tensor_layout="NHD",+ q_tile_override=self.q_tile_override,+ sparse_q_block_tokens=self.sparse_q_block_tokens,+ sparse_k_block_tokens=self.sparse_k_block_tokens,+ return_sparsity=True,+ )+ else:+ hidden_states, sparsity = ark.sparge_sage2_attn_meansim_topk_xpu(+ query,+ key,+ value,+ attn_mask=mask,+ is_causal=False,+ smooth_k=self.smooth_k,+ simthreshd1=-1.0,+ topk=self.topk,+ attention_sink=self.attention_sink,+ tensor_layout="NHD",+ q_tile_override=self.q_tile_override,+ sparse_q_block_tokens=self.sparse_q_block_tokens,+ sparse_k_block_tokens=self.sparse_k_block_tokens,+ return_sparsity=True,+ )self.stats.sparse_calls += 1self.stats.sparse_sparsity_sum += float(sparsity)@@ -270,8 +303,15 @@ def patch_flux_sparse_attention(q_tile_override: int = 0,sparse_q_block_tokens: int | None = None,sparse_k_block_tokens: int | None = None,+ sparse_kernel: str = "int8",):- ensure_ark_sparse_binding()+ ensure_ark_sparse_binding(+ required_symbols=(+ ("block_sparse_sdpa", "sage_dynamic_quant_layout")+ if sparse_kernel == "bf16"+ else ("sage_sparse", "sage_dynamic_quant_layout")+ )+ )originals: list[tuple[FluxAttention, object]] = []stats = FluxSparseAttentionStats()@@ -289,6 +329,7 @@ def patch_flux_sparse_attention(q_tile_override=q_tile_override,sparse_q_block_tokens=sparse_q_block_tokens,sparse_k_block_tokens=sparse_k_block_tokens,+ sparse_kernel=sparse_kernel,))originals.append((module, original))@@ -310,4 +351,5 @@ def patch_flux_sparse_attention_from_env(transformer):q_tile_override=int(os.getenv("FLUX_SPARSE_Q_TILE_OVERRIDE", "0")),sparse_q_block_tokens=_parse_optional_int_env("FLUX_SPARSE_Q_BLOCK_TOKENS"),sparse_k_block_tokens=_parse_optional_int_env("FLUX_SPARSE_K_BLOCK_TOKENS"),+ sparse_kernel=_parse_sparse_kernel_env(),)diff --git a/auto_round_extension/ark/examples/run_flux.py b/auto_round_extension/ark/examples/run_flux.pyindex 0c1bd3544..57fcf902d 100644--- a/auto_round_extension/ark/examples/run_flux.py+++ b/auto_round_extension/ark/examples/run_flux.py@@ -24,6 +24,19 @@ def env_flag(name, default="0"):return os.getenv(name, default).lower() not in {"0", "false", "no", "off"}+def env_sparse_kernel(name="FLUX_SPARSE_KERNEL", default="int8"):+ value = os.getenv(name, default).strip().lower()+ aliases = {+ "int8": "int8",+ "qks8": "int8",+ "bf16": "bf16",+ }+ normalized = aliases.get(value)+ if normalized is None:+ raise ValueError(f"{name} must be one of: int8, bf16")+ return normalized++benchmark_enabled = env_flag("FLUX_BENCHMARK_ENABLE", "0")profiler_enabled = env_flag("FLUX_PROFILER_ENABLE", "0")benchmark_scope = os.getenv("FLUX_BENCHMARK_SCOPE", "full").strip().lower()@@ -41,10 +54,10 @@ cpu_offload_enabled = env_flag("FLUX_ENABLE_CPU_OFFLOAD", "0" if benchmark_scopemodel_id = os.getenv("FLUX_MODEL", "~/workspace/models/black-forest-labs/FLUX.1-dev/")pipe = FluxPipeline.from_pretrained(model_id, torch_dtype=dtype)-if benchmark_scope != "block":- pipe.to(device)if cpu_offload_enabled:- pipe.enable_model_cpu_offload()+ pipe.enable_model_cpu_offload(device=device)+elif benchmark_scope != "block":+ pipe.to(device)height = int(os.getenv("FLUX_HEIGHT", "1024"))width = int(os.getenv("FLUX_WIDTH", "1024"))@@ -53,11 +66,15 @@ guidance_scale = float(os.getenv("FLUX_GUIDANCE_SCALE", "3.5"))max_sequence_length = int(os.getenv("FLUX_MAX_SEQUENCE_LENGTH", "512"))seed = int(os.getenv("FLUX_SEED", "0"))use_sparse = os.getenv("FLUX_USE_SPARSE", "1").lower() not in {"0", "false", "no", "off"}+sparse_kernel = env_sparse_kernel()-output_file = (- f"flux_output_{height}x{width}_{num_inference_steps}steps_"- f"{guidance_scale}gs_sparse{os.getenv('FLUX_SPARSE_TOPK', '0.5')}.png"-)+if use_sparse:+ output_file = (+ f"flux_output_{height}x{width}_{num_inference_steps}steps_"+ f"{guidance_scale}gs_sparse_{sparse_kernel}_topk{os.getenv('FLUX_SPARSE_TOPK', '0.5')}.png"+ )+else:+ output_file = f"flux_output_{height}x{width}_{num_inference_steps}steps_{guidance_scale}gs_dense.png"output_path = os.getenv("FLUX_OUTPUT", output_file)prompt = os.getenv("FLUX_PROMPT", "A cat holding a sign that says hello world")@@ -195,7 +212,8 @@ def sparse_patch_context():if not use_sparse:return contextlib.nullcontext(None)print(- f"[flux_sparse] enabled attention sparse patch: topk={os.getenv('FLUX_SPARSE_TOPK', '0.5')} "+ f"[flux_sparse] enabled attention sparse patch: kernel={sparse_kernel}"+ f" topk={os.getenv('FLUX_SPARSE_TOPK', '0.5')} "f"smooth_k={os.getenv('FLUX_SPARSE_SMOOTH_K', '1')}"f" q_tile={os.getenv('FLUX_SPARSE_Q_TILE_OVERRIDE', '0')}"f" q_block={os.getenv('FLUX_SPARSE_Q_BLOCK_TOKENS', 'default')}"@@ -524,6 +542,7 @@ def run_block_benchmark(current_run_tag):"max_sequence_length": max_sequence_length,"seed": seed,"use_sparse": use_sparse,+ "sparse_kernel": sparse_kernel if use_sparse else None,"cpu_offload_enabled": cpu_offload_enabled,"block_kind": block_state["block_kind"],"block_index": block_state["block_index"],@@ -543,7 +562,7 @@ def run_block_benchmark(current_run_tag):return result-run_tag = "sparse" if use_sparse else "dense"+run_tag = f"sparse_{sparse_kernel}" if use_sparse else "dense"def maybe_run_with_profiler(run_callable, current_run_tag, record_label="flux_generate"):@@ -615,6 +634,7 @@ def run_benchmark(run_callable, current_run_tag, benchmark_scope):"max_sequence_length": max_sequence_length,"seed": seed,"use_sparse": use_sparse,+ "sparse_kernel": sparse_kernel if use_sparse else None,"sparse_topk": os.getenv("FLUX_SPARSE_TOPK", "0.5"),"sparse_smooth_k": os.getenv("FLUX_SPARSE_SMOOTH_K", "1"),"output_path": output_path,@@ -704,6 +724,7 @@ def run_denoising_benchmark(current_run_tag):"max_sequence_length": max_sequence_length,"seed": seed,"use_sparse": use_sparse,+ "sparse_kernel": sparse_kernel if use_sparse else None,"sparse_topk": os.getenv("FLUX_SPARSE_TOPK", "0.5"),"sparse_smooth_k": os.getenv("FLUX_SPARSE_SMOOTH_K", "1"),"output_path": output_path,diff --git a/auto_round_extension/ark/examples/wan_sparse_patch.py b/auto_round_extension/ark/examples/wan_sparse_patch.pyindex 2e61ca6cd..0d82071f8 100644--- a/auto_round_extension/ark/examples/wan_sparse_patch.py+++ b/auto_round_extension/ark/examples/wan_sparse_patch.py@@ -36,10 +36,18 @@ def _parse_optional_int_env(name: str) -> int | None:return None if parsed == 0 else parsed-def ensure_ark_sparse_binding() -> None:+def ensure_ark_sparse_binding(+ *, required_symbols: tuple[str, ...] = ("sage_sparse", "sage_dynamic_quant_layout")+) -> None:ensure_xpu_lib(- required_symbols=("sage_sparse", "sage_dynamic_quant_layout"),- search_roots=(KERNEL_DIR, KERNEL_DIR / "xbuild", KERNEL_DIR / "xbuild_diffuser"),+ required_symbols=required_symbols,+ search_roots=(+ KERNEL_DIR,+ KERNEL_DIR / "xbuild_diffuser",+ KERNEL_DIR / "xbuild",+ KERNEL_DIR / "xbuild_bf16_v2",+ KERNEL_DIR / "ark-xbuild",+ ),)