PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add smooth K for sagev1 (#1806)
Luo Yu017ed92c6fa1
diff --git a/auto_round_extension/ark/.gitignore b/auto_round_extension/ark/.gitignorenew file mode 100644index 000000000..d95db0037--- /dev/null+++ b/auto_round_extension/ark/.gitignore@@ -0,0 +1,3 @@+build+xbuild+*.csv\ No newline at end of filediff --git a/auto_round_extension/ark/README.md b/auto_round_extension/ark/README.mdindex a36eef2b3..c087147d9 100644--- a/auto_round_extension/ark/README.md+++ b/auto_round_extension/ark/README.md@@ -82,3 +82,24 @@ y = qlinear(x)#### A Weight-Only ExampleA runnable end-to-end example is available in [test_weightonly.py](test/test_weightonly.py). It demonstrates how to prepare quantized weights and scales, call repack_quantized_weight to build ARK-packed weights, verify correctness with unpack_weight, and run woqgemm on CPU and XPU.++#### Replace torch SDPA and run lm-eval+ ARK also exposes an XPU SDPA kernel through `ARK.sdpa(...)`. If you want to replace `torch.nn.functional.scaled_dot_product_attention` globally for evaluation without editing model code, use the helper launcher in [tools/lm_eval_with_ark_sdpa.py](tools/lm_eval_with_ark_sdpa.py).++ Example:+```bash+cd /path/to/auto_round_extension/ark+PYTHONPATH=$PWD python tools/lm_eval_with_ark_sdpa.py \+ --model hf \+ --model_args pretrained=/path/to/model,trust_remote_code=True,dtype=bfloat16 \+ --tasks hellaswag,piqa,winogrande \+ --device xpu:0 \+ --batch_size 1+```++ Notes:+ * The patch only routes calls to ARK on XPU when the inputs match ARK kernel constraints; otherwise it falls back to the original torch SDPA.+ * Supported Q/K/V dtypes are FP16 and BF16.+ * Supported head dimensions are 64, 96, 128, and 192.+ * `dropout_p` must be 0.0 for the ARK path.+ * Additive masks are supported when they can be normalized to `[B, 1, Sq, Skv]`; boolean masks fall back to torch.diff --git a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txtindex 1d799d098..5891bbb5c 100755--- a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt+++ b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt@@ -162,8 +162,25 @@ target_link_libraries(${PY_NAME} PRIVATE ${libs})if(ARK_UT)set(TEST_NAME test_${ARK_TYPE})- add_executable(${TEST_NAME} wrapper/test/test_main.cpp)+ set(TEST_SRCS wrapper/test/test_main.cpp)+ if(ARK_XPU AND ARK_SYCL_TLA)+ list(APPEND TEST_SRCS sdpa.cpp)+ list(APPEND TEST_SRCS ${SDPA_GENERATED_SRCS})+ endif()+ add_executable(${TEST_NAME} ${TEST_SRCS})target_compile_features(${TEST_NAME} PRIVATE cxx_std_17)target_compile_definitions(${TEST_NAME} PRIVATE ${ARK_TYPE}=1)+ target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/wrapper/include)+ if(ARK_XPU AND ARK_SYCL_TLA)+ target_compile_definitions(${TEST_NAME} PRIVATE ARK_SYCL_TLA=1 CUTLASS_ENABLE_SYCL=1 SYCL_INTEL_TARGET=1)+ target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/generated/sdpa)+ foreach(_inc_dir IN LISTS _sycl_tla_include_dirs)+ if(EXISTS "${_inc_dir}")+ target_include_directories(${TEST_NAME} SYSTEM PRIVATE "${_inc_dir}")+ endif()+ endforeach()+ target_compile_options(${TEST_NAME} PRIVATE -fsycl -fno-sycl-instrument-device-code)+ target_link_options(${TEST_NAME} PRIVATE ${SYCL_TLA_LINK_FLAGS})+ endif()target_link_libraries(${TEST_NAME} PRIVATE ${libs})endif()diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex 48778aff3..831d3714a 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -675,6 +675,7 @@ class ARK:lib.sage_dynamic_quant(stream,query.data_ptr(),+ 0,q_i8.data_ptr(),q_scale.data_ptr(),q_num_rows,@@ -690,6 +691,7 @@ class ARK:lib.sage_dynamic_quant(stream,key.data_ptr(),+ 0,k_i8.data_ptr(),k_scale.data_ptr(),k_num_rows,@@ -859,7 +861,19 @@ if __name__ == "__main__":pack_unpack()-__all__ = ["ARK"]+def patch_torch_sdpa(*args, **kwargs):+ from .torch_sdpa_patch import patch_torch_sdpa_with_ark++ return patch_torch_sdpa_with_ark(*args, **kwargs)+++def unpatch_torch_sdpa():+ from .torch_sdpa_patch import unpatch_torch_sdpa_with_ark++ return unpatch_torch_sdpa_with_ark()+++__all__ = ["ARK", "patch_torch_sdpa", "unpatch_torch_sdpa"]# -----------------------------------------------------------------------------@@ -937,7 +951,15 @@ def repack_quantized_weight(*args, **kwargs):def unpack_weight(- blob: torch.Tensor, out_dtype: torch.dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym+ blob: torch.Tensor,+ out_dtype: torch.dtype,+ n,+ k,+ groupsize,+ compute_type,+ weight_type,+ scale_type,+ asym,):return _ark_instance().unpack_weight(blob, out_dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym)diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppindex cd5c1ab03..86d983f90 100755--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -153,10 +153,11 @@ static void moe_gemm_wrapper(torch_ptr stream, torch_ptr activations, torch_ptr(void*)outputs, (BTLA_DTYPE)(dtype), N, K, (int*)num_tokens_per_expert, num_experts);}-static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr output, torch_ptr scale_out, int num_rows,- int head_dim, int block_size) {+static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr bias, torch_ptr output, torch_ptr scale_out,+ int num_rows, int head_dim, int block_size) {auto* q = (sycl::queue*)stream;auto* in_ptr = (sycl::half*)input;+ auto* bias_ptr = bias ? (sycl::half*)bias : nullptr;auto* out_ptr = (int8_t*)output;auto* scale_ptr = (float*)scale_out;@@ -181,6 +182,7 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr outpint tid = item.get_local_id(0);auto wg = item.get_group();auto* block_in = in_ptr + (size_t)block_id * elems_per_block;+ auto* block_bias = bias_ptr ? bias_ptr + (size_t)block_id * elems_per_block : nullptr;auto* block_out = out_ptr + (size_t)block_id * elems_per_block;// Phase 1: compute absmax across entire block@@ -189,6 +191,9 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr outplocal_max_vec = sycl::vec<sycl::half, Unroll>(0.0f);for (int i = tid * Unroll; i < elems_per_block; i += wg_size * Unroll) {local_data = *(sycl::vec<sycl::half, Unroll>*)(&block_in[i]);+ if (block_bias) {+ local_data = local_data - *(sycl::vec<sycl::half, Unroll>*)(&block_bias[i]);+ }local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));}for (int i = 0; i < Unroll; ++i) {@@ -208,7 +213,11 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr outpfor (int i = tid * Unroll; i < elems_per_block; i += wg_size * Unroll) {#pragma unrollfor (int j = 0; j < Unroll; ++j) {- float val = static_cast<float>(block_in[i + j]) * inv_scale;+ float val = static_cast<float>(block_in[i + j]);+ if (block_bias) {+ val -= static_cast<float>(block_bias[i + j]);+ }+ val *= inv_scale;int iv = static_cast<int>(val + (val >= 0.0f ? 0.5f : -0.5f));iv = sycl::clamp(iv, -127, 127);block_out[i + j] = static_cast<int8_t>(iv);@@ -223,6 +232,7 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr outpint tid = item.get_local_id(0);auto wg = item.get_group();auto* block_in = in_ptr + (size_t)block_id * elems_per_block;+ auto* block_bias = bias_ptr ? bias_ptr + (size_t)block_id * elems_per_block : nullptr;auto* block_out = out_ptr + (size_t)block_id * elems_per_block;// Phase 1: compute absmax across entire block@@ -232,6 +242,9 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr outpint local_i = 0;for (int i = tid * Unroll; i < elems_per_block; i += wg_size * Unroll, local_i++) {local_data[local_i] = *(sycl::vec<sycl::half, Unroll>*)&block_in[i];+ if (block_bias) {+ local_data[local_i] = local_data[local_i] - *(sycl::vec<sycl::half, Unroll>*)(&block_bias[i]);+ }local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));}#pragma unrolldiff --git a/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.pynew file mode 100644index 000000000..7b51f96db--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py@@ -0,0 +1,238 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++from __future__ import annotations++import logging++import torch++from . import ARK++logger = logging.getLogger(__name__)++_ARK_SDPA_PATCHED = False+_ORIG_SDPA = None+_PATCH_CONFIG = None+_FALLBACK_WARNING_LOGGED = False+++def _log_fallback_warning_once(backend: str, error: Exception) -> None:+ global _FALLBACK_WARNING_LOGGED++ if _FALLBACK_WARNING_LOGGED:+ return++ logger.warning(+ "ARK %s patch failed and fell back to torch SDPA. Subsequent failures will be suppressed. Error: %s",+ backend,+ error,+ )+ _FALLBACK_WARNING_LOGGED = True+++def _validate_backend(backend: str, quant_block_size: int) -> str:+ backend = backend.lower()+ if backend not in {"sdpa", "sagev1"}:+ raise ValueError(f"Unsupported ARK attention backend: {backend}")+ if backend == "sagev1" and quant_block_size <= 0:+ raise ValueError("quant_block_size must be > 0 when backend='sagev1'")+ return backend+++def _normalize_attn_mask(attn_mask: torch.Tensor, batch: int, seq_q: int, seq_kv: int) -> torch.Tensor:+ if attn_mask.dtype == torch.bool:+ raise ValueError("Boolean attention masks are not supported by the ARK SDPA patch")++ if attn_mask.ndim == 2:+ if attn_mask.shape != (seq_q, seq_kv):+ raise ValueError("Unsupported 2D attention mask shape")+ attn_mask = attn_mask.view(1, 1, seq_q, seq_kv)+ elif attn_mask.ndim == 3:+ if attn_mask.shape != (batch, seq_q, seq_kv):+ raise ValueError("Unsupported 3D attention mask shape")+ attn_mask = attn_mask.unsqueeze(1)+ elif attn_mask.ndim == 4:+ if attn_mask.shape[-2:] != (seq_q, seq_kv):+ raise ValueError("Unsupported 4D attention mask shape")+ if attn_mask.shape[1] != 1:+ raise ValueError("Only attention masks with head dimension 1 are supported")+ if attn_mask.shape[0] == 1 and batch != 1:+ attn_mask = attn_mask.expand(batch, -1, -1, -1)+ elif attn_mask.shape[0] != batch:+ raise ValueError("Unsupported batch dimension in attention mask")+ else:+ raise ValueError("Unsupported attention mask rank")++ return attn_mask.contiguous().to(torch.float32)+++def _is_pure_causal_mask(attn_mask: torch.Tensor) -> bool:+ seq_q = attn_mask.shape[-2]+ seq_kv = attn_mask.shape[-1]+ if seq_q != seq_kv:+ return False++ mask_2d = attn_mask.reshape(-1, seq_q, seq_kv)[0]+ tri_up = torch.triu(torch.ones(seq_q, seq_kv, dtype=torch.bool, device=attn_mask.device), 1)+ return bool(torch.isinf(mask_2d[tri_up]).all().item()) and bool((mask_2d[~tri_up] == 0).all().item())+++def _can_use_ark_attention(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None,+ dropout_p: float,+ backend: str,+) -> bool:+ if query.device.type != "xpu":+ return False+ if query.requires_grad or key.requires_grad or value.requires_grad:+ return False+ if query.dtype not in (torch.float16, torch.bfloat16):+ return False+ if key.dtype != query.dtype or value.dtype != query.dtype:+ return False+ if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:+ return False+ if query.shape[0] != key.shape[0] or query.shape[0] != value.shape[0]:+ return False+ if key.shape != value.shape:+ return False+ if query.shape[-1] != key.shape[-1]:+ return False+ if dropout_p != 0.0:+ return False+ if attn_mask is not None and attn_mask.device.type != "xpu":+ return False+ if backend == "sdpa":+ return query.dtype in (torch.float16, torch.bfloat16) and query.shape[-1] in (+ 64,+ 96,+ 128,+ 192,+ )+ if backend == "sagev1":+ return query.dtype == torch.float16 and query.shape[-1] in (64, 128)+ return True+++def patch_torch_sdpa_with_ark(*, strict: bool = False, backend: str = "sdpa", quant_block_size: int = 64) -> bool:+ global _ARK_SDPA_PATCHED, _ORIG_SDPA, _PATCH_CONFIG++ backend = _validate_backend(backend, quant_block_size)+ new_config = {"backend": backend, "quant_block_size": int(quant_block_size)}++ if _ARK_SDPA_PATCHED:+ if _PATCH_CONFIG == new_config:+ return True+ unpatch_torch_sdpa_with_ark()++ if not (hasattr(torch, "xpu") and torch.xpu.is_available()):+ if strict:+ raise RuntimeError("XPU is not available")+ return False++ ark = ARK()+ if ark.xpu_lib is None or not hasattr(ark.xpu_lib, backend):+ if strict:+ raise RuntimeError(f"ARK XPU {backend} kernel is not available")+ return False++ _ORIG_SDPA = torch.nn.functional.scaled_dot_product_attention+ orig_sdpa = _ORIG_SDPA++ def _patched_sdpa(+ query,+ key,+ value,+ attn_mask=None,+ dropout_p=0.0,+ is_causal=False,+ scale=None,+ **kwargs,+ ):+ if not _can_use_ark_attention(query, key, value, attn_mask, dropout_p, backend):+ return orig_sdpa(+ query,+ key,+ value,+ attn_mask=attn_mask,+ dropout_p=dropout_p,+ is_causal=is_causal,+ scale=scale,+ **kwargs,+ )++ normalized_mask = None+ if attn_mask is not None:+ try:+ normalized_mask = _normalize_attn_mask(attn_mask, query.shape[0], query.shape[-2], key.shape[-2])+ except ValueError:+ return orig_sdpa(+ query,+ key,+ value,+ attn_mask=attn_mask,+ dropout_p=dropout_p,+ is_causal=is_causal,+ scale=scale,+ **kwargs,+ )+ if not is_causal and _is_pure_causal_mask(normalized_mask):+ normalized_mask = None+ is_causal = True++ try:+ if backend == "sdpa":+ return ark.sdpa(+ query.contiguous(),+ key.contiguous(),+ value.contiguous(),+ attn_mask=normalized_mask,+ dropout_p=dropout_p,+ is_causal=is_causal,+ scale=scale,+ )+ return ark.sagev1(+ query.contiguous(),+ key.contiguous(),+ value.contiguous(),+ attn_mask=normalized_mask,+ dropout_p=dropout_p,+ is_causal=is_causal,+ scale=scale,+ quant_block_size=quant_block_size,+ )+ except (NotImplementedError, RuntimeError, ValueError) as error:+ _log_fallback_warning_once(backend, error)+ return orig_sdpa(+ query,+ key,+ value,+ attn_mask=attn_mask,+ dropout_p=dropout_p,+ is_causal=is_causal,+ scale=scale,+ **kwargs,+ )++ torch.nn.functional.scaled_dot_product_attention = _patched_sdpa+ _ARK_SDPA_PATCHED = True+ _PATCH_CONFIG = new_config+ return True+++def unpatch_torch_sdpa_with_ark() -> bool:+ global _ARK_SDPA_PATCHED, _ORIG_SDPA, _PATCH_CONFIG, _FALLBACK_WARNING_LOGGED++ if not _ARK_SDPA_PATCHED or _ORIG_SDPA is None:+ return False++ torch.nn.functional.scaled_dot_product_attention = _ORIG_SDPA+ _ARK_SDPA_PATCHED = False+ _ORIG_SDPA = None+ _PATCH_CONFIG = None+ _FALLBACK_WARNING_LOGGED = False+ return Truediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hppindex cc60e5d7a..8b50ee769 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp@@ -32,6 +32,8 @@ namespace ark {struct env_params {int verbose = 2; // trace 0/ debug 1/ info 2/ warn 3/ error 4/ critical 5/ off 6int auto_s8 = 0;+ int sage_use_mean_bias = 1;+ int sage_print_kbias = 0;static env_params* Instance() {static env_params instance;return &instance;@@ -39,6 +41,8 @@ struct env_params {env_params() {env_i("ARK_VERBOSE", verbose);env_i("ARK_AUTO_S8", auto_s8);+ env_i("ARK_SAGE_USE_MEAN_BIAS", sage_use_mean_bias);+ env_i("ARK_SAGE_PRINT_KBIAS", sage_print_kbias);}static inline void env_i(const char* envstr, int& default_) {const char* log_level_env = getenv(envstr);diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hppindex 682a9085b..a7dec5b4b 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp@@ -11,8 +11,11 @@//#pragma once+#include <algorithm>#include <cmath>#include <cstdio>+#include <limits>+#include <vector>#include "utils.hpp"#include "dnnl_wrapper.hpp"#include "sycl_tla_common.hpp"@@ -693,11 +696,79 @@ class XpuWrapper {}}+ template <typename T>+ static void compute_seq_mean_bias(sycl::queue* q, const T* in_ptr, T* bias_ptr, int num_rows, int seq, int head_dim) {+ constexpr int SG_SIZE = 32;+ constexpr int WG_SIZE = 512;+ constexpr int SG_PER_WG = WG_SIZE / SG_SIZE;+ constexpr int SeqUnroll = 4;+ size_t total = size_t(num_rows) * size_t(head_dim);+ size_t num_groups = (total + SG_PER_WG - 1) / SG_PER_WG;+ size_t global = num_groups * WG_SIZE;+ q->parallel_for(sycl::nd_range<1>(global, WG_SIZE),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ size_t group_id = item.get_group(0);+ int local_id = item.get_local_id(0);+ int subgroup_id = local_id / SG_SIZE;+ int lane_id = local_id % SG_SIZE;+ size_t index = group_id * SG_PER_WG + subgroup_id;+ if (index >= total) {+ return;+ }++ int row_id = index / head_dim;+ int dim_id = index % head_dim;+ const T* row_ptr = in_ptr + size_t(row_id) * seq * head_dim + dim_id;+ auto sg = item.get_sub_group();++ float sum = 0.0f;+ int token = lane_id;+ for (; token + (SeqUnroll - 1) * SG_SIZE < seq; token += SeqUnroll * SG_SIZE) {+ #pragma unroll+ for (int unroll_idx = 0; unroll_idx < SeqUnroll; ++unroll_idx) {+ sum += static_cast<float>(row_ptr[size_t(token + unroll_idx * SG_SIZE) * head_dim]);+ }+ }+ for (; token < seq; token += SG_SIZE) {+ sum += static_cast<float>(row_ptr[size_t(token) * head_dim]);+ }+ sum = sycl::reduce_over_group(sg, sum, sycl::plus<float>{});+ if (lane_id == 0) {+ bias_ptr[index] = static_cast<T>(sum / static_cast<float>(seq));+ }+ });+ }++ template <typename T>+ static void print_value_distribution(sycl::queue* q, const T* dev_ptr, size_t count, const char* name) {+ if (dev_ptr == nullptr || count == 0) {+ std::printf("[%s] empty\n", name);+ return;+ }++ std::vector<T> host_values(count);+ q->memcpy(host_values.data(), dev_ptr, count * sizeof(T)).wait();++ float min_value = std::numeric_limits<float>::max();+ float max_value = std::numeric_limits<float>::lowest();+ double sum = 0.0;++ for (size_t i = 0; i < count; ++i) {+ float value = static_cast<float>(host_values[i]);+ min_value = std::min(min_value, value);+ max_value = std::max(max_value, value);+ sum += value;+ }++ double mean = sum / static_cast<double>(count);+ std::printf("[%s] min=%f max=%f mean=%f\n", name, min_value, max_value, static_cast<float>(mean));+ }+// input: num_rows x head_dim matrix, output: int8 quantized matrix + scale (per block)// scale: num_rows // block_sizetemplate <typename T>static void sage_dynamic_quant(sycl::queue* q, const T* in_ptr, int8_t* out_ptr, float* scale_ptr, int num_rows,- int seq, int n_seq_blk, int head_dim, int block_size) {+ int seq, int n_seq_blk, int head_dim, int block_size, const T* bias_ptr = nullptr) {int num_blocks = num_rows * n_seq_blk;int elems_per_block = block_size * head_dim;@@ -708,127 +779,299 @@ class XpuWrapper {constexpr int MAX_Reg = 64;constexpr int MAX_WG_SIZE = 256;constexpr int Unroll = 4;+ bool fast_vector_bias = bias_ptr && (head_dim % Unroll == 0);+ bool power_of_two_head_dim = fast_vector_bias && ((head_dim & (head_dim - 1)) == 0);+ int bias_mask = head_dim - 1;if (elems_per_block > MAX_Reg * MAX_WG_SIZE) {int wg_size = (elems_per_block <= 256) ? SG_SIZE : 256;// Ensure wg_size is a multiple of SG_SIZEwg_size = ((wg_size + SG_SIZE - 1) / SG_SIZE) * SG_SIZE;- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {- int block_id = item.get_group(0);- int row_id = block_id / n_seq_blk;- int seq_id = block_id % n_seq_blk;- int tid = item.get_local_id(0);- auto wg = item.get_group();- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;-- int elems_this_wg =- (seq_id + 1) * block_size <= seq ? elems_per_block : (seq - seq_id * block_size) * head_dim;-- // Phase 1: compute absmax across entire block- float local_max = 0.0f;- sycl::vec<T, Unroll> local_data, local_max_vec;- local_max_vec = sycl::vec<T, Unroll>(0.0f);- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {- local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));- }- for (int i = 0; i < Unroll; ++i) {- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));- }- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ if (bias_ptr) {+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int block_id = item.get_group(0);+ int row_id = block_id / n_seq_blk;+ int seq_id = block_id % n_seq_blk;+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;+ auto* row_bias = bias_ptr + (size_t)row_id * head_dim;- // Compute scale- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? elems_per_block+ : (seq - seq_id * block_size) * head_dim;- // Store scale (one thread writes)- if (tid == 0) {- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;- }+ float local_max = 0.0f;+ sycl::vec<T, Unroll> local_data, local_max_vec;+ local_max_vec = sycl::vec<T, Unroll>(0.0f);+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {+ local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);+ if (fast_vector_bias) {+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);+ local_data = local_data - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);+ } else {+#pragma unroll+ for (int j = 0; j < Unroll; ++j) {+ local_data[j] = local_data[j] - row_bias[(i + j) % head_dim];+ }+ }+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));+ }+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;- // Phase 2: fused quantize- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {+ if (tid == 0) {+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;+ }++ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {+ sycl::vec<T, Unroll> quant_input = *(sycl::vec<T, Unroll>*)(&block_in[i]);+ if (fast_vector_bias) {+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);+ quant_input = quant_input - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);+ } else {#pragma unroll- for (int j = 0; j < Unroll; ++j) {- float val = static_cast<float>(block_in[i + j]) * inv_scale;- int iv = static_cast<int>(val + (val >= 0.0f ? 0.5f : -0.5f));- iv = sycl::clamp(iv, -127, 127);- block_out[i + j] = static_cast<int8_t>(iv);+ for (int j = 0; j < Unroll; ++j) {+ quant_input[j] = quant_input[j] - row_bias[(i + j) % head_dim];+ }+ }+ sycl::vec<float, Unroll> val =+ quant_input.template convert<float, sycl::rounding_mode::automatic>();+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv =+ val.template convert<int8_t, sycl::rounding_mode::automatic>();+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;}- }- });+ });+ } else {+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int block_id = item.get_group(0);+ int row_id = block_id / n_seq_blk;+ int seq_id = block_id % n_seq_blk;+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;++ int elems_this_wg = (seq_id + 1) * block_size <= seq ? elems_per_block+ : (seq - seq_id * block_size) * head_dim;++ float local_max = 0.0f;+ sycl::vec<T, Unroll> local_data, local_max_vec;+ local_max_vec = sycl::vec<T, Unroll>(0.0f);+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {+ local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));+ }+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;++ if (tid == 0) {+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;+ }++ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {+ sycl::vec<T, Unroll> quant_input = *(sycl::vec<T, Unroll>*)(&block_in[i]);+ sycl::vec<float, Unroll> val =+ quant_input.template convert<float, sycl::rounding_mode::automatic>();+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv =+ val.template convert<int8_t, sycl::rounding_mode::automatic>();+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;+ }+ });+ }} else {int wg_size = MAX_WG_SIZE;- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {- int seq_id = item.get_group(0);- int row_id = seq_id / n_seq_blk;- seq_id = seq_id % n_seq_blk;- int tid = item.get_local_id(0);- auto wg = item.get_group();- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;-- // Phase 1: compute absmax across entire block- float local_max = 0.0f;- sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;- local_max_vec = sycl::vec<T, Unroll>(0.0f);- int local_i = 0;- int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim- : (seq - seq_id * block_size) * head_dim;- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {- local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));- }+ if (bias_ptr) {+ if (fast_vector_bias) {+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int seq_id = item.get_group(0);+ int row_id = seq_id / n_seq_blk;+ seq_id = seq_id % n_seq_blk;+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;+ auto* row_bias = bias_ptr + (size_t)row_id * head_dim;++ float local_max = 0.0f;+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;+ local_max_vec = sycl::vec<T, Unroll>(0.0f);+ int local_i = 0;+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim+ : (seq - seq_id * block_size) * head_dim;+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);+ local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];+ local_data[local_i] = local_data[local_i] - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));+ }#pragma unroll- for (int i = 0; i < Unroll; ++i) {- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));- }- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;- // Compute scale- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;+ if (tid == 0) {+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;+ }- // Store scale (one thread writes)- if (tid == 0) {- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;- }+ local_i = 0;+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {+ sycl::vec<float, Unroll> val =+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv =+ val.template convert<int8_t, sycl::rounding_mode::automatic>();+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;+ }+ });+ } else {+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int seq_id = item.get_group(0);+ int row_id = seq_id / n_seq_blk;+ seq_id = seq_id % n_seq_blk;+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;+ auto* row_bias = bias_ptr + (size_t)row_id * head_dim;- // Phase 2: fused quantize- local_i = 0;- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {- sycl::vec<float, Unroll> val =- local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();- val = val * inv_scale;- val = sycl::round(val);- val = sycl::clamp(val, -127, 127);- sycl::vec<int8_t, Unroll> qv = val.template convert<int8_t, sycl::rounding_mode::automatic>();- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;- }- });+ float local_max = 0.0f;+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;+ local_max_vec = sycl::vec<T, Unroll>(0.0f);+ int local_i = 0;+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim+ : (seq - seq_id * block_size) * head_dim;+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {+ local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];+#pragma unroll+ for (int j = 0; j < Unroll; ++j) {+ local_data[local_i][j] = local_data[local_i][j] - row_bias[(i + j) % head_dim];+ }+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));+ }+#pragma unroll+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;++ if (tid == 0) {+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;+ }++ local_i = 0;+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {+ sycl::vec<float, Unroll> val =+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv =+ val.template convert<int8_t, sycl::rounding_mode::automatic>();+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;+ }+ });+ }+ } else {+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int seq_id = item.get_group(0);+ int row_id = seq_id / n_seq_blk;+ seq_id = seq_id % n_seq_blk;+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;++ float local_max = 0.0f;+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;+ local_max_vec = sycl::vec<T, Unroll>(0.0f);+ int local_i = 0;+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim+ : (seq - seq_id * block_size) * head_dim;+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {+ local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));+ }+#pragma unroll+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;++ if (tid == 0) {+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;+ }++ local_i = 0;+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {+ sycl::vec<float, Unroll> val =+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv =+ val.template convert<int8_t, sycl::rounding_mode::automatic>();+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;+ }+ });+ }}}static void sagev1(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,int head_dim, float softmax_scale, bool is_causal) {+ bool use_mean_bias = env_params::Instance()->sage_use_mean_bias != 0;size_t seq_q_blk = (seq_len_q + scale_block_size - 1) / scale_block_size;size_t seq_kv_blk = (seq_len_kv + scale_block_size - 1) / scale_block_size;size_t q_size = num_heads_q * seq_len_q * head_dim * batch;size_t q_scale_size = num_heads_q * seq_q_blk * batch;size_t k_size = num_heads_kv * seq_len_kv * head_dim * batch;size_t k_scale_size = num_heads_kv * seq_kv_blk * batch;- size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float);+ size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(sycl::half) : 0;+ size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + k_bias_size;auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);auto q_out_ptr = (int8_t*)ptr;auto k_out_ptr = (int8_t*)ptr + q_size;auto qscale = (float*)((int8_t*)ptr + q_size + k_size);auto kscale = (float*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float));+ auto kbias = use_mean_bias+ ? (sycl::half*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) ++ k_scale_size * sizeof(float))+ : nullptr;sage_dynamic_quant<sycl::half>(q, (sycl::half*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch * num_heads_q,- seq_len_q, seq_q_blk, head_dim, scale_block_size);+ seq_len_q, seq_q_blk, head_dim, scale_block_size);+ if (use_mean_bias) {+ compute_seq_mean_bias<sycl::half>(q, (sycl::half*)K_ptr, kbias, batch * num_heads_kv, seq_len_kv, head_dim);+ if (env_params::Instance()->sage_print_kbias != 0) {+ print_value_distribution(q, kbias, size_t(batch) * num_heads_kv * head_dim, "kbias");+ }+ }sage_dynamic_quant<sycl::half>(q, (sycl::half*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,- seq_len_kv, seq_kv_blk, head_dim, scale_block_size);+ seq_len_kv, seq_kv_blk, head_dim, scale_block_size, kbias);ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, batch,num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);}diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hppindex ae5a17d33..2ea29dc47 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hpp@@ -62,6 +62,4 @@ struct TestGemm {ctx->deallocate(C);ctx->deallocate(scaleB);}-};--static TestGemm sTestGemm;\ No newline at end of file+};\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cppindex 9e067c3f0..1281b16da 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cpp@@ -1,7 +1,12 @@#include <iostream>#include "test_gemm.hpp"+#include "test_quant.hpp"+#include "test_sdpa.hpp"int main() {printf("Welcome to ARK TEST\n");- return 1;+ // TestGemm test_gemm;+ // TestQuant test_quant;+ TestSDPA test_sdpa;+ return 0;}\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_quant.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_quant.hppnew file mode 100644index 000000000..29d9fbe95--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_quant.hpp@@ -0,0 +1,337 @@+#pragma once++#include <algorithm>+#include <chrono>+#include <cmath>+#include <cstdint>+#include <iomanip>+#include <iostream>+#include <random>+#include <stdexcept>+#include <string>+#include <vector>++#include "common.hpp"+#include "../include/xpu_wrapper.hpp"++struct TestQuant {+ TestQuant() {+#ifdef ARK_XPU+ test_compute_seq_mean_bias_accuracy("k_mean_bias", 3, 111, 128);+ test_compute_seq_mean_bias_perf("k_mean_bias_bw", 16, 2048, 128, 20, 100);+ test_sage_dynamic_quant_accuracy("no_bias_small_block", 4, 128, 128, 64, false);+ test_sage_dynamic_quant_accuracy("no_bias_unaligned_seq", 5, 96, 128, 64, false);+ test_sage_dynamic_quant_accuracy("bias_large_block", 3, 512, 128, 256, true);+ test_sage_dynamic_quant_accuracy("bias_unaligned_seq", 3, 112, 128, 64, true);+ test_sage_dynamic_quant_perf("bw_no_bias", 16, 2048, 128, 64, false, 20, 100);+ test_sage_dynamic_quant_perf("bw_with_bias", 16, 2048, 128, 64, true, 20, 100);+#else+ std::cout << "[sage_dynamic_quant] skipped: ARK_XPU is disabled\n";+#endif+ }++#ifdef ARK_XPU+ static int ceil_div(int value, int divisor) { return (value + divisor - 1) / divisor; }++ static sycl::half f32_to_f16(float value) { return sycl::half(value); }++ static std::vector<float> make_random_vector(size_t count, float low, float high, uint32_t seed) {+ std::mt19937 rng(seed);+ std::uniform_real_distribution<float> dist(low, high);+ std::vector<float> values(count);+ for (auto& value : values) {+ value = dist(rng);+ }+ return values;+ }++ static std::vector<sycl::half> to_fp16_vector(const std::vector<float>& src) {+ std::vector<sycl::half> dst(src.size());+ for (size_t index = 0; index < src.size(); ++index) {+ dst[index] = f32_to_f16(src[index]);+ }+ return dst;+ }++ static std::vector<float> to_f32_vector(const std::vector<sycl::half>& src) {+ std::vector<float> dst(src.size());+ for (size_t index = 0; index < src.size(); ++index) {+ dst[index] = static_cast<float>(src[index]);+ }+ return dst;+ }++ static std::vector<float> reference_seq_mean_bias(const std::vector<float>& input, int num_rows, int seq,+ int head_dim) {+ std::vector<float> bias(size_t(num_rows) * size_t(head_dim), 0.0f);+ for (int row_id = 0; row_id < num_rows; ++row_id) {+ for (int dim = 0; dim < head_dim; ++dim) {+ float sum = 0.0f;+ for (int token = 0; token < seq; ++token) {+ size_t offset = (size_t(row_id) * seq + size_t(token)) * head_dim + size_t(dim);+ sum += input[offset];+ }+ bias[size_t(row_id) * head_dim + size_t(dim)] = sum / float(seq);+ }+ }+ return bias;+ }++ void test_compute_seq_mean_bias_accuracy(const std::string& name, int num_rows, int seq, int head_dim) {+ GETQ();+ LOG_LINE();+ size_t input_count = size_t(num_rows) * size_t(seq) * size_t(head_dim);+ size_t bias_count = size_t(num_rows) * size_t(head_dim);++ auto host_input_f32 = make_random_vector(input_count, -2.0f, 2.0f, 2025u + uint32_t(seq));+ auto host_input = to_fp16_vector(host_input_f32);+ auto ref_input_f32 = to_f32_vector(host_input);+ auto ref_bias = reference_seq_mean_bias(ref_input_f32, num_rows, seq, head_dim);++ auto* dev_input = reinterpret_cast<sycl::half*>(ctx->allocate(input_count * sizeof(sycl::half)));+ auto* dev_bias = reinterpret_cast<sycl::half*>(ctx->allocate(bias_count * sizeof(sycl::half)));++ q->memcpy(dev_input, host_input.data(), input_count * sizeof(sycl::half)).wait();+ ark::XpuWrapper::compute_seq_mean_bias<sycl::half>(q, dev_input, dev_bias, num_rows, seq, head_dim);+ q->wait();++ std::vector<sycl::half> host_bias(bias_count);+ q->memcpy(host_bias.data(), dev_bias, bias_count * sizeof(sycl::half)).wait();++ float max_diff = 0.0f;+ for (size_t index = 0; index < bias_count; ++index) {+ max_diff = std::max(max_diff, std::fabs(static_cast<float>(host_bias[index]) - ref_bias[index]));+ }++ std::cout << "[compute_seq_mean_bias][accuracy] " << name << " max_diff=" << max_diff << "\n";+ if (max_diff > 1e-3f) {+ throw std::runtime_error("compute_seq_mean_bias accuracy check failed: " + name);+ }++ ctx->deallocate(dev_input);+ ctx->deallocate(dev_bias);+ }++ void test_compute_seq_mean_bias_perf(const std::string& name, int num_rows, int seq, int head_dim, int warmup,+ int iters) {+ GETQ();+ LOG_LINE();+ size_t input_count = size_t(num_rows) * size_t(seq) * size_t(head_dim);+ size_t bias_count = size_t(num_rows) * size_t(head_dim);++ auto host_input = to_fp16_vector(make_random_vector(input_count, -2.0f, 2.0f, 5051u));++ auto* dev_input = reinterpret_cast<sycl::half*>(ctx->allocate(input_count * sizeof(sycl::half)));+ auto* dev_bias = reinterpret_cast<sycl::half*>(ctx->allocate(bias_count * sizeof(sycl::half)));++ q->memcpy(dev_input, host_input.data(), input_count * sizeof(sycl::half)).wait();++ for (int iter = 0; iter < warmup; ++iter) {+ ark::XpuWrapper::compute_seq_mean_bias<sycl::half>(q, dev_input, dev_bias, num_rows, seq, head_dim);+ }+ q->wait();++ auto start = std::chrono::steady_clock::now();+ for (int iter = 0; iter < iters; ++iter) {+ ark::XpuWrapper::compute_seq_mean_bias<sycl::half>(q, dev_input, dev_bias, num_rows, seq, head_dim);+ }+ q->wait();+ auto end = std::chrono::steady_clock::now();++ double total_ms = std::chrono::duration<double, std::milli>(end - start).count();+ double avg_ms = total_ms / double(iters);+ double input_bytes = double(input_count * sizeof(sycl::half));+ double output_bytes = double(bias_count * sizeof(sycl::half));+ double logical_io_bytes = input_bytes + output_bytes;+ double estimated_kernel_bytes = input_bytes + output_bytes;+ double logical_io_gbps = logical_io_bytes / (avg_ms * 1e-3) / 1e9;+ double estimated_kernel_gbps = estimated_kernel_bytes / (avg_ms * 1e-3) / 1e9;++ std::cout << std::fixed << std::setprecision(3) << "[compute_seq_mean_bias][perf] " << name+ << " avg_ms=" << avg_ms << " input_MB=" << (input_bytes / (1024.0 * 1024.0))+ << " output_MB=" << (output_bytes / (1024.0 * 1024.0))+ << " logical_io_GBps=" << logical_io_gbps << " est_kernel_GBps=" << estimated_kernel_gbps << "\n";++ ctx->deallocate(dev_input);+ ctx->deallocate(dev_bias);+ }++ static void reference_quant(const std::vector<float>& input, const std::vector<float>* bias, int num_rows, int seq,+ int head_dim, int block_size, std::vector<int8_t>& output, std::vector<float>& scales) {+ int n_seq_blk = ceil_div(seq, block_size);+ output.assign(input.size(), int8_t(0));+ scales.assign(size_t(num_rows) * size_t(n_seq_blk), 0.0f);+ for (int row_id = 0; row_id < num_rows; ++row_id) {+ for (int seq_blk = 0; seq_blk < n_seq_blk; ++seq_blk) {+ int seq_begin = seq_blk * block_size;+ int seq_end = std::min(seq_begin + block_size, seq);+ float absmax = 0.0f;+ for (int token = seq_begin; token < seq_end; ++token) {+ for (int dim = 0; dim < head_dim; ++dim) {+ size_t offset = (size_t(row_id) * seq + size_t(token)) * head_dim + size_t(dim);+ float value = input[offset];+ if (bias) {+ value -= (*bias)[size_t(row_id) * head_dim + size_t(dim)];+ }+ absmax = std::max(absmax, std::fabs(value));+ }+ }+ scales[size_t(row_id) * n_seq_blk + size_t(seq_blk)] = absmax / 127.0f;+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;+ for (int token = seq_begin; token < seq_end; ++token) {+ for (int dim = 0; dim < head_dim; ++dim) {+ size_t offset = (size_t(row_id) * seq + size_t(token)) * head_dim + size_t(dim);+ float value = input[offset];+ if (bias) {+ value -= (*bias)[size_t(row_id) * head_dim + size_t(dim)];+ }+ value *= inv_scale;+ int quantized = static_cast<int>(std::round(value));+ quantized = std::max(-127, std::min(127, quantized));+ output[offset] = static_cast<int8_t>(quantized);+ }+ }+ }+ }+ }++ static void validate_results(const std::string& name, const std::vector<int8_t>& actual_q,+ const std::vector<int8_t>& ref_q, const std::vector<float>& actual_scales,+ const std::vector<float>& ref_scales) {+ size_t q_mismatches = 0;+ int max_q_diff = 0;+ for (size_t index = 0; index < actual_q.size(); ++index) {+ int diff = std::abs(int(actual_q[index]) - int(ref_q[index]));+ max_q_diff = std::max(max_q_diff, diff);+ if (diff != 0) {+ ++q_mismatches;+ }+ }++ float max_scale_diff = 0.0f;+ for (size_t index = 0; index < actual_scales.size(); ++index) {+ max_scale_diff = std::max(max_scale_diff, std::fabs(actual_scales[index] - ref_scales[index]));+ }++ std::cout << "[sage_dynamic_quant][accuracy] " << name << " q_mismatches=" << q_mismatches+ << " max_q_diff=" << max_q_diff << " max_scale_diff=" << max_scale_diff << "\n";++ if (max_q_diff > 1 || max_scale_diff > 1e-4f) {+ throw std::runtime_error("sage_dynamic_quant accuracy check failed: " + name);+ }+ }++ void test_sage_dynamic_quant_accuracy(const std::string& name, int num_rows, int seq, int head_dim, int block_size,+ bool use_bias) {+ GETQ();+ LOG_LINE();+ int n_seq_blk = ceil_div(seq, block_size);+ size_t input_count = size_t(num_rows) * size_t(seq) * size_t(head_dim);+ size_t bias_count = size_t(num_rows) * size_t(head_dim);+ size_t scale_count = size_t(num_rows) * size_t(n_seq_blk);++ auto host_input_f32 = make_random_vector(input_count, -2.0f, 2.0f, 2026u + uint32_t(block_size));+ auto host_input = to_fp16_vector(host_input_f32);+ auto ref_input_f32 = to_f32_vector(host_input);+ std::vector<float> host_bias_f32;+ std::vector<sycl::half> host_bias;+ std::vector<float> ref_bias_f32;+ if (use_bias) {+ host_bias_f32 = make_random_vector(bias_count, -0.5f, 0.5f, 3039u + uint32_t(num_rows));+ host_bias = to_fp16_vector(host_bias_f32);+ ref_bias_f32 = to_f32_vector(host_bias);+ }++ auto* dev_input = reinterpret_cast<sycl::half*>(ctx->allocate(input_count * sizeof(sycl::half)));+ auto* dev_bias = use_bias ? reinterpret_cast<sycl::half*>(ctx->allocate(bias_count * sizeof(sycl::half))) : nullptr;+ auto* dev_output = reinterpret_cast<int8_t*>(ctx->allocate(input_count * sizeof(int8_t)));+ auto* dev_scales = reinterpret_cast<float*>(ctx->allocate(scale_count * sizeof(float)));++ q->memcpy(dev_input, host_input.data(), input_count * sizeof(sycl::half)).wait();+ if (use_bias) {+ q->memcpy(dev_bias, host_bias.data(), bias_count * sizeof(sycl::half)).wait();+ }++ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_input, dev_output, dev_scales, num_rows, seq, n_seq_blk,+ head_dim, block_size, dev_bias);+ q->wait();++ std::vector<int8_t> host_output(input_count);+ std::vector<float> host_scales(scale_count);+ q->memcpy(host_output.data(), dev_output, input_count * sizeof(int8_t)).wait();+ q->memcpy(host_scales.data(), dev_scales, scale_count * sizeof(float)).wait();++ std::vector<int8_t> ref_output;+ std::vector<float> ref_scales;+ reference_quant(ref_input_f32, use_bias ? &ref_bias_f32 : nullptr, num_rows, seq, head_dim, block_size,+ ref_output, ref_scales);+ validate_results(name, host_output, ref_output, host_scales, ref_scales);++ ctx->deallocate(dev_input);+ ctx->deallocate(dev_bias);+ ctx->deallocate(dev_output);+ ctx->deallocate(dev_scales);+ }++ void test_sage_dynamic_quant_perf(const std::string& name, int num_rows, int seq, int head_dim, int block_size,+ bool use_bias, int warmup, int iters) {+ GETQ();+ LOG_LINE();+ int n_seq_blk = ceil_div(seq, block_size);+ size_t input_count = size_t(num_rows) * size_t(seq) * size_t(head_dim);+ size_t bias_count = size_t(num_rows) * size_t(head_dim);+ size_t scale_count = size_t(num_rows) * size_t(n_seq_blk);++ auto host_input = to_fp16_vector(make_random_vector(input_count, -2.0f, 2.0f, 4041u));+ std::vector<sycl::half> host_bias;+ if (use_bias) {+ host_bias = to_fp16_vector(make_random_vector(bias_count, -0.25f, 0.25f, 4042u));+ }++ auto* dev_input = reinterpret_cast<sycl::half*>(ctx->allocate(input_count * sizeof(sycl::half)));+ auto* dev_bias = use_bias ? reinterpret_cast<sycl::half*>(ctx->allocate(bias_count * sizeof(sycl::half))) : nullptr;+ auto* dev_output = reinterpret_cast<int8_t*>(ctx->allocate(input_count * sizeof(int8_t)));+ auto* dev_scales = reinterpret_cast<float*>(ctx->allocate(scale_count * sizeof(float)));++ q->memcpy(dev_input, host_input.data(), input_count * sizeof(sycl::half)).wait();+ if (use_bias) {+ q->memcpy(dev_bias, host_bias.data(), bias_count * sizeof(sycl::half)).wait();+ }++ for (int iter = 0; iter < warmup; ++iter) {+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_input, dev_output, dev_scales, num_rows, seq, n_seq_blk,+ head_dim, block_size, dev_bias);+ }+ q->wait();++ auto start = std::chrono::steady_clock::now();+ for (int iter = 0; iter < iters; ++iter) {+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_input, dev_output, dev_scales, num_rows, seq, n_seq_blk,+ head_dim, block_size, dev_bias);+ }+ q->wait();+ auto end = std::chrono::steady_clock::now();++ double total_ms = std::chrono::duration<double, std::milli>(end - start).count();+ double avg_ms = total_ms / double(iters);+ double input_bytes = double(input_count * sizeof(sycl::half)) + double(use_bias ? bias_count * sizeof(sycl::half) : 0);+ double output_bytes = double(input_count * sizeof(int8_t)) + double(scale_count * sizeof(float));+ double logical_io_bytes = input_bytes + output_bytes;+ double estimated_kernel_bytes =+ double(input_count * sizeof(sycl::half) * 2) ++ double(use_bias ? bias_count * sizeof(sycl::half) * 2 : 0) + output_bytes;+ double logical_io_gbps = logical_io_bytes / (avg_ms * 1e-3) / 1e9;+ double estimated_kernel_gbps = estimated_kernel_bytes / (avg_ms * 1e-3) / 1e9;++ std::cout << std::fixed << std::setprecision(3)+ << "[sage_dynamic_quant][perf] " << name << " avg_ms=" << avg_ms << " input_MB="+ << (input_bytes / (1024.0 * 1024.0)) << " output_MB=" << (output_bytes / (1024.0 * 1024.0))+ << " logical_io_GBps=" << logical_io_gbps << " est_kernel_GBps=" << estimated_kernel_gbps << "\n";++ ctx->deallocate(dev_input);+ ctx->deallocate(dev_bias);+ ctx->deallocate(dev_output);+ ctx->deallocate(dev_scales);+ }+#endif+};\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hppnew file mode 100644index 000000000..b2ee33c4d--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp@@ -0,0 +1,229 @@+#pragma once++#include <algorithm>+#include <chrono>+#include <cmath>+#include <cstdint>+#include <iomanip>+#include <iostream>+#include <functional>+#include <random>+#include <stdexcept>+#include <string>+#include <vector>++#include "common.hpp"+#include "../include/xpu_wrapper.hpp"++struct TestSDPA {+ TestSDPA() {+#if defined(ARK_XPU) && defined(ARK_SYCL_TLA) && ARK_XPU && ARK_SYCL_TLA+ // test_sagev1_accuracy("prefill_gqa_128_bias", 1, 32, 8, 256, 512, 128, 128, false, true);+ // test_sagev1_accuracy("prefill_gqa_128_no_bias", 1, 32, 8, 256, 512, 128, 128, false, false);+ benchmark_sagev1("bench_prefill_gqa_128", 1, 32, 8, 4096, 4096, 128, 128, false, 5, 20);+#else+ std::cout << "[sagev1] skipped: requires ARK_XPU and ARK_SYCL_TLA\n";+#endif+ }++#if defined(ARK_XPU) && defined(ARK_SYCL_TLA) && ARK_XPU && ARK_SYCL_TLA+ static std::vector<float> make_random_vector(size_t count, float low, float high, uint32_t seed) {+ std::mt19937 rng(seed);+ std::uniform_real_distribution<float> dist(low, high);+ std::vector<float> values(count);+ for (auto& value : values) {+ value = dist(rng);+ }+ return values;+ }++ static std::vector<sycl::half> to_fp16_vector(const std::vector<float>& src) {+ std::vector<sycl::half> dst(src.size());+ for (size_t index = 0; index < src.size(); ++index) {+ dst[index] = sycl::half(src[index]);+ }+ return dst;+ }++ static double compute_sdpa_flops(int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,+ int head_dim) {+ int group = num_heads_q / num_heads_kv;+ double qk = double(group) * num_heads_kv * seq_len_q * head_dim * seq_len_kv * 2.0;+ double softmax = double(group) * num_heads_kv * seq_len_q * seq_len_kv * 2.0;+ double pv = double(group) * num_heads_kv * seq_len_q * seq_len_kv * head_dim * 2.0;+ return double(batch) * (qk + softmax + pv);+ }++ static double compute_sdpa_io_bytes(int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,+ int head_dim) {+ double q_bytes = double(batch) * num_heads_q * seq_len_q * head_dim * sizeof(sycl::half);+ double k_bytes = double(batch) * num_heads_kv * seq_len_kv * head_dim * sizeof(sycl::half);+ double v_bytes = double(batch) * num_heads_kv * seq_len_kv * head_dim * sizeof(sycl::half);+ double o_bytes = double(batch) * num_heads_q * seq_len_q * head_dim * sizeof(sycl::half);+ return q_bytes + k_bytes + v_bytes + o_bytes;+ }++ static double run_bench(const std::function<void()>& fn, sycl::queue* q, int warmup, int iters) {+ for (int iter = 0; iter < warmup; ++iter) {+ fn();+ }+ q->wait();+ auto start = std::chrono::steady_clock::now();+ for (int iter = 0; iter < iters; ++iter) {+ fn();+ }+ q->wait();+ auto end = std::chrono::steady_clock::now();+ return std::chrono::duration<double, std::milli>(end - start).count() / double(iters);+ }++ void test_sagev1_accuracy(const std::string& name, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, int scale_block_size, bool is_causal,+ bool use_mean_bias) {+ GETQ();+ LOG_LINE();+ size_t q_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;+ size_t k_count = size_t(batch) * num_heads_kv * seq_len_kv * head_dim;+ size_t o_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;+ size_t q_scale_count = size_t(batch) * num_heads_q * ((seq_len_q + scale_block_size - 1) / scale_block_size);+ size_t k_scale_count = size_t(batch) * num_heads_kv * ((seq_len_kv + scale_block_size - 1) / scale_block_size);+ size_t k_bias_count = size_t(batch) * num_heads_kv * head_dim;+ float softmax_scale = 1.0f / std::sqrt(float(head_dim));+ int previous_use_mean_bias = ark::env_params::Instance()->sage_use_mean_bias;+ ark::env_params::Instance()->sage_use_mean_bias = use_mean_bias ? 1 : 0;++ auto host_q = to_fp16_vector(make_random_vector(q_count, -1.0f, 1.0f, 601u + uint32_t(seq_len_q)));+ auto host_k = to_fp16_vector(make_random_vector(k_count, -1.0f, 1.0f, 701u + uint32_t(seq_len_kv)));+ auto host_v = to_fp16_vector(make_random_vector(k_count, -1.0f, 1.0f, 801u + uint32_t(head_dim)));++ auto* dev_q = reinterpret_cast<sycl::half*>(ctx->allocate(q_count * sizeof(sycl::half)));+ auto* dev_k = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));+ auto* dev_v = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));+ auto* dev_out = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));+ auto* dev_ref = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));+ auto* dev_qi8 = reinterpret_cast<int8_t*>(ctx->allocate(q_count * sizeof(int8_t)));+ auto* dev_ki8 = reinterpret_cast<int8_t*>(ctx->allocate(k_count * sizeof(int8_t)));+ auto* dev_qscale = reinterpret_cast<float*>(ctx->allocate(q_scale_count * sizeof(float)));+ auto* dev_kscale = reinterpret_cast<float*>(ctx->allocate(k_scale_count * sizeof(float)));+ auto* dev_kbias = use_mean_bias ? reinterpret_cast<sycl::half*>(ctx->allocate(k_bias_count * sizeof(sycl::half))) : nullptr;++ try {+ q->memcpy(dev_q, host_q.data(), q_count * sizeof(sycl::half)).wait();+ q->memcpy(dev_k, host_k.data(), k_count * sizeof(sycl::half)).wait();+ q->memcpy(dev_v, host_v.data(), k_count * sizeof(sycl::half)).wait();++ ark::XpuWrapper::sagev1(q, dev_q, dev_k, dev_v, dev_out, nullptr, scale_block_size, batch, num_heads_q,+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);++ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_q, dev_qi8, dev_qscale, batch * num_heads_q, seq_len_q,+ (seq_len_q + scale_block_size - 1) / scale_block_size, head_dim,+ scale_block_size);+ if (use_mean_bias) {+ ark::XpuWrapper::compute_seq_mean_bias<sycl::half>(q, dev_k, dev_kbias, batch * num_heads_kv, seq_len_kv,+ head_dim);+ }+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_k, dev_ki8, dev_kscale, batch * num_heads_kv, seq_len_kv,+ (seq_len_kv + scale_block_size - 1) / scale_block_size, head_dim,+ scale_block_size, dev_kbias);+ ark::sdpa_impl_qks8_pvhalf(q, dev_qi8, dev_ki8, dev_v, dev_ref, nullptr, scale_block_size, dev_qscale,+ dev_kscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,+ softmax_scale, is_causal);+ q->wait();++ std::vector<sycl::half> host_out(o_count);+ std::vector<sycl::half> host_ref(o_count);+ q->memcpy(host_out.data(), dev_out, o_count * sizeof(sycl::half)).wait();+ q->memcpy(host_ref.data(), dev_ref, o_count * sizeof(sycl::half)).wait();++ float max_diff = 0.0f;+ double mean_diff = 0.0;+ for (size_t index = 0; index < o_count; ++index) {+ float out_value = float(host_out[index]);+ float ref_value = float(host_ref[index]);+ if (!std::isfinite(out_value) || !std::isfinite(ref_value)) {+ throw std::runtime_error("sagev1 accuracy test produced non-finite output: " + name);+ }+ float diff = std::fabs(out_value - ref_value);+ max_diff = std::max(max_diff, diff);+ mean_diff += diff;+ }+ mean_diff /= double(o_count);++ std::cout << std::fixed << std::setprecision(6) << "[sagev1][accuracy] " << name << " max_diff=" << max_diff+ << " mean_diff=" << mean_diff << " use_mean_bias=" << (use_mean_bias ? 1 : 0) << "\n";++ if (max_diff > 5e-3f || mean_diff > 5e-4f) {+ throw std::runtime_error("sagev1 accuracy mismatch: " + name);+ }+ } catch (...) {+ ark::env_params::Instance()->sage_use_mean_bias = previous_use_mean_bias;+ ctx->deallocate(dev_q);+ ctx->deallocate(dev_k);+ ctx->deallocate(dev_v);+ ctx->deallocate(dev_out);+ ctx->deallocate(dev_ref);+ ctx->deallocate(dev_qi8);+ ctx->deallocate(dev_ki8);+ ctx->deallocate(dev_qscale);+ ctx->deallocate(dev_kscale);+ ctx->deallocate(dev_kbias);+ throw;+ }++ ark::env_params::Instance()->sage_use_mean_bias = previous_use_mean_bias;++ ctx->deallocate(dev_q);+ ctx->deallocate(dev_k);+ ctx->deallocate(dev_v);+ ctx->deallocate(dev_out);+ ctx->deallocate(dev_ref);+ ctx->deallocate(dev_qi8);+ ctx->deallocate(dev_ki8);+ ctx->deallocate(dev_qscale);+ ctx->deallocate(dev_kscale);+ ctx->deallocate(dev_kbias);+ }++ void benchmark_sagev1(const std::string& name, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, int scale_block_size, bool is_causal, int warmup, int iters) {+ GETQ();+ LOG_LINE();+ size_t q_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;+ size_t k_count = size_t(batch) * num_heads_kv * seq_len_kv * head_dim;+ size_t o_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;+ float softmax_scale = 1.0f / std::sqrt(float(head_dim));++ auto host_q = to_fp16_vector(make_random_vector(q_count, -1.0f, 1.0f, 901u + uint32_t(seq_len_q)));+ auto host_k = to_fp16_vector(make_random_vector(k_count, -1.0f, 1.0f, 1001u + uint32_t(seq_len_kv)));+ auto host_v = to_fp16_vector(make_random_vector(k_count, -1.0f, 1.0f, 1101u + uint32_t(head_dim)));++ auto* dev_q = reinterpret_cast<sycl::half*>(ctx->allocate(q_count * sizeof(sycl::half)));+ auto* dev_k = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));+ auto* dev_v = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));+ auto* dev_out = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));+ q->memcpy(dev_q, host_q.data(), q_count * sizeof(sycl::half)).wait();+ q->memcpy(dev_k, host_k.data(), k_count * sizeof(sycl::half)).wait();+ q->memcpy(dev_v, host_v.data(), k_count * sizeof(sycl::half)).wait();++ double sage_ms = run_bench(+ [&]() {+ ark::XpuWrapper::sagev1(q, dev_q, dev_k, dev_v, dev_out, nullptr, scale_block_size, batch, num_heads_q,+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ },+ q, warmup, iters);++ double flops = compute_sdpa_flops(batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim);+ double io_bytes = compute_sdpa_io_bytes(batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim);+ double sage_tflops = flops / (sage_ms * 1e-3) / 1e12;+ double sage_gbps = io_bytes / (sage_ms * 1e-3) / 1e9;++ std::cout << std::fixed << std::setprecision(3) << "[sagev1][bench] " << name << " sage_ms=" << sage_ms+ << " sage_TFLOPS=" << sage_tflops << " sage_GBps=" << sage_gbps << "\n";++ ctx->deallocate(dev_q);+ ctx->deallocate(dev_k);+ ctx->deallocate(dev_v);+ ctx->deallocate(dev_out);+ }+#endif+};\ No newline at end of filediff --git a/auto_round_extension/ark/tools/lm_eval_with_ark_sdpa.py b/auto_round_extension/ark/tools/lm_eval_with_ark_sdpa.pynew file mode 100644index 000000000..0d2852da9--- /dev/null+++ b/auto_round_extension/ark/tools/lm_eval_with_ark_sdpa.py@@ -0,0 +1,26 @@+#!/usr/bin/env python++# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++from __future__ import annotations++import os+import runpy++from auto_round_kernel import patch_torch_sdpa+++def main() -> None:+ backend = os.environ.get("ARK_TORCH_SDPA_BACKEND", "sdpa")+ quant_block_size = int(os.environ.get("ARK_SAGEV1_QUANT_BLOCK_SIZE", "64"))+ patch_torch_sdpa(+ strict=True,+ backend=backend,+ quant_block_size=quant_block_size,+ )+ runpy.run_module("lm_eval.__main__", run_name="__main__")+++if __name__ == "__main__":+ main()