Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add smooth K for sagev1 (#1806)

Luo Yu017ed92c6fa1
diff --git a/auto_round_extension/ark/.gitignore b/auto_round_extension/ark/.gitignore
new file mode 100644
index 000000000..d95db0037
--- /dev/null
+++ b/auto_round_extension/ark/.gitignore
@@ -0,0 +1,3 @@
+build
+xbuild
+*.csv
\ No newline at end of file
diff --git a/auto_round_extension/ark/README.md b/auto_round_extension/ark/README.md
index a36eef2b3..c087147d9 100644
--- a/auto_round_extension/ark/README.md
+++ b/auto_round_extension/ark/README.md
@@ -82,3 +82,24 @@ y = qlinear(x)
#### A Weight-Only Example
A runnable end-to-end example is available in [test_weightonly.py](test/test_weightonly.py). It demonstrates how to prepare quantized weights and scales, call repack_quantized_weight to build ARK-packed weights, verify correctness with unpack_weight, and run woqgemm on CPU and XPU.
+
+#### Replace torch SDPA and run lm-eval
+ ARK also exposes an XPU SDPA kernel through `ARK.sdpa(...)`. If you want to replace `torch.nn.functional.scaled_dot_product_attention` globally for evaluation without editing model code, use the helper launcher in [tools/lm_eval_with_ark_sdpa.py](tools/lm_eval_with_ark_sdpa.py).
+
+ Example:
+```bash
+cd /path/to/auto_round_extension/ark
+PYTHONPATH=$PWD python tools/lm_eval_with_ark_sdpa.py \
+ --model hf \
+ --model_args pretrained=/path/to/model,trust_remote_code=True,dtype=bfloat16 \
+ --tasks hellaswag,piqa,winogrande \
+ --device xpu:0 \
+ --batch_size 1
+```
+
+ Notes:
+ * The patch only routes calls to ARK on XPU when the inputs match ARK kernel constraints; otherwise it falls back to the original torch SDPA.
+ * Supported Q/K/V dtypes are FP16 and BF16.
+ * Supported head dimensions are 64, 96, 128, and 192.
+ * `dropout_p` must be 0.0 for the ARK path.
+ * Additive masks are supported when they can be normalized to `[B, 1, Sq, Skv]`; boolean masks fall back to torch.
diff --git a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt
index 1d799d098..5891bbb5c 100755
--- a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt
+++ b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt
@@ -162,8 +162,25 @@ target_link_libraries(${PY_NAME} PRIVATE ${libs})
if(ARK_UT)
set(TEST_NAME test_${ARK_TYPE})
- add_executable(${TEST_NAME} wrapper/test/test_main.cpp)
+ set(TEST_SRCS wrapper/test/test_main.cpp)
+ if(ARK_XPU AND ARK_SYCL_TLA)
+ list(APPEND TEST_SRCS sdpa.cpp)
+ list(APPEND TEST_SRCS ${SDPA_GENERATED_SRCS})
+ endif()
+ add_executable(${TEST_NAME} ${TEST_SRCS})
target_compile_features(${TEST_NAME} PRIVATE cxx_std_17)
target_compile_definitions(${TEST_NAME} PRIVATE ${ARK_TYPE}=1)
+ target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/wrapper/include)
+ if(ARK_XPU AND ARK_SYCL_TLA)
+ target_compile_definitions(${TEST_NAME} PRIVATE ARK_SYCL_TLA=1 CUTLASS_ENABLE_SYCL=1 SYCL_INTEL_TARGET=1)
+ target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/generated/sdpa)
+ foreach(_inc_dir IN LISTS _sycl_tla_include_dirs)
+ if(EXISTS "${_inc_dir}")
+ target_include_directories(${TEST_NAME} SYSTEM PRIVATE "${_inc_dir}")
+ endif()
+ endforeach()
+ target_compile_options(${TEST_NAME} PRIVATE -fsycl -fno-sycl-instrument-device-code)
+ target_link_options(${TEST_NAME} PRIVATE ${SYCL_TLA_LINK_FLAGS})
+ endif()
target_link_libraries(${TEST_NAME} PRIVATE ${libs})
endif()
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.py
index 48778aff3..831d3714a 100644
--- a/auto_round_extension/ark/auto_round_kernel/__init__.py
+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py
@@ -675,6 +675,7 @@ class ARK:
lib.sage_dynamic_quant(
stream,
query.data_ptr(),
+ 0,
q_i8.data_ptr(),
q_scale.data_ptr(),
q_num_rows,
@@ -690,6 +691,7 @@ class ARK:
lib.sage_dynamic_quant(
stream,
key.data_ptr(),
+ 0,
k_i8.data_ptr(),
k_scale.data_ptr(),
k_num_rows,
@@ -859,7 +861,19 @@ if __name__ == "__main__":
pack_unpack()
-__all__ = ["ARK"]
+def patch_torch_sdpa(*args, **kwargs):
+ from .torch_sdpa_patch import patch_torch_sdpa_with_ark
+
+ return patch_torch_sdpa_with_ark(*args, **kwargs)
+
+
+def unpatch_torch_sdpa():
+ from .torch_sdpa_patch import unpatch_torch_sdpa_with_ark
+
+ return unpatch_torch_sdpa_with_ark()
+
+
+__all__ = ["ARK", "patch_torch_sdpa", "unpatch_torch_sdpa"]
# -----------------------------------------------------------------------------
@@ -937,7 +951,15 @@ def repack_quantized_weight(*args, **kwargs):
def unpack_weight(
- blob: torch.Tensor, out_dtype: torch.dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym
+ blob: torch.Tensor,
+ out_dtype: torch.dtype,
+ n,
+ k,
+ groupsize,
+ compute_type,
+ weight_type,
+ scale_type,
+ asym,
):
return _ark_instance().unpack_weight(blob, out_dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym)
diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cpp
index cd5c1ab03..86d983f90 100755
--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp
@@ -153,10 +153,11 @@ static void moe_gemm_wrapper(torch_ptr stream, torch_ptr activations, torch_ptr
(void*)outputs, (BTLA_DTYPE)(dtype), N, K, (int*)num_tokens_per_expert, num_experts);
}
-static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr output, torch_ptr scale_out, int num_rows,
- int head_dim, int block_size) {
+static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr bias, torch_ptr output, torch_ptr scale_out,
+ int num_rows, int head_dim, int block_size) {
auto* q = (sycl::queue*)stream;
auto* in_ptr = (sycl::half*)input;
+ auto* bias_ptr = bias ? (sycl::half*)bias : nullptr;
auto* out_ptr = (int8_t*)output;
auto* scale_ptr = (float*)scale_out;
@@ -181,6 +182,7 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr outp
int tid = item.get_local_id(0);
auto wg = item.get_group();
auto* block_in = in_ptr + (size_t)block_id * elems_per_block;
+ auto* block_bias = bias_ptr ? bias_ptr + (size_t)block_id * elems_per_block : nullptr;
auto* block_out = out_ptr + (size_t)block_id * elems_per_block;
// Phase 1: compute absmax across entire block
@@ -189,6 +191,9 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr outp
local_max_vec = sycl::vec<sycl::half, Unroll>(0.0f);
for (int i = tid * Unroll; i < elems_per_block; i += wg_size * Unroll) {
local_data = *(sycl::vec<sycl::half, Unroll>*)(&block_in[i]);
+ if (block_bias) {
+ local_data = local_data - *(sycl::vec<sycl::half, Unroll>*)(&block_bias[i]);
+ }
local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));
}
for (int i = 0; i < Unroll; ++i) {
@@ -208,7 +213,11 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr outp
for (int i = tid * Unroll; i < elems_per_block; i += wg_size * Unroll) {
#pragma unroll
for (int j = 0; j < Unroll; ++j) {
- float val = static_cast<float>(block_in[i + j]) * inv_scale;
+ float val = static_cast<float>(block_in[i + j]);
+ if (block_bias) {
+ val -= static_cast<float>(block_bias[i + j]);
+ }
+ val *= inv_scale;
int iv = static_cast<int>(val + (val >= 0.0f ? 0.5f : -0.5f));
iv = sycl::clamp(iv, -127, 127);
block_out[i + j] = static_cast<int8_t>(iv);
@@ -223,6 +232,7 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr outp
int tid = item.get_local_id(0);
auto wg = item.get_group();
auto* block_in = in_ptr + (size_t)block_id * elems_per_block;
+ auto* block_bias = bias_ptr ? bias_ptr + (size_t)block_id * elems_per_block : nullptr;
auto* block_out = out_ptr + (size_t)block_id * elems_per_block;
// Phase 1: compute absmax across entire block
@@ -232,6 +242,9 @@ static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr outp
int local_i = 0;
for (int i = tid * Unroll; i < elems_per_block; i += wg_size * Unroll, local_i++) {
local_data[local_i] = *(sycl::vec<sycl::half, Unroll>*)&block_in[i];
+ if (block_bias) {
+ local_data[local_i] = local_data[local_i] - *(sycl::vec<sycl::half, Unroll>*)(&block_bias[i]);
+ }
local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));
}
#pragma unroll
diff --git a/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py
new file mode 100644
index 000000000..7b51f96db
--- /dev/null
+++ b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py
@@ -0,0 +1,238 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import logging
+
+import torch
+
+from . import ARK
+
+logger = logging.getLogger(__name__)
+
+_ARK_SDPA_PATCHED = False
+_ORIG_SDPA = None
+_PATCH_CONFIG = None
+_FALLBACK_WARNING_LOGGED = False
+
+
+def _log_fallback_warning_once(backend: str, error: Exception) -> None:
+ global _FALLBACK_WARNING_LOGGED
+
+ if _FALLBACK_WARNING_LOGGED:
+ return
+
+ logger.warning(
+ "ARK %s patch failed and fell back to torch SDPA. Subsequent failures will be suppressed. Error: %s",
+ backend,
+ error,
+ )
+ _FALLBACK_WARNING_LOGGED = True
+
+
+def _validate_backend(backend: str, quant_block_size: int) -> str:
+ backend = backend.lower()
+ if backend not in {"sdpa", "sagev1"}:
+ raise ValueError(f"Unsupported ARK attention backend: {backend}")
+ if backend == "sagev1" and quant_block_size <= 0:
+ raise ValueError("quant_block_size must be > 0 when backend='sagev1'")
+ return backend
+
+
+def _normalize_attn_mask(attn_mask: torch.Tensor, batch: int, seq_q: int, seq_kv: int) -> torch.Tensor:
+ if attn_mask.dtype == torch.bool:
+ raise ValueError("Boolean attention masks are not supported by the ARK SDPA patch")
+
+ if attn_mask.ndim == 2:
+ if attn_mask.shape != (seq_q, seq_kv):
+ raise ValueError("Unsupported 2D attention mask shape")
+ attn_mask = attn_mask.view(1, 1, seq_q, seq_kv)
+ elif attn_mask.ndim == 3:
+ if attn_mask.shape != (batch, seq_q, seq_kv):
+ raise ValueError("Unsupported 3D attention mask shape")
+ attn_mask = attn_mask.unsqueeze(1)
+ elif attn_mask.ndim == 4:
+ if attn_mask.shape[-2:] != (seq_q, seq_kv):
+ raise ValueError("Unsupported 4D attention mask shape")
+ if attn_mask.shape[1] != 1:
+ raise ValueError("Only attention masks with head dimension 1 are supported")
+ if attn_mask.shape[0] == 1 and batch != 1:
+ attn_mask = attn_mask.expand(batch, -1, -1, -1)
+ elif attn_mask.shape[0] != batch:
+ raise ValueError("Unsupported batch dimension in attention mask")
+ else:
+ raise ValueError("Unsupported attention mask rank")
+
+ return attn_mask.contiguous().to(torch.float32)
+
+
+def _is_pure_causal_mask(attn_mask: torch.Tensor) -> bool:
+ seq_q = attn_mask.shape[-2]
+ seq_kv = attn_mask.shape[-1]
+ if seq_q != seq_kv:
+ return False
+
+ mask_2d = attn_mask.reshape(-1, seq_q, seq_kv)[0]
+ tri_up = torch.triu(torch.ones(seq_q, seq_kv, dtype=torch.bool, device=attn_mask.device), 1)
+ return bool(torch.isinf(mask_2d[tri_up]).all().item()) and bool((mask_2d[~tri_up] == 0).all().item())
+
+
+def _can_use_ark_attention(
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attn_mask: torch.Tensor | None,
+ dropout_p: float,
+ backend: str,
+) -> bool:
+ if query.device.type != "xpu":
+ return False
+ if query.requires_grad or key.requires_grad or value.requires_grad:
+ return False
+ if query.dtype not in (torch.float16, torch.bfloat16):
+ return False
+ if key.dtype != query.dtype or value.dtype != query.dtype:
+ return False
+ if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:
+ return False
+ if query.shape[0] != key.shape[0] or query.shape[0] != value.shape[0]:
+ return False
+ if key.shape != value.shape:
+ return False
+ if query.shape[-1] != key.shape[-1]:
+ return False
+ if dropout_p != 0.0:
+ return False
+ if attn_mask is not None and attn_mask.device.type != "xpu":
+ return False
+ if backend == "sdpa":
+ return query.dtype in (torch.float16, torch.bfloat16) and query.shape[-1] in (
+ 64,
+ 96,
+ 128,
+ 192,
+ )
+ if backend == "sagev1":
+ return query.dtype == torch.float16 and query.shape[-1] in (64, 128)
+ return True
+
+
+def patch_torch_sdpa_with_ark(*, strict: bool = False, backend: str = "sdpa", quant_block_size: int = 64) -> bool:
+ global _ARK_SDPA_PATCHED, _ORIG_SDPA, _PATCH_CONFIG
+
+ backend = _validate_backend(backend, quant_block_size)
+ new_config = {"backend": backend, "quant_block_size": int(quant_block_size)}
+
+ if _ARK_SDPA_PATCHED:
+ if _PATCH_CONFIG == new_config:
+ return True
+ unpatch_torch_sdpa_with_ark()
+
+ if not (hasattr(torch, "xpu") and torch.xpu.is_available()):
+ if strict:
+ raise RuntimeError("XPU is not available")
+ return False
+
+ ark = ARK()
+ if ark.xpu_lib is None or not hasattr(ark.xpu_lib, backend):
+ if strict:
+ raise RuntimeError(f"ARK XPU {backend} kernel is not available")
+ return False
+
+ _ORIG_SDPA = torch.nn.functional.scaled_dot_product_attention
+ orig_sdpa = _ORIG_SDPA
+
+ def _patched_sdpa(
+ query,
+ key,
+ value,
+ attn_mask=None,
+ dropout_p=0.0,
+ is_causal=False,
+ scale=None,
+ **kwargs,
+ ):
+ if not _can_use_ark_attention(query, key, value, attn_mask, dropout_p, backend):
+ return orig_sdpa(
+ query,
+ key,
+ value,
+ attn_mask=attn_mask,
+ dropout_p=dropout_p,
+ is_causal=is_causal,
+ scale=scale,
+ **kwargs,
+ )
+
+ normalized_mask = None
+ if attn_mask is not None:
+ try:
+ normalized_mask = _normalize_attn_mask(attn_mask, query.shape[0], query.shape[-2], key.shape[-2])
+ except ValueError:
+ return orig_sdpa(
+ query,
+ key,
+ value,
+ attn_mask=attn_mask,
+ dropout_p=dropout_p,
+ is_causal=is_causal,
+ scale=scale,
+ **kwargs,
+ )
+ if not is_causal and _is_pure_causal_mask(normalized_mask):
+ normalized_mask = None
+ is_causal = True
+
+ try:
+ if backend == "sdpa":
+ return ark.sdpa(
+ query.contiguous(),
+ key.contiguous(),
+ value.contiguous(),
+ attn_mask=normalized_mask,
+ dropout_p=dropout_p,
+ is_causal=is_causal,
+ scale=scale,
+ )
+ return ark.sagev1(
+ query.contiguous(),
+ key.contiguous(),
+ value.contiguous(),
+ attn_mask=normalized_mask,
+ dropout_p=dropout_p,
+ is_causal=is_causal,
+ scale=scale,
+ quant_block_size=quant_block_size,
+ )
+ except (NotImplementedError, RuntimeError, ValueError) as error:
+ _log_fallback_warning_once(backend, error)
+ return orig_sdpa(
+ query,
+ key,
+ value,
+ attn_mask=attn_mask,
+ dropout_p=dropout_p,
+ is_causal=is_causal,
+ scale=scale,
+ **kwargs,
+ )
+
+ torch.nn.functional.scaled_dot_product_attention = _patched_sdpa
+ _ARK_SDPA_PATCHED = True
+ _PATCH_CONFIG = new_config
+ return True
+
+
+def unpatch_torch_sdpa_with_ark() -> bool:
+ global _ARK_SDPA_PATCHED, _ORIG_SDPA, _PATCH_CONFIG, _FALLBACK_WARNING_LOGGED
+
+ if not _ARK_SDPA_PATCHED or _ORIG_SDPA is None:
+ return False
+
+ torch.nn.functional.scaled_dot_product_attention = _ORIG_SDPA
+ _ARK_SDPA_PATCHED = False
+ _ORIG_SDPA = None
+ _PATCH_CONFIG = None
+ _FALLBACK_WARNING_LOGGED = False
+ return True
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
index cc60e5d7a..8b50ee769 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
@@ -32,6 +32,8 @@ namespace ark {
struct env_params {
int verbose = 2; // trace 0/ debug 1/ info 2/ warn 3/ error 4/ critical 5/ off 6
int auto_s8 = 0;
+ int sage_use_mean_bias = 1;
+ int sage_print_kbias = 0;
static env_params* Instance() {
static env_params instance;
return &instance;
@@ -39,6 +41,8 @@ struct env_params {
env_params() {
env_i("ARK_VERBOSE", verbose);
env_i("ARK_AUTO_S8", auto_s8);
+ env_i("ARK_SAGE_USE_MEAN_BIAS", sage_use_mean_bias);
+ env_i("ARK_SAGE_PRINT_KBIAS", sage_print_kbias);
}
static inline void env_i(const char* envstr, int& default_) {
const char* log_level_env = getenv(envstr);
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
index 682a9085b..a7dec5b4b 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
@@ -11,8 +11,11 @@
//
#pragma once
+#include <algorithm>
#include <cmath>
#include <cstdio>
+#include <limits>
+#include <vector>
#include "utils.hpp"
#include "dnnl_wrapper.hpp"
#include "sycl_tla_common.hpp"
@@ -693,11 +696,79 @@ class XpuWrapper {
}
}
+ template <typename T>
+ static void compute_seq_mean_bias(sycl::queue* q, const T* in_ptr, T* bias_ptr, int num_rows, int seq, int head_dim) {
+ constexpr int SG_SIZE = 32;
+ constexpr int WG_SIZE = 512;
+ constexpr int SG_PER_WG = WG_SIZE / SG_SIZE;
+ constexpr int SeqUnroll = 4;
+ size_t total = size_t(num_rows) * size_t(head_dim);
+ size_t num_groups = (total + SG_PER_WG - 1) / SG_PER_WG;
+ size_t global = num_groups * WG_SIZE;
+ q->parallel_for(sycl::nd_range<1>(global, WG_SIZE),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ size_t group_id = item.get_group(0);
+ int local_id = item.get_local_id(0);
+ int subgroup_id = local_id / SG_SIZE;
+ int lane_id = local_id % SG_SIZE;
+ size_t index = group_id * SG_PER_WG + subgroup_id;
+ if (index >= total) {
+ return;
+ }
+
+ int row_id = index / head_dim;
+ int dim_id = index % head_dim;
+ const T* row_ptr = in_ptr + size_t(row_id) * seq * head_dim + dim_id;
+ auto sg = item.get_sub_group();
+
+ float sum = 0.0f;
+ int token = lane_id;
+ for (; token + (SeqUnroll - 1) * SG_SIZE < seq; token += SeqUnroll * SG_SIZE) {
+ #pragma unroll
+ for (int unroll_idx = 0; unroll_idx < SeqUnroll; ++unroll_idx) {
+ sum += static_cast<float>(row_ptr[size_t(token + unroll_idx * SG_SIZE) * head_dim]);
+ }
+ }
+ for (; token < seq; token += SG_SIZE) {
+ sum += static_cast<float>(row_ptr[size_t(token) * head_dim]);
+ }
+ sum = sycl::reduce_over_group(sg, sum, sycl::plus<float>{});
+ if (lane_id == 0) {
+ bias_ptr[index] = static_cast<T>(sum / static_cast<float>(seq));
+ }
+ });
+ }
+
+ template <typename T>
+ static void print_value_distribution(sycl::queue* q, const T* dev_ptr, size_t count, const char* name) {
+ if (dev_ptr == nullptr || count == 0) {
+ std::printf("[%s] empty\n", name);
+ return;
+ }
+
+ std::vector<T> host_values(count);
+ q->memcpy(host_values.data(), dev_ptr, count * sizeof(T)).wait();
+
+ float min_value = std::numeric_limits<float>::max();
+ float max_value = std::numeric_limits<float>::lowest();
+ double sum = 0.0;
+
+ for (size_t i = 0; i < count; ++i) {
+ float value = static_cast<float>(host_values[i]);
+ min_value = std::min(min_value, value);
+ max_value = std::max(max_value, value);
+ sum += value;
+ }
+
+ double mean = sum / static_cast<double>(count);
+ std::printf("[%s] min=%f max=%f mean=%f\n", name, min_value, max_value, static_cast<float>(mean));
+ }
+
// input: num_rows x head_dim matrix, output: int8 quantized matrix + scale (per block)
// scale: num_rows // block_size
template <typename T>
static void sage_dynamic_quant(sycl::queue* q, const T* in_ptr, int8_t* out_ptr, float* scale_ptr, int num_rows,
- int seq, int n_seq_blk, int head_dim, int block_size) {
+ int seq, int n_seq_blk, int head_dim, int block_size, const T* bias_ptr = nullptr) {
int num_blocks = num_rows * n_seq_blk;
int elems_per_block = block_size * head_dim;
@@ -708,127 +779,299 @@ class XpuWrapper {
constexpr int MAX_Reg = 64;
constexpr int MAX_WG_SIZE = 256;
constexpr int Unroll = 4;
+ bool fast_vector_bias = bias_ptr && (head_dim % Unroll == 0);
+ bool power_of_two_head_dim = fast_vector_bias && ((head_dim & (head_dim - 1)) == 0);
+ int bias_mask = head_dim - 1;
if (elems_per_block > MAX_Reg * MAX_WG_SIZE) {
int wg_size = (elems_per_block <= 256) ? SG_SIZE : 256;
// Ensure wg_size is a multiple of SG_SIZE
wg_size = ((wg_size + SG_SIZE - 1) / SG_SIZE) * SG_SIZE;
- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
- int block_id = item.get_group(0);
- int row_id = block_id / n_seq_blk;
- int seq_id = block_id % n_seq_blk;
- int tid = item.get_local_id(0);
- auto wg = item.get_group();
- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
-
- int elems_this_wg =
- (seq_id + 1) * block_size <= seq ? elems_per_block : (seq - seq_id * block_size) * head_dim;
-
- // Phase 1: compute absmax across entire block
- float local_max = 0.0f;
- sycl::vec<T, Unroll> local_data, local_max_vec;
- local_max_vec = sycl::vec<T, Unroll>(0.0f);
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
- local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);
- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));
- }
- for (int i = 0; i < Unroll; ++i) {
- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
- }
- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ if (bias_ptr) {
+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ int block_id = item.get_group(0);
+ int row_id = block_id / n_seq_blk;
+ int seq_id = block_id % n_seq_blk;
+ int tid = item.get_local_id(0);
+ auto wg = item.get_group();
+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
+ auto* row_bias = bias_ptr + (size_t)row_id * head_dim;
- // Compute scale
- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? elems_per_block
+ : (seq - seq_id * block_size) * head_dim;
- // Store scale (one thread writes)
- if (tid == 0) {
- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
- }
+ float local_max = 0.0f;
+ sycl::vec<T, Unroll> local_data, local_max_vec;
+ local_max_vec = sycl::vec<T, Unroll>(0.0f);
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
+ local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);
+ if (fast_vector_bias) {
+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);
+ local_data = local_data - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);
+ } else {
+#pragma unroll
+ for (int j = 0; j < Unroll; ++j) {
+ local_data[j] = local_data[j] - row_bias[(i + j) % head_dim];
+ }
+ }
+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));
+ }
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
+ }
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
- // Phase 2: fused quantize
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
+ if (tid == 0) {
+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
+ }
+
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
+ sycl::vec<T, Unroll> quant_input = *(sycl::vec<T, Unroll>*)(&block_in[i]);
+ if (fast_vector_bias) {
+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);
+ quant_input = quant_input - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);
+ } else {
#pragma unroll
- for (int j = 0; j < Unroll; ++j) {
- float val = static_cast<float>(block_in[i + j]) * inv_scale;
- int iv = static_cast<int>(val + (val >= 0.0f ? 0.5f : -0.5f));
- iv = sycl::clamp(iv, -127, 127);
- block_out[i + j] = static_cast<int8_t>(iv);
+ for (int j = 0; j < Unroll; ++j) {
+ quant_input[j] = quant_input[j] - row_bias[(i + j) % head_dim];
+ }
+ }
+ sycl::vec<float, Unroll> val =
+ quant_input.template convert<float, sycl::rounding_mode::automatic>();
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127, 127);
+ sycl::vec<int8_t, Unroll> qv =
+ val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
}
- }
- });
+ });
+ } else {
+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ int block_id = item.get_group(0);
+ int row_id = block_id / n_seq_blk;
+ int seq_id = block_id % n_seq_blk;
+ int tid = item.get_local_id(0);
+ auto wg = item.get_group();
+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
+
+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? elems_per_block
+ : (seq - seq_id * block_size) * head_dim;
+
+ float local_max = 0.0f;
+ sycl::vec<T, Unroll> local_data, local_max_vec;
+ local_max_vec = sycl::vec<T, Unroll>(0.0f);
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
+ local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);
+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));
+ }
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
+ }
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
+
+ if (tid == 0) {
+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
+ }
+
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {
+ sycl::vec<T, Unroll> quant_input = *(sycl::vec<T, Unroll>*)(&block_in[i]);
+ sycl::vec<float, Unroll> val =
+ quant_input.template convert<float, sycl::rounding_mode::automatic>();
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127, 127);
+ sycl::vec<int8_t, Unroll> qv =
+ val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
+ }
+ });
+ }
} else {
int wg_size = MAX_WG_SIZE;
- q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
- [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
- int seq_id = item.get_group(0);
- int row_id = seq_id / n_seq_blk;
- seq_id = seq_id % n_seq_blk;
- int tid = item.get_local_id(0);
- auto wg = item.get_group();
- auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
- auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
-
- // Phase 1: compute absmax across entire block
- float local_max = 0.0f;
- sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;
- local_max_vec = sycl::vec<T, Unroll>(0.0f);
- int local_i = 0;
- int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim
- : (seq - seq_id * block_size) * head_dim;
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
- local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];
- local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));
- }
+ if (bias_ptr) {
+ if (fast_vector_bias) {
+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ int seq_id = item.get_group(0);
+ int row_id = seq_id / n_seq_blk;
+ seq_id = seq_id % n_seq_blk;
+ int tid = item.get_local_id(0);
+ auto wg = item.get_group();
+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
+ auto* row_bias = bias_ptr + (size_t)row_id * head_dim;
+
+ float local_max = 0.0f;
+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;
+ local_max_vec = sycl::vec<T, Unroll>(0.0f);
+ int local_i = 0;
+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim
+ : (seq - seq_id * block_size) * head_dim;
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
+ int bias_offset = power_of_two_head_dim ? (i & bias_mask) : (i % head_dim);
+ local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];
+ local_data[local_i] = local_data[local_i] - *(sycl::vec<T, Unroll>*)(&row_bias[bias_offset]);
+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));
+ }
#pragma unroll
- for (int i = 0; i < Unroll; ++i) {
- local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
- }
- float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
+ }
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
- // Compute scale
- float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
+ if (tid == 0) {
+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
+ }
- // Store scale (one thread writes)
- if (tid == 0) {
- scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
- }
+ local_i = 0;
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
+ sycl::vec<float, Unroll> val =
+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127, 127);
+ sycl::vec<int8_t, Unroll> qv =
+ val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
+ }
+ });
+ } else {
+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ int seq_id = item.get_group(0);
+ int row_id = seq_id / n_seq_blk;
+ seq_id = seq_id % n_seq_blk;
+ int tid = item.get_local_id(0);
+ auto wg = item.get_group();
+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
+ auto* row_bias = bias_ptr + (size_t)row_id * head_dim;
- // Phase 2: fused quantize
- local_i = 0;
- for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
- sycl::vec<float, Unroll> val =
- local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();
- val = val * inv_scale;
- val = sycl::round(val);
- val = sycl::clamp(val, -127, 127);
- sycl::vec<int8_t, Unroll> qv = val.template convert<int8_t, sycl::rounding_mode::automatic>();
- *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
- }
- });
+ float local_max = 0.0f;
+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;
+ local_max_vec = sycl::vec<T, Unroll>(0.0f);
+ int local_i = 0;
+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim
+ : (seq - seq_id * block_size) * head_dim;
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
+ local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];
+#pragma unroll
+ for (int j = 0; j < Unroll; ++j) {
+ local_data[local_i][j] = local_data[local_i][j] - row_bias[(i + j) % head_dim];
+ }
+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));
+ }
+#pragma unroll
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
+ }
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
+
+ if (tid == 0) {
+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
+ }
+
+ local_i = 0;
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
+ sycl::vec<float, Unroll> val =
+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127, 127);
+ sycl::vec<int8_t, Unroll> qv =
+ val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
+ }
+ });
+ }
+ } else {
+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {
+ int seq_id = item.get_group(0);
+ int row_id = seq_id / n_seq_blk;
+ seq_id = seq_id % n_seq_blk;
+ int tid = item.get_local_id(0);
+ auto wg = item.get_group();
+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;
+
+ float local_max = 0.0f;
+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;
+ local_max_vec = sycl::vec<T, Unroll>(0.0f);
+ int local_i = 0;
+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim
+ : (seq - seq_id * block_size) * head_dim;
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
+ local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];
+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));
+ }
+#pragma unroll
+ for (int i = 0; i < Unroll; ++i) {
+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));
+ }
+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});
+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;
+
+ if (tid == 0) {
+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;
+ }
+
+ local_i = 0;
+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {
+ sycl::vec<float, Unroll> val =
+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();
+ val = val * inv_scale;
+ val = sycl::round(val);
+ val = sycl::clamp(val, -127, 127);
+ sycl::vec<int8_t, Unroll> qv =
+ val.template convert<int8_t, sycl::rounding_mode::automatic>();
+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;
+ }
+ });
+ }
}
}
static void sagev1(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
int head_dim, float softmax_scale, bool is_causal) {
+ bool use_mean_bias = env_params::Instance()->sage_use_mean_bias != 0;
size_t seq_q_blk = (seq_len_q + scale_block_size - 1) / scale_block_size;
size_t seq_kv_blk = (seq_len_kv + scale_block_size - 1) / scale_block_size;
size_t q_size = num_heads_q * seq_len_q * head_dim * batch;
size_t q_scale_size = num_heads_q * seq_q_blk * batch;
size_t k_size = num_heads_kv * seq_len_kv * head_dim * batch;
size_t k_scale_size = num_heads_kv * seq_kv_blk * batch;
- size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float);
+ size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(sycl::half) : 0;
+ size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + k_bias_size;
auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);
auto q_out_ptr = (int8_t*)ptr;
auto k_out_ptr = (int8_t*)ptr + q_size;
auto qscale = (float*)((int8_t*)ptr + q_size + k_size);
auto kscale = (float*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float));
+ auto kbias = use_mean_bias
+ ? (sycl::half*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) +
+ k_scale_size * sizeof(float))
+ : nullptr;
sage_dynamic_quant<sycl::half>(q, (sycl::half*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch * num_heads_q,
- seq_len_q, seq_q_blk, head_dim, scale_block_size);
+ seq_len_q, seq_q_blk, head_dim, scale_block_size);
+ if (use_mean_bias) {
+ compute_seq_mean_bias<sycl::half>(q, (sycl::half*)K_ptr, kbias, batch * num_heads_kv, seq_len_kv, head_dim);
+ if (env_params::Instance()->sage_print_kbias != 0) {
+ print_value_distribution(q, kbias, size_t(batch) * num_heads_kv * head_dim, "kbias");
+ }
+ }
sage_dynamic_quant<sycl::half>(q, (sycl::half*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,
- seq_len_kv, seq_kv_blk, head_dim, scale_block_size);
+ seq_len_kv, seq_kv_blk, head_dim, scale_block_size, kbias);
ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, batch,
num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
}
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hpp
index ae5a17d33..2ea29dc47 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hpp
@@ -62,6 +62,4 @@ struct TestGemm {
ctx->deallocate(C);
ctx->deallocate(scaleB);
}
-};
-
-static TestGemm sTestGemm;
\ No newline at end of file
+};
\ No newline at end of file
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cpp
index 9e067c3f0..1281b16da 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cpp
@@ -1,7 +1,12 @@
#include <iostream>
#include "test_gemm.hpp"
+#include "test_quant.hpp"
+#include "test_sdpa.hpp"
int main() {
printf("Welcome to ARK TEST\n");
- return 1;
+ // TestGemm test_gemm;
+ // TestQuant test_quant;
+ TestSDPA test_sdpa;
+ return 0;
}
\ No newline at end of file
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_quant.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_quant.hpp
new file mode 100644
index 000000000..29d9fbe95
--- /dev/null
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_quant.hpp
@@ -0,0 +1,337 @@
+#pragma once
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdint>
+#include <iomanip>
+#include <iostream>
+#include <random>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "common.hpp"
+#include "../include/xpu_wrapper.hpp"
+
+struct TestQuant {
+ TestQuant() {
+#ifdef ARK_XPU
+ test_compute_seq_mean_bias_accuracy("k_mean_bias", 3, 111, 128);
+ test_compute_seq_mean_bias_perf("k_mean_bias_bw", 16, 2048, 128, 20, 100);
+ test_sage_dynamic_quant_accuracy("no_bias_small_block", 4, 128, 128, 64, false);
+ test_sage_dynamic_quant_accuracy("no_bias_unaligned_seq", 5, 96, 128, 64, false);
+ test_sage_dynamic_quant_accuracy("bias_large_block", 3, 512, 128, 256, true);
+ test_sage_dynamic_quant_accuracy("bias_unaligned_seq", 3, 112, 128, 64, true);
+ test_sage_dynamic_quant_perf("bw_no_bias", 16, 2048, 128, 64, false, 20, 100);
+ test_sage_dynamic_quant_perf("bw_with_bias", 16, 2048, 128, 64, true, 20, 100);
+#else
+ std::cout << "[sage_dynamic_quant] skipped: ARK_XPU is disabled\n";
+#endif
+ }
+
+#ifdef ARK_XPU
+ static int ceil_div(int value, int divisor) { return (value + divisor - 1) / divisor; }
+
+ static sycl::half f32_to_f16(float value) { return sycl::half(value); }
+
+ static std::vector<float> make_random_vector(size_t count, float low, float high, uint32_t seed) {
+ std::mt19937 rng(seed);
+ std::uniform_real_distribution<float> dist(low, high);
+ std::vector<float> values(count);
+ for (auto& value : values) {
+ value = dist(rng);
+ }
+ return values;
+ }
+
+ static std::vector<sycl::half> to_fp16_vector(const std::vector<float>& src) {
+ std::vector<sycl::half> dst(src.size());
+ for (size_t index = 0; index < src.size(); ++index) {
+ dst[index] = f32_to_f16(src[index]);
+ }
+ return dst;
+ }
+
+ static std::vector<float> to_f32_vector(const std::vector<sycl::half>& src) {
+ std::vector<float> dst(src.size());
+ for (size_t index = 0; index < src.size(); ++index) {
+ dst[index] = static_cast<float>(src[index]);
+ }
+ return dst;
+ }
+
+ static std::vector<float> reference_seq_mean_bias(const std::vector<float>& input, int num_rows, int seq,
+ int head_dim) {
+ std::vector<float> bias(size_t(num_rows) * size_t(head_dim), 0.0f);
+ for (int row_id = 0; row_id < num_rows; ++row_id) {
+ for (int dim = 0; dim < head_dim; ++dim) {
+ float sum = 0.0f;
+ for (int token = 0; token < seq; ++token) {
+ size_t offset = (size_t(row_id) * seq + size_t(token)) * head_dim + size_t(dim);
+ sum += input[offset];
+ }
+ bias[size_t(row_id) * head_dim + size_t(dim)] = sum / float(seq);
+ }
+ }
+ return bias;
+ }
+
+ void test_compute_seq_mean_bias_accuracy(const std::string& name, int num_rows, int seq, int head_dim) {
+ GETQ();
+ LOG_LINE();
+ size_t input_count = size_t(num_rows) * size_t(seq) * size_t(head_dim);
+ size_t bias_count = size_t(num_rows) * size_t(head_dim);
+
+ auto host_input_f32 = make_random_vector(input_count, -2.0f, 2.0f, 2025u + uint32_t(seq));
+ auto host_input = to_fp16_vector(host_input_f32);
+ auto ref_input_f32 = to_f32_vector(host_input);
+ auto ref_bias = reference_seq_mean_bias(ref_input_f32, num_rows, seq, head_dim);
+
+ auto* dev_input = reinterpret_cast<sycl::half*>(ctx->allocate(input_count * sizeof(sycl::half)));
+ auto* dev_bias = reinterpret_cast<sycl::half*>(ctx->allocate(bias_count * sizeof(sycl::half)));
+
+ q->memcpy(dev_input, host_input.data(), input_count * sizeof(sycl::half)).wait();
+ ark::XpuWrapper::compute_seq_mean_bias<sycl::half>(q, dev_input, dev_bias, num_rows, seq, head_dim);
+ q->wait();
+
+ std::vector<sycl::half> host_bias(bias_count);
+ q->memcpy(host_bias.data(), dev_bias, bias_count * sizeof(sycl::half)).wait();
+
+ float max_diff = 0.0f;
+ for (size_t index = 0; index < bias_count; ++index) {
+ max_diff = std::max(max_diff, std::fabs(static_cast<float>(host_bias[index]) - ref_bias[index]));
+ }
+
+ std::cout << "[compute_seq_mean_bias][accuracy] " << name << " max_diff=" << max_diff << "\n";
+ if (max_diff > 1e-3f) {
+ throw std::runtime_error("compute_seq_mean_bias accuracy check failed: " + name);
+ }
+
+ ctx->deallocate(dev_input);
+ ctx->deallocate(dev_bias);
+ }
+
+ void test_compute_seq_mean_bias_perf(const std::string& name, int num_rows, int seq, int head_dim, int warmup,
+ int iters) {
+ GETQ();
+ LOG_LINE();
+ size_t input_count = size_t(num_rows) * size_t(seq) * size_t(head_dim);
+ size_t bias_count = size_t(num_rows) * size_t(head_dim);
+
+ auto host_input = to_fp16_vector(make_random_vector(input_count, -2.0f, 2.0f, 5051u));
+
+ auto* dev_input = reinterpret_cast<sycl::half*>(ctx->allocate(input_count * sizeof(sycl::half)));
+ auto* dev_bias = reinterpret_cast<sycl::half*>(ctx->allocate(bias_count * sizeof(sycl::half)));
+
+ q->memcpy(dev_input, host_input.data(), input_count * sizeof(sycl::half)).wait();
+
+ for (int iter = 0; iter < warmup; ++iter) {
+ ark::XpuWrapper::compute_seq_mean_bias<sycl::half>(q, dev_input, dev_bias, num_rows, seq, head_dim);
+ }
+ q->wait();
+
+ auto start = std::chrono::steady_clock::now();
+ for (int iter = 0; iter < iters; ++iter) {
+ ark::XpuWrapper::compute_seq_mean_bias<sycl::half>(q, dev_input, dev_bias, num_rows, seq, head_dim);
+ }
+ q->wait();
+ auto end = std::chrono::steady_clock::now();
+
+ double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
+ double avg_ms = total_ms / double(iters);
+ double input_bytes = double(input_count * sizeof(sycl::half));
+ double output_bytes = double(bias_count * sizeof(sycl::half));
+ double logical_io_bytes = input_bytes + output_bytes;
+ double estimated_kernel_bytes = input_bytes + output_bytes;
+ double logical_io_gbps = logical_io_bytes / (avg_ms * 1e-3) / 1e9;
+ double estimated_kernel_gbps = estimated_kernel_bytes / (avg_ms * 1e-3) / 1e9;
+
+ std::cout << std::fixed << std::setprecision(3) << "[compute_seq_mean_bias][perf] " << name
+ << " avg_ms=" << avg_ms << " input_MB=" << (input_bytes / (1024.0 * 1024.0))
+ << " output_MB=" << (output_bytes / (1024.0 * 1024.0))
+ << " logical_io_GBps=" << logical_io_gbps << " est_kernel_GBps=" << estimated_kernel_gbps << "\n";
+
+ ctx->deallocate(dev_input);
+ ctx->deallocate(dev_bias);
+ }
+
+ static void reference_quant(const std::vector<float>& input, const std::vector<float>* bias, int num_rows, int seq,
+ int head_dim, int block_size, std::vector<int8_t>& output, std::vector<float>& scales) {
+ int n_seq_blk = ceil_div(seq, block_size);
+ output.assign(input.size(), int8_t(0));
+ scales.assign(size_t(num_rows) * size_t(n_seq_blk), 0.0f);
+ for (int row_id = 0; row_id < num_rows; ++row_id) {
+ for (int seq_blk = 0; seq_blk < n_seq_blk; ++seq_blk) {
+ int seq_begin = seq_blk * block_size;
+ int seq_end = std::min(seq_begin + block_size, seq);
+ float absmax = 0.0f;
+ for (int token = seq_begin; token < seq_end; ++token) {
+ for (int dim = 0; dim < head_dim; ++dim) {
+ size_t offset = (size_t(row_id) * seq + size_t(token)) * head_dim + size_t(dim);
+ float value = input[offset];
+ if (bias) {
+ value -= (*bias)[size_t(row_id) * head_dim + size_t(dim)];
+ }
+ absmax = std::max(absmax, std::fabs(value));
+ }
+ }
+ scales[size_t(row_id) * n_seq_blk + size_t(seq_blk)] = absmax / 127.0f;
+ float inv_scale = absmax > 0.0f ? 127.0f / absmax : 0.0f;
+ for (int token = seq_begin; token < seq_end; ++token) {
+ for (int dim = 0; dim < head_dim; ++dim) {
+ size_t offset = (size_t(row_id) * seq + size_t(token)) * head_dim + size_t(dim);
+ float value = input[offset];
+ if (bias) {
+ value -= (*bias)[size_t(row_id) * head_dim + size_t(dim)];
+ }
+ value *= inv_scale;
+ int quantized = static_cast<int>(std::round(value));
+ quantized = std::max(-127, std::min(127, quantized));
+ output[offset] = static_cast<int8_t>(quantized);
+ }
+ }
+ }
+ }
+ }
+
+ static void validate_results(const std::string& name, const std::vector<int8_t>& actual_q,
+ const std::vector<int8_t>& ref_q, const std::vector<float>& actual_scales,
+ const std::vector<float>& ref_scales) {
+ size_t q_mismatches = 0;
+ int max_q_diff = 0;
+ for (size_t index = 0; index < actual_q.size(); ++index) {
+ int diff = std::abs(int(actual_q[index]) - int(ref_q[index]));
+ max_q_diff = std::max(max_q_diff, diff);
+ if (diff != 0) {
+ ++q_mismatches;
+ }
+ }
+
+ float max_scale_diff = 0.0f;
+ for (size_t index = 0; index < actual_scales.size(); ++index) {
+ max_scale_diff = std::max(max_scale_diff, std::fabs(actual_scales[index] - ref_scales[index]));
+ }
+
+ std::cout << "[sage_dynamic_quant][accuracy] " << name << " q_mismatches=" << q_mismatches
+ << " max_q_diff=" << max_q_diff << " max_scale_diff=" << max_scale_diff << "\n";
+
+ if (max_q_diff > 1 || max_scale_diff > 1e-4f) {
+ throw std::runtime_error("sage_dynamic_quant accuracy check failed: " + name);
+ }
+ }
+
+ void test_sage_dynamic_quant_accuracy(const std::string& name, int num_rows, int seq, int head_dim, int block_size,
+ bool use_bias) {
+ GETQ();
+ LOG_LINE();
+ int n_seq_blk = ceil_div(seq, block_size);
+ size_t input_count = size_t(num_rows) * size_t(seq) * size_t(head_dim);
+ size_t bias_count = size_t(num_rows) * size_t(head_dim);
+ size_t scale_count = size_t(num_rows) * size_t(n_seq_blk);
+
+ auto host_input_f32 = make_random_vector(input_count, -2.0f, 2.0f, 2026u + uint32_t(block_size));
+ auto host_input = to_fp16_vector(host_input_f32);
+ auto ref_input_f32 = to_f32_vector(host_input);
+ std::vector<float> host_bias_f32;
+ std::vector<sycl::half> host_bias;
+ std::vector<float> ref_bias_f32;
+ if (use_bias) {
+ host_bias_f32 = make_random_vector(bias_count, -0.5f, 0.5f, 3039u + uint32_t(num_rows));
+ host_bias = to_fp16_vector(host_bias_f32);
+ ref_bias_f32 = to_f32_vector(host_bias);
+ }
+
+ auto* dev_input = reinterpret_cast<sycl::half*>(ctx->allocate(input_count * sizeof(sycl::half)));
+ auto* dev_bias = use_bias ? reinterpret_cast<sycl::half*>(ctx->allocate(bias_count * sizeof(sycl::half))) : nullptr;
+ auto* dev_output = reinterpret_cast<int8_t*>(ctx->allocate(input_count * sizeof(int8_t)));
+ auto* dev_scales = reinterpret_cast<float*>(ctx->allocate(scale_count * sizeof(float)));
+
+ q->memcpy(dev_input, host_input.data(), input_count * sizeof(sycl::half)).wait();
+ if (use_bias) {
+ q->memcpy(dev_bias, host_bias.data(), bias_count * sizeof(sycl::half)).wait();
+ }
+
+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_input, dev_output, dev_scales, num_rows, seq, n_seq_blk,
+ head_dim, block_size, dev_bias);
+ q->wait();
+
+ std::vector<int8_t> host_output(input_count);
+ std::vector<float> host_scales(scale_count);
+ q->memcpy(host_output.data(), dev_output, input_count * sizeof(int8_t)).wait();
+ q->memcpy(host_scales.data(), dev_scales, scale_count * sizeof(float)).wait();
+
+ std::vector<int8_t> ref_output;
+ std::vector<float> ref_scales;
+ reference_quant(ref_input_f32, use_bias ? &ref_bias_f32 : nullptr, num_rows, seq, head_dim, block_size,
+ ref_output, ref_scales);
+ validate_results(name, host_output, ref_output, host_scales, ref_scales);
+
+ ctx->deallocate(dev_input);
+ ctx->deallocate(dev_bias);
+ ctx->deallocate(dev_output);
+ ctx->deallocate(dev_scales);
+ }
+
+ void test_sage_dynamic_quant_perf(const std::string& name, int num_rows, int seq, int head_dim, int block_size,
+ bool use_bias, int warmup, int iters) {
+ GETQ();
+ LOG_LINE();
+ int n_seq_blk = ceil_div(seq, block_size);
+ size_t input_count = size_t(num_rows) * size_t(seq) * size_t(head_dim);
+ size_t bias_count = size_t(num_rows) * size_t(head_dim);
+ size_t scale_count = size_t(num_rows) * size_t(n_seq_blk);
+
+ auto host_input = to_fp16_vector(make_random_vector(input_count, -2.0f, 2.0f, 4041u));
+ std::vector<sycl::half> host_bias;
+ if (use_bias) {
+ host_bias = to_fp16_vector(make_random_vector(bias_count, -0.25f, 0.25f, 4042u));
+ }
+
+ auto* dev_input = reinterpret_cast<sycl::half*>(ctx->allocate(input_count * sizeof(sycl::half)));
+ auto* dev_bias = use_bias ? reinterpret_cast<sycl::half*>(ctx->allocate(bias_count * sizeof(sycl::half))) : nullptr;
+ auto* dev_output = reinterpret_cast<int8_t*>(ctx->allocate(input_count * sizeof(int8_t)));
+ auto* dev_scales = reinterpret_cast<float*>(ctx->allocate(scale_count * sizeof(float)));
+
+ q->memcpy(dev_input, host_input.data(), input_count * sizeof(sycl::half)).wait();
+ if (use_bias) {
+ q->memcpy(dev_bias, host_bias.data(), bias_count * sizeof(sycl::half)).wait();
+ }
+
+ for (int iter = 0; iter < warmup; ++iter) {
+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_input, dev_output, dev_scales, num_rows, seq, n_seq_blk,
+ head_dim, block_size, dev_bias);
+ }
+ q->wait();
+
+ auto start = std::chrono::steady_clock::now();
+ for (int iter = 0; iter < iters; ++iter) {
+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_input, dev_output, dev_scales, num_rows, seq, n_seq_blk,
+ head_dim, block_size, dev_bias);
+ }
+ q->wait();
+ auto end = std::chrono::steady_clock::now();
+
+ double total_ms = std::chrono::duration<double, std::milli>(end - start).count();
+ double avg_ms = total_ms / double(iters);
+ double input_bytes = double(input_count * sizeof(sycl::half)) + double(use_bias ? bias_count * sizeof(sycl::half) : 0);
+ double output_bytes = double(input_count * sizeof(int8_t)) + double(scale_count * sizeof(float));
+ double logical_io_bytes = input_bytes + output_bytes;
+ double estimated_kernel_bytes =
+ double(input_count * sizeof(sycl::half) * 2) +
+ double(use_bias ? bias_count * sizeof(sycl::half) * 2 : 0) + output_bytes;
+ double logical_io_gbps = logical_io_bytes / (avg_ms * 1e-3) / 1e9;
+ double estimated_kernel_gbps = estimated_kernel_bytes / (avg_ms * 1e-3) / 1e9;
+
+ std::cout << std::fixed << std::setprecision(3)
+ << "[sage_dynamic_quant][perf] " << name << " avg_ms=" << avg_ms << " input_MB="
+ << (input_bytes / (1024.0 * 1024.0)) << " output_MB=" << (output_bytes / (1024.0 * 1024.0))
+ << " logical_io_GBps=" << logical_io_gbps << " est_kernel_GBps=" << estimated_kernel_gbps << "\n";
+
+ ctx->deallocate(dev_input);
+ ctx->deallocate(dev_bias);
+ ctx->deallocate(dev_output);
+ ctx->deallocate(dev_scales);
+ }
+#endif
+};
\ No newline at end of file
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
new file mode 100644
index 000000000..b2ee33c4d
--- /dev/null
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
@@ -0,0 +1,229 @@
+#pragma once
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdint>
+#include <iomanip>
+#include <iostream>
+#include <functional>
+#include <random>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "common.hpp"
+#include "../include/xpu_wrapper.hpp"
+
+struct TestSDPA {
+ TestSDPA() {
+#if defined(ARK_XPU) && defined(ARK_SYCL_TLA) && ARK_XPU && ARK_SYCL_TLA
+ // test_sagev1_accuracy("prefill_gqa_128_bias", 1, 32, 8, 256, 512, 128, 128, false, true);
+ // test_sagev1_accuracy("prefill_gqa_128_no_bias", 1, 32, 8, 256, 512, 128, 128, false, false);
+ benchmark_sagev1("bench_prefill_gqa_128", 1, 32, 8, 4096, 4096, 128, 128, false, 5, 20);
+#else
+ std::cout << "[sagev1] skipped: requires ARK_XPU and ARK_SYCL_TLA\n";
+#endif
+ }
+
+#if defined(ARK_XPU) && defined(ARK_SYCL_TLA) && ARK_XPU && ARK_SYCL_TLA
+ static std::vector<float> make_random_vector(size_t count, float low, float high, uint32_t seed) {
+ std::mt19937 rng(seed);
+ std::uniform_real_distribution<float> dist(low, high);
+ std::vector<float> values(count);
+ for (auto& value : values) {
+ value = dist(rng);
+ }
+ return values;
+ }
+
+ static std::vector<sycl::half> to_fp16_vector(const std::vector<float>& src) {
+ std::vector<sycl::half> dst(src.size());
+ for (size_t index = 0; index < src.size(); ++index) {
+ dst[index] = sycl::half(src[index]);
+ }
+ return dst;
+ }
+
+ static double compute_sdpa_flops(int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
+ int head_dim) {
+ int group = num_heads_q / num_heads_kv;
+ double qk = double(group) * num_heads_kv * seq_len_q * head_dim * seq_len_kv * 2.0;
+ double softmax = double(group) * num_heads_kv * seq_len_q * seq_len_kv * 2.0;
+ double pv = double(group) * num_heads_kv * seq_len_q * seq_len_kv * head_dim * 2.0;
+ return double(batch) * (qk + softmax + pv);
+ }
+
+ static double compute_sdpa_io_bytes(int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
+ int head_dim) {
+ double q_bytes = double(batch) * num_heads_q * seq_len_q * head_dim * sizeof(sycl::half);
+ double k_bytes = double(batch) * num_heads_kv * seq_len_kv * head_dim * sizeof(sycl::half);
+ double v_bytes = double(batch) * num_heads_kv * seq_len_kv * head_dim * sizeof(sycl::half);
+ double o_bytes = double(batch) * num_heads_q * seq_len_q * head_dim * sizeof(sycl::half);
+ return q_bytes + k_bytes + v_bytes + o_bytes;
+ }
+
+ static double run_bench(const std::function<void()>& fn, sycl::queue* q, int warmup, int iters) {
+ for (int iter = 0; iter < warmup; ++iter) {
+ fn();
+ }
+ q->wait();
+ auto start = std::chrono::steady_clock::now();
+ for (int iter = 0; iter < iters; ++iter) {
+ fn();
+ }
+ q->wait();
+ auto end = std::chrono::steady_clock::now();
+ return std::chrono::duration<double, std::milli>(end - start).count() / double(iters);
+ }
+
+ void test_sagev1_accuracy(const std::string& name, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
+ int seq_len_kv, int head_dim, int scale_block_size, bool is_causal,
+ bool use_mean_bias) {
+ GETQ();
+ LOG_LINE();
+ size_t q_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;
+ size_t k_count = size_t(batch) * num_heads_kv * seq_len_kv * head_dim;
+ size_t o_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;
+ size_t q_scale_count = size_t(batch) * num_heads_q * ((seq_len_q + scale_block_size - 1) / scale_block_size);
+ size_t k_scale_count = size_t(batch) * num_heads_kv * ((seq_len_kv + scale_block_size - 1) / scale_block_size);
+ size_t k_bias_count = size_t(batch) * num_heads_kv * head_dim;
+ float softmax_scale = 1.0f / std::sqrt(float(head_dim));
+ int previous_use_mean_bias = ark::env_params::Instance()->sage_use_mean_bias;
+ ark::env_params::Instance()->sage_use_mean_bias = use_mean_bias ? 1 : 0;
+
+ auto host_q = to_fp16_vector(make_random_vector(q_count, -1.0f, 1.0f, 601u + uint32_t(seq_len_q)));
+ auto host_k = to_fp16_vector(make_random_vector(k_count, -1.0f, 1.0f, 701u + uint32_t(seq_len_kv)));
+ auto host_v = to_fp16_vector(make_random_vector(k_count, -1.0f, 1.0f, 801u + uint32_t(head_dim)));
+
+ auto* dev_q = reinterpret_cast<sycl::half*>(ctx->allocate(q_count * sizeof(sycl::half)));
+ auto* dev_k = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));
+ auto* dev_v = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));
+ auto* dev_out = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));
+ auto* dev_ref = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));
+ auto* dev_qi8 = reinterpret_cast<int8_t*>(ctx->allocate(q_count * sizeof(int8_t)));
+ auto* dev_ki8 = reinterpret_cast<int8_t*>(ctx->allocate(k_count * sizeof(int8_t)));
+ auto* dev_qscale = reinterpret_cast<float*>(ctx->allocate(q_scale_count * sizeof(float)));
+ auto* dev_kscale = reinterpret_cast<float*>(ctx->allocate(k_scale_count * sizeof(float)));
+ auto* dev_kbias = use_mean_bias ? reinterpret_cast<sycl::half*>(ctx->allocate(k_bias_count * sizeof(sycl::half))) : nullptr;
+
+ try {
+ q->memcpy(dev_q, host_q.data(), q_count * sizeof(sycl::half)).wait();
+ q->memcpy(dev_k, host_k.data(), k_count * sizeof(sycl::half)).wait();
+ q->memcpy(dev_v, host_v.data(), k_count * sizeof(sycl::half)).wait();
+
+ ark::XpuWrapper::sagev1(q, dev_q, dev_k, dev_v, dev_out, nullptr, scale_block_size, batch, num_heads_q,
+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+
+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_q, dev_qi8, dev_qscale, batch * num_heads_q, seq_len_q,
+ (seq_len_q + scale_block_size - 1) / scale_block_size, head_dim,
+ scale_block_size);
+ if (use_mean_bias) {
+ ark::XpuWrapper::compute_seq_mean_bias<sycl::half>(q, dev_k, dev_kbias, batch * num_heads_kv, seq_len_kv,
+ head_dim);
+ }
+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_k, dev_ki8, dev_kscale, batch * num_heads_kv, seq_len_kv,
+ (seq_len_kv + scale_block_size - 1) / scale_block_size, head_dim,
+ scale_block_size, dev_kbias);
+ ark::sdpa_impl_qks8_pvhalf(q, dev_qi8, dev_ki8, dev_v, dev_ref, nullptr, scale_block_size, dev_qscale,
+ dev_kscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
+ softmax_scale, is_causal);
+ q->wait();
+
+ std::vector<sycl::half> host_out(o_count);
+ std::vector<sycl::half> host_ref(o_count);
+ q->memcpy(host_out.data(), dev_out, o_count * sizeof(sycl::half)).wait();
+ q->memcpy(host_ref.data(), dev_ref, o_count * sizeof(sycl::half)).wait();
+
+ float max_diff = 0.0f;
+ double mean_diff = 0.0;
+ for (size_t index = 0; index < o_count; ++index) {
+ float out_value = float(host_out[index]);
+ float ref_value = float(host_ref[index]);
+ if (!std::isfinite(out_value) || !std::isfinite(ref_value)) {
+ throw std::runtime_error("sagev1 accuracy test produced non-finite output: " + name);
+ }
+ float diff = std::fabs(out_value - ref_value);
+ max_diff = std::max(max_diff, diff);
+ mean_diff += diff;
+ }
+ mean_diff /= double(o_count);
+
+ std::cout << std::fixed << std::setprecision(6) << "[sagev1][accuracy] " << name << " max_diff=" << max_diff
+ << " mean_diff=" << mean_diff << " use_mean_bias=" << (use_mean_bias ? 1 : 0) << "\n";
+
+ if (max_diff > 5e-3f || mean_diff > 5e-4f) {
+ throw std::runtime_error("sagev1 accuracy mismatch: " + name);
+ }
+ } catch (...) {
+ ark::env_params::Instance()->sage_use_mean_bias = previous_use_mean_bias;
+ ctx->deallocate(dev_q);
+ ctx->deallocate(dev_k);
+ ctx->deallocate(dev_v);
+ ctx->deallocate(dev_out);
+ ctx->deallocate(dev_ref);
+ ctx->deallocate(dev_qi8);
+ ctx->deallocate(dev_ki8);
+ ctx->deallocate(dev_qscale);
+ ctx->deallocate(dev_kscale);
+ ctx->deallocate(dev_kbias);
+ throw;
+ }
+
+ ark::env_params::Instance()->sage_use_mean_bias = previous_use_mean_bias;
+
+ ctx->deallocate(dev_q);
+ ctx->deallocate(dev_k);
+ ctx->deallocate(dev_v);
+ ctx->deallocate(dev_out);
+ ctx->deallocate(dev_ref);
+ ctx->deallocate(dev_qi8);
+ ctx->deallocate(dev_ki8);
+ ctx->deallocate(dev_qscale);
+ ctx->deallocate(dev_kscale);
+ ctx->deallocate(dev_kbias);
+ }
+
+ void benchmark_sagev1(const std::string& name, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
+ int seq_len_kv, int head_dim, int scale_block_size, bool is_causal, int warmup, int iters) {
+ GETQ();
+ LOG_LINE();
+ size_t q_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;
+ size_t k_count = size_t(batch) * num_heads_kv * seq_len_kv * head_dim;
+ size_t o_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;
+ float softmax_scale = 1.0f / std::sqrt(float(head_dim));
+
+ auto host_q = to_fp16_vector(make_random_vector(q_count, -1.0f, 1.0f, 901u + uint32_t(seq_len_q)));
+ auto host_k = to_fp16_vector(make_random_vector(k_count, -1.0f, 1.0f, 1001u + uint32_t(seq_len_kv)));
+ auto host_v = to_fp16_vector(make_random_vector(k_count, -1.0f, 1.0f, 1101u + uint32_t(head_dim)));
+
+ auto* dev_q = reinterpret_cast<sycl::half*>(ctx->allocate(q_count * sizeof(sycl::half)));
+ auto* dev_k = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));
+ auto* dev_v = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));
+ auto* dev_out = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));
+ q->memcpy(dev_q, host_q.data(), q_count * sizeof(sycl::half)).wait();
+ q->memcpy(dev_k, host_k.data(), k_count * sizeof(sycl::half)).wait();
+ q->memcpy(dev_v, host_v.data(), k_count * sizeof(sycl::half)).wait();
+
+ double sage_ms = run_bench(
+ [&]() {
+ ark::XpuWrapper::sagev1(q, dev_q, dev_k, dev_v, dev_out, nullptr, scale_block_size, batch, num_heads_q,
+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ },
+ q, warmup, iters);
+
+ double flops = compute_sdpa_flops(batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim);
+ double io_bytes = compute_sdpa_io_bytes(batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim);
+ double sage_tflops = flops / (sage_ms * 1e-3) / 1e12;
+ double sage_gbps = io_bytes / (sage_ms * 1e-3) / 1e9;
+
+ std::cout << std::fixed << std::setprecision(3) << "[sagev1][bench] " << name << " sage_ms=" << sage_ms
+ << " sage_TFLOPS=" << sage_tflops << " sage_GBps=" << sage_gbps << "\n";
+
+ ctx->deallocate(dev_q);
+ ctx->deallocate(dev_k);
+ ctx->deallocate(dev_v);
+ ctx->deallocate(dev_out);
+ }
+#endif
+};
\ No newline at end of file
diff --git a/auto_round_extension/ark/tools/lm_eval_with_ark_sdpa.py b/auto_round_extension/ark/tools/lm_eval_with_ark_sdpa.py
new file mode 100644
index 000000000..0d2852da9
--- /dev/null
+++ b/auto_round_extension/ark/tools/lm_eval_with_ark_sdpa.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import os
+import runpy
+
+from auto_round_kernel import patch_torch_sdpa
+
+
+def main() -> None:
+ backend = os.environ.get("ARK_TORCH_SDPA_BACKEND", "sdpa")
+ quant_block_size = int(os.environ.get("ARK_SAGEV1_QUANT_BLOCK_SIZE", "64"))
+ patch_torch_sdpa(
+ strict=True,
+ backend=backend,
+ quant_block_size=quant_block_size,
+ )
+ runpy.run_module("lm_eval.__main__", run_name="__main__")
+
+
+if __name__ == "__main__":
+ main()