Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Decoupling oneDNN (#2049)

Zhenzhong Xu54b4b84cdfcb
diff --git a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt
index 5d40df7a4..6cdf56e12 100755
--- a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt
+++ b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt
@@ -4,7 +4,12 @@ project(ark LANGUAGES CXX)
option(ARK_XPU "Build XPU kernels" OFF)
option(ARK_UT "Build XPU kernels UT" OFF)
option(ARK_RESCALE "Experimental" OFF)
-set(ARK_DNNL_BUILD_SOURCE ON CACHE BOOL "Build oneDNN from source" FORCE)
+if(ARK_XPU)
+ option(ARK_DNNL "Build oneDNN-backed kernels" OFF)
+ option(ARK_JOINT_MATRIX "Build BestLA joint_matrix S8 backend" OFF)
+else()
+ option(ARK_DNNL "Build oneDNN-backed kernels" ON)
+endif()
# ARK_SYCL_TLA defaults to ON when ARK_XPU is enabled
if(ARK_XPU AND NOT DEFINED ARK_SYCL_TLA)
@@ -13,6 +18,18 @@ else()
option(ARK_SYCL_TLA "Build SYCL TLA" OFF)
endif()
+if(ARK_SYCL_TLA AND NOT ARK_XPU)
+ message(FATAL_ERROR "ARK_SYCL_TLA requires ARK_XPU=ON")
+endif()
+
+if(ARK_XPU AND NOT ARK_DNNL AND NOT ARK_SYCL_TLA)
+ message(FATAL_ERROR "XPU build requires ARK_SYCL_TLA=ON or ARK_DNNL=ON")
+endif()
+
+if(ARK_XPU AND NOT ARK_SYCL_TLA AND NOT ARK_JOINT_MATRIX)
+ message(FATAL_ERROR "XPU S8 WOQ requires ARK_SYCL_TLA=ON unless ARK_JOINT_MATRIX=ON")
+endif()
+
include(FetchContent)
FetchContent_Declare(
pybind11
@@ -25,40 +42,43 @@ include(FindOpenMP)
set(BTLA_ENABLE_OPENMP ON CACHE BOOL "BesTLA enable compiling OpenMP threading")
-FetchContent_Declare(
- dnnl
- GIT_REPOSITORY https://github.com/uxlfoundation/oneDNN.git
- GIT_TAG v3.10.2
-)
-set(libs dnnl)
-set(DNNL_LIBRARY_TYPE "STATIC" CACHE INTERNAL "")
-set(
- DNNL_ENABLE_PRIMITIVE
- "MATMUL;ELTWISE;REDUCTION;REORDER"
- CACHE STRING "oneDNN primitives used by ARK production code"
-)
+set(libs)
-set(DNNL_BUILD_EXAMPLES OFF CACHE INTERNAL "")
-set(DNNL_BUILD_TESTS OFF CACHE INTERNAL "")
-set(DNNL_BUILD_DOC OFF CACHE INTERNAL "")
-set(ONEDNN_BUILD_GRAPH OFF CACHE INTERNAL "")
-set(ONEDNN_ENABLE_WORKLOAD "INFERENCE" CACHE INTERNAL "")
+if(ARK_DNNL)
+ set(ARK_DNNL_BUILD_SOURCE ON CACHE BOOL "Build oneDNN from source" FORCE)
+ FetchContent_Declare(
+ dnnl
+ GIT_REPOSITORY https://github.com/uxlfoundation/oneDNN.git
+ GIT_TAG v3.10.2
+ )
+ list(APPEND libs dnnl)
+ set(DNNL_LIBRARY_TYPE "STATIC" CACHE INTERNAL "")
+ set(DNNL_ENABLE_PRIMITIVE "MATMUL;ELTWISE;REDUCTION;REORDER" CACHE STRING "oneDNN primitives used by ARK production code")
+ set(DNNL_BUILD_EXAMPLES OFF CACHE INTERNAL "")
+ set(DNNL_BUILD_TESTS OFF CACHE INTERNAL "")
+ set(DNNL_BUILD_DOC OFF CACHE INTERNAL "")
+ set(ONEDNN_BUILD_GRAPH OFF CACHE INTERNAL "")
+ set(ONEDNN_ENABLE_WORKLOAD "INFERENCE" CACHE INTERNAL "")
+endif()
if(ARK_XPU)
- # Skip oneDNN CPU engine compilation for the XPU module build.
- set(DNNL_CPU_RUNTIME "NONE" CACHE INTERNAL "")
- set(DNNL_GPU_RUNTIME "SYCL" CACHE INTERNAL "")
- FetchContent_MakeAvailable(dnnl)
+ if(ARK_DNNL)
+ set(DNNL_CPU_RUNTIME "NONE" CACHE INTERNAL "")
+ set(DNNL_GPU_RUNTIME "SYCL" CACHE INTERNAL "")
+ FetchContent_MakeAvailable(dnnl)
+ endif()
set(BTLA_SYCL ON CACHE BOOL "BesTLA with SYCL")
set(PY_NAME auto_round_kernel_xpu)
set(ARK_TYPE ARK_XPU)
find_package(IntelSYCL REQUIRED)
list(APPEND libs IntelSYCL::SYCL_CXX)
else()
- set(DNNL_CPU_RUNTIME "OMP" CACHE INTERNAL "")
- set(DNNL_CPU_THREADING_RUNTIME "OMP")
- set(DNNL_GPU_RUNTIME "NONE" CACHE INTERNAL "")
- FetchContent_MakeAvailable(dnnl)
+ if(ARK_DNNL)
+ set(DNNL_CPU_RUNTIME "OMP" CACHE INTERNAL "")
+ set(DNNL_CPU_THREADING_RUNTIME "OMP")
+ set(DNNL_GPU_RUNTIME "NONE" CACHE INTERNAL "")
+ FetchContent_MakeAvailable(dnnl)
+ endif()
set(BTLA_SYCL OFF CACHE BOOL "BesTLA without SYCL")
set(PY_NAME auto_round_kernel_cpu)
set(ARK_TYPE ARK_CPU)
@@ -134,9 +154,19 @@ endif()
pybind11_add_module(${PY_NAME} ${SRCS} ${HEADERS} ${SDPA_GENERATED_SRCS})
target_compile_features(${PY_NAME} PRIVATE cxx_std_17)
target_compile_definitions(${PY_NAME} PRIVATE PY_NAME=${PY_NAME} ${ARK_TYPE}=1)
+
+if(ARK_DNNL)
+ target_compile_definitions(${PY_NAME} PRIVATE ARK_DNNL=1)
+endif()
+
+if(ARK_XPU AND ARK_JOINT_MATRIX)
+ target_compile_definitions(${PY_NAME} PRIVATE ARK_JOINT_MATRIX=1)
+endif()
+
if(ARK_RESCALE)
target_compile_definitions(${PY_NAME} PRIVATE ARK_RESCALE=1)
endif()
+
if(ARK_XPU AND ARK_SYCL_TLA)
target_compile_definitions(${PY_NAME} PRIVATE ARK_SYCL_TLA=1 CUTLASS_ENABLE_SYCL=1 SYCL_INTEL_TARGET=1)
target_include_directories(${PY_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/wrapper/include)
@@ -175,5 +205,11 @@ if(ARK_UT)
target_compile_options(${TEST_NAME} PRIVATE -fsycl -fno-sycl-instrument-device-code)
target_link_options(${TEST_NAME} PRIVATE ${SYCL_TLA_LINK_FLAGS})
endif()
+ if(ARK_DNNL)
+ target_compile_definitions(${TEST_NAME} PRIVATE ARK_DNNL=1)
+ endif()
+ if(ARK_XPU AND ARK_JOINT_MATRIX)
+ target_compile_definitions(${TEST_NAME} PRIVATE ARK_JOINT_MATRIX=1)
+ endif()
target_link_libraries(${TEST_NAME} PRIVATE ${libs})
endif()
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.py
index 512fab6a5..1f38e8aa7 100644
--- a/auto_round_extension/ark/auto_round_kernel/__init__.py
+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py
@@ -244,8 +244,8 @@ def matmul_sycl_tla(A: torch.Tensor, B: torch.Tensor, bias: Optional[torch.Tenso
raise ValueError("A and B must be 2D tensors")
if A.device != B.device:
raise ValueError("A and B must be on the same device")
- if A.dtype not in (torch.float16, torch.bfloat16):
- raise ValueError("matmul_sycl_tla only supports torch.float16 and torch.bfloat16")
+ if A.dtype not in (torch.float32, torch.float16, torch.bfloat16):
+ raise ValueError("matmul_sycl_tla only supports torch.float32, torch.float16 and torch.bfloat16")
if B.dtype != A.dtype:
raise ValueError("A and B must have the same dtype")
@@ -345,7 +345,7 @@ def igemm_s8s8s32(A: torch.Tensor, B: torch.Tensor):
# A: mxk:DT, B: nxk:s8, scaleB: n:DT
# return: mxn:DT
-def woqgemm_s8(A: torch.Tensor, B: torch.Tensor, scaleB: torch.Tensor, bias: torch.Tensor):
+def woqgemm_s8(A: torch.Tensor, B: torch.Tensor, scaleB: torch.Tensor, bias: torch.Tensor | None = None):
m = A.shape[0]
n = B.shape[0]
k = B.shape[1]
@@ -362,7 +362,7 @@ def woqgemm_s8(A: torch.Tensor, B: torch.Tensor, scaleB: torch.Tensor, bias: tor
cvt_dtype(A.dtype),
B.contiguous().data_ptr(),
C.contiguous().data_ptr(),
- bias.contiguous().data_ptr(),
+ 0 if bias is None or bias.numel() == 0 else bias.contiguous().data_ptr(),
True,
scaleB.contiguous().data_ptr(),
)
diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cpp
index 279083b26..210a421d7 100755
--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp
@@ -21,6 +21,7 @@ typedef uintptr_t torch_ptr;
#if ARK_XPU
#include <sycl/sycl.hpp>
#include "xpu_wrapper.hpp"
+#include "sycl_s8_wrapper.hpp"
#if ARK_SYCL_TLA
// Only include declarations, implementations are in separate .cpp files
#include "sycl_tla_common.hpp"
@@ -34,17 +35,26 @@ typedef uintptr_t torch_ptr;
#include "cpu_wrapper.hpp"
#endif
+#if ARK_DNNL
#include "dnnl_wrapper.hpp"
+#endif
namespace ark {
static void matmul(torch_ptr stream, int m, int n, int k, torch_ptr A, int Adt, torch_ptr B, int Bdt, torch_ptr C,
int Cdt, torch_ptr bias, bool BT) {
+#ifdef ARK_XPU
+#if ARK_DNNL
auto dt = ark::to_dt((BTLA_DTYPE)Adt);
auto cdt = dt;
if (Adt == (int)BTLA_DTYPE::S8) cdt = dnnl::memory::data_type::s32;
-#ifdef ARK_XPU
ark::DnnlWrapper::gemm((sycl::queue*)stream, m, n, k, (void*)A, dt, (void*)B, dt, BT, (void*)C, cdt, (void*)bias);
+#elif ARK_SYCL_TLA
+ ark::sycl_tla_dense_gemm((sycl::queue*)stream, m, n, k, (void*)A, (BTLA_DTYPE)Adt, (void*)B, (BTLA_DTYPE)Bdt,
+ (void*)C, (BTLA_DTYPE)Cdt, (void*)bias, BT);
+#else
+ throw std::runtime_error("ark::matmul on XPU requires ARK_DNNL=ON or ARK_SYCL_TLA=ON");
+#endif
#else
CpuWrapper::gemm(m, n, k, (void*)A, (BTLA_DTYPE)Adt, (void*)B, BT, (float*)C, (const float*)bias);
#endif
@@ -52,9 +62,16 @@ static void matmul(torch_ptr stream, int m, int n, int k, torch_ptr A, int Adt,
static void woqgemm_s8(torch_ptr stream, int m, int n, int k, torch_ptr A, int ACdt, torch_ptr B, torch_ptr C,
torch_ptr bias, bool BT, torch_ptr scaleb) {
+#if ARK_XPU
+ ark::SyclS8Wrapper::woq_s8((sycl::queue*)stream, m, n, k, (void*)A, (void*)B, BT, (void*)C,
+ (BTLA_DTYPE)ACdt, (void*)scaleb, (void*)bias, k);
+#elif ARK_DNNL
auto dt = ark::to_dt((BTLA_DTYPE)ACdt);
ark::DnnlWrapper::woq_s8((sycl::queue*)stream, m, n, k, (void*)A, (void*)B, BT, (void*)C, dt, (void*)scaleb,
(void*)bias, k);
+#else
+ throw std::runtime_error("ark::woqgemm_s8 requires ARK_XPU or ARK_DNNL");
+#endif
}
static void woqgemm(torch_ptr stream, int m, int n, int k, torch_ptr A, int ACdt, torch_ptr BlobB, torch_ptr C,
@@ -729,5 +746,5 @@ PYBIND11_MODULE(PY_NAME, m) {
m.def("moe_gemm_prefill_fp8_dpas", &ark::moe_gemm_prefill_fp8_dpas_wrapper);
m.def("moe_gemm_prefill_int_dpas", &ark::moe_gemm_prefill_int_dpas_wrapper);
m.def("matmul_sycl_tla", &ark::matmul_sycl_tla);
-#endif
+#endif // ARK_SYCL_TLA
}
\ No newline at end of file
diff --git a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
index 10afcf646..2d9b5f424 100644
--- a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
@@ -449,9 +449,9 @@ void sage_prefill_varlen(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr,
compat::set_default_queue(*q);
- // Zero-filled workspace for cu_seqlens_kv_cache via DnnlContext scratch pool.
+ // Zero-filled workspace for cu_seqlens_kv_cache via DeviceMemoryPool scratch pool.
int* zero_cu_buf = static_cast<int*>(
- DnnlContext::Instance()->get_scratch_mem((batch + 1) * sizeof(int), 3, q));
+ DeviceMemoryPool::Instance()->get_scratch_mem((batch + 1) * sizeof(int), 3, q));
q->memset(zero_cu_buf, 0, (batch + 1) * sizeof(int));
options.cu_seqlens_kv_cache = zero_cu_buf;
options.use_tensor_strides = true;
@@ -512,10 +512,10 @@ void sdpa_varlen_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, voi
// When isVarLen=true, the kernel's apply_variable_length accesses
// cumulative_length for ALL three fields. Even with max_seqlen_kv_cache=0,
- // the pointer must be non-null and device-accessible. Use the DnnlContext
+ // the pointer must be non-null and device-accessible. Use the DeviceMemoryPool
// scratch pool (reuses allocation across calls, only grows when needed).
int* zero_cu_buf = static_cast<int*>(
- DnnlContext::Instance()->get_scratch_mem((batch + 1) * sizeof(int), 4, q));
+ DeviceMemoryPool::Instance()->get_scratch_mem((batch + 1) * sizeof(int), 4, q));
q->memset(zero_cu_buf, 0, (batch + 1) * sizeof(int));
options.cu_seqlens_kv_cache = zero_cu_buf;
options.use_tensor_strides = true;
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hpp
index 91f842bb5..559722158 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hpp
@@ -616,7 +616,7 @@ class CpuWrapper {
auto bsize = bestla::BTLAGemmNPackBSize(n, k, dt);
auto wssize = bsize;
wssize += BT ? n * k * bestla::utils::bestla_dtype_bytes(dt) : 0;
- auto ptr = DnnlContext::Instance()->get_scratch_mem(wssize, 0, nullptr);
+ auto ptr = DeviceMemoryPool::Instance()->get_scratch_mem(wssize, 0, nullptr);
auto packwptr = ptr;
auto bntptr = (int8_t*)packwptr + bsize;
auto bptr = B;
@@ -631,7 +631,7 @@ class CpuWrapper {
static void woq_gemm(int m, const void* a, const void* b, void* c, const void* bias, BTLA_DTYPE acdt, QuantParam* p,
size_t blob_count = 0) {
auto wssize = bestla::BTLAWOQGemmForwardWorkspace((void*)b, m, p->n, p->k, p->k, p->n, blob_count);
- auto ptr = DnnlContext::Instance()->get_scratch_mem(wssize, 0, nullptr);
+ auto ptr = DeviceMemoryPool::Instance()->get_scratch_mem(wssize, 0, nullptr);
bestla::BTLAWOQGemmFp32Forward((void*)b, (const float*)a, (float*)c, (const float*)bias, m, p->n, p->k, p->k, p->n,
ptr, get_threading(), blob_count);
}
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_s8_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_s8_wrapper.hpp
new file mode 100644
index 000000000..a77dcf2c7
--- /dev/null
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_s8_wrapper.hpp
@@ -0,0 +1,139 @@
+//
+// MIT license
+// Copyright (C) 2026 Intel Corporation
+// SPDX-License-Identifier: MIT
+//
+
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+#pragma once
+
+#include "utils.hpp"
+
+#if ARK_XPU
+
+#if ARK_SYCL_TLA
+#include "sycl_tla_s8_gemm.hpp"
+#endif
+
+namespace ark {
+
+class SyclS8Wrapper {
+ public:
+ static inline void prepare_qa_and_quantize(sycl::queue* q, int m, int k, const void* a, BTLA_DTYPE act,
+ int8_t*& qa_ptr, int8_t*& scalea_ptr) {
+ size_t qa_size = size_t(m) * size_t(k);
+ size_t scalea_offset = (qa_size + alignof(float) - 1) & ~(size_t(alignof(float)) - 1);
+ size_t tmp_size = scalea_offset + size_t(m) * sizeof(float);
+
+ auto tmp_ptr = static_cast<int8_t*>(DeviceMemoryPool::Instance()->get_scratch_mem(tmp_size, 1, q));
+ qa_ptr = tmp_ptr;
+ scalea_ptr = tmp_ptr + scalea_offset;
+
+ dyn_quant_s8(q, m, k, a, act, qa_ptr, scalea_ptr, 0);
+ }
+
+ static void dyn_quant_s8(sycl::queue* q, int m, int k, const void* a, BTLA_DTYPE adt, int8_t* qa, void* scalea,
+ int mask) {
+ if (adt == BTLA_DTYPE::F32) {
+ using T = float;
+ using Pro = bestla::sycl_prologue_a::ActivationBase<T>;
+ Pro::template quant_s8<typename Pro::CfgQuantF32>(m, k, mask, {(T*)a, k}, qa, (T*)scalea, q);
+ } else if (adt == BTLA_DTYPE::F16) {
+ using T = sycl::half;
+ using Pro = bestla::sycl_prologue_a::ActivationBase<T>;
+ Pro::template quant_s8<typename Pro::CfgQuantF32>(m, k, mask, {(T*)a, k}, qa, (T*)scalea, q);
+ } else if (adt == BTLA_DTYPE::BF16) {
+ using T = sycl::ext::oneapi::bfloat16;
+ using Pro = bestla::sycl_prologue_a::ActivationBase<T>;
+ Pro::template quant_s8<typename Pro::CfgQuantF32>(m, k, mask, {(T*)a, k}, qa, (T*)scalea, q);
+ } else {
+ throw std::invalid_argument("SyclS8Wrapper::dyn_quant_s8: unsupported activation dtype");
+ }
+ }
+
+ static void igemm_s8s8(sycl::queue* q, int m, int n, int k, const void* a, const void* b, bool BT, void* c,
+ BTLA_DTYPE ct, void* scale_a, void* scale_b, void* bias, int blocksize) {
+ if (!BT) {
+ throw std::invalid_argument("SyclS8Wrapper::igemm_s8s8: only B as n x k is supported");
+ }
+
+#if ARK_JOINT_MATRIX
+ igemm_s8s8_joint_matrix(q, m, n, k, a, b, BT, c, ct, scale_a, scale_b, bias, blocksize);
+#elif ARK_SYCL_TLA
+ igemm_s8s8_sycl_tla(q, m, n, k, a, b, BT, c, ct, scale_a, scale_b, bias, blocksize);
+#else
+ throw std::runtime_error("SyclS8Wrapper::igemm_s8s8 requires ARK_SYCL_TLA=ON or ARK_JOINT_MATRIX=ON");
+#endif
+ }
+
+#if ARK_SYCL_TLA
+ static void igemm_s8s8_sycl_tla(sycl::queue* q, int m, int n, int k, const void* a, const void* b, bool BT, void* c,
+ BTLA_DTYPE ct, void* scale_a, void* scale_b, void* bias, int blocksize) {
+ if (!BT) {
+ throw std::invalid_argument("SyclS8Wrapper::igemm_s8s8: only B as n x k is supported");
+ }
+
+ ark::sycl_tla_igemm_s8s8_dequant(q, m, n, k, a, b, c, ct, scale_a, scale_b, bias, blocksize);
+ }
+#endif // ARK_SYCL_TLA
+
+
+#if ARK_JOINT_MATRIX
+ static void igemm_s8s8_joint_matrix(sycl::queue* q, int m, int n, int k, const void* a, const void* b, bool BT, void* c,
+ BTLA_DTYPE ct, void* scale_a, void* scale_b, void* bias, int blocksize) {
+ if (!BT) {
+ throw std::invalid_argument("SyclS8Wrapper::igemm_s8s8: only B as n x k is supported");
+ }
+
+ using namespace bestla::sycl_gemm;
+
+ if (blocksize == k || blocksize == -1) {
+ if (ct == BTLA_DTYPE::F32) {
+ using T = float;
+ Launcher<xmx::IGemmDQCfg<T>, xmx::IGemmDQCore>::run(
+ q, {(void*)a, (void*)b, c, m, n, k, k, k, n, bias, scale_a, scale_b});
+ } else if (ct == BTLA_DTYPE::F16) {
+ using T = sycl::half;
+ Launcher<xmx::IGemmDQCfg<T>, xmx::IGemmDQCore>::run(
+ q, {(void*)a, (void*)b, c, m, n, k, k, k, n, bias, scale_a, scale_b});
+ } else if (ct == BTLA_DTYPE::BF16) {
+ using T = sycl::ext::oneapi::bfloat16;
+ Launcher<xmx::IGemmDQCfg<T>, xmx::IGemmDQCore>::run(
+ q, {(void*)a, (void*)b, c, m, n, k, k, k, n, bias, scale_a, scale_b});
+ } else {
+ throw std::invalid_argument("SyclS8Wrapper::igemm_s8s8: unsupported output dtype");
+ }
+ return;
+ }
+
+ if (ct == BTLA_DTYPE::F32) {
+ using T = float;
+ Launcher<xmx::IKblockGemmDQCfg<T>, xmx::IKblockGemmDQCore>::run(
+ q, {(void*)a, (void*)b, c, m, n, k, k, k, n, bias, scale_a, scale_b, blocksize});
+ } else if (ct == BTLA_DTYPE::F16) {
+ using T = sycl::half;
+ Launcher<xmx::IKblockGemmDQCfg<T>, xmx::IKblockGemmDQCore>::run(
+ q, {(void*)a, (void*)b, c, m, n, k, k, k, n, bias, scale_a, scale_b, blocksize});
+ } else {
+ throw std::invalid_argument("SyclS8Wrapper::igemm_s8s8: k-block path supports only F32/F16 output");
+ }
+ }
+#endif // ARK_JOINT_MATRIX
+
+ static void woq_s8(sycl::queue* q, int m, int n, int k, const void* a, const void* b, bool BT, void* c,
+ BTLA_DTYPE act, void* scale_b, void* bias, int blocksize) {
+
+ int8_t *qa_ptr, *scalea_ptr;
+ prepare_qa_and_quantize(q, m, k, a, act, qa_ptr, scalea_ptr);
+ igemm_s8s8(q, m, n, k, qa_ptr, b, BT, c, act, scalea_ptr, scale_b, bias, blocksize);
+ }
+
+};
+
+} // namespace ark
+
+#endif // ARK_XPU
\ No newline at end of file
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_dense_gemm.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_dense_gemm.hpp
index 383bce4a4..8db797453 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_dense_gemm.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_dense_gemm.hpp
@@ -39,27 +39,30 @@ namespace dense_gemm_detail {
using namespace cute;
-template <class ATensor, class BTensor, class CTensor, class TiledMMA, class BiasElement>
+template <bool HasBias, class ATensor, class BTensor, class CTensor, class TiledMMA, class BiasElement>
void gemm_device_impl(ATensor const& A, BTensor const& B, CTensor& C, TiledMMA const& mma,
const BiasElement* bias_ptr) {
+ /* Get workgroup and local IDs */
auto item = sycl::ext::oneapi::this_work_item::get_nd_item<2>();
auto wg_m = int(item.get_group(1));
auto wg_n = int(item.get_group(0));
auto local_id = int(item.get_local_id(0));
- Tensor cA = make_identity_tensor(A.shape());
- Tensor cB = make_identity_tensor(B.shape());
- Tensor cC = make_identity_tensor(C.shape());
+ /* Create proxy coordinate tensors for each global tensor */
+ Tensor cA = make_identity_tensor(A.shape()); // (M,K)
+ Tensor cB = make_identity_tensor(B.shape()); // (N,K)
+ Tensor cC = make_identity_tensor(C.shape()); // (M,N)
+ /* Split GEMM into workgroup tiles, and identify our workgroup's tile (wg_coord) */
auto wg_tile = mma.tile_mnk();
auto wg_coord = make_coord(wg_m, wg_n, 0);
- Tensor gA = local_tile(cA, select<0, 2>(wg_tile), make_coord(wg_m, _));
- Tensor gB = local_tile(cB, select<1, 2>(wg_tile), make_coord(wg_n, _));
- Tensor gC = local_tile(cC, wg_tile, wg_coord, Step<_1, _1, X>{});
+ Tensor gA = local_tile(cA, select<0, 2>(wg_tile), make_coord(wg_m, _)); // (BLK_M,BLK_K,k)
+ Tensor gB = local_tile(cB, select<1, 2>(wg_tile), make_coord(wg_n, _)); // (BLK_N,BLK_K,k)
+ Tensor gC = local_tile(cC, wg_tile, wg_coord, Step<_1, _1, X>{}); // (BLK_M,BLK_N)
- auto copy_a = make_block_2d_copy_A(XE_LOAD_2D<16, 8, 32, 32>{}, mma, A);
- auto copy_b = make_block_2d_copy_B(XE_LOAD_2D_TRANSPOSE<32, 16, 8>{}, mma, B);
+ auto copy_a = make_block_2d_copy_A(mma, A);
+ auto copy_b = make_block_2d_copy_B(mma, B);
auto copy_c = make_block_2d_copy_D(mma, C);
auto thr_mma = mma.get_slice(local_id);
@@ -118,30 +121,41 @@ void gemm_device_impl(ATensor const& A, BTensor const& B, CTensor& C, TiledMMA c
}
using ElementOutput = typename CTensor::element_type;
- if constexpr (is_same_v<ElementOutput, float>) {
+
+ if constexpr (!HasBias && is_same_v<ElementOutput, float>) {
copy(copy_c, tCrC, tCgC);
- } else {
- Tensor tCrD = make_tensor_like<ElementOutput>(tCrC);
+ return;
+ }
+
+ Tensor tCrD = make_tensor_like<ElementOutput>(tCrC);
+ if constexpr (HasBias) {
CUTE_UNROLL
for (int i = 0; i < size(tCrC); ++i) {
auto coord = tCgC(i);
int col = int(get<1>(coord));
float value = static_cast<float>(tCrC(i));
- if (bias_ptr != nullptr && col < int(shape<1>(C))) {
+ if (col < int(shape<1>(C))) {
value += static_cast<float>(bias_ptr[col]);
}
tCrD(i) = static_cast<ElementOutput>(value);
}
-
- copy(copy_c, tCrD, tCgC);
+ } else {
+ CUTE_UNROLL
+ for (int i = 0; i < size(tCrC); ++i) {
+ tCrD(i) = static_cast<ElementOutput>(tCrC(i));
+ }
}
+
+ copy(copy_c, tCrD, tCgC);
}
template <typename TA, typename TB, typename TC>
auto choose_mma_op() {
if constexpr (is_complete_v<XE_DPAS_TT<8, TC, TA, TB>>) {
return XE_DPAS_TT<8, TC, TA, TB>{};
+ } else if constexpr (is_same_v<TA, float> && is_same_v<TB, float>) {
+ return XE_DPAS_TT<8, float, cute::tfloat32_t>{};
} else if constexpr (is_same_v<TA, cute::bfloat16_t>) {
return XE_DPAS_TT<8, float, cute::bfloat16_t>{};
} else {
@@ -169,10 +183,10 @@ auto choose_tiled_mma_tile(ATensor const& A, BTensor const& B, CTensor const&) {
return MMA{};
}
-template <class, class, class, int, int, class, char, char>
+template <bool, class, class, class, int, int, class, char, char>
class DenseGemmStoreKernelName;
-template <int TileM, int TileN, class SGLayout, class ATensor, class BTensor, class CTensor, typename TA,
+template <bool HasBias, int TileM, int TileN, class SGLayout, class ATensor, class BTensor, class CTensor, typename TA,
typename TB, typename BiasElement, char layoutA, char layoutB>
void gemm_cute_store_tile(sycl::queue* q, ATensor const& A, BTensor const& B, CTensor& C,
const BiasElement* bias_ptr) {
@@ -190,15 +204,14 @@ void gemm_cute_store_tile(sycl::queue* q, ATensor const& A, BTensor const& B, CT
syclex::properties kernel_props{syclex::sub_group_size<16>, intelex::grf_size<256>};
- auto event = q->parallel_for<
- DenseGemmStoreKernelName<TA, TB, BiasElement, TileM, TileN, SGLayout, layoutA, layoutB>>(
+ q->parallel_for<DenseGemmStoreKernelName<HasBias, TA, TB, BiasElement, TileM, TileN, SGLayout, layoutA, layoutB>>(
sycl::nd_range<2>(global, local), kernel_props,
- [=](auto) { gemm_device_impl(A, B, C, mma, bias_ptr); });
+ [=](auto) { gemm_device_impl<HasBias>(A, B, C, mma, bias_ptr); });
}
-template <typename Element>
+template <bool HasBias, typename Element>
void run_dense_gemm_tuned_tile(sycl::queue* q, int m, int n, int k, const Element* a_ptr, const Element* b_ptr,
Element* c_ptr, const Element* bias_ptr) {
auto A = make_tensor(make_gmem_ptr(const_cast<Element*>(a_ptr)), make_shape(m, k), make_stride(k, _1{}));
@@ -211,16 +224,16 @@ void run_dense_gemm_tuned_tile(sycl::queue* q, int m, int n, int k, const Elemen
using LargeTileSG = Layout<Shape<_8, _4, _1>, Stride<_4, _1, _0>>;
if (m < 32) {
- gemm_cute_store_tile<8, 128, SmallTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
+ gemm_cute_store_tile<HasBias, 8, 128, SmallTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
'R', 'R'>(q, A, B, C, bias_ptr);
} else if (m < 128) {
- gemm_cute_store_tile<64, 128, SmallMidTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
+ gemm_cute_store_tile<HasBias, 64, 128, SmallMidTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
'R', 'R'>(q, A, B, C, bias_ptr);
} else if (m <= 1024) {
- gemm_cute_store_tile<128, 128, MediumTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
+ gemm_cute_store_tile<HasBias, 128, 128, MediumTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
'R', 'R'>(q, A, B, C, bias_ptr);
} else {
- gemm_cute_store_tile<256, 256, LargeTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
+ gemm_cute_store_tile<HasBias, 256, 256, LargeTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
'R', 'R'>(q, A, B, C, bias_ptr);
}
}
@@ -228,7 +241,11 @@ void run_dense_gemm_tuned_tile(sycl::queue* q, int m, int n, int k, const Elemen
template <typename Element>
void run_dense_gemm(sycl::queue* q, int m, int n, int k, const Element* a_ptr, const Element* b_ptr,
Element* c_ptr, const Element* bias_ptr) {
- run_dense_gemm_tuned_tile(q, m, n, k, a_ptr, b_ptr, c_ptr, bias_ptr);
+ if (bias_ptr != nullptr) {
+ run_dense_gemm_tuned_tile<true>(q, m, n, k, a_ptr, b_ptr, c_ptr, bias_ptr);
+ } else {
+ run_dense_gemm_tuned_tile<false>(q, m, n, k, a_ptr, b_ptr, c_ptr, bias_ptr);
+ }
}
} // namespace dense_gemm_detail
@@ -256,6 +273,12 @@ inline void sycl_tla_dense_gemm(sycl::queue* q, int m, int n, int k, const void*
}
switch (at) {
+ case BTLA_DTYPE::F32:
+ dense_gemm_detail::run_dense_gemm(
+ q, m, n, k, static_cast<const float*>(a), static_cast<const float*>(b),
+ static_cast<float*>(c), static_cast<const float*>(bias));
+ return;
+
case BTLA_DTYPE::F16:
dense_gemm_detail::run_dense_gemm(
q, m, n, k, static_cast<const cute::half_t*>(a), static_cast<const cute::half_t*>(b),
@@ -269,7 +292,7 @@ inline void sycl_tla_dense_gemm(sycl::queue* q, int m, int n, int k, const void*
return;
default:
- throw std::invalid_argument("sycl_tla_dense_gemm: only FP16 and BF16 are supported");
+ throw std::invalid_argument("sycl_tla_dense_gemm: only FP32, FP16 and BF16 are supported");
}
}
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp
new file mode 100644
index 000000000..51fe8fac5
--- /dev/null
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp
@@ -0,0 +1,313 @@
+// SYCL-TLA S8 GEMM Wrapper
+//
+// Copyright (C) 2026 Intel Corporation
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <cstdint>
+#include <stdexcept>
+
+#ifdef ARK_XPU
+#include <sycl/sycl.hpp>
+#endif
+
+#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)
+#include <sycl/ext/intel/experimental/grf_size_properties.hpp>
+#include "cute/tensor.hpp"
+#include "cute/util/compat.hpp"
+#endif
+
+#include "utils.hpp"
+
+namespace ark {
+
+#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)
+
+namespace sycl_tla_s8_detail {
+
+using namespace cute;
+
+// template <bool AccumBlock, class ElementOut, int TileM, int TileN, class SGLayout>
+// class S8DequantKernelName;
+
+template <bool AccumBlock, bool HasBias, bool FullTile, class ElementOut, int TileM, int TileN, class SGLayout>
+class S8DequantKernelName;
+
+template <class ElementOut>
+class S8FinalizeKernelName;
+
+template <int TileM, int TileN, class SGLayout>
+class S8AccumKernelName;
+
+template <bool AccumBlock, bool HasBias, bool FullTile, class ATensor, class BTensor, class TiledMMA, class ElementOut>
+void igemm_device_impl(ATensor const& A, BTensor const& B, TiledMMA const& mma, ElementOut* c, float* accum,
+ const ElementOut* scale_a, const ElementOut* scale_b, const ElementOut* bias, int m, int n,
+ int block_idx, int scale_b_stride) {
+ auto item = sycl::ext::oneapi::this_work_item::get_nd_item<2>();
+ int wg_m = int(item.get_group(1));
+ int wg_n = int(item.get_group(0));
+ int local_id = int(item.get_local_id(0));
+
+ Tensor cA = make_identity_tensor(A.shape());
+ Tensor cB = make_identity_tensor(B.shape());
+ Tensor cC = make_identity_tensor(make_shape(m, n));
+
+ auto wg_tile = mma.tile_mnk();
+ auto wg_coord = make_coord(wg_m, wg_n, 0);
+
+ Tensor gA = local_tile(cA, select<0, 2>(wg_tile), make_coord(wg_m, _));
+ Tensor gB = local_tile(cB, select<1, 2>(wg_tile), make_coord(wg_n, _));
+ Tensor gC = local_tile(cC, wg_tile, wg_coord, Step<_1, _1, X>{});
+
+ auto copy_a = make_block_2d_copy_A(mma, A);
+ auto copy_b = make_block_2d_copy_B(mma, B);
+
+ auto thr_mma = mma.get_slice(local_id);
+ auto thr_copy_a = copy_a.get_slice(local_id);
+ auto thr_copy_b = copy_b.get_slice(local_id);
+
+ auto tCrA = thr_mma.partition_sg_fragment_A(gA(_, _, 0));
+ auto tCrB = thr_mma.partition_sg_fragment_B(gB(_, _, 0));
+
+ auto tArA = thr_copy_a.partition_sg_fragment_D(gA(_, _, 0));
+ auto tBrB = thr_copy_b.partition_sg_fragment_D(gB(_, _, 0));
+
+ Tensor tAgA = thr_copy_a.partition_S(gA);
+ Tensor tBgB = thr_copy_b.partition_S(gB);
+
+ Tensor tCrC = partition_fragment_C(mma, select<0, 1>(wg_tile));
+ Tensor tCgC = thr_mma.partition_C(gC);
+
+ constexpr SPIRVScope barrier_scope = ScopeWorkgroup;
+ int k_tile_count = ceil_div(shape<1>(A), get<2>(wg_tile));
+
+ clear(tCrC);
+ auto prefetch_a = make_block_2d_prefetch(copy_a);
+ auto prefetch_b = make_block_2d_prefetch(copy_b);
+
+ auto thr_prefetch_A = prefetch_a.get_slice(local_id);
+ auto thr_prefetch_B = prefetch_b.get_slice(local_id);
+
+ auto pAgA = thr_prefetch_A.partition_S(gA);
+ auto pBgB = thr_prefetch_B.partition_S(gB);
+
+ constexpr int prefetch_dist = 3;
+ int k_tile_prefetch = 0;
+
+ CUTE_UNROLL
+ for (; k_tile_prefetch < prefetch_dist; ++k_tile_prefetch) {
+ prefetch(prefetch_a, pAgA(_, _, _, k_tile_prefetch));
+ prefetch(prefetch_b, pBgB(_, _, _, k_tile_prefetch));
+ }
+
+ for (int k_tile = 0; k_tile < k_tile_count; ++k_tile, ++k_tile_prefetch) {
+ barrier_arrive(barrier_scope);
+
+ copy(copy_a, tAgA(_, _, _, k_tile), tArA);
+ copy(copy_b, tBgB(_, _, _, k_tile), tBrB);
+
+ prefetch(prefetch_a, pAgA(_, _, _, k_tile_prefetch));
+ prefetch(prefetch_b, pBgB(_, _, _, k_tile_prefetch));
+
+ reorder(tArA, tCrA);
+ reorder(tBrB, tCrB);
+ gemm(mma, tCrA, tCrB, tCrC);
+
+ barrier_wait(barrier_scope);
+ }
+
+ CUTE_UNROLL
+ for (int i = 0; i < size(tCrC); ++i) {
+ auto coord = tCgC(i);
+ int row = int(get<0>(coord));
+ int col = int(get<1>(coord));
+
+ if constexpr (!FullTile) {
+ if (row >= m || col >= n) continue;
+ }
+
+ float acc = static_cast<float>(tCrC(i));
+ if constexpr (AccumBlock) {
+ float sb = static_cast<float>(scale_b[col * scale_b_stride + block_idx]);
+ accum[row * n + col] += acc * sb;
+ } else {
+ float sa = static_cast<float>(scale_a[row]);
+ float sb = static_cast<float>(scale_b[col]);
+ float value = acc * sa * sb;
+ if constexpr (HasBias) {
+ value += static_cast<float>(bias[col]);
+ }
+ c[row * n + col] = static_cast<ElementOut>(value);
+ }
+ }
+}
+
+
+template <bool AccumBlock, class ElementOut, int TileM, int TileN, class SGLayout>
+void launch_igemm_tile(sycl::queue* q, int m, int n, int gemm_k, int lda, int ldb, const int8_t* a, const int8_t* b,
+ ElementOut* c, float* accum, const ElementOut* scale_a, const ElementOut* scale_b,
+ const ElementOut* bias, int block_idx, int scale_b_stride) {
+ compat::set_default_queue(*q);
+
+ auto A = make_tensor(make_gmem_ptr(const_cast<int8_t*>(a)), make_shape(m, gemm_k), make_stride(lda, _1{}));
+ auto B = make_tensor(make_gmem_ptr(const_cast<int8_t*>(b)), make_shape(n, gemm_k), make_stride(ldb, _1{}));
+
+ using Op = XE_DPAS_TT<8, int32_t, int8_t, int8_t>;
+ using WGTile = Shape<Int<TileM>, Int<TileN>, _64>;
+ using MMA = typename TiledMMAHelper<MMA_Atom<Op>, Layout<WGTile>, SGLayout>::TiledMMA;
+ MMA mma{};
+
+ sycl::range<2> local = {size(mma), 1};
+ sycl::range<2> global = {local[0] * ceil_div(n, get<1>(mma.tile_mnk())),
+ local[1] * ceil_div(m, get<0>(mma.tile_mnk()))};
+
+ namespace syclex = sycl::ext::oneapi::experimental;
+ namespace intelex = sycl::ext::intel::experimental;
+ syclex::properties props{syclex::sub_group_size<16>, intelex::grf_size<256>};
+
+ bool full_tile = (m % TileM == 0) && (n % TileN == 0);
+ bool has_bias = (!AccumBlock) && (bias != nullptr);
+
+ if (full_tile) {
+ if (has_bias) {
+ q->parallel_for<S8DequantKernelName<AccumBlock, true, true, ElementOut, TileM, TileN, SGLayout>>(
+ sycl::nd_range<2>(global, local), props, [=](auto) {
+ igemm_device_impl<AccumBlock, true, true>(A, B, mma, c, accum, scale_a, scale_b, bias, m, n, block_idx,
+ scale_b_stride);
+ });
+ } else {
+ q->parallel_for<S8DequantKernelName<AccumBlock, false, true, ElementOut, TileM, TileN, SGLayout>>(
+ sycl::nd_range<2>(global, local), props, [=](auto) {
+ igemm_device_impl<AccumBlock, false, true>(A, B, mma, c, accum, scale_a, scale_b, bias, m, n, block_idx,
+ scale_b_stride);
+ });
+ }
+ } else {
+ if (has_bias) {
+ q->parallel_for<S8DequantKernelName<AccumBlock, true, false, ElementOut, TileM, TileN, SGLayout>>(
+ sycl::nd_range<2>(global, local), props, [=](auto) {
+ igemm_device_impl<AccumBlock, true, false>(A, B, mma, c, accum, scale_a, scale_b, bias, m, n, block_idx,
+ scale_b_stride);
+ });
+ } else {
+ q->parallel_for<S8DequantKernelName<AccumBlock, false, false, ElementOut, TileM, TileN, SGLayout>>(
+ sycl::nd_range<2>(global, local), props, [=](auto) {
+ igemm_device_impl<AccumBlock, false, false>(A, B, mma, c, accum, scale_a, scale_b, bias, m, n, block_idx,
+ scale_b_stride);
+ });
+ }
+ }
+}
+
+
+template <bool AccumBlock, class ElementOut>
+void launch_igemm(sycl::queue* q, int m, int n, int gemm_k, int lda, int ldb,
+ const int8_t* a, const int8_t* b, ElementOut* c, float* accum,
+ const ElementOut* scale_a, const ElementOut* scale_b,
+ const ElementOut* bias, int block_idx, int scale_b_stride) {
+ using SmallTileSG = Layout<Shape<_1, _4, _1>, Stride<_0, _1, _0>>;
+ using SmallMidTileSG = Layout<Shape<_2, _4, _1>, Stride<_4, _1, _0>>;
+ using MediumTileSG = Layout<Shape<_4, _4, _1>, Stride<_4, _1, _0>>;
+ using LargeTileSG = Layout<Shape<_8, _4, _1>, Stride<_4, _1, _0>>;
+
+ if (m < 16) {
+ launch_igemm_tile<AccumBlock, ElementOut, 8, 128, SmallTileSG>(
+ q, m, n, gemm_k, lda, ldb, a, b, c, accum, scale_a, scale_b, bias, block_idx, scale_b_stride);
+ } else if (m < 128) {
+ launch_igemm_tile<AccumBlock, ElementOut, 64, 128, SmallMidTileSG>(
+ q, m, n, gemm_k, lda, ldb, a, b, c, accum, scale_a, scale_b, bias, block_idx, scale_b_stride);
+ } else if (m <= 1024) {
+ launch_igemm_tile<AccumBlock, ElementOut, 128, 128, MediumTileSG>(
+ q, m, n, gemm_k, lda, ldb, a, b, c, accum, scale_a, scale_b, bias, block_idx, scale_b_stride);
+ } else {
+ launch_igemm_tile<AccumBlock, ElementOut, 256, 256, LargeTileSG>(
+ q, m, n, gemm_k, lda, ldb, a, b, c, accum, scale_a, scale_b, bias, block_idx, scale_b_stride);
+ }
+}
+
+template <class ElementOut>
+void finalize_block_output(sycl::queue* q, int m, int n, const float* accum, ElementOut* c,
+ const ElementOut* scale_a, const ElementOut* bias) {
+ q->parallel_for<S8FinalizeKernelName<ElementOut>>(sycl::range<1>(size_t(m) * size_t(n)), [=](sycl::id<1> id) {
+ size_t idx = id[0];
+ int row = int(idx / n);
+ int col = int(idx - size_t(row) * size_t(n));
+ float value = accum[idx] * static_cast<float>(scale_a[row]);
+ if (bias) value += static_cast<float>(bias[col]);
+ c[idx] = static_cast<ElementOut>(value);
+ });
+}
+
+template <class ElementOut>
+void run_typed(sycl::queue* q, int m, int n, int k, const int8_t* a, const int8_t* b, ElementOut* c,
+ const ElementOut* scale_a, const ElementOut* scale_b, const ElementOut* bias, int blocksize) {
+ bool k_block = !(blocksize == k || blocksize == -1);
+
+ if (!k_block) {
+ launch_igemm<false, ElementOut>(q, m, n, k, k, k, a, b, c, nullptr, scale_a, scale_b, bias, 0, 1);
+ return;
+ }
+
+ if (blocksize <= 0 || k % blocksize != 0) {
+ throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: blocksize must divide k");
+ }
+
+ int blks = k / blocksize;
+ size_t bytes = size_t(m) * size_t(n) * sizeof(float);
+ auto* accum = static_cast<float*>(DeviceMemoryPool::Instance()->get_scratch_mem(bytes, 3, q));
+ q->memset(accum, 0, bytes);
+
+ for (int ib = 0; ib < blks; ++ib) {
+ const int8_t* a_blk = a + size_t(ib) * size_t(blocksize);
+ const int8_t* b_blk = b + size_t(ib) * size_t(blocksize);
+ launch_igemm<true, ElementOut>(q, m, n, blocksize, k, k, a_blk, b_blk, nullptr, accum, nullptr, scale_b, nullptr,
+ ib, blks);
+ }
+
+ finalize_block_output(q, m, n, accum, c, scale_a, bias);
+}
+
+} // namespace sycl_tla_s8_detail
+
+inline void sycl_tla_igemm_s8s8_dequant(sycl::queue* q, int m, int n, int k, const void* a, const void* b, void* c,
+ BTLA_DTYPE ct, const void* scale_a, const void* scale_b, const void* bias,
+ int blocksize) {
+ if (!q) throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: queue must not be null");
+ if (!a || !b || !c || !scale_a || !scale_b) {
+ throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: input pointers must not be null");
+ }
+ if (m <= 0 || n <= 0 || k <= 0) return;
+
+ bool k_block = !(blocksize == k || blocksize == -1);
+
+ switch (ct) {
+ case BTLA_DTYPE::F32:
+ sycl_tla_s8_detail::run_typed(q, m, n, k, static_cast<const int8_t*>(a), static_cast<const int8_t*>(b),
+ static_cast<float*>(c), static_cast<const float*>(scale_a),
+ static_cast<const float*>(scale_b), static_cast<const float*>(bias), blocksize);
+ return;
+ case BTLA_DTYPE::F16:
+ sycl_tla_s8_detail::run_typed(q, m, n, k, static_cast<const int8_t*>(a), static_cast<const int8_t*>(b),
+ static_cast<cute::half_t*>(c), static_cast<const cute::half_t*>(scale_a),
+ static_cast<const cute::half_t*>(scale_b), static_cast<const cute::half_t*>(bias),
+ blocksize);
+ return;
+ case BTLA_DTYPE::BF16:
+ if (k_block) {
+ throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: k-block path supports only F32/F16 output");
+ }
+ sycl_tla_s8_detail::run_typed(q, m, n, k, static_cast<const int8_t*>(a), static_cast<const int8_t*>(b),
+ static_cast<cute::bfloat16_t*>(c),
+ static_cast<const cute::bfloat16_t*>(scale_a),
+ static_cast<const cute::bfloat16_t*>(scale_b),
+ static_cast<const cute::bfloat16_t*>(bias), blocksize);
+ return;
+ default:
+ throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: unsupported output dtype");
+ }
+}
+
+#endif // ARK_XPU && ARK_SYCL_TLA
+
+} // namespace ark
\ No newline at end of file
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
index bbf2f2bad..ac182fe2a 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
@@ -11,20 +11,34 @@
//
#pragma once
-#include <cstdlib>
+
#include <array>
+#include <cstdlib>
+#include <cstdint>
+#include <cstdio>
+#include <stdexcept>
+#include <string>
+#include <type_traits>
#include <unordered_map>
+
+#if ARK_DNNL
#include <dnnl.hpp>
+#endif
+
#include "bestla/bestla_wrapper.h"
+
#if ARK_XPU
#include "bestla/sycl/sycl_wrapper.h"
-#include <dnnl_sycl.hpp>
#else
namespace sycl {
typedef void queue;
}
#endif
+#if ARK_XPU && ARK_DNNL
+#include <dnnl_sycl.hpp>
+#endif
+
#define LOG_LINE() printf("%s:L%d\n", __FUNCTION__, __LINE__);
namespace ark {
@@ -35,10 +49,12 @@ struct env_params {
int sage_use_mean_bias = 1;
int sage_print_kbias = 0;
int sage_disable_packed_hnd_fast = 0;
+
static env_params* Instance() {
static env_params instance;
return &instance;
}
+
env_params() {
env_i("ARK_VERBOSE", verbose);
env_i("ARK_AUTO_S8", auto_s8);
@@ -46,22 +62,21 @@ struct env_params {
env_i("ARK_SAGE_PRINT_KBIAS", sage_print_kbias);
env_i("ARK_SAGE_DISABLE_PACKED_HND_FAST", sage_disable_packed_hnd_fast);
}
+
static inline void env_i(const char* envstr, int& default_) {
- const char* log_level_env = getenv(envstr);
+ const char* log_level_env = std::getenv(envstr);
if (log_level_env != nullptr) default_ = std::stoi(log_level_env);
}
};
using UUIDArray = std::array<unsigned char, 16>;
-// 高性能 UUID 哈希器
+
struct UUIDHasher {
size_t operator()(const UUIDArray& uuid) const {
- // 将 16 字节视为两个 64 位整数进行处理,性能远高于逐字节遍历
const uint64_t* p = reinterpret_cast<const uint64_t*>(uuid.data());
uint64_t h1 = p[0];
uint64_t h2 = p[1];
- // 使用类似 splitmix64 的混合逻辑
h1 ^= h1 >> 33;
h1 *= 0xff51afd7ed558ccdLLU;
h2 ^= h2 >> 33;
@@ -71,20 +86,103 @@ struct UUIDHasher {
}
};
+class DeviceMemoryPool {
+ public:
+ static DeviceMemoryPool* Instance() {
+ static DeviceMemoryPool instance;
+ return &instance;
+ }
+
+ size_t get_device_key(sycl::queue* q) {
+#if ARK_XPU
+ if (q != nullptr) {
+ auto uuid = q->get_device().get_info<sycl::ext::intel::info::device::uuid>();
+ return UUIDHasher{}(uuid);
+ }
+#endif
+ return 0;
+ }
+
+ void* get_scratch_mem(size_t size, size_t buf_loc, sycl::queue* q) {
+ auto key = get_device_key(q);
+ return get_scratch_ptr(size, buf_loc, q, key);
+ }
+
+ void* get_scratch_ptr(size_t size, size_t buf_loc, sycl::queue* q, size_t key) {
+ if (size == 0 || buf_loc >= MaxLocNum) return nullptr;
+
+ auto it = dev_mem_ptr_map[buf_loc].find(key);
+ if (it == dev_mem_ptr_map[buf_loc].end()) {
+ auto newptr = allocate(size, q);
+ dev_mem_size_map[buf_loc][key] = size;
+ dev_mem_ptr_map[buf_loc][key] = newptr;
+ return newptr;
+ }
+
+ auto old_size = dev_mem_size_map[buf_loc][key];
+ if (old_size < size) {
+ release(it->second, q);
+ auto newptr = allocate(size, q);
+ dev_mem_size_map[buf_loc][key] = size;
+ dev_mem_ptr_map[buf_loc][key] = newptr;
+ return newptr;
+ }
+
+ return it->second;
+ }
+
+ private:
+ static constexpr int MaxLocNum = 8;
+ using SizeMap = std::unordered_map<size_t, size_t>;
+ using PtrMap = std::unordered_map<size_t, int8_t*>;
+
+ int8_t* allocate(size_t size, sycl::queue* q) {
+#if ARK_XPU
+ if (q == nullptr) {
+ throw std::invalid_argument("DeviceMemoryPool: XPU allocation requires a non-null SYCL queue");
+ }
+ return sycl::aligned_alloc_device<int8_t>(128, size, *q);
+#else
+ return static_cast<int8_t*>(std::malloc(size));
+#endif
+ }
+
+ void release(void* ptr, sycl::queue* q) {
+ if (ptr == nullptr) return;
+#if ARK_XPU
+ if (q == nullptr) {
+ throw std::invalid_argument("DeviceMemoryPool: XPU free requires a non-null SYCL queue");
+ }
+ sycl::free(ptr, *q);
+#else
+ std::free(ptr);
+#endif
+ }
+
+ std::array<SizeMap, MaxLocNum> dev_mem_size_map;
+ std::array<PtrMap, MaxLocNum> dev_mem_ptr_map;
+};
+
+#if ARK_DNNL
+
+template <typename>
+struct always_false : std::false_type {};
+
template <typename T>
static inline constexpr dnnl::memory::data_type to_dt() {
- if constexpr (std::is_same_v<T, float>)
+ if constexpr (std::is_same_v<T, float>) {
return dnnl::memory::data_type::f32;
- else if constexpr (std::is_same_v<T, bestla::utils::fp16>)
+ } else if constexpr (std::is_same_v<T, bestla::utils::fp16>) {
return dnnl::memory::data_type::f16;
- else if constexpr (std::is_same_v<T, int8_t>)
+ } else if constexpr (std::is_same_v<T, int8_t>) {
return dnnl::memory::data_type::s8;
- else if constexpr (std::is_same_v<T, uint8_t>)
+ } else if constexpr (std::is_same_v<T, uint8_t>) {
return dnnl::memory::data_type::u8;
- else if constexpr (std::is_same_v<T, bestla::utils::bf16>)
+ } else if constexpr (std::is_same_v<T, bestla::utils::bf16>) {
return dnnl::memory::data_type::bf16;
- else
- static_assert(0);
+ } else {
+ static_assert(always_false<T>::value, "unsupported dnnl dtype");
+ }
}
static inline constexpr dnnl::memory::data_type to_dt(BTLA_DTYPE bt) {
@@ -123,6 +221,7 @@ class DnnlContext {
size_t check_dnnl_device(sycl::queue* q) {
size_t key = 0;
+
if (q == nullptr) {
if (dev_engine_map.find(key) == dev_engine_map.end()) {
dev_engine_map[key] = dnnl::engine(dnnl::engine::kind::cpu, 0);
@@ -130,75 +229,46 @@ class DnnlContext {
}
return key;
}
+
#if ARK_XPU
- auto uuid = q->get_device().get_info<sycl::ext::intel::info::device::uuid>();
- key = UUIDHasher{}(uuid);
+ key = DeviceMemoryPool::Instance()->get_device_key(q);
if (dev_engine_map.find(key) == dev_engine_map.end()) {
sycl::device dev = q->get_device();
- // Get the context associated with the queue
sycl::context ctx = q->get_context();
dev_engine_map[key] = dnnl::sycl_interop::make_engine(dev, ctx);
dev_stream_map[key] = dnnl::sycl_interop::make_stream(dev_engine_map[key], *q);
}
+#else
+ if (dev_engine_map.find(key) == dev_engine_map.end()) {
+ dev_engine_map[key] = dnnl::engine(dnnl::engine::kind::cpu, 0);
+ dev_stream_map[key] = dnnl::stream(dev_engine_map[key]);
+ }
#endif
+
return key;
}
- dnnl::memory get_scratch_mem(dnnl::memory::desc _md, sycl::queue* q) {
+ dnnl::memory get_scratch_mem(dnnl::memory::desc md, sycl::queue* q) {
auto key = check_dnnl_device(q);
- auto ptr = get_scratch_ptr(_md.get_size(), 0, q, key);
- return dnnl::memory(_md, dev_engine_map[key], ptr);
+ auto ptr = DeviceMemoryPool::Instance()->get_scratch_ptr(md.get_size(), 0, q, key);
+ return dnnl::memory(md, dev_engine_map[key], ptr);
}
- void* get_scratch_mem(size_t _size, size_t buf_loc, sycl::queue* q) {
- auto key = check_dnnl_device(q);
- auto ptr = get_scratch_ptr(_size, buf_loc, q, key);
- return ptr;
+ void* get_scratch_mem(size_t size, size_t buf_loc, sycl::queue* q) {
+ return DeviceMemoryPool::Instance()->get_scratch_mem(size, buf_loc, q);
}
- void* get_scratch_ptr(size_t _size, size_t buf_loc, sycl::queue* q, size_t key) {
- if (_size <= 0 || buf_loc >= MaxLocNum) return nullptr;
-
- auto it = dev_mem_ptr_map[buf_loc].find(key);
- if (it == dev_mem_ptr_map[buf_loc].end()) {
- dev_mem_size_map[buf_loc][key] = _size;
-#if ARK_XPU
- auto newptr = sycl::aligned_alloc_device<int8_t>(128, _size, *q);
-#else
- auto newptr = (int8_t*)malloc(_size);
-#endif
- dev_mem_ptr_map[buf_loc][key] = newptr;
- return newptr;
- } else {
- auto pre_size = dev_mem_size_map[buf_loc][key];
- if (pre_size < _size) {
- auto pre_ptr = it->second;
-#if ARK_XPU
- sycl::free(pre_ptr, *q);
-#else
- free(pre_ptr);
-#endif
- dev_mem_size_map[buf_loc][key] = _size;
-#if ARK_XPU
- auto newptr = sycl::aligned_alloc_device<int8_t>(128, _size, *q);
-#else
- auto newptr = (int8_t*)malloc(_size);
-#endif
- dev_mem_ptr_map[buf_loc][key] = newptr;
- return newptr;
- }
- }
- return it->second;
+ void* get_scratch_ptr(size_t size, size_t buf_loc, sycl::queue* q, size_t key) {
+ return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, key);
}
- typedef std::unordered_map<size_t, size_t> msize_t;
- typedef std::unordered_map<size_t, int8_t*> mptr_t;
- static constexpr int MaxLocNum = 8;
- std::array<msize_t, MaxLocNum> dev_mem_size_map;
- std::array<mptr_t, MaxLocNum> dev_mem_ptr_map;
+
+ private:
std::unordered_map<size_t, dnnl::engine> dev_engine_map;
std::unordered_map<size_t, dnnl::stream> dev_stream_map;
};
+#endif // ARK_DNNL
+
struct QuantParam {
int n;
int k;
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
index 09fd1ccac..f67774ab5 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
@@ -17,7 +17,18 @@
#include <limits>
#include <vector>
#include "utils.hpp"
+#if ARK_DNNL
#include "dnnl_wrapper.hpp"
+#endif
+
+#if ARK_SYCL_TLA
+#include "sycl_tla_dense_gemm.hpp"
+#endif
+
+#if ARK_XPU
+#include "sycl_s8_wrapper.hpp"
+#endif
+
#include "sycl_tla_common.hpp"
namespace ark {
@@ -696,14 +707,19 @@ class XpuWrapper {
}
auto ret = woq_gemv(q, m, p, a, b, c, bias, acdt);
if (ret) {
- auto dnnl_dt = to_dt(acdt);
+
check_compute_type(p);
if (p->compute_type != BTLA_DTYPE::S8) {
size_t elesize = bestla::utils::bestla_dtype_bytes(acdt);
size_t total_size = elesize * p->k * p->n;
- auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);
+ auto ptr = DeviceMemoryPool::Instance()->get_scratch_mem(total_size, 1, q);
unpackq(acdt, (int8_t*)b, ptr, p, q);
+#if ARK_DNNL
+ auto dnnl_dt = to_dt(acdt);
DnnlWrapper::gemm(q, m, p->n, p->k, a, dnnl_dt, ptr, dnnl_dt, true, c, dnnl_dt, bias);
+#elif ARK_SYCL_TLA
+ ark::sycl_tla_dense_gemm(q, m, p->n, p->k, a, acdt, ptr, acdt, c, acdt, bias, true);
+#endif
} else {
auto bptr = (int8_t*)b;
bool _rescale = rescale(p);
@@ -713,12 +729,12 @@ class XpuWrapper {
} else {
size_t elesize = 1;
size_t total_size = elesize * p->k * p->n;
- bptr = (int8_t*)DnnlContext::Instance()->get_scratch_mem(total_size, 2, q);
+ bptr = (int8_t*)DeviceMemoryPool::Instance()->get_scratch_mem(total_size, 2, q);
unpackq(BTLA_DTYPE::S8, (int8_t*)b, bptr, p, q);
scaleb_ptr = _rescale ? (int8_t*)b + get_scalext_offset(p) : (int8_t*)b + get_scale_offset(p);
}
auto blocksize = rescale_blocksize(p);
- DnnlWrapper::woq_s8(q, m, p->n, p->k, a, bptr, true, c, dnnl_dt, scaleb_ptr, (void*)bias, blocksize);
+ ark::SyclS8Wrapper::woq_s8(q, m, p->n, p->k, a, bptr, true, c, acdt, scaleb_ptr, (void*)bias, blocksize);
}
}
}
@@ -819,7 +835,7 @@ class XpuWrapper {
template <typename T>
static void print_value_distribution(sycl::queue* q, const T* dev_ptr, size_t count, const char* name) {
if (dev_ptr == nullptr || count == 0) {
- std::printf("[%s] empty\n", name);
+ std::fprintf(stdout, "[%s] empty\n", name);
return;
}
@@ -838,7 +854,7 @@ class XpuWrapper {
}
double mean = sum / static_cast<double>(count);
- std::printf("[%s] min=%f max=%f mean=%f\n", name, min_value, max_value, static_cast<float>(mean));
+ std::fprintf(stdout, "[%s] min=%f max=%f mean=%f\n", name, min_value, max_value, static_cast<float>(mean));
}
// input: num_rows x head_dim matrix, output: int8 quantized matrix + scale (per block)
@@ -1495,7 +1511,7 @@ class XpuWrapper {
size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(T) : 0;
size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size +
v_scale_size + k_bias_size;
- auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);
+ auto ptr = DeviceMemoryPool::Instance()->get_scratch_mem(total_size, 1, q);
auto q_out_ptr = (int8_t*)ptr;
auto k_out_ptr = (int8_t*)ptr + q_size;
auto qscale = (float*)((int8_t*)ptr + q_size + k_size);
@@ -1598,7 +1614,7 @@ class XpuWrapper {
size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * sizeof(T) : 0;
size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size +
v_scale_size + k_bias_size;
- auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);
+ auto ptr = DeviceMemoryPool::Instance()->get_scratch_mem(total_size, 1, q);
auto q_out_ptr = (int8_t*)ptr;
auto k_out_ptr = (int8_t*)ptr + q_size;
auto qscale = (float*)((int8_t*)ptr + q_size + k_size);
diff --git a/auto_round_extension/ark/setup.py b/auto_round_extension/ark/setup.py
index fb957b4ac..8d7eba1d5 100644
--- a/auto_round_extension/ark/setup.py
+++ b/auto_round_extension/ark/setup.py
@@ -48,7 +48,10 @@ def get_build_version():
def fetch_requirements(path):
requirements = []
with open(path, "r") as fd:
- requirements = [r.strip() for r in fd]
+ for r in fd:
+ line = r.strip()
+ if line and not line.startswith("-"):
+ requirements.append(line)
return requirements
@@ -89,8 +92,18 @@ if not oneapi_version:
"and that the 'icx' compiler is in your PATH."
)
+
+def env_flag(name, default=False):
+ value = os.environ.get(name)
+ if value is None:
+ return default
+ return value.strip().lower() in ("1", "on", "true", "yes")
+
+
requirements = fetch_requirements("requirements.txt")
enable_sycl_tla = parse_major_minor(oneapi_version) >= (2025, 3)
+enable_dnnl = env_flag("ARK_DNNL", default=not enable_sycl_tla)
+enable_joint_matrix = env_flag("ARK_JOINT_MATRIX", default=False)
def get_system_memory_gb():
@@ -181,6 +194,8 @@ class CMakeBuild(build_ext):
"-DCMAKE_BUILD_TYPE=Release",
"-DCMAKE_CXX_COMPILER=icx",
"-DARK_XPU=ON",
+ f"-DARK_DNNL={'ON' if enable_dnnl else 'OFF'}",
+ f"-DARK_JOINT_MATRIX={'ON' if enable_joint_matrix else 'OFF'}",
f"-DARK_SYCL_TLA={'ON' if enable_sycl_tla else 'OFF'}",
]
if sys.platform == "win32":
diff --git a/auto_round_extension/ark/test/test_matmul.py b/auto_round_extension/ark/test/test_matmul.py
index 4d4aee98c..86f06a8ee 100644
--- a/auto_round_extension/ark/test/test_matmul.py
+++ b/auto_round_extension/ark/test/test_matmul.py
@@ -56,7 +56,7 @@ def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device, op
diff = abs(tar_dst - ref_dst)
print(f" Max Diff: {diff.max().item():.5f}, Mean Diff: {diff.mean().item():.5f}", end="", flush=True)
if dt == torch.float32:
- atol = 0.001
+ atol = 0.01
rtol = 0.03
if dt == torch.float16:
atol = 0.1
@@ -166,7 +166,7 @@ def woqgemm(m, k, n, dt, batch_size, runs, record_property, device):
@pytest.mark.parametrize("m", [4096])
@pytest.mark.parametrize("k, n", [(4096, 4096)])
[email protected]("dt", [torch.float16, torch.float32])
[email protected]("dt", [torch.float32, torch.float16], ids=["float32", "float16"])
@pytest.mark.parametrize("batch_size", [1])
@pytest.mark.parametrize("runs", [1])
def test_woqgemm_xpu(m, k, n, dt, batch_size, runs, record_property):
@@ -179,7 +179,7 @@ def test_woqgemm_xpu(m, k, n, dt, batch_size, runs, record_property):
@pytest.mark.parametrize("m", [1, 8, 16, 32, 64, 128, 256, 1024])
@pytest.mark.parametrize("k, n", [(4096, 4096)])
[email protected]("dt", [torch.float32, torch.float16, torch.bfloat16])
[email protected]("dt", [torch.float32, torch.float16, torch.bfloat16], ids=["float32", "float16", "bfloat16"])
@pytest.mark.parametrize("batch_size", [1])
@pytest.mark.parametrize("runs", [1])
def test_xpu(m, k, n, dt, batch_size, runs, record_property):
@@ -192,7 +192,7 @@ def test_xpu(m, k, n, dt, batch_size, runs, record_property):
@pytest.mark.parametrize("m", [1, 8, 16, 32, 64, 128, 256, 1024])
@pytest.mark.parametrize("k, n", [(4096, 4096)])
[email protected]("dt", [torch.float32])
[email protected]("dt", [torch.float32], ids=["float32"])
@pytest.mark.parametrize("batch_size", [1])
@pytest.mark.parametrize("runs", [1])
def test_cpu(m, k, n, dt, batch_size, runs, record_property):
@@ -231,7 +231,7 @@ def test_xpu_sycl_tla(m, k, n, dt, batch_size, runs, record_property):
@pytest.mark.parametrize("m", [1, 8, 16, 32, 64, 128, 256, 1024, 2048, 4096])
@pytest.mark.parametrize("k, n", [(4096, 4096)])
[email protected]("dt", [torch.float16, torch.bfloat16])
[email protected]("dt", [torch.float16, torch.bfloat16], ids=["float16", "bfloat16"])
@pytest.mark.parametrize("batch_size", [1])
@pytest.mark.parametrize("runs", [1])
def test_xpu_sycl_tla_no_bias(m, k, n, dt, batch_size, runs, record_property):
@@ -260,25 +260,19 @@ def test_xpu_sycl_tla_no_bias(m, k, n, dt, batch_size, runs, record_property):
# pytest -vs auto_round_extension/ark/test/test_matmul.py -k compare_dnnl_vs_sycl_tla
+ os.environ.get("ARK_DNNL", "0") not in ["1", "ON", "true", "True"],
+ reason="Skipped because ARK_DNNL is not enabled in environment variables",
+)
@pytest.mark.parametrize("m", [1, 8, 16, 32, 128, 1024, 2048, 4096])
@pytest.mark.parametrize("k, n", [(4096, 4096)])
[email protected]("dt", [torch.float16, torch.bfloat16])
-def test_xpu_compare_dnnl_vs_sycl_tla(m, k, n, dt):
- warmup = 100
- runs = 1000
-
- compare_matmul_backends(m, k, n, dt, warmup, runs, "xpu")
- torch.xpu.empty_cache()
-
-
[email protected]("m", [1, 8, 16, 32, 128, 1024, 2048, 4096])
[email protected]("k, n", [(4096, 4096)])
[email protected]("dt", [torch.float16, torch.bfloat16])
-def test_xpu_compare_dnnl_vs_sycl_tla_no_bias(m, k, n, dt):
[email protected]("dt", [torch.float32, torch.float16, torch.bfloat16], ids=["float32", "float16", "bfloat16"])
[email protected]("has_bias", [True, False], ids=["with_bias", "no_bias"])
+def test_xpu_compare_dnnl_vs_sycl_tla(m, k, n, dt, has_bias):
warmup = 100
runs = 1000
- compare_matmul_backends(m, k, n, dt, warmup, runs, "xpu", has_bias=False)
+ compare_matmul_backends(m, k, n, dt, warmup, runs, "xpu", has_bias=has_bias)
torch.xpu.empty_cache()
@@ -288,7 +282,8 @@ def compare_matmul_backends(m, k, n, dt, warmup, runs, device="xpu", has_bias=Tr
def _matmul_tolerance(dt):
if dt == torch.float32:
- return 0.001, 0.03
+ # SYCL-TLA fp32 GEMM uses TF32 DPAS internally.
+ return 0.01, 0.06
if dt == torch.float16:
return 0.1, 0.06
if dt == torch.bfloat16:
@@ -318,6 +313,8 @@ def compare_matmul_backends(m, k, n, dt, warmup, runs, device="xpu", has_bias=Tr
dnnl_out = ark.matmul(activation, wei, bias)
tla_out = ark.matmul_sycl_tla(activation, wei, bias)
+ diff = abs(dnnl_out - tla_out)
+ print(f" Max Diff: {diff.max().item():.5f}, Mean Diff: {diff.mean().item():.5f}", end="", flush=True)
atol, rtol = _matmul_tolerance(dt)
ok = torch.allclose(dnnl_out, tla_out, atol=atol, rtol=rtol)
diff --git a/auto_round_extension/ark/test/test_weightonly.py b/auto_round_extension/ark/test/test_weightonly.py
index c81aed0cd..97363d7ef 100644
--- a/auto_round_extension/ark/test/test_weightonly.py
+++ b/auto_round_extension/ark/test/test_weightonly.py
@@ -21,7 +21,14 @@ import time
import auto_round_kernel
import pytest
import torch
-from ut_utils import decode_fp8_to_float, gen_weis8, get_torch_dt, sample_valid_fp8, sample_valid_fp8_e8m0_xpu_safe
+from ut_utils import (
+ decode_fp8_to_float,
+ gen_weis8,
+ get_torch_dt,
+ make_fp8_consistent_weight,
+ sample_valid_fp8,
+ sample_valid_fp8_e8m0_xpu_safe,
+)
ark = auto_round_kernel
@@ -64,14 +71,9 @@ def main_op(m, n, k, blocksize, compute_type, weight_type, scale_type, asym, dev
raw_qweight = sample_valid_fp8((k, n), weight_type, device)
decoded = decode_fp8_to_float(raw_qweight.cpu(), weight_type).to(dtype).to(device)
if scale_type == "fp8_e8m0":
- # Narrow exponent range for fp16 to avoid inf/nan in matmul and kernel output.
- if compute_type == "fp16":
- scale = torch.randint(-4, 4, (k // blocksize, n), dtype=torch.int32, device=device).to(torch.float32)
- else:
- scale = torch.randint(-8, 8, (k // blocksize, n), dtype=torch.int32, device=device).to(torch.float32)
- scale_re = torch.pow(torch.tensor(2.0, device=device, dtype=torch.float32), scale)
- scale_re = scale_re.repeat_interleave(repeats=blocksize, dim=0)
- ref_dst = decoded * scale_re.to(dtype)
+ raw_qweight, scale, ref_dst = make_fp8_consistent_weight(
+ k, n, blocksize, weight_type, scale_type, dtype, device
+ )
else:
scale = torch.randn(k // blocksize, n, dtype=stype, device=device) / 100 + 0.01
scale_re = scale.repeat_interleave(repeats=blocksize, dim=0)
diff --git a/auto_round_extension/ark/test/ut_utils.py b/auto_round_extension/ark/test/ut_utils.py
index 5a1e6e856..c1a93d247 100644
--- a/auto_round_extension/ark/test/ut_utils.py
+++ b/auto_round_extension/ark/test/ut_utils.py
@@ -119,6 +119,54 @@ def decode_fp8_to_float(raw_bytes: torch.Tensor, fmt: str) -> torch.Tensor:
return val * sign_mask
+def make_fp8_consistent_weight(k, n, blocksize, weight_type, scale_type, dtype, device):
+ if k % blocksize != 0:
+ raise ValueError("blocksize must divide k for this test")
+
+ if weight_type == "fp8_e4m3":
+ fp8_dtype = torch.float8_e4m3fn
+ emax = 8
+ elif weight_type == "fp8_e5m2":
+ fp8_dtype = torch.float8_e5m2
+ emax = 15
+ else:
+ raise ValueError(f"unsupported FP8 weight_type: {weight_type}")
+
+ cpu = torch.device("cpu")
+
+ # Realistic source first. Payload and scale are derived from the same block.
+ source = torch.randn(k, n, dtype=torch.float32, device=cpu) * 0.5
+ grouped = source.view(k // blocksize, blocksize, n)
+ block_absmax = grouped.abs().amax(dim=1).clamp_min(torch.finfo(torch.float32).tiny)
+
+ fp8_max = torch.finfo(fp8_dtype).max
+
+ if scale_type == "fp8_e8m0":
+ # E8M0 stores exponent. Real scale is 2 ** exponent.
+ scale = torch.floor(torch.log2(block_absmax)) - float(emax)
+ scale = scale.clamp(-127, 127).to(torch.float32)
+ scale_re = torch.pow(torch.tensor(2.0, dtype=torch.float32, device=cpu), scale)
+ scale_for_pack = scale
+ elif scale_type in {"fp16", "fp32"}:
+ # FP16/FP32 scales store the real multiplier directly.
+ scale = (block_absmax / fp8_max).clamp_min(torch.finfo(torch.float32).tiny)
+ scale_re = scale
+ scale_for_pack = scale.to(torch.float16 if scale_type == "fp16" else torch.float32)
+ else:
+ raise ValueError(f"unsupported FP8 scale_type: {scale_type}")
+
+ scale_re_full = scale_re.repeat_interleave(repeats=blocksize, dim=0)
+ scaled = (source / scale_re_full).clamp(-fp8_max, fp8_max)
+
+ raw_qweight = scaled.to(fp8_dtype).view(torch.uint8).view(torch.int8).to(device)
+ scale_for_pack = scale_for_pack.to(device)
+
+ decoded = decode_fp8_to_float(raw_qweight.cpu(), weight_type).to(dtype).to(device)
+ ref_dst = decoded * scale_re_full.to(device).to(dtype)
+
+ return raw_qweight, scale_for_pack, ref_dst
+
+
def sample_valid_fp8(shape, fmt: str, device):
"""Generate FP8 payload bytes that avoid NaN encodings."""
if isinstance(device, str):