Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[ARK] Support Dense Gemm using sycl-tla (#1968)

Zhenzhong Xu478224299b57
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.py
index 545dcb53f..c5af8ae74 100644
--- a/auto_round_extension/ark/auto_round_kernel/__init__.py
+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py
@@ -15,7 +15,6 @@
import os
from typing import Optional
import torch
-import sys
# Intel GPU compiler (IGC) environment variable: ensure implicit local IDs are
# not removed by the compiler even when unused. This guarantees consistent
@@ -237,6 +236,58 @@ def get_lib(A: torch.Tensor):
return lib
+# A: mxk, B: nxk, bias: n or [1, n]
+def matmul_sycl_tla(A: torch.Tensor, B: torch.Tensor, bias: Optional[torch.Tensor] = None):
+ if A.device.type != "xpu" or B.device.type != "xpu":
+ raise NotImplementedError("matmul_sycl_tla is only supported on XPU")
+ if A.ndim != 2 or B.ndim != 2:
+ raise ValueError("A and B must be 2D tensors")
+ if A.device != B.device:
+ raise ValueError("A and B must be on the same device")
+ if A.dtype not in (torch.float16, torch.bfloat16):
+ raise ValueError("matmul_sycl_tla only supports torch.float16 and torch.bfloat16")
+ if B.dtype != A.dtype:
+ raise ValueError("A and B must have the same dtype")
+
+ m, k = A.shape
+ n, kb = B.shape
+ if k != kb:
+ raise ValueError(f"Shape mismatch: A.shape={tuple(A.shape)}, B.shape={tuple(B.shape)}")
+
+ lib = get_lib(A)
+ if lib is None or not hasattr(lib, "matmul_sycl_tla"):
+ raise NotImplementedError("Current XPU build does not expose matmul_sycl_tla")
+
+ A_arg = A.contiguous()
+ B_arg = B.contiguous()
+ C = torch.empty(m, n, dtype=A.dtype, device=A.device)
+
+ bias_ptr = 0
+ bias_arg = None
+ if bias is not None and bias.numel() > 0:
+ bias_arg = bias.to(dtype=C.dtype, device=A.device).contiguous().view(-1)
+ if bias_arg.numel() != n:
+ raise ValueError(f"bias must have {n} elements, got {bias_arg.numel()}")
+ bias_ptr = bias_arg.data_ptr()
+
+ stream = get_stream(A_arg)
+ lib.matmul_sycl_tla(
+ stream,
+ m,
+ n,
+ k,
+ A_arg.data_ptr(),
+ cvt_dtype(A_arg.dtype),
+ B_arg.data_ptr(),
+ cvt_dtype(B_arg.dtype),
+ C.data_ptr(),
+ cvt_dtype(C.dtype),
+ bias_ptr,
+ True,
+ )
+ return C
+
+
# A: mxk, B: nxk, bias: n
def matmul(A: torch.Tensor, B: torch.Tensor, bias: torch.Tensor):
m = A.shape[0]
diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cpp
index 3716d79ef..df06dc2c0 100755
--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp
@@ -26,6 +26,7 @@ typedef uintptr_t torch_ptr;
#include "sycl_tla_common.hpp"
#include "sycl_tla_moe.hpp"
#include "sycl_tla_sdpa.hpp"
+#include "sycl_tla_dense_gemm.hpp"
#endif
#else
#include "cpu_wrapper.hpp"
@@ -101,6 +102,12 @@ static size_t packed_weight_size(torch_ptr stream, int n, int k, int blocksize,
#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)
+static void matmul_sycl_tla(torch_ptr stream, int m, int n, int k, torch_ptr A, int Adt, torch_ptr B,
+ int Bdt, torch_ptr C, int Cdt, torch_ptr bias, bool BT) {
+ ark::sycl_tla_dense_gemm((sycl::queue*)stream, m, n, k, (void*)A, (BTLA_DTYPE)Adt, (void*)B,
+ (BTLA_DTYPE)Bdt, (void*)C, (BTLA_DTYPE)Cdt, (void*)bias, BT);
+}
+
// Tensor layout codes passed from Python (tensor_layout argument).
constexpr int TENSOR_LAYOUT_HND = 0; // [B, H, S, D]
constexpr int TENSOR_LAYOUT_NHD = 1; // [B, S, H, D]
@@ -669,5 +676,6 @@ PYBIND11_MODULE(PY_NAME, m) {
m.def("sage_dynamic_quant_layout", &ark::sage_dynamic_quant_layout);
m.def("sage_dynamic_quant_v_layout", &ark::sage_dynamic_quant_v_layout);
m.def("moe_gemm", &ark::moe_gemm_wrapper);
+ m.def("matmul_sycl_tla", &ark::matmul_sycl_tla);
#endif
}
\ No newline at end of file
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_dense_gemm.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_dense_gemm.hpp
new file mode 100644
index 000000000..383bce4a4
--- /dev/null
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_dense_gemm.hpp
@@ -0,0 +1,278 @@
+// SYCL-TLA Dense GEMM Wrapper
+// Based on sycl-tla/examples/cute/tutorial/xe_gemm.cpp
+//
+// Copyright (C) 2026 Intel Corporation
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <stdexcept>
+#include <type_traits>
+
+#ifdef ARK_XPU
+#include <sycl/sycl.hpp>
+#endif
+
+#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)
+#include <sycl/ext/intel/experimental/grf_size_properties.hpp>
+
+#include "cute/tensor.hpp"
+#include "cute/util/compat.hpp"
+
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wpass-failed"
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+#endif
+
+#include "utils.hpp"
+
+namespace ark {
+
+#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)
+
+namespace dense_gemm_detail {
+
+using namespace cute;
+
+template <class ATensor, class BTensor, class CTensor, class TiledMMA, class BiasElement>
+void gemm_device_impl(ATensor const& A, BTensor const& B, CTensor& C, TiledMMA const& mma,
+ const BiasElement* bias_ptr) {
+ auto item = sycl::ext::oneapi::this_work_item::get_nd_item<2>();
+ auto wg_m = int(item.get_group(1));
+ auto wg_n = int(item.get_group(0));
+ auto local_id = int(item.get_local_id(0));
+
+ Tensor cA = make_identity_tensor(A.shape());
+ Tensor cB = make_identity_tensor(B.shape());
+ Tensor cC = make_identity_tensor(C.shape());
+
+ auto wg_tile = mma.tile_mnk();
+ auto wg_coord = make_coord(wg_m, wg_n, 0);
+
+ Tensor gA = local_tile(cA, select<0, 2>(wg_tile), make_coord(wg_m, _));
+ Tensor gB = local_tile(cB, select<1, 2>(wg_tile), make_coord(wg_n, _));
+ Tensor gC = local_tile(cC, wg_tile, wg_coord, Step<_1, _1, X>{});
+
+ auto copy_a = make_block_2d_copy_A(XE_LOAD_2D<16, 8, 32, 32>{}, mma, A);
+ auto copy_b = make_block_2d_copy_B(XE_LOAD_2D_TRANSPOSE<32, 16, 8>{}, mma, B);
+ auto copy_c = make_block_2d_copy_D(mma, C);
+
+ auto thr_mma = mma.get_slice(local_id);
+ auto thr_copy_a = copy_a.get_slice(local_id);
+ auto thr_copy_b = copy_b.get_slice(local_id);
+
+ auto tCrA = thr_mma.partition_sg_fragment_A(gA(_, _, 0));
+ auto tCrB = thr_mma.partition_sg_fragment_B(gB(_, _, 0));
+
+ auto tArA = thr_copy_a.partition_sg_fragment_D(gA(_, _, 0));
+ auto tBrB = thr_copy_b.partition_sg_fragment_D(gB(_, _, 0));
+
+ Tensor tAgA = thr_copy_a.partition_S(gA);
+ Tensor tBgB = thr_copy_b.partition_S(gB);
+
+ Tensor tCrC = partition_fragment_C(mma, select<0, 1>(wg_tile));
+ Tensor tCgC = thr_mma.partition_C(gC);
+
+ auto prefetch_a = make_block_2d_prefetch(copy_a);
+ auto prefetch_b = make_block_2d_prefetch(copy_b);
+
+ auto thr_prefetch_A = prefetch_a.get_slice(local_id);
+ auto thr_prefetch_B = prefetch_b.get_slice(local_id);
+
+ auto pAgA = thr_prefetch_A.partition_S(gA);
+ auto pBgB = thr_prefetch_B.partition_S(gB);
+
+ constexpr int prefetch_dist = 3;
+ constexpr SPIRVScope barrier_scope = ScopeWorkgroup;
+
+ int k_tile_count = ceil_div(shape<1>(A), get<2>(wg_tile));
+ int k_tile_prefetch = 0;
+
+ clear(tCrC);
+
+ CUTE_UNROLL
+ for (; k_tile_prefetch < prefetch_dist; ++k_tile_prefetch) {
+ prefetch(prefetch_a, pAgA(_, _, _, k_tile_prefetch));
+ prefetch(prefetch_b, pBgB(_, _, _, k_tile_prefetch));
+ }
+
+ for (int k_tile = 0; k_tile < k_tile_count; ++k_tile, ++k_tile_prefetch) {
+ barrier_arrive(barrier_scope);
+
+ copy(copy_a, tAgA(_, _, _, k_tile), tArA);
+ copy(copy_b, tBgB(_, _, _, k_tile), tBrB);
+
+ prefetch(prefetch_a, pAgA(_, _, _, k_tile_prefetch));
+ prefetch(prefetch_b, pBgB(_, _, _, k_tile_prefetch));
+
+ reorder(tArA, tCrA);
+ reorder(tBrB, tCrB);
+ gemm(mma, tCrA, tCrB, tCrC);
+
+ barrier_wait(barrier_scope);
+ }
+
+ using ElementOutput = typename CTensor::element_type;
+ if constexpr (is_same_v<ElementOutput, float>) {
+ copy(copy_c, tCrC, tCgC);
+ } else {
+ Tensor tCrD = make_tensor_like<ElementOutput>(tCrC);
+
+ CUTE_UNROLL
+ for (int i = 0; i < size(tCrC); ++i) {
+ auto coord = tCgC(i);
+ int col = int(get<1>(coord));
+ float value = static_cast<float>(tCrC(i));
+ if (bias_ptr != nullptr && col < int(shape<1>(C))) {
+ value += static_cast<float>(bias_ptr[col]);
+ }
+ tCrD(i) = static_cast<ElementOutput>(value);
+ }
+
+ copy(copy_c, tCrD, tCgC);
+ }
+}
+
+template <typename TA, typename TB, typename TC>
+auto choose_mma_op() {
+ if constexpr (is_complete_v<XE_DPAS_TT<8, TC, TA, TB>>) {
+ return XE_DPAS_TT<8, TC, TA, TB>{};
+ } else if constexpr (is_same_v<TA, cute::bfloat16_t>) {
+ return XE_DPAS_TT<8, float, cute::bfloat16_t>{};
+ } else {
+ return XE_DPAS_TT<8, float, cute::half_t>{};
+ }
+}
+
+template <int TileM, int TileN, class SGLayout, class ATensor, class BTensor, class CTensor>
+auto choose_tiled_mma_tile(ATensor const& A, BTensor const& B, CTensor const&) {
+ using TA = typename ATensor::element_type;
+ using TB = typename BTensor::element_type;
+ using TC = typename CTensor::element_type;
+
+ auto op = choose_mma_op<TA, TB, TC>();
+
+ constexpr bool byte = (cute::max(sizeof_bits_v<TA>, sizeof_bits_v<TB>) <= 8);
+ constexpr bool a_t = is_constant_v<1, decltype(stride<0>(A))>;
+ constexpr bool b_n = is_constant_v<1, decltype(stride<0>(B))>;
+ constexpr bool use_1x_dpas_per_k = a_t || (byte && b_n);
+
+ using _K = conditional_t<use_1x_dpas_per_k, C<op.K>, C<op.K * 2>>;
+ using WGTile = Shape<Int<TileM>, Int<TileN>, _K>;
+ using MMA = typename TiledMMAHelper<MMA_Atom<decltype(op)>, Layout<WGTile>, SGLayout>::TiledMMA;
+
+ return MMA{};
+}
+
+template <class, class, class, int, int, class, char, char>
+class DenseGemmStoreKernelName;
+
+template <int TileM, int TileN, class SGLayout, class ATensor, class BTensor, class CTensor, typename TA,
+ typename TB, typename BiasElement, char layoutA, char layoutB>
+void gemm_cute_store_tile(sycl::queue* q, ATensor const& A, BTensor const& B, CTensor& C,
+ const BiasElement* bias_ptr) {
+ compat::set_default_queue(*q);
+
+ auto C_accum = make_tensor(make_gmem_ptr(static_cast<float*>(nullptr)), C.shape(), C.stride());
+ auto mma = choose_tiled_mma_tile<TileM, TileN, SGLayout>(A, B, C_accum);
+
+ sycl::range<2> local = {size(mma), 1};
+ sycl::range<2> global = {local[0] * ceil_div(shape<0>(B), get<1>(mma.tile_mnk())),
+ local[1] * ceil_div(shape<0>(A), get<0>(mma.tile_mnk()))};
+
+ namespace syclex = sycl::ext::oneapi::experimental;
+ namespace intelex = sycl::ext::intel::experimental;
+
+ syclex::properties kernel_props{syclex::sub_group_size<16>, intelex::grf_size<256>};
+
+ auto event = q->parallel_for<
+ DenseGemmStoreKernelName<TA, TB, BiasElement, TileM, TileN, SGLayout, layoutA, layoutB>>(
+ sycl::nd_range<2>(global, local), kernel_props,
+ [=](auto) { gemm_device_impl(A, B, C, mma, bias_ptr); });
+
+}
+
+
+template <typename Element>
+void run_dense_gemm_tuned_tile(sycl::queue* q, int m, int n, int k, const Element* a_ptr, const Element* b_ptr,
+ Element* c_ptr, const Element* bias_ptr) {
+ auto A = make_tensor(make_gmem_ptr(const_cast<Element*>(a_ptr)), make_shape(m, k), make_stride(k, _1{}));
+ auto B = make_tensor(make_gmem_ptr(const_cast<Element*>(b_ptr)), make_shape(n, k), make_stride(k, _1{}));
+ auto C = make_tensor(make_gmem_ptr(c_ptr), make_shape(m, n), make_stride(n, _1{}));
+
+ using SmallTileSG = Layout<Shape<_1, _4, _1>, Stride<_0, _1, _0>>;
+ using SmallMidTileSG = Layout<Shape<_2, _4, _1>, Stride<_4, _1, _0>>;
+ using MediumTileSG = Layout<Shape<_4, _4, _1>, Stride<_4, _1, _0>>;
+ using LargeTileSG = Layout<Shape<_8, _4, _1>, Stride<_4, _1, _0>>;
+
+ if (m < 32) {
+ gemm_cute_store_tile<8, 128, SmallTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
+ 'R', 'R'>(q, A, B, C, bias_ptr);
+ } else if (m < 128) {
+ gemm_cute_store_tile<64, 128, SmallMidTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
+ 'R', 'R'>(q, A, B, C, bias_ptr);
+ } else if (m <= 1024) {
+ gemm_cute_store_tile<128, 128, MediumTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
+ 'R', 'R'>(q, A, B, C, bias_ptr);
+ } else {
+ gemm_cute_store_tile<256, 256, LargeTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,
+ 'R', 'R'>(q, A, B, C, bias_ptr);
+ }
+}
+
+template <typename Element>
+void run_dense_gemm(sycl::queue* q, int m, int n, int k, const Element* a_ptr, const Element* b_ptr,
+ Element* c_ptr, const Element* bias_ptr) {
+ run_dense_gemm_tuned_tile(q, m, n, k, a_ptr, b_ptr, c_ptr, bias_ptr);
+}
+
+} // namespace dense_gemm_detail
+
+inline void sycl_tla_dense_gemm(sycl::queue* q, int m, int n, int k, const void* a, BTLA_DTYPE at,
+ const void* b, BTLA_DTYPE bt, void* c, BTLA_DTYPE ct, const void* bias,
+ bool BT) {
+ if (!q) {
+ throw std::invalid_argument("sycl_tla_dense_gemm: stream must be a valid SYCL queue");
+ }
+ if (!BT) {
+ throw std::invalid_argument("sycl_tla_dense_gemm: only the A @ B.T contract is supported");
+ }
+ if (!a || !b || !c) {
+ throw std::invalid_argument("sycl_tla_dense_gemm: input and output pointers must not be null");
+ }
+ if (m <= 0 || n <= 0 || k <= 0) {
+ return;
+ }
+ if (at != bt) {
+ throw std::invalid_argument("sycl_tla_dense_gemm: A and B must use the same dtype");
+ }
+ if (ct != at) {
+ throw std::invalid_argument("sycl_tla_dense_gemm: output dtype must match input dtype");
+ }
+
+ switch (at) {
+ case BTLA_DTYPE::F16:
+ dense_gemm_detail::run_dense_gemm(
+ q, m, n, k, static_cast<const cute::half_t*>(a), static_cast<const cute::half_t*>(b),
+ static_cast<cute::half_t*>(c), static_cast<const cute::half_t*>(bias));
+ return;
+
+ case BTLA_DTYPE::BF16:
+ dense_gemm_detail::run_dense_gemm(
+ q, m, n, k, static_cast<const cute::bfloat16_t*>(a), static_cast<const cute::bfloat16_t*>(b),
+ static_cast<cute::bfloat16_t*>(c), static_cast<const cute::bfloat16_t*>(bias));
+ return;
+
+ default:
+ throw std::invalid_argument("sycl_tla_dense_gemm: only FP16 and BF16 are supported");
+ }
+}
+
+#endif // ARK_XPU && ARK_SYCL_TLA
+
+} // namespace ark
\ No newline at end of file
diff --git a/auto_round_extension/ark/test/test_matmul.py b/auto_round_extension/ark/test/test_matmul.py
index 6b26aa337..4d4aee98c 100644
--- a/auto_round_extension/ark/test/test_matmul.py
+++ b/auto_round_extension/ark/test/test_matmul.py
@@ -25,8 +25,9 @@ import torch
ark = auto_round_kernel
-def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):
- print(f"\n m={m}, k={k}, n={n}, dt={dt}, batch={batch_size}")
+def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device, op=None, op_name="matmul"):
+ op = op or ark.matmul
+ print(f"\n op={op_name}, m={m}, k={k}, n={n}, dt={dt}, batch={batch_size}")
torch.manual_seed(0)
cdt = dt
if device == "cpu":
@@ -48,13 +49,12 @@ def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):
print(f"OOM during basic init: {e}")
return
- tar_dst = ark.matmul(activation, wei, bias)
+ tar_dst = op(activation, wei, bias)
ref_dst = torch.matmul(activation, wei.T).to(cdt)
if has_bias:
ref_dst = ref_dst + bias
diff = abs(tar_dst - ref_dst)
- # print(f" abs(ref_dst).max(): {abs(ref_dst).max()}, abs(ref_dst).mean(): {abs(ref_dst).mean()}, abs(tar_dst).max(): {abs(tar_dst).max()}, abs(tar_dst).mean(): {abs(tar_dst).mean()}")
- print(f" Max Diff: {diff.max().item():.6f}, Mean Diff: {diff.mean().item():.6f}")
+ print(f" Max Diff: {diff.max().item():.5f}, Mean Diff: {diff.mean().item():.5f}", end="", flush=True)
if dt == torch.float32:
atol = 0.001
rtol = 0.03
@@ -80,14 +80,14 @@ def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):
for i in range(runs):
idx = i % batch_size
- tar_dst = ark.matmul(act_set[idx], wei_set[idx], bias)
+ tar_dst = op(act_set[idx], wei_set[idx], bias)
if device == "xpu":
torch.xpu.synchronize()
st = time.perf_counter()
for i in range(runs):
idx = i % batch_size
- tar_dst = ark.matmul(act_set[idx], wei_set[idx], bias)
+ tar_dst = op(act_set[idx], wei_set[idx], bias)
if device == "xpu":
torch.xpu.synchronize()
dur = (time.perf_counter() - st) / runs
@@ -106,9 +106,7 @@ def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):
record_property("GFLOPS", round(gflops, 2))
record_property("Bandwidth_GBs", round(bandwidth, 2))
- print(f"[Performance] Time: {dur*1000:.4f} ms")
- print(f" GFLOPS: {gflops:.2f}")
- print(f" Bandwidth: {bandwidth:.2f} GB/s")
+ print(f"[Performance] Time: {dur*1000:.4f} ms, GFLOPS: {gflops:.2f}, Bandwidth: {bandwidth:.2f} GB/s")
def woqgemm(m, k, n, dt, batch_size, runs, record_property, device):
@@ -129,8 +127,8 @@ def woqgemm(m, k, n, dt, batch_size, runs, record_property, device):
DB = B[0].to(dt) * scaleB
ref = torch.matmul(DA, DB.T)
ref = ref + bias
- dff = abs(C - ref)
- print(f" Max Diff: {dff.max().item():.6f}, Mean Diff: {dff.mean().item():.6f}")
+ diff = abs(C - ref)
+ print(f" Max Diff: {diff.max().item():.5f}, Mean Diff: {diff.mean().item():.5f}", end="", flush=True)
# QDQ tolerance
assert torch.allclose(C, ref, atol=1, rtol=0.1), "Verification Failed!"
print(" -> Verification PASS")
@@ -163,9 +161,7 @@ def woqgemm(m, k, n, dt, batch_size, runs, record_property, device):
record_property("GFLOPS", round(gflops, 2))
record_property("Bandwidth_GBs", round(bandwidth, 2))
- print(f"[Performance] Time: {dur*1000:.4f} ms")
- print(f" GFLOPS: {gflops:.2f}")
- print(f" Bandwidth: {bandwidth:.2f} GB/s")
+ print(f"[Performance] Time: {dur*1000:.4f} ms, GFLOPS: {gflops:.2f}, Bandwidth: {bandwidth:.2f} GB/s")
@pytest.mark.parametrize("m", [4096])
@@ -203,6 +199,140 @@ def test_cpu(m, k, n, dt, batch_size, runs, record_property):
main_op(m, k, n, dt, batch_size, runs, True, record_property, "cpu")
[email protected]("m", [1, 8, 16, 32, 64, 128, 256, 1024, 2048, 4096])
[email protected]("k, n", [(4096, 4096)])
[email protected]("dt", [torch.float16, torch.bfloat16])
[email protected]("batch_size", [1])
[email protected]("runs", [1])
+def test_xpu_sycl_tla(m, k, n, dt, batch_size, runs, record_property):
+ if not torch.xpu.is_available():
+ pytest.skip("No XPU Device")
+ if not hasattr(ark, "matmul_sycl_tla"):
+ pytest.skip("Python wrapper for matmul_sycl_tla is not available")
+ if getattr(ark, "xpu_lib", None) is None or not hasattr(ark.xpu_lib, "matmul_sycl_tla"):
+ pytest.skip("SYCL-TLA matmul is not available in this build")
+
+ main_op(
+ m,
+ k,
+ n,
+ dt,
+ batch_size,
+ runs,
+ True,
+ record_property,
+ "xpu",
+ op=ark.matmul_sycl_tla,
+ op_name="matmul_sycl_tla",
+ )
+
+ torch.xpu.empty_cache()
+
+
[email protected]("m", [1, 8, 16, 32, 64, 128, 256, 1024, 2048, 4096])
[email protected]("k, n", [(4096, 4096)])
[email protected]("dt", [torch.float16, torch.bfloat16])
[email protected]("batch_size", [1])
[email protected]("runs", [1])
+def test_xpu_sycl_tla_no_bias(m, k, n, dt, batch_size, runs, record_property):
+ if not torch.xpu.is_available():
+ pytest.skip("No XPU Device")
+ if not hasattr(ark, "matmul_sycl_tla"):
+ pytest.skip("Python wrapper for matmul_sycl_tla is not available")
+ if getattr(ark, "xpu_lib", None) is None or not hasattr(ark.xpu_lib, "matmul_sycl_tla"):
+ pytest.skip("SYCL-TLA matmul is not available in this build")
+
+ main_op(
+ m,
+ k,
+ n,
+ dt,
+ batch_size,
+ runs,
+ False,
+ record_property,
+ "xpu",
+ op=ark.matmul_sycl_tla,
+ op_name="matmul_sycl_tla_no_bias",
+ )
+
+ torch.xpu.empty_cache()
+
+
+# pytest -vs auto_round_extension/ark/test/test_matmul.py -k compare_dnnl_vs_sycl_tla
[email protected]("m", [1, 8, 16, 32, 128, 1024, 2048, 4096])
[email protected]("k, n", [(4096, 4096)])
[email protected]("dt", [torch.float16, torch.bfloat16])
+def test_xpu_compare_dnnl_vs_sycl_tla(m, k, n, dt):
+ warmup = 100
+ runs = 1000
+
+ compare_matmul_backends(m, k, n, dt, warmup, runs, "xpu")
+ torch.xpu.empty_cache()
+
+
[email protected]("m", [1, 8, 16, 32, 128, 1024, 2048, 4096])
[email protected]("k, n", [(4096, 4096)])
[email protected]("dt", [torch.float16, torch.bfloat16])
+def test_xpu_compare_dnnl_vs_sycl_tla_no_bias(m, k, n, dt):
+ warmup = 100
+ runs = 1000
+
+ compare_matmul_backends(m, k, n, dt, warmup, runs, "xpu", has_bias=False)
+ torch.xpu.empty_cache()
+
+
+def compare_matmul_backends(m, k, n, dt, warmup, runs, device="xpu", has_bias=True):
+ if device != "xpu":
+ raise ValueError("compare_matmul_backends only supports XPU")
+
+ def _matmul_tolerance(dt):
+ if dt == torch.float32:
+ return 0.001, 0.03
+ if dt == torch.float16:
+ return 0.1, 0.06
+ if dt == torch.bfloat16:
+ return 0.6, 0.1
+ raise ValueError(f"Unsupported dtype: {dt}")
+
+ def _benchmark_op(op, A, B, bias, runs, warmup):
+ for _ in range(warmup):
+ _ = op(A, B, bias)
+ if A.device.type == "xpu":
+ torch.xpu.synchronize()
+
+ st = time.perf_counter()
+ for _ in range(runs):
+ _ = op(A, B, bias)
+ if A.device.type == "xpu":
+ torch.xpu.synchronize()
+
+ return (time.perf_counter() - st) / runs
+
+ torch.manual_seed(0)
+
+ activation = torch.rand(m, k, dtype=dt, device=device) - 0.5
+ wei = torch.rand(n, k, dtype=dt, device=device) - 0.5
+ bias = torch.rand(1, n, dtype=dt, device=device) - 0.5 if has_bias else torch.Tensor()
+
+ dnnl_out = ark.matmul(activation, wei, bias)
+ tla_out = ark.matmul_sycl_tla(activation, wei, bias)
+
+ atol, rtol = _matmul_tolerance(dt)
+ ok = torch.allclose(dnnl_out, tla_out, atol=atol, rtol=rtol)
+
+ assert ok, "oneDNN vs sycl-tla mismatch"
+
+ dnnl_dur = _benchmark_op(ark.matmul, activation, wei, bias, runs=runs, warmup=warmup)
+ tla_dur = _benchmark_op(ark.matmul_sycl_tla, activation, wei, bias, runs=runs, warmup=warmup)
+
+ ops = m * n * k * 2
+ print(f"\n [oneDNN] : {dnnl_dur*1000:8.3f} ms {ops / dnnl_dur / 1e12:7.3f} TFLOPS")
+ print(
+ f" [matmul_sycl_tla] : {tla_dur*1000:8.3f} ms {ops / tla_dur / 1e12:7.3f} TFLOPS speedup={dnnl_dur / tla_dur:5.2f}x"
+ )
+
+
# LOCAL_TEST: python test_matmul.py
# CI pytest: pytest -v test_matmul.py -k 'test_xpu'
# pytest -s -v test_matmul.py -k 'test_xpu' --csv results_matmul_all.csv --csv-columns file,function,parameters_as_columns,properties_as_columns