PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[ARK] Support Dense Gemm using sycl-tla (#1968)
Zhenzhong Xu478224299b57
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex 545dcb53f..c5af8ae74 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -15,7 +15,6 @@import osfrom typing import Optionalimport torch-import sys# Intel GPU compiler (IGC) environment variable: ensure implicit local IDs are# not removed by the compiler even when unused. This guarantees consistent@@ -237,6 +236,58 @@ def get_lib(A: torch.Tensor):return lib+# A: mxk, B: nxk, bias: n or [1, n]+def matmul_sycl_tla(A: torch.Tensor, B: torch.Tensor, bias: Optional[torch.Tensor] = None):+ if A.device.type != "xpu" or B.device.type != "xpu":+ raise NotImplementedError("matmul_sycl_tla is only supported on XPU")+ if A.ndim != 2 or B.ndim != 2:+ raise ValueError("A and B must be 2D tensors")+ if A.device != B.device:+ raise ValueError("A and B must be on the same device")+ if A.dtype not in (torch.float16, torch.bfloat16):+ raise ValueError("matmul_sycl_tla only supports torch.float16 and torch.bfloat16")+ if B.dtype != A.dtype:+ raise ValueError("A and B must have the same dtype")++ m, k = A.shape+ n, kb = B.shape+ if k != kb:+ raise ValueError(f"Shape mismatch: A.shape={tuple(A.shape)}, B.shape={tuple(B.shape)}")++ lib = get_lib(A)+ if lib is None or not hasattr(lib, "matmul_sycl_tla"):+ raise NotImplementedError("Current XPU build does not expose matmul_sycl_tla")++ A_arg = A.contiguous()+ B_arg = B.contiguous()+ C = torch.empty(m, n, dtype=A.dtype, device=A.device)++ bias_ptr = 0+ bias_arg = None+ if bias is not None and bias.numel() > 0:+ bias_arg = bias.to(dtype=C.dtype, device=A.device).contiguous().view(-1)+ if bias_arg.numel() != n:+ raise ValueError(f"bias must have {n} elements, got {bias_arg.numel()}")+ bias_ptr = bias_arg.data_ptr()++ stream = get_stream(A_arg)+ lib.matmul_sycl_tla(+ stream,+ m,+ n,+ k,+ A_arg.data_ptr(),+ cvt_dtype(A_arg.dtype),+ B_arg.data_ptr(),+ cvt_dtype(B_arg.dtype),+ C.data_ptr(),+ cvt_dtype(C.dtype),+ bias_ptr,+ True,+ )+ return C++# A: mxk, B: nxk, bias: ndef matmul(A: torch.Tensor, B: torch.Tensor, bias: torch.Tensor):m = A.shape[0]diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppindex 3716d79ef..df06dc2c0 100755--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -26,6 +26,7 @@ typedef uintptr_t torch_ptr;#include "sycl_tla_common.hpp"#include "sycl_tla_moe.hpp"#include "sycl_tla_sdpa.hpp"+#include "sycl_tla_dense_gemm.hpp"#endif#else#include "cpu_wrapper.hpp"@@ -101,6 +102,12 @@ static size_t packed_weight_size(torch_ptr stream, int n, int k, int blocksize,#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)+static void matmul_sycl_tla(torch_ptr stream, int m, int n, int k, torch_ptr A, int Adt, torch_ptr B,+ int Bdt, torch_ptr C, int Cdt, torch_ptr bias, bool BT) {+ ark::sycl_tla_dense_gemm((sycl::queue*)stream, m, n, k, (void*)A, (BTLA_DTYPE)Adt, (void*)B,+ (BTLA_DTYPE)Bdt, (void*)C, (BTLA_DTYPE)Cdt, (void*)bias, BT);+}+// Tensor layout codes passed from Python (tensor_layout argument).constexpr int TENSOR_LAYOUT_HND = 0; // [B, H, S, D]constexpr int TENSOR_LAYOUT_NHD = 1; // [B, S, H, D]@@ -669,5 +676,6 @@ PYBIND11_MODULE(PY_NAME, m) {m.def("sage_dynamic_quant_layout", &ark::sage_dynamic_quant_layout);m.def("sage_dynamic_quant_v_layout", &ark::sage_dynamic_quant_v_layout);m.def("moe_gemm", &ark::moe_gemm_wrapper);+ m.def("matmul_sycl_tla", &ark::matmul_sycl_tla);#endif}\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_dense_gemm.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_dense_gemm.hppnew file mode 100644index 000000000..383bce4a4--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_dense_gemm.hpp@@ -0,0 +1,278 @@+// SYCL-TLA Dense GEMM Wrapper+// Based on sycl-tla/examples/cute/tutorial/xe_gemm.cpp+//+// Copyright (C) 2026 Intel Corporation+// SPDX-License-Identifier: MIT++#pragma once++#include <cstddef>+#include <cstdint>+#include <stdexcept>+#include <type_traits>++#ifdef ARK_XPU+#include <sycl/sycl.hpp>+#endif++#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)+#include <sycl/ext/intel/experimental/grf_size_properties.hpp>++#include "cute/tensor.hpp"+#include "cute/util/compat.hpp"++#if defined(__clang__)+#pragma clang diagnostic ignored "-Wpass-failed"+#pragma clang diagnostic ignored "-Wdeprecated-declarations"+#elif defined(__GNUC__)+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"+#endif+#endif++#include "utils.hpp"++namespace ark {++#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)++namespace dense_gemm_detail {++using namespace cute;++template <class ATensor, class BTensor, class CTensor, class TiledMMA, class BiasElement>+void gemm_device_impl(ATensor const& A, BTensor const& B, CTensor& C, TiledMMA const& mma,+ const BiasElement* bias_ptr) {+ auto item = sycl::ext::oneapi::this_work_item::get_nd_item<2>();+ auto wg_m = int(item.get_group(1));+ auto wg_n = int(item.get_group(0));+ auto local_id = int(item.get_local_id(0));++ Tensor cA = make_identity_tensor(A.shape());+ Tensor cB = make_identity_tensor(B.shape());+ Tensor cC = make_identity_tensor(C.shape());++ auto wg_tile = mma.tile_mnk();+ auto wg_coord = make_coord(wg_m, wg_n, 0);++ Tensor gA = local_tile(cA, select<0, 2>(wg_tile), make_coord(wg_m, _));+ Tensor gB = local_tile(cB, select<1, 2>(wg_tile), make_coord(wg_n, _));+ Tensor gC = local_tile(cC, wg_tile, wg_coord, Step<_1, _1, X>{});++ auto copy_a = make_block_2d_copy_A(XE_LOAD_2D<16, 8, 32, 32>{}, mma, A);+ auto copy_b = make_block_2d_copy_B(XE_LOAD_2D_TRANSPOSE<32, 16, 8>{}, mma, B);+ auto copy_c = make_block_2d_copy_D(mma, C);++ auto thr_mma = mma.get_slice(local_id);+ auto thr_copy_a = copy_a.get_slice(local_id);+ auto thr_copy_b = copy_b.get_slice(local_id);++ auto tCrA = thr_mma.partition_sg_fragment_A(gA(_, _, 0));+ auto tCrB = thr_mma.partition_sg_fragment_B(gB(_, _, 0));++ auto tArA = thr_copy_a.partition_sg_fragment_D(gA(_, _, 0));+ auto tBrB = thr_copy_b.partition_sg_fragment_D(gB(_, _, 0));++ Tensor tAgA = thr_copy_a.partition_S(gA);+ Tensor tBgB = thr_copy_b.partition_S(gB);++ Tensor tCrC = partition_fragment_C(mma, select<0, 1>(wg_tile));+ Tensor tCgC = thr_mma.partition_C(gC);++ auto prefetch_a = make_block_2d_prefetch(copy_a);+ auto prefetch_b = make_block_2d_prefetch(copy_b);++ auto thr_prefetch_A = prefetch_a.get_slice(local_id);+ auto thr_prefetch_B = prefetch_b.get_slice(local_id);++ auto pAgA = thr_prefetch_A.partition_S(gA);+ auto pBgB = thr_prefetch_B.partition_S(gB);++ constexpr int prefetch_dist = 3;+ constexpr SPIRVScope barrier_scope = ScopeWorkgroup;++ int k_tile_count = ceil_div(shape<1>(A), get<2>(wg_tile));+ int k_tile_prefetch = 0;++ clear(tCrC);++ CUTE_UNROLL+ for (; k_tile_prefetch < prefetch_dist; ++k_tile_prefetch) {+ prefetch(prefetch_a, pAgA(_, _, _, k_tile_prefetch));+ prefetch(prefetch_b, pBgB(_, _, _, k_tile_prefetch));+ }++ for (int k_tile = 0; k_tile < k_tile_count; ++k_tile, ++k_tile_prefetch) {+ barrier_arrive(barrier_scope);++ copy(copy_a, tAgA(_, _, _, k_tile), tArA);+ copy(copy_b, tBgB(_, _, _, k_tile), tBrB);++ prefetch(prefetch_a, pAgA(_, _, _, k_tile_prefetch));+ prefetch(prefetch_b, pBgB(_, _, _, k_tile_prefetch));++ reorder(tArA, tCrA);+ reorder(tBrB, tCrB);+ gemm(mma, tCrA, tCrB, tCrC);++ barrier_wait(barrier_scope);+ }++ using ElementOutput = typename CTensor::element_type;+ if constexpr (is_same_v<ElementOutput, float>) {+ copy(copy_c, tCrC, tCgC);+ } else {+ Tensor tCrD = make_tensor_like<ElementOutput>(tCrC);++ CUTE_UNROLL+ for (int i = 0; i < size(tCrC); ++i) {+ auto coord = tCgC(i);+ int col = int(get<1>(coord));+ float value = static_cast<float>(tCrC(i));+ if (bias_ptr != nullptr && col < int(shape<1>(C))) {+ value += static_cast<float>(bias_ptr[col]);+ }+ tCrD(i) = static_cast<ElementOutput>(value);+ }++ copy(copy_c, tCrD, tCgC);+ }+}++template <typename TA, typename TB, typename TC>+auto choose_mma_op() {+ if constexpr (is_complete_v<XE_DPAS_TT<8, TC, TA, TB>>) {+ return XE_DPAS_TT<8, TC, TA, TB>{};+ } else if constexpr (is_same_v<TA, cute::bfloat16_t>) {+ return XE_DPAS_TT<8, float, cute::bfloat16_t>{};+ } else {+ return XE_DPAS_TT<8, float, cute::half_t>{};+ }+}++template <int TileM, int TileN, class SGLayout, class ATensor, class BTensor, class CTensor>+auto choose_tiled_mma_tile(ATensor const& A, BTensor const& B, CTensor const&) {+ using TA = typename ATensor::element_type;+ using TB = typename BTensor::element_type;+ using TC = typename CTensor::element_type;++ auto op = choose_mma_op<TA, TB, TC>();++ constexpr bool byte = (cute::max(sizeof_bits_v<TA>, sizeof_bits_v<TB>) <= 8);+ constexpr bool a_t = is_constant_v<1, decltype(stride<0>(A))>;+ constexpr bool b_n = is_constant_v<1, decltype(stride<0>(B))>;+ constexpr bool use_1x_dpas_per_k = a_t || (byte && b_n);++ using _K = conditional_t<use_1x_dpas_per_k, C<op.K>, C<op.K * 2>>;+ using WGTile = Shape<Int<TileM>, Int<TileN>, _K>;+ using MMA = typename TiledMMAHelper<MMA_Atom<decltype(op)>, Layout<WGTile>, SGLayout>::TiledMMA;++ return MMA{};+}++template <class, class, class, int, int, class, char, char>+class DenseGemmStoreKernelName;++template <int TileM, int TileN, class SGLayout, class ATensor, class BTensor, class CTensor, typename TA,+ typename TB, typename BiasElement, char layoutA, char layoutB>+void gemm_cute_store_tile(sycl::queue* q, ATensor const& A, BTensor const& B, CTensor& C,+ const BiasElement* bias_ptr) {+ compat::set_default_queue(*q);++ auto C_accum = make_tensor(make_gmem_ptr(static_cast<float*>(nullptr)), C.shape(), C.stride());+ auto mma = choose_tiled_mma_tile<TileM, TileN, SGLayout>(A, B, C_accum);++ sycl::range<2> local = {size(mma), 1};+ sycl::range<2> global = {local[0] * ceil_div(shape<0>(B), get<1>(mma.tile_mnk())),+ local[1] * ceil_div(shape<0>(A), get<0>(mma.tile_mnk()))};++ namespace syclex = sycl::ext::oneapi::experimental;+ namespace intelex = sycl::ext::intel::experimental;++ syclex::properties kernel_props{syclex::sub_group_size<16>, intelex::grf_size<256>};++ auto event = q->parallel_for<+ DenseGemmStoreKernelName<TA, TB, BiasElement, TileM, TileN, SGLayout, layoutA, layoutB>>(+ sycl::nd_range<2>(global, local), kernel_props,+ [=](auto) { gemm_device_impl(A, B, C, mma, bias_ptr); });++}+++template <typename Element>+void run_dense_gemm_tuned_tile(sycl::queue* q, int m, int n, int k, const Element* a_ptr, const Element* b_ptr,+ Element* c_ptr, const Element* bias_ptr) {+ auto A = make_tensor(make_gmem_ptr(const_cast<Element*>(a_ptr)), make_shape(m, k), make_stride(k, _1{}));+ auto B = make_tensor(make_gmem_ptr(const_cast<Element*>(b_ptr)), make_shape(n, k), make_stride(k, _1{}));+ auto C = make_tensor(make_gmem_ptr(c_ptr), make_shape(m, n), make_stride(n, _1{}));++ using SmallTileSG = Layout<Shape<_1, _4, _1>, Stride<_0, _1, _0>>;+ using SmallMidTileSG = Layout<Shape<_2, _4, _1>, Stride<_4, _1, _0>>;+ using MediumTileSG = Layout<Shape<_4, _4, _1>, Stride<_4, _1, _0>>;+ using LargeTileSG = Layout<Shape<_8, _4, _1>, Stride<_4, _1, _0>>;++ if (m < 32) {+ gemm_cute_store_tile<8, 128, SmallTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,+ 'R', 'R'>(q, A, B, C, bias_ptr);+ } else if (m < 128) {+ gemm_cute_store_tile<64, 128, SmallMidTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,+ 'R', 'R'>(q, A, B, C, bias_ptr);+ } else if (m <= 1024) {+ gemm_cute_store_tile<128, 128, MediumTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,+ 'R', 'R'>(q, A, B, C, bias_ptr);+ } else {+ gemm_cute_store_tile<256, 256, LargeTileSG, decltype(A), decltype(B), decltype(C), Element, Element, Element,+ 'R', 'R'>(q, A, B, C, bias_ptr);+ }+}++template <typename Element>+void run_dense_gemm(sycl::queue* q, int m, int n, int k, const Element* a_ptr, const Element* b_ptr,+ Element* c_ptr, const Element* bias_ptr) {+ run_dense_gemm_tuned_tile(q, m, n, k, a_ptr, b_ptr, c_ptr, bias_ptr);+}++} // namespace dense_gemm_detail++inline void sycl_tla_dense_gemm(sycl::queue* q, int m, int n, int k, const void* a, BTLA_DTYPE at,+ const void* b, BTLA_DTYPE bt, void* c, BTLA_DTYPE ct, const void* bias,+ bool BT) {+ if (!q) {+ throw std::invalid_argument("sycl_tla_dense_gemm: stream must be a valid SYCL queue");+ }+ if (!BT) {+ throw std::invalid_argument("sycl_tla_dense_gemm: only the A @ B.T contract is supported");+ }+ if (!a || !b || !c) {+ throw std::invalid_argument("sycl_tla_dense_gemm: input and output pointers must not be null");+ }+ if (m <= 0 || n <= 0 || k <= 0) {+ return;+ }+ if (at != bt) {+ throw std::invalid_argument("sycl_tla_dense_gemm: A and B must use the same dtype");+ }+ if (ct != at) {+ throw std::invalid_argument("sycl_tla_dense_gemm: output dtype must match input dtype");+ }++ switch (at) {+ case BTLA_DTYPE::F16:+ dense_gemm_detail::run_dense_gemm(+ q, m, n, k, static_cast<const cute::half_t*>(a), static_cast<const cute::half_t*>(b),+ static_cast<cute::half_t*>(c), static_cast<const cute::half_t*>(bias));+ return;++ case BTLA_DTYPE::BF16:+ dense_gemm_detail::run_dense_gemm(+ q, m, n, k, static_cast<const cute::bfloat16_t*>(a), static_cast<const cute::bfloat16_t*>(b),+ static_cast<cute::bfloat16_t*>(c), static_cast<const cute::bfloat16_t*>(bias));+ return;++ default:+ throw std::invalid_argument("sycl_tla_dense_gemm: only FP16 and BF16 are supported");+ }+}++#endif // ARK_XPU && ARK_SYCL_TLA++} // namespace ark\ No newline at end of filediff --git a/auto_round_extension/ark/test/test_matmul.py b/auto_round_extension/ark/test/test_matmul.pyindex 6b26aa337..4d4aee98c 100644--- a/auto_round_extension/ark/test/test_matmul.py+++ b/auto_round_extension/ark/test/test_matmul.py@@ -25,8 +25,9 @@ import torchark = auto_round_kernel-def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):- print(f"\n m={m}, k={k}, n={n}, dt={dt}, batch={batch_size}")+def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device, op=None, op_name="matmul"):+ op = op or ark.matmul+ print(f"\n op={op_name}, m={m}, k={k}, n={n}, dt={dt}, batch={batch_size}")torch.manual_seed(0)cdt = dtif device == "cpu":@@ -48,13 +49,12 @@ def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):print(f"OOM during basic init: {e}")return- tar_dst = ark.matmul(activation, wei, bias)+ tar_dst = op(activation, wei, bias)ref_dst = torch.matmul(activation, wei.T).to(cdt)if has_bias:ref_dst = ref_dst + biasdiff = abs(tar_dst - ref_dst)- # print(f" abs(ref_dst).max(): {abs(ref_dst).max()}, abs(ref_dst).mean(): {abs(ref_dst).mean()}, abs(tar_dst).max(): {abs(tar_dst).max()}, abs(tar_dst).mean(): {abs(tar_dst).mean()}")- print(f" Max Diff: {diff.max().item():.6f}, Mean Diff: {diff.mean().item():.6f}")+ print(f" Max Diff: {diff.max().item():.5f}, Mean Diff: {diff.mean().item():.5f}", end="", flush=True)if dt == torch.float32:atol = 0.001rtol = 0.03@@ -80,14 +80,14 @@ def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):for i in range(runs):idx = i % batch_size- tar_dst = ark.matmul(act_set[idx], wei_set[idx], bias)+ tar_dst = op(act_set[idx], wei_set[idx], bias)if device == "xpu":torch.xpu.synchronize()st = time.perf_counter()for i in range(runs):idx = i % batch_size- tar_dst = ark.matmul(act_set[idx], wei_set[idx], bias)+ tar_dst = op(act_set[idx], wei_set[idx], bias)if device == "xpu":torch.xpu.synchronize()dur = (time.perf_counter() - st) / runs@@ -106,9 +106,7 @@ def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):record_property("GFLOPS", round(gflops, 2))record_property("Bandwidth_GBs", round(bandwidth, 2))- print(f"[Performance] Time: {dur*1000:.4f} ms")- print(f" GFLOPS: {gflops:.2f}")- print(f" Bandwidth: {bandwidth:.2f} GB/s")+ print(f"[Performance] Time: {dur*1000:.4f} ms, GFLOPS: {gflops:.2f}, Bandwidth: {bandwidth:.2f} GB/s")def woqgemm(m, k, n, dt, batch_size, runs, record_property, device):@@ -129,8 +127,8 @@ def woqgemm(m, k, n, dt, batch_size, runs, record_property, device):DB = B[0].to(dt) * scaleBref = torch.matmul(DA, DB.T)ref = ref + bias- dff = abs(C - ref)- print(f" Max Diff: {dff.max().item():.6f}, Mean Diff: {dff.mean().item():.6f}")+ diff = abs(C - ref)+ print(f" Max Diff: {diff.max().item():.5f}, Mean Diff: {diff.mean().item():.5f}", end="", flush=True)# QDQ toleranceassert torch.allclose(C, ref, atol=1, rtol=0.1), "Verification Failed!"print(" -> Verification PASS")@@ -163,9 +161,7 @@ def woqgemm(m, k, n, dt, batch_size, runs, record_property, device):record_property("GFLOPS", round(gflops, 2))record_property("Bandwidth_GBs", round(bandwidth, 2))- print(f"[Performance] Time: {dur*1000:.4f} ms")- print(f" GFLOPS: {gflops:.2f}")- print(f" Bandwidth: {bandwidth:.2f} GB/s")+ print(f"[Performance] Time: {dur*1000:.4f} ms, GFLOPS: {gflops:.2f}, Bandwidth: {bandwidth:.2f} GB/s")@pytest.mark.parametrize("m", [4096])@@ -203,6 +199,140 @@ def test_cpu(m, k, n, dt, batch_size, runs, record_property):main_op(m, k, n, dt, batch_size, runs, True, record_property, "cpu")[email protected]("m", [1, 8, 16, 32, 64, 128, 256, 1024, 2048, 4096])[email protected]("k, n", [(4096, 4096)])[email protected]("dt", [torch.float16, torch.bfloat16])[email protected]("batch_size", [1])[email protected]("runs", [1])+def test_xpu_sycl_tla(m, k, n, dt, batch_size, runs, record_property):+ if not torch.xpu.is_available():+ pytest.skip("No XPU Device")+ if not hasattr(ark, "matmul_sycl_tla"):+ pytest.skip("Python wrapper for matmul_sycl_tla is not available")+ if getattr(ark, "xpu_lib", None) is None or not hasattr(ark.xpu_lib, "matmul_sycl_tla"):+ pytest.skip("SYCL-TLA matmul is not available in this build")++ main_op(+ m,+ k,+ n,+ dt,+ batch_size,+ runs,+ True,+ record_property,+ "xpu",+ op=ark.matmul_sycl_tla,+ op_name="matmul_sycl_tla",+ )++ torch.xpu.empty_cache()++[email protected]("m", [1, 8, 16, 32, 64, 128, 256, 1024, 2048, 4096])[email protected]("k, n", [(4096, 4096)])[email protected]("dt", [torch.float16, torch.bfloat16])[email protected]("batch_size", [1])[email protected]("runs", [1])+def test_xpu_sycl_tla_no_bias(m, k, n, dt, batch_size, runs, record_property):+ if not torch.xpu.is_available():+ pytest.skip("No XPU Device")+ if not hasattr(ark, "matmul_sycl_tla"):+ pytest.skip("Python wrapper for matmul_sycl_tla is not available")+ if getattr(ark, "xpu_lib", None) is None or not hasattr(ark.xpu_lib, "matmul_sycl_tla"):+ pytest.skip("SYCL-TLA matmul is not available in this build")++ main_op(+ m,+ k,+ n,+ dt,+ batch_size,+ runs,+ False,+ record_property,+ "xpu",+ op=ark.matmul_sycl_tla,+ op_name="matmul_sycl_tla_no_bias",+ )++ torch.xpu.empty_cache()+++# pytest -vs auto_round_extension/ark/test/test_matmul.py -k compare_dnnl_vs_sycl_tla[email protected]("m", [1, 8, 16, 32, 128, 1024, 2048, 4096])[email protected]("k, n", [(4096, 4096)])[email protected]("dt", [torch.float16, torch.bfloat16])+def test_xpu_compare_dnnl_vs_sycl_tla(m, k, n, dt):+ warmup = 100+ runs = 1000++ compare_matmul_backends(m, k, n, dt, warmup, runs, "xpu")+ torch.xpu.empty_cache()++[email protected]("m", [1, 8, 16, 32, 128, 1024, 2048, 4096])[email protected]("k, n", [(4096, 4096)])[email protected]("dt", [torch.float16, torch.bfloat16])+def test_xpu_compare_dnnl_vs_sycl_tla_no_bias(m, k, n, dt):+ warmup = 100+ runs = 1000++ compare_matmul_backends(m, k, n, dt, warmup, runs, "xpu", has_bias=False)+ torch.xpu.empty_cache()+++def compare_matmul_backends(m, k, n, dt, warmup, runs, device="xpu", has_bias=True):+ if device != "xpu":+ raise ValueError("compare_matmul_backends only supports XPU")++ def _matmul_tolerance(dt):+ if dt == torch.float32:+ return 0.001, 0.03+ if dt == torch.float16:+ return 0.1, 0.06+ if dt == torch.bfloat16:+ return 0.6, 0.1+ raise ValueError(f"Unsupported dtype: {dt}")++ def _benchmark_op(op, A, B, bias, runs, warmup):+ for _ in range(warmup):+ _ = op(A, B, bias)+ if A.device.type == "xpu":+ torch.xpu.synchronize()++ st = time.perf_counter()+ for _ in range(runs):+ _ = op(A, B, bias)+ if A.device.type == "xpu":+ torch.xpu.synchronize()++ return (time.perf_counter() - st) / runs++ torch.manual_seed(0)++ activation = torch.rand(m, k, dtype=dt, device=device) - 0.5+ wei = torch.rand(n, k, dtype=dt, device=device) - 0.5+ bias = torch.rand(1, n, dtype=dt, device=device) - 0.5 if has_bias else torch.Tensor()++ dnnl_out = ark.matmul(activation, wei, bias)+ tla_out = ark.matmul_sycl_tla(activation, wei, bias)++ atol, rtol = _matmul_tolerance(dt)+ ok = torch.allclose(dnnl_out, tla_out, atol=atol, rtol=rtol)++ assert ok, "oneDNN vs sycl-tla mismatch"++ dnnl_dur = _benchmark_op(ark.matmul, activation, wei, bias, runs=runs, warmup=warmup)+ tla_dur = _benchmark_op(ark.matmul_sycl_tla, activation, wei, bias, runs=runs, warmup=warmup)++ ops = m * n * k * 2+ print(f"\n [oneDNN] : {dnnl_dur*1000:8.3f} ms {ops / dnnl_dur / 1e12:7.3f} TFLOPS")+ print(+ f" [matmul_sycl_tla] : {tla_dur*1000:8.3f} ms {ops / tla_dur / 1e12:7.3f} TFLOPS speedup={dnnl_dur / tla_dur:5.2f}x"+ )++# LOCAL_TEST: python test_matmul.py# CI pytest: pytest -v test_matmul.py -k 'test_xpu'# pytest -s -v test_matmul.py -k 'test_xpu' --csv results_matmul_all.csv --csv-columns file,function,parameters_as_columns,properties_as_columns