PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[ARK] Optimize SYCL-TLA woqgemm_s8 on B70 (#2133)
Zhenzhong Xu2ea60e555569
diff --git a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txtindex 86d54ce2c..0d915d7f3 100755--- a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt+++ b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt@@ -118,9 +118,14 @@ if(ARK_XPU AND ARK_SYCL_TLA))# Map target to device name for -Xs flag- set(DPCPP_SYCL_TARGET "intel_gpu_bmg_g21" CACHE STRING "SYCL target (intel_gpu_pvc, intel_gpu_bmg_g21)")- if(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g21" OR DPCPP_SYCL_TARGET STREQUAL "bmg")- set(SYCL_DEVICE_NAME "bmg_g21")+ set(DPCPP_SYCL_TARGET "intel_gpu_bmg_g21" CACHE STRING "SYCL target (intel_gpu_pvc, intel_gpu_bmg_g31, intel_gpu_bmg_g21)")++ if(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g21")+ set(SYCL_DEVICE_NAME "bmg-g21")+ elseif(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g31")+ set(SYCL_DEVICE_NAME "bmg-g31")+ elseif(DPCPP_SYCL_TARGET STREQUAL "bmg")+ set(SYCL_DEVICE_NAME "bmg-g21,bmg-g31")elseif(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_pvc" OR DPCPP_SYCL_TARGET STREQUAL "pvc")set(SYCL_DEVICE_NAME "pvc")else()diff --git a/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txt b/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txtindex dbf70c5e2..a22b83b14 100644--- a/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txt+++ b/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txt@@ -208,12 +208,16 @@ if(BTLA_UT_BENCHMARK)endforeach()# AOT compile target for Intel GPUs- # Use intel_gpu_pvc for Data Center GPU Max series, or intel_gpu_bmg_g21 for Battlemage- set(DPCPP_SYCL_TARGET "intel_gpu_bmg_g21" CACHE STRING "SYCL target (intel_gpu_pvc, intel_gpu_bmg_g21)")-+ # Use intel_gpu_pvc for Data Center GPU Max series, or intel_gpu_bmg_g31 for Battlemage+ set(DPCPP_SYCL_TARGET "intel_gpu_bmg_g21" CACHE STRING "SYCL target (intel_gpu_pvc, intel_gpu_bmg_g31, intel_gpu_bmg_g21)")+# Map target to device name for -Xs flag- if(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g21" OR DPCPP_SYCL_TARGET STREQUAL "bmg")- set(SYCL_DEVICE_NAME "bmg_g21")+ if(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g21")+ set(SYCL_DEVICE_NAME "bmg-g21")+ elseif(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g31")+ set(SYCL_DEVICE_NAME "bmg-g31")+ elseif(DPCPP_SYCL_TARGET STREQUAL "bmg")+ set(SYCL_DEVICE_NAME "bmg-g21,bmg-g31")elseif(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_pvc" OR DPCPP_SYCL_TARGET STREQUAL "pvc")set(SYCL_DEVICE_NAME "pvc")else()diff --git a/auto_round_extension/ark/auto_round_kernel/qlinear.py b/auto_round_extension/ark/auto_round_kernel/qlinear.pyindex 8384f29ef..749d2db1c 100644--- a/auto_round_extension/ark/auto_round_kernel/qlinear.py+++ b/auto_round_extension/ark/auto_round_kernel/qlinear.py@@ -17,8 +17,6 @@ import mathimport torchimport torch.nn as nn-from .utils import fallback_compute_type_if_needed-try:import auto_round_kernel@@ -209,7 +207,6 @@ class QuantLinear(nn.Module):self.sdt = "fp16"self.cdt = "int8"self.torch_dt = torch.float16- self.cdt = fallback_compute_type_if_needed(self.cdt, self.qweight.device.index)else:self.sdt = "fp32"self.cdt = "auto"diff --git a/auto_round_extension/ark/auto_round_kernel/utils.py b/auto_round_extension/ark/auto_round_kernel/utils.pyindex 35b8791a1..17c464260 100644--- a/auto_round_extension/ark/auto_round_kernel/utils.py+++ b/auto_round_extension/ark/auto_round_kernel/utils.py@@ -49,12 +49,3 @@ def is_b70(device: int = 0) -> bool:return Falsereturn any(identifier in pro.name.lower() for identifier in B70_IDENTIFIERS) or hex(pro.device_id) == B70_DEVICE_ID---@lru_cache(maxsize=None)-def fallback_compute_type_if_needed(compute_dtype: str, device: int = 0) -> str:- if compute_dtype.lower() == "int8" and is_b70(device) and not is_oneapi_ge_2026():- logger.warning("XMX int8 is not supported on B70 with oneAPI < 2026. Falling back to fp16.")- return "fp16"-- return compute_dtypediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hppindex 51fe8fac5..5a6dec850 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp@@ -28,17 +28,150 @@ namespace sycl_tla_s8_detail {using namespace cute;-// template <bool AccumBlock, class ElementOut, int TileM, int TileN, class SGLayout>-// class S8DequantKernelName;-template <bool AccumBlock, bool HasBias, bool FullTile, class ElementOut, int TileM, int TileN, class SGLayout>class S8DequantKernelName;-template <class ElementOut>-class S8FinalizeKernelName;+template <bool HasBias, bool FullTile, class ElementOut, int TileM, int TileN, class SGLayout>+class S8KBlockDequantKernelName;++static constexpr int S8_GEMM_TILE_K = 64;++using SmallTileSG = Layout<Shape<_1, _4, _1>, Stride<_0, _1, _0>>;+using SmallMidTileSG = Layout<Shape<_2, _4, _1>, Stride<_4, _1, _0>>;+using MediumTileSG = Layout<Shape<_4, _4, _1>, Stride<_4, _1, _0>>;+using LargeTileSG = Layout<Shape<_8, _4, _1>, Stride<_4, _1, _0>>;++template <bool HasBias, bool FullTile, class TiledMMA, class ElementOut>+void igemm_kblock_device_impl(TiledMMA const& mma, const int8_t* a, const int8_t* b, ElementOut* c,+ const ElementOut* scale_a, const ElementOut* scale_b, const ElementOut* bias, int m,+ int n, int k, int blocksize, int blks) {+ auto item = sycl::ext::oneapi::this_work_item::get_nd_item<2>();+ int wg_m = int(item.get_group(1));+ int wg_n = int(item.get_group(0));+ int local_id = int(item.get_local_id(0));++ auto wg_tile = mma.tile_mnk();+ auto wg_coord = make_coord(wg_m, wg_n, 0);++ Tensor cC = make_identity_tensor(make_shape(m, n));+ Tensor gC = local_tile(cC, wg_tile, wg_coord, Step<_1, _1, X>{});++ auto thr_mma = mma.get_slice(local_id);++ Tensor tCrC = partition_fragment_C(mma, select<0, 1>(wg_tile));+ Tensor tFrC = make_tensor_like<float>(tCrC);+ Tensor tCgC = thr_mma.partition_C(gC);++ CUTE_UNROLL+ for (int i = 0; i < size(tFrC); ++i) {+ tFrC(i) = 0.0f;+ }++ constexpr SPIRVScope barrier_scope = ScopeWorkgroup;+ constexpr int prefetch_dist = 3;++ constexpr int k_tile_size = S8_GEMM_TILE_K;++ auto A = make_tensor(make_gmem_ptr(const_cast<int8_t*>(a)), make_shape(m, k), make_stride(k, _1{}));+ auto B = make_tensor(make_gmem_ptr(const_cast<int8_t*>(b)), make_shape(n, k), make_stride(k, _1{}));++ Tensor cA = make_identity_tensor(A.shape());+ Tensor cB = make_identity_tensor(B.shape());++ Tensor gA = local_tile(cA, select<0, 2>(wg_tile), make_coord(wg_m, _));+ Tensor gB = local_tile(cB, select<1, 2>(wg_tile), make_coord(wg_n, _));++ auto copy_a = make_block_2d_copy_A(mma, A);+ auto copy_b = make_block_2d_copy_B(mma, B);++ auto thr_copy_a = copy_a.get_slice(local_id);+ auto thr_copy_b = copy_b.get_slice(local_id);++ auto tCrA = thr_mma.partition_sg_fragment_A(gA(_, _, 0));+ auto tCrB = thr_mma.partition_sg_fragment_B(gB(_, _, 0));++ auto tArA = thr_copy_a.partition_sg_fragment_D(gA(_, _, 0));+ auto tBrB = thr_copy_b.partition_sg_fragment_D(gB(_, _, 0));++ Tensor tAgA = thr_copy_a.partition_S(gA);+ Tensor tBgB = thr_copy_b.partition_S(gB);++ auto prefetch_a = make_block_2d_prefetch(copy_a);+ auto prefetch_b = make_block_2d_prefetch(copy_b);++ auto thr_prefetch_A = prefetch_a.get_slice(local_id);+ auto thr_prefetch_B = prefetch_b.get_slice(local_id);++ auto pAgA = thr_prefetch_A.partition_S(gA);+ auto pBgB = thr_prefetch_B.partition_S(gB);++ int k_tiles_per_block = blocksize / k_tile_size;+ int k_tile_count = blks * k_tiles_per_block;+ int k_tile_prefetch = 0;++ CUTE_UNROLL+ for (; k_tile_prefetch < prefetch_dist && k_tile_prefetch < k_tile_count; ++k_tile_prefetch) {+ prefetch(prefetch_a, pAgA(_, _, _, k_tile_prefetch));+ prefetch(prefetch_b, pBgB(_, _, _, k_tile_prefetch));+ }++ for (int ib = 0; ib < blks; ++ib) {+ clear(tCrC);++ for (int bk = 0; bk < k_tiles_per_block; ++bk) {+ int k_tile = ib * k_tiles_per_block + bk;++ barrier_arrive(barrier_scope);++ copy(copy_a, tAgA(_, _, _, k_tile), tArA);+ copy(copy_b, tBgB(_, _, _, k_tile), tBrB);++ if (k_tile_prefetch < k_tile_count) {+ prefetch(prefetch_a, pAgA(_, _, _, k_tile_prefetch));+ prefetch(prefetch_b, pBgB(_, _, _, k_tile_prefetch));+ }+ ++k_tile_prefetch;++ reorder(tArA, tCrA);+ reorder(tBrB, tCrB);+ gemm(mma, tCrA, tCrB, tCrC);++ barrier_wait(barrier_scope);+ }++ CUTE_UNROLL+ for (int i = 0; i < size(tCrC); ++i) {+ auto coord = tCgC(i);+ int row = int(get<0>(coord));+ int col = int(get<1>(coord));-template <int TileM, int TileN, class SGLayout>-class S8AccumKernelName;+ if constexpr (!FullTile) {+ if (row >= m || col >= n) continue;+ }++ float sb = static_cast<float>(scale_b[col * blks + ib]);+ tFrC(i) += static_cast<float>(tCrC(i)) * sb;+ }+ }+++ CUTE_UNROLL+ for (int i = 0; i < size(tFrC); ++i) {+ auto coord = tCgC(i);+ int row = int(get<0>(coord));+ int col = int(get<1>(coord));++ if constexpr (!FullTile) {+ if (row >= m || col >= n) continue;+ }++ float value = tFrC(i) * static_cast<float>(scale_a[row]);+ if constexpr (HasBias) {+ value += static_cast<float>(bias[col]);+ }+ c[row * n + col] = static_cast<ElementOut>(value);+ }+}template <bool AccumBlock, bool HasBias, bool FullTile, class ATensor, class BTensor, class TiledMMA, class ElementOut>void igemm_device_impl(ATensor const& A, BTensor const& B, TiledMMA const& mma, ElementOut* c, float* accum,@@ -143,6 +276,39 @@ void igemm_device_impl(ATensor const& A, BTensor const& B, TiledMMA const& mma,}}+template <bool HasBias, class ElementOut, int TileM, int TileN, class SGLayout>+void launch_igemm_kblock_tile(sycl::queue* q, int m, int n, int k, const int8_t* a, const int8_t* b, ElementOut* c,+ const ElementOut* scale_a, const ElementOut* scale_b, const ElementOut* bias,+ int blocksize, int blks) {+ compat::set_default_queue(*q);++ using Op = XE_DPAS_TT<8, int32_t, int8_t, int8_t>;+ using WGTile = Shape<Int<TileM>, Int<TileN>, _64>;+ using MMA = typename TiledMMAHelper<MMA_Atom<Op>, Layout<WGTile>, SGLayout>::TiledMMA;+ MMA mma{};++ sycl::range<2> local = {size(mma), 1};+ sycl::range<2> global = {local[0] * ceil_div(n, get<1>(mma.tile_mnk())),+ local[1] * ceil_div(m, get<0>(mma.tile_mnk()))};++ namespace syclex = sycl::ext::oneapi::experimental;+ namespace intelex = sycl::ext::intel::experimental;+ syclex::properties props{syclex::sub_group_size<16>, intelex::grf_size<256>};++ bool full_tile = (m % TileM == 0) && (n % TileN == 0);++ if (full_tile) {+ q->parallel_for<S8KBlockDequantKernelName<HasBias, true, ElementOut, TileM, TileN, SGLayout>>(+ sycl::nd_range<2>(global, local), props, [=](auto) {+ igemm_kblock_device_impl<HasBias, true>(mma, a, b, c, scale_a, scale_b, bias, m, n, k, blocksize, blks);+ });+ } else {+ q->parallel_for<S8KBlockDequantKernelName<HasBias, false, ElementOut, TileM, TileN, SGLayout>>(+ sycl::nd_range<2>(global, local), props, [=](auto) {+ igemm_kblock_device_impl<HasBias, false>(mma, a, b, c, scale_a, scale_b, bias, m, n, k, blocksize, blks);+ });+ }+}template <bool AccumBlock, class ElementOut, int TileM, int TileN, class SGLayout>void launch_igemm_tile(sycl::queue* q, int m, int n, int gemm_k, int lda, int ldb, const int8_t* a, const int8_t* b,@@ -200,17 +366,44 @@ void launch_igemm_tile(sycl::queue* q, int m, int n, int gemm_k, int lda, int ld}}+template <class ElementOut, int TileM, int TileN, class SGLayout>+void launch_igemm_kblock_tile_dispatch(sycl::queue* q, int m, int n, int k, const int8_t* a, const int8_t* b,+ ElementOut* c, const ElementOut* scale_a, const ElementOut* scale_b,+ const ElementOut* bias, int blocksize, int blks) {+ if (bias) {+ launch_igemm_kblock_tile<true, ElementOut, TileM, TileN, SGLayout>(+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);+ } else {+ launch_igemm_kblock_tile<false, ElementOut, TileM, TileN, SGLayout>(+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);+ }+}++template <class ElementOut>+void launch_igemm_kblock(sycl::queue* q, int m, int n, int k, const int8_t* a, const int8_t* b, ElementOut* c,+ const ElementOut* scale_a, const ElementOut* scale_b, const ElementOut* bias, int blocksize,+ int blks) {+ if (m < 16) {+ launch_igemm_kblock_tile_dispatch<ElementOut, 8, 128, SmallTileSG>(+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);+ } else if (m < 128) {+ launch_igemm_kblock_tile_dispatch<ElementOut, 64, 128, SmallMidTileSG>(+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);+ } else if (m <= 1024) {+ launch_igemm_kblock_tile_dispatch<ElementOut, 128, 128, MediumTileSG>(+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);+ } else {+ launch_igemm_kblock_tile_dispatch<ElementOut, 256, 128, LargeTileSG>(+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);+ }+}+template <bool AccumBlock, class ElementOut>void launch_igemm(sycl::queue* q, int m, int n, int gemm_k, int lda, int ldb,const int8_t* a, const int8_t* b, ElementOut* c, float* accum,const ElementOut* scale_a, const ElementOut* scale_b,const ElementOut* bias, int block_idx, int scale_b_stride) {- using SmallTileSG = Layout<Shape<_1, _4, _1>, Stride<_0, _1, _0>>;- using SmallMidTileSG = Layout<Shape<_2, _4, _1>, Stride<_4, _1, _0>>;- using MediumTileSG = Layout<Shape<_4, _4, _1>, Stride<_4, _1, _0>>;- using LargeTileSG = Layout<Shape<_8, _4, _1>, Stride<_4, _1, _0>>;-if (m < 16) {launch_igemm_tile<AccumBlock, ElementOut, 8, 128, SmallTileSG>(q, m, n, gemm_k, lda, ldb, a, b, c, accum, scale_a, scale_b, bias, block_idx, scale_b_stride);@@ -226,17 +419,13 @@ void launch_igemm(sycl::queue* q, int m, int n, int gemm_k, int lda, int ldb,}}-template <class ElementOut>-void finalize_block_output(sycl::queue* q, int m, int n, const float* accum, ElementOut* c,- const ElementOut* scale_a, const ElementOut* bias) {- q->parallel_for<S8FinalizeKernelName<ElementOut>>(sycl::range<1>(size_t(m) * size_t(n)), [=](sycl::id<1> id) {- size_t idx = id[0];- int row = int(idx / n);- int col = int(idx - size_t(row) * size_t(n));- float value = accum[idx] * static_cast<float>(scale_a[row]);- if (bias) value += static_cast<float>(bias[col]);- c[idx] = static_cast<ElementOut>(value);- });+static inline void validate_s8_kblock_args(int k, int blocksize) {+ if (blocksize <= 0 || k % blocksize != 0) {+ throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: blocksize must divide k");+ }+ if (blocksize % S8_GEMM_TILE_K != 0) {+ throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: k-block blocksize must be a multiple of 64");+ }}template <class ElementOut>@@ -249,23 +438,10 @@ void run_typed(sycl::queue* q, int m, int n, int k, const int8_t* a, const int8_return;}- if (blocksize <= 0 || k % blocksize != 0) {- throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: blocksize must divide k");- }+ validate_s8_kblock_args(k, blocksize);int blks = k / blocksize;- size_t bytes = size_t(m) * size_t(n) * sizeof(float);- auto* accum = static_cast<float*>(DeviceMemoryPool::Instance()->get_scratch_mem(bytes, 3, q));- q->memset(accum, 0, bytes);-- for (int ib = 0; ib < blks; ++ib) {- const int8_t* a_blk = a + size_t(ib) * size_t(blocksize);- const int8_t* b_blk = b + size_t(ib) * size_t(blocksize);- launch_igemm<true, ElementOut>(q, m, n, blocksize, k, k, a_blk, b_blk, nullptr, accum, nullptr, scale_b, nullptr,- ib, blks);- }-- finalize_block_output(q, m, n, accum, c, scale_a, bias);+ launch_igemm_kblock<ElementOut>(q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);}} // namespace sycl_tla_s8_detaildiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hppindex e702fe5a2..863d52227 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp@@ -45,7 +45,7 @@ namespace ark {struct env_params {int verbose = 2; // trace 0/ debug 1/ info 2/ warn 3/ error 4/ critical 5/ off 6- int auto_s8 = 0;+ int auto_s8 = -1;int sage_use_mean_bias = 1;int sage_print_kbias = 0;int sage_disable_packed_hnd_fast = 0;diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hppindex f67774ab5..804198ba3 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp@@ -70,7 +70,7 @@ class XpuWrapper {static inline size_t get_scalext_size(QuantParam* p) {using namespace bestla::utils;- if (env_params::Instance()->auto_s8 == 0) return 0;+ if (env_params::Instance()->auto_s8 == 0 or !rescale(p)) return 0;size_t nblk = 1;if (env_params::Instance()->auto_s8 != -1) {nblk *= p->k / env_params::Instance()->auto_s8;@@ -245,6 +245,7 @@ class XpuWrapper {auto st = p->scale_type;if (st == BTLA_DTYPE::F32) {using T = float;+ const float weight_fullrange = get_weight_fullrange(p);auto ker2 = [&](sycl::handler& cgh) {cgh.parallel_for(sycl::nd_range<2>({1, n}, {1, SG_SIZE}),[=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]@@ -260,7 +261,7 @@ class XpuWrapper {for (int i = start_blk; i < end_blk; i += 1) {maxv = std::max(abs(sptr[i * n + g_1]), maxv);}- sxtptr[g_1 * newblks + j] = maxv * get_weight_fullrange(p) / 127.f;+ sxtptr[g_1 * newblks + j] = maxv * weight_fullrange / 127.f;}});};@@ -268,6 +269,7 @@ class XpuWrapper {}if (st == BTLA_DTYPE::F16) {using T = sycl::half;+ const float weight_fullrange = get_weight_fullrange(p);auto ker2 = [&](sycl::handler& cgh) {cgh.parallel_for(sycl::nd_range<2>({1, n}, {1, SG_SIZE}),[=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]@@ -283,7 +285,7 @@ class XpuWrapper {for (int i = start_blk; i < end_blk; i += 1) {maxv = std::max(abs((float)sptr[i * n + g_1]), maxv);}- sxtptr[g_1 * newblks + j] = maxv * get_weight_fullrange(p) / 127.f;+ sxtptr[g_1 * newblks + j] = maxv * weight_fullrange / 127.f;}});};@@ -669,6 +671,7 @@ class XpuWrapper {static inline bool rescale(QuantParam* p) {#ifdef ARK_RESCALE+ if (p->weight_type != BTLA_DTYPE::S2 && p->weight_type != BTLA_DTYPE::S4) return false;if (env_params::Instance()->auto_s8 == -1) return true;if (env_params::Instance()->auto_s8 > p->blocksize && p->k % env_params::Instance()->auto_s8 == 0) return true;#endif@@ -677,6 +680,7 @@ class XpuWrapper {static inline int rescale_blocksize(QuantParam* p) {#ifdef ARK_RESCALE+ if (!rescale(p)) return p->blocksize;if (env_params::Instance()->auto_s8 == 0) return p->blocksize;return env_params::Instance()->auto_s8 == -1 ? p->k : env_params::Instance()->auto_s8;#elsediff --git a/auto_round_extension/ark/setup.py b/auto_round_extension/ark/setup.pyindex 8d7eba1d5..84d6e7b13 100644--- a/auto_round_extension/ark/setup.py+++ b/auto_round_extension/ark/setup.py@@ -106,6 +106,73 @@ enable_dnnl = env_flag("ARK_DNNL", default=not enable_sycl_tla)enable_joint_matrix = env_flag("ARK_JOINT_MATRIX", default=False)+def detect_sycl_target():+ def _read_command_output(cmd):+ try:+ result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, check=True)+ return result.stdout or "", None+ except (FileNotFoundError, subprocess.CalledProcessError) as error:+ return "", str(error)++ def _map_detected_device_to_sycl_target(device_text):+ text = device_text.lower()++ if "b70" in text or "0xe223" in text or "g31" in text:+ return "intel_gpu_bmg_g31"++ if "b60" in text or "b580" in text or "g21" in text:+ return "intel_gpu_bmg_g21"++ if "pvc" in text or "max 1550" in text or "max 1100" in text:+ return "intel_gpu_pvc"++ return None++ override = os.environ.get("ARK_SYCL_TARGET")+ if override:+ print(f"Using ARK_SYCL_TARGET override: {override}")+ return override++ detection_failures = []++ try:+ import torch++ if hasattr(torch, "xpu") and torch.xpu.is_available():+ props = torch.xpu.get_device_properties(0)+ detected = _map_detected_device_to_sycl_target(f"{props.name} {hex(props.device_id)}")+ if detected:+ print(f"Detected SYCL target from torch.xpu: {detected}")+ return detected+ detection_failures.append("torch.xpu returned an unrecognized XPU device")+ else:+ detection_failures.append("torch.xpu is unavailable")+ except Exception as error:+ detection_failures.append(f"torch.xpu detection failed: {error}")++ for cmd in (["xpu-smi", "discovery", "-d", "0"], ["sycl-ls"]):+ output, error = _read_command_output(cmd)+ detected = _map_detected_device_to_sycl_target(output)+ if detected:+ print(f"Detected SYCL target from {' '.join(cmd)}: {detected}")+ return detected++ if error:+ detection_failures.append(f"{' '.join(cmd)} failed: {error}")+ elif output:+ detection_failures.append(f"{' '.join(cmd)} returned an unrecognized device")++ print("Warning: unable to determine exact SYCL target.")+ for failure in detection_failures:+ print(f"Warning: {failure}")+ print("Warning: falling back to bmg")++ return "bmg"+++sycl_target = detect_sycl_target() if enable_sycl_tla else None++def get_system_memory_gb():if hasattr(os, "sysconf"):page_size_names = ("SC_PAGE_SIZE", "SC_PAGESIZE")@@ -194,10 +261,13 @@ class CMakeBuild(build_ext):"-DCMAKE_BUILD_TYPE=Release","-DCMAKE_CXX_COMPILER=icx","-DARK_XPU=ON",+ "-DARK_RESCALE=ON",f"-DARK_DNNL={'ON' if enable_dnnl else 'OFF'}",f"-DARK_JOINT_MATRIX={'ON' if enable_joint_matrix else 'OFF'}",f"-DARK_SYCL_TLA={'ON' if enable_sycl_tla else 'OFF'}",]+ if sycl_target:+ cmake_cmd.append(f"-DDPCPP_SYCL_TARGET={sycl_target}")if sys.platform == "win32":cmake_cmd.append("-GNinja")xpu_n_job = get_sycl_tla_job_count(n_job) if enable_sycl_tla else n_jobdiff --git a/auto_round_extension/ark/test/test_weightonly.py b/auto_round_extension/ark/test/test_weightonly.pyindex 97363d7ef..d2fa042ac 100644--- a/auto_round_extension/ark/test/test_weightonly.py+++ b/auto_round_extension/ark/test/test_weightonly.py@@ -220,6 +220,15 @@ def test_xpu(m, n, k, blocksize, compute_type, weight_type, scale_type, asym):torch.xpu.empty_cache()[email protected]("m, n, k", [(1280, 256, 256), (1281, 288, 256)])+def test_xpu_s8_large_tile(m, n, k):+ if not torch.xpu.is_available():+ pytest.skip("No XPU Device")+ with torch.no_grad():+ main_op(m, n, k, 128, "int8", "int8", "fp16", False, "xpu", True)+ torch.xpu.empty_cache()++@pytest.mark.parametrize("m", [1, 1024])@pytest.mark.parametrize("n, k", [(1024, 768)])@pytest.mark.parametrize("blocksize", [32, 128])