Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[ARK] Optimize SYCL-TLA woqgemm_s8 on B70 (#2133)

Zhenzhong Xu2ea60e555569
diff --git a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt
index 86d54ce2c..0d915d7f3 100755
--- a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt
+++ b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt
@@ -118,9 +118,14 @@ if(ARK_XPU AND ARK_SYCL_TLA)
)
# Map target to device name for -Xs flag
- set(DPCPP_SYCL_TARGET "intel_gpu_bmg_g21" CACHE STRING "SYCL target (intel_gpu_pvc, intel_gpu_bmg_g21)")
- if(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g21" OR DPCPP_SYCL_TARGET STREQUAL "bmg")
- set(SYCL_DEVICE_NAME "bmg_g21")
+ set(DPCPP_SYCL_TARGET "intel_gpu_bmg_g21" CACHE STRING "SYCL target (intel_gpu_pvc, intel_gpu_bmg_g31, intel_gpu_bmg_g21)")
+
+ if(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g21")
+ set(SYCL_DEVICE_NAME "bmg-g21")
+ elseif(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g31")
+ set(SYCL_DEVICE_NAME "bmg-g31")
+ elseif(DPCPP_SYCL_TARGET STREQUAL "bmg")
+ set(SYCL_DEVICE_NAME "bmg-g21,bmg-g31")
elseif(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_pvc" OR DPCPP_SYCL_TARGET STREQUAL "pvc")
set(SYCL_DEVICE_NAME "pvc")
else()
diff --git a/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txt b/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txt
index dbf70c5e2..a22b83b14 100644
--- a/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txt
+++ b/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txt
@@ -208,12 +208,16 @@ if(BTLA_UT_BENCHMARK)
endforeach()
# AOT compile target for Intel GPUs
- # Use intel_gpu_pvc for Data Center GPU Max series, or intel_gpu_bmg_g21 for Battlemage
- set(DPCPP_SYCL_TARGET "intel_gpu_bmg_g21" CACHE STRING "SYCL target (intel_gpu_pvc, intel_gpu_bmg_g21)")
-
+ # Use intel_gpu_pvc for Data Center GPU Max series, or intel_gpu_bmg_g31 for Battlemage
+ set(DPCPP_SYCL_TARGET "intel_gpu_bmg_g21" CACHE STRING "SYCL target (intel_gpu_pvc, intel_gpu_bmg_g31, intel_gpu_bmg_g21)")
+
# Map target to device name for -Xs flag
- if(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g21" OR DPCPP_SYCL_TARGET STREQUAL "bmg")
- set(SYCL_DEVICE_NAME "bmg_g21")
+ if(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g21")
+ set(SYCL_DEVICE_NAME "bmg-g21")
+ elseif(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g31")
+ set(SYCL_DEVICE_NAME "bmg-g31")
+ elseif(DPCPP_SYCL_TARGET STREQUAL "bmg")
+ set(SYCL_DEVICE_NAME "bmg-g21,bmg-g31")
elseif(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_pvc" OR DPCPP_SYCL_TARGET STREQUAL "pvc")
set(SYCL_DEVICE_NAME "pvc")
else()
diff --git a/auto_round_extension/ark/auto_round_kernel/qlinear.py b/auto_round_extension/ark/auto_round_kernel/qlinear.py
index 8384f29ef..749d2db1c 100644
--- a/auto_round_extension/ark/auto_round_kernel/qlinear.py
+++ b/auto_round_extension/ark/auto_round_kernel/qlinear.py
@@ -17,8 +17,6 @@ import math
import torch
import torch.nn as nn
-from .utils import fallback_compute_type_if_needed
-
try:
import auto_round_kernel
@@ -209,7 +207,6 @@ class QuantLinear(nn.Module):
self.sdt = "fp16"
self.cdt = "int8"
self.torch_dt = torch.float16
- self.cdt = fallback_compute_type_if_needed(self.cdt, self.qweight.device.index)
else:
self.sdt = "fp32"
self.cdt = "auto"
diff --git a/auto_round_extension/ark/auto_round_kernel/utils.py b/auto_round_extension/ark/auto_round_kernel/utils.py
index 35b8791a1..17c464260 100644
--- a/auto_round_extension/ark/auto_round_kernel/utils.py
+++ b/auto_round_extension/ark/auto_round_kernel/utils.py
@@ -49,12 +49,3 @@ def is_b70(device: int = 0) -> bool:
return False
return any(identifier in pro.name.lower() for identifier in B70_IDENTIFIERS) or hex(pro.device_id) == B70_DEVICE_ID
-
-
-@lru_cache(maxsize=None)
-def fallback_compute_type_if_needed(compute_dtype: str, device: int = 0) -> str:
- if compute_dtype.lower() == "int8" and is_b70(device) and not is_oneapi_ge_2026():
- logger.warning("XMX int8 is not supported on B70 with oneAPI < 2026. Falling back to fp16.")
- return "fp16"
-
- return compute_dtype
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp
index 51fe8fac5..5a6dec850 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_s8_gemm.hpp
@@ -28,17 +28,150 @@ namespace sycl_tla_s8_detail {
using namespace cute;
-// template <bool AccumBlock, class ElementOut, int TileM, int TileN, class SGLayout>
-// class S8DequantKernelName;
-
template <bool AccumBlock, bool HasBias, bool FullTile, class ElementOut, int TileM, int TileN, class SGLayout>
class S8DequantKernelName;
-template <class ElementOut>
-class S8FinalizeKernelName;
+template <bool HasBias, bool FullTile, class ElementOut, int TileM, int TileN, class SGLayout>
+class S8KBlockDequantKernelName;
+
+static constexpr int S8_GEMM_TILE_K = 64;
+
+using SmallTileSG = Layout<Shape<_1, _4, _1>, Stride<_0, _1, _0>>;
+using SmallMidTileSG = Layout<Shape<_2, _4, _1>, Stride<_4, _1, _0>>;
+using MediumTileSG = Layout<Shape<_4, _4, _1>, Stride<_4, _1, _0>>;
+using LargeTileSG = Layout<Shape<_8, _4, _1>, Stride<_4, _1, _0>>;
+
+template <bool HasBias, bool FullTile, class TiledMMA, class ElementOut>
+void igemm_kblock_device_impl(TiledMMA const& mma, const int8_t* a, const int8_t* b, ElementOut* c,
+ const ElementOut* scale_a, const ElementOut* scale_b, const ElementOut* bias, int m,
+ int n, int k, int blocksize, int blks) {
+ auto item = sycl::ext::oneapi::this_work_item::get_nd_item<2>();
+ int wg_m = int(item.get_group(1));
+ int wg_n = int(item.get_group(0));
+ int local_id = int(item.get_local_id(0));
+
+ auto wg_tile = mma.tile_mnk();
+ auto wg_coord = make_coord(wg_m, wg_n, 0);
+
+ Tensor cC = make_identity_tensor(make_shape(m, n));
+ Tensor gC = local_tile(cC, wg_tile, wg_coord, Step<_1, _1, X>{});
+
+ auto thr_mma = mma.get_slice(local_id);
+
+ Tensor tCrC = partition_fragment_C(mma, select<0, 1>(wg_tile));
+ Tensor tFrC = make_tensor_like<float>(tCrC);
+ Tensor tCgC = thr_mma.partition_C(gC);
+
+ CUTE_UNROLL
+ for (int i = 0; i < size(tFrC); ++i) {
+ tFrC(i) = 0.0f;
+ }
+
+ constexpr SPIRVScope barrier_scope = ScopeWorkgroup;
+ constexpr int prefetch_dist = 3;
+
+ constexpr int k_tile_size = S8_GEMM_TILE_K;
+
+ auto A = make_tensor(make_gmem_ptr(const_cast<int8_t*>(a)), make_shape(m, k), make_stride(k, _1{}));
+ auto B = make_tensor(make_gmem_ptr(const_cast<int8_t*>(b)), make_shape(n, k), make_stride(k, _1{}));
+
+ Tensor cA = make_identity_tensor(A.shape());
+ Tensor cB = make_identity_tensor(B.shape());
+
+ Tensor gA = local_tile(cA, select<0, 2>(wg_tile), make_coord(wg_m, _));
+ Tensor gB = local_tile(cB, select<1, 2>(wg_tile), make_coord(wg_n, _));
+
+ auto copy_a = make_block_2d_copy_A(mma, A);
+ auto copy_b = make_block_2d_copy_B(mma, B);
+
+ auto thr_copy_a = copy_a.get_slice(local_id);
+ auto thr_copy_b = copy_b.get_slice(local_id);
+
+ auto tCrA = thr_mma.partition_sg_fragment_A(gA(_, _, 0));
+ auto tCrB = thr_mma.partition_sg_fragment_B(gB(_, _, 0));
+
+ auto tArA = thr_copy_a.partition_sg_fragment_D(gA(_, _, 0));
+ auto tBrB = thr_copy_b.partition_sg_fragment_D(gB(_, _, 0));
+
+ Tensor tAgA = thr_copy_a.partition_S(gA);
+ Tensor tBgB = thr_copy_b.partition_S(gB);
+
+ auto prefetch_a = make_block_2d_prefetch(copy_a);
+ auto prefetch_b = make_block_2d_prefetch(copy_b);
+
+ auto thr_prefetch_A = prefetch_a.get_slice(local_id);
+ auto thr_prefetch_B = prefetch_b.get_slice(local_id);
+
+ auto pAgA = thr_prefetch_A.partition_S(gA);
+ auto pBgB = thr_prefetch_B.partition_S(gB);
+
+ int k_tiles_per_block = blocksize / k_tile_size;
+ int k_tile_count = blks * k_tiles_per_block;
+ int k_tile_prefetch = 0;
+
+ CUTE_UNROLL
+ for (; k_tile_prefetch < prefetch_dist && k_tile_prefetch < k_tile_count; ++k_tile_prefetch) {
+ prefetch(prefetch_a, pAgA(_, _, _, k_tile_prefetch));
+ prefetch(prefetch_b, pBgB(_, _, _, k_tile_prefetch));
+ }
+
+ for (int ib = 0; ib < blks; ++ib) {
+ clear(tCrC);
+
+ for (int bk = 0; bk < k_tiles_per_block; ++bk) {
+ int k_tile = ib * k_tiles_per_block + bk;
+
+ barrier_arrive(barrier_scope);
+
+ copy(copy_a, tAgA(_, _, _, k_tile), tArA);
+ copy(copy_b, tBgB(_, _, _, k_tile), tBrB);
+
+ if (k_tile_prefetch < k_tile_count) {
+ prefetch(prefetch_a, pAgA(_, _, _, k_tile_prefetch));
+ prefetch(prefetch_b, pBgB(_, _, _, k_tile_prefetch));
+ }
+ ++k_tile_prefetch;
+
+ reorder(tArA, tCrA);
+ reorder(tBrB, tCrB);
+ gemm(mma, tCrA, tCrB, tCrC);
+
+ barrier_wait(barrier_scope);
+ }
+
+ CUTE_UNROLL
+ for (int i = 0; i < size(tCrC); ++i) {
+ auto coord = tCgC(i);
+ int row = int(get<0>(coord));
+ int col = int(get<1>(coord));
-template <int TileM, int TileN, class SGLayout>
-class S8AccumKernelName;
+ if constexpr (!FullTile) {
+ if (row >= m || col >= n) continue;
+ }
+
+ float sb = static_cast<float>(scale_b[col * blks + ib]);
+ tFrC(i) += static_cast<float>(tCrC(i)) * sb;
+ }
+ }
+
+
+ CUTE_UNROLL
+ for (int i = 0; i < size(tFrC); ++i) {
+ auto coord = tCgC(i);
+ int row = int(get<0>(coord));
+ int col = int(get<1>(coord));
+
+ if constexpr (!FullTile) {
+ if (row >= m || col >= n) continue;
+ }
+
+ float value = tFrC(i) * static_cast<float>(scale_a[row]);
+ if constexpr (HasBias) {
+ value += static_cast<float>(bias[col]);
+ }
+ c[row * n + col] = static_cast<ElementOut>(value);
+ }
+}
template <bool AccumBlock, bool HasBias, bool FullTile, class ATensor, class BTensor, class TiledMMA, class ElementOut>
void igemm_device_impl(ATensor const& A, BTensor const& B, TiledMMA const& mma, ElementOut* c, float* accum,
@@ -143,6 +276,39 @@ void igemm_device_impl(ATensor const& A, BTensor const& B, TiledMMA const& mma,
}
}
+template <bool HasBias, class ElementOut, int TileM, int TileN, class SGLayout>
+void launch_igemm_kblock_tile(sycl::queue* q, int m, int n, int k, const int8_t* a, const int8_t* b, ElementOut* c,
+ const ElementOut* scale_a, const ElementOut* scale_b, const ElementOut* bias,
+ int blocksize, int blks) {
+ compat::set_default_queue(*q);
+
+ using Op = XE_DPAS_TT<8, int32_t, int8_t, int8_t>;
+ using WGTile = Shape<Int<TileM>, Int<TileN>, _64>;
+ using MMA = typename TiledMMAHelper<MMA_Atom<Op>, Layout<WGTile>, SGLayout>::TiledMMA;
+ MMA mma{};
+
+ sycl::range<2> local = {size(mma), 1};
+ sycl::range<2> global = {local[0] * ceil_div(n, get<1>(mma.tile_mnk())),
+ local[1] * ceil_div(m, get<0>(mma.tile_mnk()))};
+
+ namespace syclex = sycl::ext::oneapi::experimental;
+ namespace intelex = sycl::ext::intel::experimental;
+ syclex::properties props{syclex::sub_group_size<16>, intelex::grf_size<256>};
+
+ bool full_tile = (m % TileM == 0) && (n % TileN == 0);
+
+ if (full_tile) {
+ q->parallel_for<S8KBlockDequantKernelName<HasBias, true, ElementOut, TileM, TileN, SGLayout>>(
+ sycl::nd_range<2>(global, local), props, [=](auto) {
+ igemm_kblock_device_impl<HasBias, true>(mma, a, b, c, scale_a, scale_b, bias, m, n, k, blocksize, blks);
+ });
+ } else {
+ q->parallel_for<S8KBlockDequantKernelName<HasBias, false, ElementOut, TileM, TileN, SGLayout>>(
+ sycl::nd_range<2>(global, local), props, [=](auto) {
+ igemm_kblock_device_impl<HasBias, false>(mma, a, b, c, scale_a, scale_b, bias, m, n, k, blocksize, blks);
+ });
+ }
+}
template <bool AccumBlock, class ElementOut, int TileM, int TileN, class SGLayout>
void launch_igemm_tile(sycl::queue* q, int m, int n, int gemm_k, int lda, int ldb, const int8_t* a, const int8_t* b,
@@ -200,17 +366,44 @@ void launch_igemm_tile(sycl::queue* q, int m, int n, int gemm_k, int lda, int ld
}
}
+template <class ElementOut, int TileM, int TileN, class SGLayout>
+void launch_igemm_kblock_tile_dispatch(sycl::queue* q, int m, int n, int k, const int8_t* a, const int8_t* b,
+ ElementOut* c, const ElementOut* scale_a, const ElementOut* scale_b,
+ const ElementOut* bias, int blocksize, int blks) {
+ if (bias) {
+ launch_igemm_kblock_tile<true, ElementOut, TileM, TileN, SGLayout>(
+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);
+ } else {
+ launch_igemm_kblock_tile<false, ElementOut, TileM, TileN, SGLayout>(
+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);
+ }
+}
+
+template <class ElementOut>
+void launch_igemm_kblock(sycl::queue* q, int m, int n, int k, const int8_t* a, const int8_t* b, ElementOut* c,
+ const ElementOut* scale_a, const ElementOut* scale_b, const ElementOut* bias, int blocksize,
+ int blks) {
+ if (m < 16) {
+ launch_igemm_kblock_tile_dispatch<ElementOut, 8, 128, SmallTileSG>(
+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);
+ } else if (m < 128) {
+ launch_igemm_kblock_tile_dispatch<ElementOut, 64, 128, SmallMidTileSG>(
+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);
+ } else if (m <= 1024) {
+ launch_igemm_kblock_tile_dispatch<ElementOut, 128, 128, MediumTileSG>(
+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);
+ } else {
+ launch_igemm_kblock_tile_dispatch<ElementOut, 256, 128, LargeTileSG>(
+ q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);
+ }
+}
+
template <bool AccumBlock, class ElementOut>
void launch_igemm(sycl::queue* q, int m, int n, int gemm_k, int lda, int ldb,
const int8_t* a, const int8_t* b, ElementOut* c, float* accum,
const ElementOut* scale_a, const ElementOut* scale_b,
const ElementOut* bias, int block_idx, int scale_b_stride) {
- using SmallTileSG = Layout<Shape<_1, _4, _1>, Stride<_0, _1, _0>>;
- using SmallMidTileSG = Layout<Shape<_2, _4, _1>, Stride<_4, _1, _0>>;
- using MediumTileSG = Layout<Shape<_4, _4, _1>, Stride<_4, _1, _0>>;
- using LargeTileSG = Layout<Shape<_8, _4, _1>, Stride<_4, _1, _0>>;
-
if (m < 16) {
launch_igemm_tile<AccumBlock, ElementOut, 8, 128, SmallTileSG>(
q, m, n, gemm_k, lda, ldb, a, b, c, accum, scale_a, scale_b, bias, block_idx, scale_b_stride);
@@ -226,17 +419,13 @@ void launch_igemm(sycl::queue* q, int m, int n, int gemm_k, int lda, int ldb,
}
}
-template <class ElementOut>
-void finalize_block_output(sycl::queue* q, int m, int n, const float* accum, ElementOut* c,
- const ElementOut* scale_a, const ElementOut* bias) {
- q->parallel_for<S8FinalizeKernelName<ElementOut>>(sycl::range<1>(size_t(m) * size_t(n)), [=](sycl::id<1> id) {
- size_t idx = id[0];
- int row = int(idx / n);
- int col = int(idx - size_t(row) * size_t(n));
- float value = accum[idx] * static_cast<float>(scale_a[row]);
- if (bias) value += static_cast<float>(bias[col]);
- c[idx] = static_cast<ElementOut>(value);
- });
+static inline void validate_s8_kblock_args(int k, int blocksize) {
+ if (blocksize <= 0 || k % blocksize != 0) {
+ throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: blocksize must divide k");
+ }
+ if (blocksize % S8_GEMM_TILE_K != 0) {
+ throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: k-block blocksize must be a multiple of 64");
+ }
}
template <class ElementOut>
@@ -249,23 +438,10 @@ void run_typed(sycl::queue* q, int m, int n, int k, const int8_t* a, const int8_
return;
}
- if (blocksize <= 0 || k % blocksize != 0) {
- throw std::invalid_argument("sycl_tla_igemm_s8s8_dequant: blocksize must divide k");
- }
+ validate_s8_kblock_args(k, blocksize);
int blks = k / blocksize;
- size_t bytes = size_t(m) * size_t(n) * sizeof(float);
- auto* accum = static_cast<float*>(DeviceMemoryPool::Instance()->get_scratch_mem(bytes, 3, q));
- q->memset(accum, 0, bytes);
-
- for (int ib = 0; ib < blks; ++ib) {
- const int8_t* a_blk = a + size_t(ib) * size_t(blocksize);
- const int8_t* b_blk = b + size_t(ib) * size_t(blocksize);
- launch_igemm<true, ElementOut>(q, m, n, blocksize, k, k, a_blk, b_blk, nullptr, accum, nullptr, scale_b, nullptr,
- ib, blks);
- }
-
- finalize_block_output(q, m, n, accum, c, scale_a, bias);
+ launch_igemm_kblock<ElementOut>(q, m, n, k, a, b, c, scale_a, scale_b, bias, blocksize, blks);
}
} // namespace sycl_tla_s8_detail
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
index e702fe5a2..863d52227 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
@@ -45,7 +45,7 @@ namespace ark {
struct env_params {
int verbose = 2; // trace 0/ debug 1/ info 2/ warn 3/ error 4/ critical 5/ off 6
- int auto_s8 = 0;
+ int auto_s8 = -1;
int sage_use_mean_bias = 1;
int sage_print_kbias = 0;
int sage_disable_packed_hnd_fast = 0;
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
index f67774ab5..804198ba3 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
@@ -70,7 +70,7 @@ class XpuWrapper {
static inline size_t get_scalext_size(QuantParam* p) {
using namespace bestla::utils;
- if (env_params::Instance()->auto_s8 == 0) return 0;
+ if (env_params::Instance()->auto_s8 == 0 or !rescale(p)) return 0;
size_t nblk = 1;
if (env_params::Instance()->auto_s8 != -1) {
nblk *= p->k / env_params::Instance()->auto_s8;
@@ -245,6 +245,7 @@ class XpuWrapper {
auto st = p->scale_type;
if (st == BTLA_DTYPE::F32) {
using T = float;
+ const float weight_fullrange = get_weight_fullrange(p);
auto ker2 = [&](sycl::handler& cgh) {
cgh.parallel_for(sycl::nd_range<2>({1, n}, {1, SG_SIZE}),
[=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]
@@ -260,7 +261,7 @@ class XpuWrapper {
for (int i = start_blk; i < end_blk; i += 1) {
maxv = std::max(abs(sptr[i * n + g_1]), maxv);
}
- sxtptr[g_1 * newblks + j] = maxv * get_weight_fullrange(p) / 127.f;
+ sxtptr[g_1 * newblks + j] = maxv * weight_fullrange / 127.f;
}
});
};
@@ -268,6 +269,7 @@ class XpuWrapper {
}
if (st == BTLA_DTYPE::F16) {
using T = sycl::half;
+ const float weight_fullrange = get_weight_fullrange(p);
auto ker2 = [&](sycl::handler& cgh) {
cgh.parallel_for(sycl::nd_range<2>({1, n}, {1, SG_SIZE}),
[=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]
@@ -283,7 +285,7 @@ class XpuWrapper {
for (int i = start_blk; i < end_blk; i += 1) {
maxv = std::max(abs((float)sptr[i * n + g_1]), maxv);
}
- sxtptr[g_1 * newblks + j] = maxv * get_weight_fullrange(p) / 127.f;
+ sxtptr[g_1 * newblks + j] = maxv * weight_fullrange / 127.f;
}
});
};
@@ -669,6 +671,7 @@ class XpuWrapper {
static inline bool rescale(QuantParam* p) {
#ifdef ARK_RESCALE
+ if (p->weight_type != BTLA_DTYPE::S2 && p->weight_type != BTLA_DTYPE::S4) return false;
if (env_params::Instance()->auto_s8 == -1) return true;
if (env_params::Instance()->auto_s8 > p->blocksize && p->k % env_params::Instance()->auto_s8 == 0) return true;
#endif
@@ -677,6 +680,7 @@ class XpuWrapper {
static inline int rescale_blocksize(QuantParam* p) {
#ifdef ARK_RESCALE
+ if (!rescale(p)) return p->blocksize;
if (env_params::Instance()->auto_s8 == 0) return p->blocksize;
return env_params::Instance()->auto_s8 == -1 ? p->k : env_params::Instance()->auto_s8;
#else
diff --git a/auto_round_extension/ark/setup.py b/auto_round_extension/ark/setup.py
index 8d7eba1d5..84d6e7b13 100644
--- a/auto_round_extension/ark/setup.py
+++ b/auto_round_extension/ark/setup.py
@@ -106,6 +106,73 @@ enable_dnnl = env_flag("ARK_DNNL", default=not enable_sycl_tla)
enable_joint_matrix = env_flag("ARK_JOINT_MATRIX", default=False)
+def detect_sycl_target():
+ def _read_command_output(cmd):
+ try:
+ result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, check=True)
+ return result.stdout or "", None
+ except (FileNotFoundError, subprocess.CalledProcessError) as error:
+ return "", str(error)
+
+ def _map_detected_device_to_sycl_target(device_text):
+ text = device_text.lower()
+
+ if "b70" in text or "0xe223" in text or "g31" in text:
+ return "intel_gpu_bmg_g31"
+
+ if "b60" in text or "b580" in text or "g21" in text:
+ return "intel_gpu_bmg_g21"
+
+ if "pvc" in text or "max 1550" in text or "max 1100" in text:
+ return "intel_gpu_pvc"
+
+ return None
+
+ override = os.environ.get("ARK_SYCL_TARGET")
+ if override:
+ print(f"Using ARK_SYCL_TARGET override: {override}")
+ return override
+
+ detection_failures = []
+
+ try:
+ import torch
+
+ if hasattr(torch, "xpu") and torch.xpu.is_available():
+ props = torch.xpu.get_device_properties(0)
+ detected = _map_detected_device_to_sycl_target(f"{props.name} {hex(props.device_id)}")
+ if detected:
+ print(f"Detected SYCL target from torch.xpu: {detected}")
+ return detected
+ detection_failures.append("torch.xpu returned an unrecognized XPU device")
+ else:
+ detection_failures.append("torch.xpu is unavailable")
+ except Exception as error:
+ detection_failures.append(f"torch.xpu detection failed: {error}")
+
+ for cmd in (["xpu-smi", "discovery", "-d", "0"], ["sycl-ls"]):
+ output, error = _read_command_output(cmd)
+ detected = _map_detected_device_to_sycl_target(output)
+ if detected:
+ print(f"Detected SYCL target from {' '.join(cmd)}: {detected}")
+ return detected
+
+ if error:
+ detection_failures.append(f"{' '.join(cmd)} failed: {error}")
+ elif output:
+ detection_failures.append(f"{' '.join(cmd)} returned an unrecognized device")
+
+ print("Warning: unable to determine exact SYCL target.")
+ for failure in detection_failures:
+ print(f"Warning: {failure}")
+ print("Warning: falling back to bmg")
+
+ return "bmg"
+
+
+sycl_target = detect_sycl_target() if enable_sycl_tla else None
+
+
def get_system_memory_gb():
if hasattr(os, "sysconf"):
page_size_names = ("SC_PAGE_SIZE", "SC_PAGESIZE")
@@ -194,10 +261,13 @@ class CMakeBuild(build_ext):
"-DCMAKE_BUILD_TYPE=Release",
"-DCMAKE_CXX_COMPILER=icx",
"-DARK_XPU=ON",
+ "-DARK_RESCALE=ON",
f"-DARK_DNNL={'ON' if enable_dnnl else 'OFF'}",
f"-DARK_JOINT_MATRIX={'ON' if enable_joint_matrix else 'OFF'}",
f"-DARK_SYCL_TLA={'ON' if enable_sycl_tla else 'OFF'}",
]
+ if sycl_target:
+ cmake_cmd.append(f"-DDPCPP_SYCL_TARGET={sycl_target}")
if sys.platform == "win32":
cmake_cmd.append("-GNinja")
xpu_n_job = get_sycl_tla_job_count(n_job) if enable_sycl_tla else n_job
diff --git a/auto_round_extension/ark/test/test_weightonly.py b/auto_round_extension/ark/test/test_weightonly.py
index 97363d7ef..d2fa042ac 100644
--- a/auto_round_extension/ark/test/test_weightonly.py
+++ b/auto_round_extension/ark/test/test_weightonly.py
@@ -220,6 +220,15 @@ def test_xpu(m, n, k, blocksize, compute_type, weight_type, scale_type, asym):
torch.xpu.empty_cache()
[email protected]("m, n, k", [(1280, 256, 256), (1281, 288, 256)])
+def test_xpu_s8_large_tile(m, n, k):
+ if not torch.xpu.is_available():
+ pytest.skip("No XPU Device")
+ with torch.no_grad():
+ main_op(m, n, k, 128, "int8", "int8", "fp16", False, "xpu", True)
+ torch.xpu.empty_cache()
+
+
@pytest.mark.parametrize("m", [1, 1024])
@pytest.mark.parametrize("n, k", [(1024, 768)])
@pytest.mark.parametrize("blocksize", [32, 128])