Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Intel XPU: fix 4bit dequantize and gemv on plain (not double-quantized) absmax (#15322)

Daniel Hand6cb541b6d12
diff --git a/tests/test_xpu_nf4_dequant_gemv.py b/tests/test_xpu_nf4_dequant_gemv.py
new file mode 100644
index 00000000000..64573a301fb
--- /dev/null
+++ b/tests/test_xpu_nf4_dequant_gemv.py
@@ -0,0 +1,202 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""XPU fast_dequantize / fast_gemv on CPU: lifted from kernels/utils.py (defined only when DEVICE_TYPE == "xpu"),
+bitsandbytes ctypes kernels replaced by torch twins."""
+
+import ast
+import ctypes
+import inspect
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from unsloth.kernels import utils as ku
+
+NF4 = torch.tensor(
+ [
+ -1.0,
+ -0.6961928009986877,
+ -0.5250730514526367,
+ -0.39491748809814453,
+ -0.28444138169288635,
+ -0.18477343022823334,
+ -0.09105003625154495,
+ 0.0,
+ 0.07958029955625534,
+ 0.16093020141124725,
+ 0.24611230194568634,
+ 0.33791524171829224,
+ 0.44070982933044434,
+ 0.5626170039176941,
+ 0.7229568362236023,
+ 1.0,
+ ]
+)
+CODE2 = torch.linspace(-1, 1, 256)
+BLOCKSIZE, BLOCKSIZE2 = 64, 256
+
+
+def _dequant_4bit(packed, absmax, code, blocksize, n):
+ p = packed.reshape(-1)
+ idx = torch.stack([p >> 4, p & 15], 1).reshape(-1)[:n].long()
+ return code[idx] * absmax.float().repeat_interleave(blocksize)[:n]
+
+
+def _quant_state(shape, dtype, nested):
+ w = torch.randn(shape) * 0.02
+ flat = w.reshape(-1, BLOCKSIZE)
+ absmax = flat.abs().amax(1).clamp_min(1e-8)
+ idx = (
+ (flat / absmax[:, None]).unsqueeze(-1).sub(NF4).abs().argmin(-1).reshape(-1).to(torch.uint8)
+ )
+ packed = ((idx[0::2] << 4) | idx[1::2]).reshape(-1, 1)
+ qs = SimpleNamespace(
+ shape = torch.Size(shape),
+ dtype = dtype,
+ blocksize = BLOCKSIZE,
+ code = NF4,
+ quant_type = "nf4",
+ offset = None,
+ state2 = None,
+ absmax = absmax,
+ )
+ if nested:
+ offset = absmax.mean()
+ n = absmax.numel()
+ centred = torch.nn.functional.pad(absmax - offset, (0, -n % BLOCKSIZE2)).reshape(
+ -1, BLOCKSIZE2
+ )
+ absmax2 = centred.abs().amax(1).clamp_min(1e-12)
+ q8 = (
+ (centred / absmax2[:, None])
+ .unsqueeze(-1)
+ .sub(CODE2)
+ .abs()
+ .argmin(-1)
+ .reshape(-1)[:n]
+ .to(torch.uint8)
+ )
+ qs.absmax, qs.offset = q8, offset
+ qs.state2 = SimpleNamespace(absmax = absmax2, code = CODE2, blocksize = BLOCKSIZE2)
+ absmax = CODE2[q8.long()] * absmax2.repeat_interleave(BLOCKSIZE2)[:n] + offset
+ ref = _dequant_4bit(packed, absmax, NF4, BLOCKSIZE, w.numel()).reshape(shape).to(dtype)
+ return packed, qs, ref
+
+
+def xpu():
+ calls = []
+ live = ["stream-0"]
+
+ def absmax_kernel(code2, absmax, absmax2, out, blocksize2, n, stream):
+ calls.append(("absmax", stream))
+ out.copy_(code2[absmax.long()] * absmax2.repeat_interleave(blocksize2.value)[: n.value])
+
+ def nf4_kernel(_, W, absmax, out, blocksize, n, stream):
+ calls.append(("nf4", stream))
+ out.view(-1).copy_(_dequant_4bit(W, absmax, NF4, blocksize.value, n.value))
+
+ def gemv_kernel(m, n, k, X, W, absmax, code, out, lda, ldb, ldc, blocksize, stream):
+ assert m.value == 1, "the XPU gemv takes m = 1, n = rows"
+ calls.append(("gemv", stream, absmax.dtype))
+ Wd = _dequant_4bit(W, absmax, code, blocksize.value, n.value * k.value).reshape(
+ n.value, k.value
+ )
+ out.view(-1).copy_((X.reshape(1, -1).float() @ Wd.t()).reshape(-1))
+
+ def bnb_dequantize_4bit(
+ W,
+ quant_state,
+ out = None,
+ ):
+ calls.append(("bitsandbytes", None))
+ return (
+ _dequant_4bit(
+ W,
+ quant_state.absmax,
+ quant_state.code,
+ quant_state.blocksize,
+ quant_state.shape.numel(),
+ )
+ .reshape(quant_state.shape)
+ .to(quant_state.dtype)
+ )
+
+ ns = dict(vars(ku))
+ ns.update(
+ get_ptr = lambda t: t,
+ torch_gpu_device = lambda device: __import__("contextlib").nullcontext(),
+ _get_tensor_stream = lambda t: live[0],
+ XPU_STREAMS = {None: "import-time"},
+ WEIGHT_BUFFERS = {None: None},
+ ABSMAX_BUFFERS = {None: None},
+ cdequantize_blockwise_fp32 = absmax_kernel,
+ cdequantize_blockwise_fp16_nf4 = nf4_kernel,
+ cdequantize_blockwise_bf16_nf4 = nf4_kernel,
+ cdequantize_blockwise_fp32_nf4 = nf4_kernel,
+ cgemm_4bit_inference_naive_fp16 = gemv_kernel,
+ cgemm_4bit_inference_naive_bf16 = gemv_kernel,
+ bnb_functional = SimpleNamespace(dequantize_4bit = bnb_dequantize_4bit),
+ DEVICE_TYPE = "xpu",
+ ctypes_c_int = ctypes.c_int,
+ ctypes_c_int32 = ctypes.c_int32,
+ )
+ tree = ast.parse(inspect.getsource(ku))
+ found = set()
+ for node in tree.body:
+ if (
+ isinstance(node, ast.If)
+ and ast.unparse(node.test) == "DEVICE_TYPE == 'xpu' and HAS_XPU_STREAM"
+ ):
+ defs = [n for n in node.body if isinstance(n, ast.FunctionDef)]
+ exec(compile(ast.Module(body = defs, type_ignores = []), ku.__file__, "exec"), ns)
+ found.update(n.name for n in defs)
+ assert {"fast_dequantize", "fast_gemv"} <= found
+ return SimpleNamespace(
+ dequantize = ns["fast_dequantize"], gemv = ns["fast_gemv"], calls = calls, live = live
+ )
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("nested", [True, False], ids = ["nested", "plain"])
[email protected]("global_buffer", [False, True])
+def test_dequantize_matches_reference(xpu, dtype, nested, global_buffer):
+ torch.manual_seed(0)
+ W, qs, ref = _quant_state((128, 256), dtype, nested)
+ out = xpu.dequantize(W, qs, use_global_buffer = global_buffer)
+ torch.testing.assert_close(out, ref, rtol = 0, atol = 0)
+ kinds = [c[0] for c in xpu.calls]
+ assert kinds == (["absmax", "nf4"] if nested else ["bitsandbytes"])
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("nested", [True, False], ids = ["nested", "plain"])
+def test_gemv_matches_reference(xpu, dtype, nested):
+ torch.manual_seed(0)
+ W, qs, ref = _quant_state((192, 256), dtype, nested)
+ X = torch.randn(1, 1, 256, dtype = dtype)
+ out = xpu.gemv(X, W, qs)
+ torch.testing.assert_close(out.float(), (X.float() @ ref.float().t()), rtol = 2e-2, atol = 2e-3)
+ assert [c[0] for c in xpu.calls] == (["absmax", "gemv"] if nested else ["gemv"])
+ assert xpu.calls[-1][2] == torch.float32
+
+
[email protected]("nested", [True, False], ids = ["nested", "plain"])
+def test_gemv_rows_not_starting_a_block_fall_back_to_dequant(xpu, nested):
+ torch.manual_seed(0)
+ W, qs, ref = _quant_state((8, 96), torch.bfloat16, nested) # 96 % 64 != 0: rows straddle blocks
+ X = torch.randn(1, 1, 96, dtype = torch.bfloat16)
+ out = xpu.gemv(X, W, qs)
+ torch.testing.assert_close(out.float(), (X @ ref.t()).float(), rtol = 2e-2, atol = 2e-3)
+ assert "gemv" not in [c[0] for c in xpu.calls]
+
+
+def test_kernels_read_the_live_stream(xpu):
+ torch.manual_seed(0)
+ W, qs, _ = _quant_state((128, 256), torch.bfloat16, True)
+ X = torch.randn(1, 1, 256, dtype = torch.bfloat16)
+ xpu.live[0] = "side-stream"
+ xpu.dequantize(W, qs)
+ xpu.gemv(X, W, qs)
+ assert {c[1] for c in xpu.calls} == {"side-stream"}
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 3d9640458c9..ad5e7bc3462 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -763,26 +763,15 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
return nvfp4_dequantize(
W, quant_state.scale, quant_state.global_scale, quant_state.dtype
)
- if type(quant_state) is not list:
- # New quant_state as a class, per TimDettmers/bitsandbytes#763.
- absmax = quant_state.absmax
- shape = quant_state.shape
- dtype = quant_state.dtype
- blocksize = quant_state.blocksize
- offset = quant_state.offset
- state2 = quant_state.state2
- absmax2 = state2.absmax
- code2 = state2.code
- blocksize2 = state2.blocksize
- else:
- # Old quant_state as a list of lists
- absmax, shape, dtype, blocksize, compressed_stats, _, _ = quant_state
- offset, state2 = compressed_stats
- absmax2, code2, blocksize2, _, _, _, _ = state2
- global XPU_STREAMS
+ absmax, shape, dtype, blocksize, _, code2, absmax2, offset, blocksize2 = (
+ _unpack_quant_state(quant_state)
+ )
+ if code2 is None:
+ return bnb_functional.dequantize_4bit(W, quant_state, out = out)
device = W.device
device_index = device.index
- XPU_STREAM = XPU_STREAMS[device_index]
+ # Live stream, as bitsandbytes' xpu ops read it; a snapshot ignores torch.xpu.stream().
+ XPU_STREAM = _get_tensor_stream(W)
n_elements_absmax = absmax.numel()
if use_global_buffer:
@@ -1124,26 +1113,14 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
return torch_matmul(X, W, out = out)
# Fast X @ W where seq_len == 1, from bitsandbytes functional.py#L1469 and TimDettmers/bitsandbytes#763.
_, q_len, hd = X.shape
-
- if type(quant_state) is not list:
- absmax = quant_state.absmax
- shape = quant_state.shape
- dtype = quant_state.dtype
- blocksize = quant_state.blocksize
- stats = quant_state.code
- offset = quant_state.offset
- state2 = quant_state.state2
- absmax2 = state2.absmax
- code2 = state2.code
- blocksize2 = state2.blocksize
- else:
- absmax, shape, dtype, blocksize, compressed_stats, quant_type, stats = quant_state
- offset, state2 = compressed_stats
- absmax2, code2, blocksize2, _, _, _, _ = state2
- global XPU_STREAMS
+ absmax, shape, dtype, blocksize, stats, code2, absmax2, offset, blocksize2 = (
+ _unpack_quant_state(quant_state)
+ )
+ if shape[1] % blocksize != 0:
+ # The gemv kernel needs each weight row to start a new block (bitsandbytes' xpu gemm_4bit checks too).
+ return torch_matmul(X, fast_dequantize(W, quant_state, dtype = X.dtype).t(), out = out)
device = W.device
- device_index = device.index
- XPU_STREAM = XPU_STREAMS[device_index]
+ XPU_STREAM = _get_tensor_stream(W)
bout = shape[0]
@@ -1158,12 +1135,9 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
device = device,
)
- if DEVICE_TYPE == "xpu":
- m = 1
- n = shape[0]
- else:
- n = 1
- m = shape[0]
+ # The xpu gemv takes m = 1, n = rows (the CUDA one is transposed).
+ m = 1
+ n = shape[0]
k = shape[1]
lda = shape[0]
ldc = shape[0]
@@ -1175,19 +1149,20 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
ldb = ctypes_c_int32(ldb)
ldc = ctypes_c_int32(ldc)
- df = torch_empty(absmax.shape, dtype = torch_float32, device = device)
with torch_gpu_device(device):
- cdequantize_blockwise_fp32(
- get_ptr(code2),
- get_ptr(absmax),
- get_ptr(absmax2),
- get_ptr(df),
- ctypes_c_int(blocksize2),
- ctypes_c_int(df.numel()),
- XPU_STREAM,
- )
- df += offset
- absmax = df
+ if code2 is not None:
+ df = torch_empty(absmax.shape, dtype = torch_float32, device = device)
+ cdequantize_blockwise_fp32(
+ get_ptr(code2),
+ get_ptr(absmax),
+ get_ptr(absmax2),
+ get_ptr(df),
+ ctypes_c_int(blocksize2),
+ ctypes_c_int(df.numel()),
+ XPU_STREAM,
+ )
+ df += offset
+ absmax = df
fx = (
cgemm_4bit_inference_naive_fp16