Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Intel XPU: fix 4bit dequantize and gemv on plain (not double-quantized) absmax (#15322)
Daniel Hand6cb541b6d12
diff --git a/tests/test_xpu_nf4_dequant_gemv.py b/tests/test_xpu_nf4_dequant_gemv.pynew file mode 100644index 00000000000..64573a301fb--- /dev/null+++ b/tests/test_xpu_nf4_dequant_gemv.py@@ -0,0 +1,202 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""XPU fast_dequantize / fast_gemv on CPU: lifted from kernels/utils.py (defined only when DEVICE_TYPE == "xpu"),+bitsandbytes ctypes kernels replaced by torch twins."""++import ast+import ctypes+import inspect+from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")++from unsloth.kernels import utils as ku++NF4 = torch.tensor(+ [+ -1.0,+ -0.6961928009986877,+ -0.5250730514526367,+ -0.39491748809814453,+ -0.28444138169288635,+ -0.18477343022823334,+ -0.09105003625154495,+ 0.0,+ 0.07958029955625534,+ 0.16093020141124725,+ 0.24611230194568634,+ 0.33791524171829224,+ 0.44070982933044434,+ 0.5626170039176941,+ 0.7229568362236023,+ 1.0,+ ]+)+CODE2 = torch.linspace(-1, 1, 256)+BLOCKSIZE, BLOCKSIZE2 = 64, 256+++def _dequant_4bit(packed, absmax, code, blocksize, n):+ p = packed.reshape(-1)+ idx = torch.stack([p >> 4, p & 15], 1).reshape(-1)[:n].long()+ return code[idx] * absmax.float().repeat_interleave(blocksize)[:n]+++def _quant_state(shape, dtype, nested):+ w = torch.randn(shape) * 0.02+ flat = w.reshape(-1, BLOCKSIZE)+ absmax = flat.abs().amax(1).clamp_min(1e-8)+ idx = (+ (flat / absmax[:, None]).unsqueeze(-1).sub(NF4).abs().argmin(-1).reshape(-1).to(torch.uint8)+ )+ packed = ((idx[0::2] << 4) | idx[1::2]).reshape(-1, 1)+ qs = SimpleNamespace(+ shape = torch.Size(shape),+ dtype = dtype,+ blocksize = BLOCKSIZE,+ code = NF4,+ quant_type = "nf4",+ offset = None,+ state2 = None,+ absmax = absmax,+ )+ if nested:+ offset = absmax.mean()+ n = absmax.numel()+ centred = torch.nn.functional.pad(absmax - offset, (0, -n % BLOCKSIZE2)).reshape(+ -1, BLOCKSIZE2+ )+ absmax2 = centred.abs().amax(1).clamp_min(1e-12)+ q8 = (+ (centred / absmax2[:, None])+ .unsqueeze(-1)+ .sub(CODE2)+ .abs()+ .argmin(-1)+ .reshape(-1)[:n]+ .to(torch.uint8)+ )+ qs.absmax, qs.offset = q8, offset+ qs.state2 = SimpleNamespace(absmax = absmax2, code = CODE2, blocksize = BLOCKSIZE2)+ absmax = CODE2[q8.long()] * absmax2.repeat_interleave(BLOCKSIZE2)[:n] + offset+ ref = _dequant_4bit(packed, absmax, NF4, BLOCKSIZE, w.numel()).reshape(shape).to(dtype)+ return packed, qs, ref+++def xpu():+ calls = []+ live = ["stream-0"]++ def absmax_kernel(code2, absmax, absmax2, out, blocksize2, n, stream):+ calls.append(("absmax", stream))+ out.copy_(code2[absmax.long()] * absmax2.repeat_interleave(blocksize2.value)[: n.value])++ def nf4_kernel(_, W, absmax, out, blocksize, n, stream):+ calls.append(("nf4", stream))+ out.view(-1).copy_(_dequant_4bit(W, absmax, NF4, blocksize.value, n.value))++ def gemv_kernel(m, n, k, X, W, absmax, code, out, lda, ldb, ldc, blocksize, stream):+ assert m.value == 1, "the XPU gemv takes m = 1, n = rows"+ calls.append(("gemv", stream, absmax.dtype))+ Wd = _dequant_4bit(W, absmax, code, blocksize.value, n.value * k.value).reshape(+ n.value, k.value+ )+ out.view(-1).copy_((X.reshape(1, -1).float() @ Wd.t()).reshape(-1))++ def bnb_dequantize_4bit(+ W,+ quant_state,+ out = None,+ ):+ calls.append(("bitsandbytes", None))+ return (+ _dequant_4bit(+ W,+ quant_state.absmax,+ quant_state.code,+ quant_state.blocksize,+ quant_state.shape.numel(),+ )+ .reshape(quant_state.shape)+ .to(quant_state.dtype)+ )++ ns = dict(vars(ku))+ ns.update(+ get_ptr = lambda t: t,+ torch_gpu_device = lambda device: __import__("contextlib").nullcontext(),+ _get_tensor_stream = lambda t: live[0],+ XPU_STREAMS = {None: "import-time"},+ WEIGHT_BUFFERS = {None: None},+ ABSMAX_BUFFERS = {None: None},+ cdequantize_blockwise_fp32 = absmax_kernel,+ cdequantize_blockwise_fp16_nf4 = nf4_kernel,+ cdequantize_blockwise_bf16_nf4 = nf4_kernel,+ cdequantize_blockwise_fp32_nf4 = nf4_kernel,+ cgemm_4bit_inference_naive_fp16 = gemv_kernel,+ cgemm_4bit_inference_naive_bf16 = gemv_kernel,+ bnb_functional = SimpleNamespace(dequantize_4bit = bnb_dequantize_4bit),+ DEVICE_TYPE = "xpu",+ ctypes_c_int = ctypes.c_int,+ ctypes_c_int32 = ctypes.c_int32,+ )+ tree = ast.parse(inspect.getsource(ku))+ found = set()+ for node in tree.body:+ if (+ isinstance(node, ast.If)+ and ast.unparse(node.test) == "DEVICE_TYPE == 'xpu' and HAS_XPU_STREAM"+ ):+ defs = [n for n in node.body if isinstance(n, ast.FunctionDef)]+ exec(compile(ast.Module(body = defs, type_ignores = []), ku.__file__, "exec"), ns)+ found.update(n.name for n in defs)+ assert {"fast_dequantize", "fast_gemv"} <= found+ return SimpleNamespace(+ dequantize = ns["fast_dequantize"], gemv = ns["fast_gemv"], calls = calls, live = live+ )++[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("nested", [True, False], ids = ["nested", "plain"])[email protected]("global_buffer", [False, True])+def test_dequantize_matches_reference(xpu, dtype, nested, global_buffer):+ torch.manual_seed(0)+ W, qs, ref = _quant_state((128, 256), dtype, nested)+ out = xpu.dequantize(W, qs, use_global_buffer = global_buffer)+ torch.testing.assert_close(out, ref, rtol = 0, atol = 0)+ kinds = [c[0] for c in xpu.calls]+ assert kinds == (["absmax", "nf4"] if nested else ["bitsandbytes"])++[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("nested", [True, False], ids = ["nested", "plain"])+def test_gemv_matches_reference(xpu, dtype, nested):+ torch.manual_seed(0)+ W, qs, ref = _quant_state((192, 256), dtype, nested)+ X = torch.randn(1, 1, 256, dtype = dtype)+ out = xpu.gemv(X, W, qs)+ torch.testing.assert_close(out.float(), (X.float() @ ref.float().t()), rtol = 2e-2, atol = 2e-3)+ assert [c[0] for c in xpu.calls] == (["absmax", "gemv"] if nested else ["gemv"])+ assert xpu.calls[-1][2] == torch.float32++[email protected]("nested", [True, False], ids = ["nested", "plain"])+def test_gemv_rows_not_starting_a_block_fall_back_to_dequant(xpu, nested):+ torch.manual_seed(0)+ W, qs, ref = _quant_state((8, 96), torch.bfloat16, nested) # 96 % 64 != 0: rows straddle blocks+ X = torch.randn(1, 1, 96, dtype = torch.bfloat16)+ out = xpu.gemv(X, W, qs)+ torch.testing.assert_close(out.float(), (X @ ref.t()).float(), rtol = 2e-2, atol = 2e-3)+ assert "gemv" not in [c[0] for c in xpu.calls]+++def test_kernels_read_the_live_stream(xpu):+ torch.manual_seed(0)+ W, qs, _ = _quant_state((128, 256), torch.bfloat16, True)+ X = torch.randn(1, 1, 256, dtype = torch.bfloat16)+ xpu.live[0] = "side-stream"+ xpu.dequantize(W, qs)+ xpu.gemv(X, W, qs)+ assert {c[1] for c in xpu.calls} == {"side-stream"}diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 3d9640458c9..ad5e7bc3462 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -763,26 +763,15 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:return nvfp4_dequantize(W, quant_state.scale, quant_state.global_scale, quant_state.dtype)- if type(quant_state) is not list:- # New quant_state as a class, per TimDettmers/bitsandbytes#763.- absmax = quant_state.absmax- shape = quant_state.shape- dtype = quant_state.dtype- blocksize = quant_state.blocksize- offset = quant_state.offset- state2 = quant_state.state2- absmax2 = state2.absmax- code2 = state2.code- blocksize2 = state2.blocksize- else:- # Old quant_state as a list of lists- absmax, shape, dtype, blocksize, compressed_stats, _, _ = quant_state- offset, state2 = compressed_stats- absmax2, code2, blocksize2, _, _, _, _ = state2- global XPU_STREAMS+ absmax, shape, dtype, blocksize, _, code2, absmax2, offset, blocksize2 = (+ _unpack_quant_state(quant_state)+ )+ if code2 is None:+ return bnb_functional.dequantize_4bit(W, quant_state, out = out)device = W.devicedevice_index = device.index- XPU_STREAM = XPU_STREAMS[device_index]+ # Live stream, as bitsandbytes' xpu ops read it; a snapshot ignores torch.xpu.stream().+ XPU_STREAM = _get_tensor_stream(W)n_elements_absmax = absmax.numel()if use_global_buffer:@@ -1124,26 +1113,14 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:return torch_matmul(X, W, out = out)# Fast X @ W where seq_len == 1, from bitsandbytes functional.py#L1469 and TimDettmers/bitsandbytes#763._, q_len, hd = X.shape-- if type(quant_state) is not list:- absmax = quant_state.absmax- shape = quant_state.shape- dtype = quant_state.dtype- blocksize = quant_state.blocksize- stats = quant_state.code- offset = quant_state.offset- state2 = quant_state.state2- absmax2 = state2.absmax- code2 = state2.code- blocksize2 = state2.blocksize- else:- absmax, shape, dtype, blocksize, compressed_stats, quant_type, stats = quant_state- offset, state2 = compressed_stats- absmax2, code2, blocksize2, _, _, _, _ = state2- global XPU_STREAMS+ absmax, shape, dtype, blocksize, stats, code2, absmax2, offset, blocksize2 = (+ _unpack_quant_state(quant_state)+ )+ if shape[1] % blocksize != 0:+ # The gemv kernel needs each weight row to start a new block (bitsandbytes' xpu gemm_4bit checks too).+ return torch_matmul(X, fast_dequantize(W, quant_state, dtype = X.dtype).t(), out = out)device = W.device- device_index = device.index- XPU_STREAM = XPU_STREAMS[device_index]+ XPU_STREAM = _get_tensor_stream(W)bout = shape[0]@@ -1158,12 +1135,9 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:device = device,)- if DEVICE_TYPE == "xpu":- m = 1- n = shape[0]- else:- n = 1- m = shape[0]+ # The xpu gemv takes m = 1, n = rows (the CUDA one is transposed).+ m = 1+ n = shape[0]k = shape[1]lda = shape[0]ldc = shape[0]@@ -1175,19 +1149,20 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:ldb = ctypes_c_int32(ldb)ldc = ctypes_c_int32(ldc)- df = torch_empty(absmax.shape, dtype = torch_float32, device = device)with torch_gpu_device(device):- cdequantize_blockwise_fp32(- get_ptr(code2),- get_ptr(absmax),- get_ptr(absmax2),- get_ptr(df),- ctypes_c_int(blocksize2),- ctypes_c_int(df.numel()),- XPU_STREAM,- )- df += offset- absmax = df+ if code2 is not None:+ df = torch_empty(absmax.shape, dtype = torch_float32, device = device)+ cdequantize_blockwise_fp32(+ get_ptr(code2),+ get_ptr(absmax),+ get_ptr(absmax2),+ get_ptr(df),+ ctypes_c_int(blocksize2),+ ctypes_c_int(df.numel()),+ XPU_STREAM,+ )+ df += offset+ absmax = dffx = (cgemm_4bit_inference_naive_fp16