Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
[XPU] Enable FP8 training on XPU (#12535)
JoshuaL3000cb67888d1b9f
diff --git a/tests/test_compressed_tensors_fp8_kernel_route.py b/tests/test_compressed_tensors_fp8_kernel_route.pyindex 23f9870f3fd..bc1dc27fcb1 100644--- a/tests/test_compressed_tensors_fp8_kernel_route.py+++ b/tests/test_compressed_tensors_fp8_kernel_route.py@@ -19,8 +19,12 @@ import typesimport pytestimport torch+from real_accelerator import has_real_cuda-pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()+dev = "cuda" if has_real_cuda() else "xpu" if xpu_available else "cpu"++pytestmark = pytest.mark.skipif(not (has_real_cuda() or xpu_available), reason = "needs CUDA or XPU")ct_quant = pytest.importorskip("compressed_tensors.quantization")@@ -63,8 +67,8 @@ def _ct_model(weight_type = "float",):torch.manual_seed(0)- lin = torch.nn.Linear(i, o, bias = bias, device = "cuda", dtype = torch.bfloat16)- Wq, s, ref = _quantize(torch.randn(o, i, device = "cuda") * 0.02, strategy, block)+ lin = torch.nn.Linear(i, o, bias = bias, device = dev, dtype = torch.bfloat16)+ Wq, s, ref = _quantize(torch.randn(o, i, device = dev) * 0.02, strategy, block)lin.weight = torch.nn.Parameter(Wq, requires_grad = False)lin.weight_scale = torch.nn.Parameter(s, requires_grad = False)kwargs = dict(num_bits = 8, type = weight_type, strategy = strategy, symmetric = True, dynamic = False)@@ -92,7 +96,7 @@ def _ct_model(def test_weight_dequant_transposed_row_scale(shape):from unsloth.kernels.fp8 import weight_dequant- Wq, s, ref = _quantize(torch.randn(*shape, device = "cuda") * 0.02, "channel")+ Wq, s, ref = _quantize(torch.randn(*shape, device = dev) * 0.02, "channel")torch.testing.assert_close(weight_dequant(Wq, s, torch.float32), ref)torch.testing.assert_close(weight_dequant(Wq.t(), s, torch.float32), ref.t())@@ -116,11 +120,11 @@ def test_fp8_modules_route_to_unsloth_kernels(strategy, shape, block, bias, monkassert _route_compressed_tensors_fp8_to_unsloth(model) == 1lin = model.linassert lin.weight.dtype == torch.float8_e4m3fn- X = torch.randn(2, 5, shape[1], device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ X = torch.randn(2, 5, shape[1], device = dev, dtype = torch.bfloat16, requires_grad = True)y = lin(X)y_ref = X.float() @ ref.t() + (lin.bias.float() if bias else 0)(dX,) = torch.autograd.grad(y.float().sum(), X)- dX_ref = torch.ones(2, 5, shape[0], device = "cuda") @ ref+ dX_ref = torch.ones(2, 5, shape[0], device = dev) @ refassert float((y.float() - y_ref).norm() / y_ref.norm()) < 0.05assert float((dX.float() - dX_ref).norm() / dX_ref.norm()) < 0.01@@ -185,9 +189,7 @@ def test_non_fp8_or_unsupported_modules_are_left_alone(monkeypatch):model.lin.weight = torch.nn.Parameter(model.lin.weight.to(torch.bfloat16), requires_grad = False)assert _route_compressed_tensors_fp8_to_unsloth(model) == 0model, _ = _ct_model(256, 256, "channel")- model.lin.weight_scale = torch.nn.Parameter(- torch.ones(3, 1, device = "cuda"), requires_grad = False- )+ model.lin.weight_scale = torch.nn.Parameter(torch.ones(3, 1, device = dev), requires_grad = False)assert _route_compressed_tensors_fp8_to_unsloth(model) == 0model, _ = _ct_model(256, 256, "channel")monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")@@ -225,7 +227,7 @@ def test_single_token_fast_path_adds_the_bias_once(strategy, block, monkeypatch)lin = model.linwith torch.no_grad():lin.bias.fill_(1.0)- X = torch.randn(1, 1, 256, device = "cuda", dtype = torch.bfloat16)+ X = torch.randn(1, 1, 256, device = dev, dtype = torch.bfloat16)y = fast_linear_forward(lin, X)y_ref = X.float() @ ref.t() + 1.0assert float((y.float() - y_ref).norm() / y_ref.norm()) < 0.05@@ -241,6 +243,7 @@ def test_a_model_mixing_fp8_and_other_schemes_is_not_partly_routed():assert not hasattr(model.lin, "_unsloth_compressed_tensors_fp8")[email protected](not has_real_cuda(), reason = "Decode GEMV needs a CUDA GPU")@pytest.mark.parametrize("shape", [(2048, 2048), (256, 2048), (11008, 2048), (2048, 11008), (100, 300), (33, 7)])@@ -252,7 +255,7 @@ def test_decode_gemv_matches_the_dequantized_matmul(shape, rows):pytest.skip("triton cannot read float8_e4m3fn on this GPU")model, ref = _ct_model(*shape, "channel")W, s = model.lin.weight, model.lin.weight_scale- X = torch.randn(*rows, shape[1], device = "cuda", dtype = torch.bfloat16)+ X = torch.randn(*rows, shape[1], device = dev, dtype = torch.bfloat16)with torch.no_grad():assert can_use_fp8_rowwise_gemv(X, W, s)y = fp8_rowwise_gemv(X, W, s)@@ -263,17 +266,18 @@ def test_decode_gemv_matches_the_dequantized_matmul(shape, rows):assert torch.equal(y, y_again)[email protected](not has_real_cuda(), reason = "Decode GEMV needs a CUDA GPU")def test_decode_gemv_refuses_what_it_cannot_compute():from unsloth.kernels.fp8 import can_use_fp8_rowwise_gemvmodel, _ = _ct_model(2048, 16384, "channel")W, s = model.lin.weight, model.lin.weight_scale- X = torch.randn(1, 16384, device = "cuda", dtype = torch.bfloat16)+ X = torch.randn(1, 16384, device = dev, dtype = torch.bfloat16)with torch.no_grad():# 128x128 block grid here has 2048 == N elements.- assert not can_use_fp8_rowwise_gemv(X, W, torch.ones(16, 128, device = "cuda"))+ assert not can_use_fp8_rowwise_gemv(X, W, torch.ones(16, 128, device = dev))assert not can_use_fp8_rowwise_gemv(- torch.randn(2, 16384, device = "cuda", dtype = torch.bfloat16), W, s+ torch.randn(2, 16384, device = dev, dtype = torch.bfloat16), W, s)assert not can_use_fp8_rowwise_gemv(X.float(), W, s)assert not can_use_fp8_rowwise_gemv(X[:, :2048], W.t(), s)@@ -282,7 +286,7 @@ def test_decode_gemv_refuses_what_it_cannot_compute():def _fp8_kernel_unsupported_here():from unsloth.kernels.fp8 import _fp8_kernel_unsupported- return _fp8_kernel_unsupported(torch.empty(1, device = "cuda", dtype = torch.float8_e4m3fn))+ return _fp8_kernel_unsupported(torch.empty(1, device = dev, dtype = torch.float8_e4m3fn))def test_fp8_weights_off_the_gpu_keep_the_compressed_tensors_path():@@ -300,13 +304,13 @@ def test_compiled_block_fp8_linear_passes_the_input_gradient():torch._dynamo.reset()torch.manual_seed(0)- W = (torch.randn(256, 256, device = "cuda") * 0.05).to(torch.float8_e4m3fn)- s = torch.rand(2, 2, device = "cuda") * 0.01 + 0.001+ W = (torch.randn(256, 256, device = dev) * 0.05).to(torch.float8_e4m3fn)+ s = torch.rand(2, 2, device = dev) * 0.01 + 0.001W.block_size = s.block_size = [128, 128]ref = W.float() * s.repeat_interleave(128, 0).repeat_interleave(128, 1)- X = torch.randn(2, 5, 256, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ X = torch.randn(2, 5, 256, device = dev, dtype = torch.bfloat16, requires_grad = True)(dX,) = torch.autograd.grad(fp8_linear(X, W, s).float().sum(), X)- dX_ref = torch.ones(2, 5, 256, device = "cuda") @ ref+ dX_ref = torch.ones(2, 5, 256, device = dev) @ refassert float((dX.float() - dX_ref).norm() / dX_ref.norm()) < 0.01@@ -315,9 +319,9 @@ def test_rowwise_dequant_fallback_follows_the_input_dtype(dtype, monkeypatch):from unsloth.kernels import fp8monkeypatch.setattr(fp8, "_has_fbgemm_rowwise", lambda: False)- W = (torch.randn(256, 512, device = "cuda") * 0.05).to(torch.float8_e4m3fn)- s = torch.rand(256, 1, device = "cuda") * 0.01 + 0.001- X = torch.randn(3, 512, device = "cuda", dtype = dtype, requires_grad = True)+ W = (torch.randn(256, 512, device = dev) * 0.05).to(torch.float8_e4m3fn)+ s = torch.rand(256, 1, device = dev) * 0.01 + 0.001+ X = torch.randn(3, 512, device = dev, dtype = dtype, requires_grad = True)y = fp8.FbgemmFp8Linear_matmul.apply(X, W, s, None)(dX,) = torch.autograd.grad(y.float().sum(), X)ref = W.float() * sdiff --git a/tests/test_fp8_compile_fullgraph.py b/tests/test_fp8_compile_fullgraph.pyindex c3f98d73529..05d124d5fed 100644--- a/tests/test_fp8_compile_fullgraph.py+++ b/tests/test_fp8_compile_fullgraph.py@@ -6,9 +6,13 @@ import pytesttorch = pytest.importorskip("torch")+cuda_available = torch.cuda.is_available()+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"+pytestmark = pytest.mark.skipif(- not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),- reason = "block-FP8 Triton kernels need CUDA sm89+",+ not ((cuda_available and torch.cuda.get_device_capability() >= (8, 9)) or xpu_available),+ reason = "block-FP8 Triton kernels need CUDA sm89+ or XPU",)@@ -25,9 +29,9 @@ def test_block_fp8_linear_compiles_fullgraph_and_matches_eager():# Several weight shapes (the second triggers automatic dynamic shapes); K = 200 takes the dequant fallback.for N, K in [(512, 256), (256, 256), (256, 768), (768, 256), (512, 200)]:torch.manual_seed(0)- X = torch.randn(256, K, device = "cuda", dtype = torch.bfloat16, requires_grad = True)- W = (torch.randn(N, K, device = "cuda") * 0.02).to(torch.float8_e4m3fn)- s = torch.rand(-(-N // 128), -(-K // 128), device = "cuda") + 0.5+ X = torch.randn(256, K, device = dev, dtype = torch.bfloat16, requires_grad = True)+ W = (torch.randn(N, K, device = dev) * 0.02).to(torch.float8_e4m3fn)+ s = torch.rand(-(-N // 128), -(-K // 128), device = dev) + 0.5s.block_size = [128, 128]compiled(X, W, s).backward()diff --git a/tests/test_fp8_device_context.py b/tests/test_fp8_device_context.pyindex 7e1569a2191..2eb99376ce0 100644--- a/tests/test_fp8_device_context.py+++ b/tests/test_fp8_device_context.py@@ -169,24 +169,27 @@ def test_fp8_triton_launches_enter_tensor_device_context() -> None:assert not (expected_launches & visitor.unguarded_launches)-def _require_two_cuda_devices():+def _require_two_accelerator_devices():torch = pytest.importorskip("torch")pytest.importorskip("triton")- if not torch.cuda.is_available() or torch.cuda.device_count() < 2:- pytest.skip("requires at least two CUDA devices")- return torch+ if torch.cuda.is_available() and torch.cuda.device_count() >= 2:+ return "cuda", torch+ if hasattr(torch, "xpu") and torch.xpu.is_available() and torch.xpu.device_count() >= 2:+ return "xpu", torch+ pytest.skip("requires at least two CUDA or XPU devices")def test_weight_dequant_block_runs_on_tensor_device_when_current_device_differs() -> None:- torch = _require_two_cuda_devices()+ device, torch = _require_two_accelerator_devices()+ accel = getattr(torch, device)from unsloth.kernels.fp8 import weight_dequant_block- previous_device = torch.cuda.current_device()+ previous_device = accel.current_device()try:- torch.cuda.set_device(0)- x = torch.arange(256 * 256, device = "cuda:1", dtype = torch.float32).reshape(256, 256)- scales = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device = "cuda:1", dtype = torch.float32)+ accel.set_device(0)+ x = torch.arange(256 * 256, device = f"{device}:1", dtype = torch.float32).reshape(256, 256)+ scales = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device = f"{device}:1", dtype = torch.float32)actual = weight_dequant_block(x, scales, block_size = 128, dtype = torch.float32)@@ -194,51 +197,54 @@ def test_weight_dequant_block_runs_on_tensor_device_when_current_device_differs(expected = x * expanded_scalesassert actual.device == x.device- assert torch.cuda.current_device() == 0+ assert accel.current_device() == 0torch.testing.assert_close(actual, expected)finally:- torch.cuda.set_device(previous_device)+ accel.set_device(previous_device)def test_act_quant_runs_on_tensor_device_when_current_device_differs() -> None:- torch = _require_two_cuda_devices()+ device, torch = _require_two_accelerator_devices()+ accel = getattr(torch, device)if not hasattr(torch, "float8_e4m3fn"):pytest.skip("requires torch.float8_e4m3fn")- if torch.cuda.get_device_capability(1)[0] < 9:+ if device == "cuda" and torch.cuda.get_device_capability(1)[0] < 9:pytest.skip("requires FP8-capable CUDA hardware")from unsloth.kernels.fp8 import act_quant- previous_device = torch.cuda.current_device()+ previous_device = accel.current_device()try:- torch.cuda.set_device(0)- x = torch.arange(256, device = "cuda:1", dtype = torch.float32).reshape(2, 128)+ accel.set_device(0)+ x = torch.arange(256, device = f"{device}:1", dtype = torch.float32).reshape(2, 128)y, scales = act_quant(x, block_size = 128)assert y.device == x.deviceassert scales.device == x.device- assert torch.cuda.current_device() == 0+ assert accel.current_device() == 0finally:- torch.cuda.set_device(previous_device)+ accel.set_device(previous_device)def test_w8a8_block_fp8_matmul_triton_runs_on_tensor_device_when_current_device_differs() -> None:- torch = _require_two_cuda_devices()+ device, torch = _require_two_accelerator_devices()+ accel = getattr(torch, device)if not hasattr(torch, "float8_e4m3fn"):pytest.skip("requires torch.float8_e4m3fn")- if torch.cuda.get_device_capability(1)[0] < 9:+ if device == "cuda" and torch.cuda.get_device_capability(1)[0] < 9:pytest.skip("requires FP8-capable CUDA hardware")from unsloth.kernels.fp8 import w8a8_block_fp8_matmul_triton- previous_device = torch.cuda.current_device()+ previous_device = accel.current_device()try:- torch.cuda.set_device(0)- A = torch.ones((128, 128), device = "cuda:1", dtype = torch.float32).to(torch.float8_e4m3fn)- B = torch.ones((128, 128), device = "cuda:1", dtype = torch.float32).to(torch.float8_e4m3fn)- As = torch.ones((128, 1), device = "cuda:1", dtype = torch.float32)- Bs = torch.ones((1, 1), device = "cuda:1", dtype = torch.float32)+ accel.set_device(0)+ dev1 = f"{device}:1"+ A = torch.ones((128, 128), device = dev1, dtype = torch.float32).to(torch.float8_e4m3fn)+ B = torch.ones((128, 128), device = dev1, dtype = torch.float32).to(torch.float8_e4m3fn)+ As = torch.ones((128, 1), device = dev1, dtype = torch.float32)+ Bs = torch.ones((1, 1), device = dev1, dtype = torch.float32)actual = w8a8_block_fp8_matmul_triton(A,@@ -249,9 +255,9 @@ def test_w8a8_block_fp8_matmul_triton_runs_on_tensor_device_when_current_device_output_dtype = torch.float32,)- expected = torch.full((128, 128), 128.0, device = "cuda:1", dtype = torch.float32)+ expected = torch.full((128, 128), 128.0, device = dev1, dtype = torch.float32)assert actual.device == A.device- assert torch.cuda.current_device() == 0+ assert accel.current_device() == 0torch.testing.assert_close(actual, expected)finally:- torch.cuda.set_device(previous_device)+ accel.set_device(previous_device)diff --git a/tests/test_fp8_device_gate.py b/tests/test_fp8_device_gate.pynew file mode 100644index 00000000000..38a6eee7f7a--- /dev/null+++ b/tests/test_fp8_device_gate.py@@ -0,0 +1,55 @@+# SPDX-License-Identifier: AGPL-3.0-only+# verify_fp8_support_if_applicable admits FP8 checkpoints on CUDA and XPU only.+from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")++import unsloth # noqa: F401+from unsloth.models import _utils+++def _config(quant_method):+ return SimpleNamespace(quantization_config = {"quant_method": quant_method})++[email protected]("quant_method", ["fp8", "fbgemm_fp8"])+def test_xpu_is_admitted(monkeypatch, quant_method):+ monkeypatch.setattr(_utils, "DEVICE_TYPE", "xpu")+ _utils.verify_fp8_support_if_applicable(_config(quant_method))++[email protected]("device_type", ["hip", "mps", "npu", "cpu"])[email protected]("quant_method", ["fp8", "fbgemm_fp8"])+def test_other_devices_are_still_refused(monkeypatch, device_type, quant_method):+ monkeypatch.setattr(_utils, "DEVICE_TYPE", device_type)+ with pytest.raises(ValueError, match = "FP8 quantization is only supported on"):+ _utils.verify_fp8_support_if_applicable(_config(quant_method))++[email protected]("device_type", ["xpu", "hip", "mps", "cpu"])+def test_non_fp8_checkpoints_load_anywhere(monkeypatch, device_type):+ monkeypatch.setattr(_utils, "DEVICE_TYPE", device_type)+ _utils.verify_fp8_support_if_applicable(_config("bitsandbytes"))+ _utils.verify_fp8_support_if_applicable(SimpleNamespace())+++ "capability, quant_method, refused",+ [+ ((8, 0), "fp8", True),+ ((8, 9), "fp8", False),+ ((8, 9), "fbgemm_fp8", True),+ ((9, 0), "fbgemm_fp8", False),+ ],+)+def test_cuda_capability_rules_are_unchanged(monkeypatch, capability, quant_method, refused):+ monkeypatch.setattr(_utils, "DEVICE_TYPE", "cuda")+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: capability)+ monkeypatch.setattr(torch.cuda, "get_device_name", lambda *a: "spoofed")+ if refused:+ with pytest.raises(ValueError):+ _utils.verify_fp8_support_if_applicable(_config(quant_method))+ else:+ _utils.verify_fp8_support_if_applicable(_config(quant_method))diff --git a/tests/test_fp8_fbgemm_block_shapes.py b/tests/test_fp8_fbgemm_block_shapes.pyindex 9a8804e2da4..2df3731f158 100644--- a/tests/test_fp8_fbgemm_block_shapes.py+++ b/tests/test_fp8_fbgemm_block_shapes.py@@ -16,10 +16,11 @@ import pytestimport torchcuda_available = torch.cuda.is_available()+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"--# Only the kernel battery needs fbgemm; the fallback tests below never reach it.-pytestmark = pytest.mark.skipif(not cuda_available, reason = "needs CUDA")+# Only the kernel battery needs fbgemm (CUDA-only); the fallback tests below never reach it.+pytestmark = pytest.mark.skipif(not (cuda_available or xpu_available), reason = "needs CUDA or XPU")def skip_without_fbgemm():@@ -119,10 +120,10 @@ def test_output_tile_grid_battery_matches_reference():(100, 136, 272),(64, 8, 16),]:- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)Wq, scale = _block_quantize_weight(W, block)scale.block_size = block- X = torch.randn(M, K, device = "cuda", dtype = torch.bfloat16)+ X = torch.randn(M, K, device = dev, dtype = torch.bfloat16)out = FP8_fbgemm_block_linear.apply(X, Wq, scale)ref = _reference(X, Wq, scale, block)@@ -136,10 +137,10 @@ def test_odd_k_uses_dequant_fallback():torch.manual_seed(0)block = [128, 128]N, K = 320, 130 # K % 16 != 0 used to crash inside the CUTLASS kernel- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)Wq, scale = _block_quantize_weight(W, block)scale.block_size = block- X = torch.randn(4, K, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ X = torch.randn(4, K, device = dev, dtype = torch.bfloat16, requires_grad = True)out = FP8_fbgemm_block_linear.apply(X, Wq, scale)assert torch.isfinite(out).all()@@ -156,10 +157,10 @@ def test_odd_n_uses_dequant_fallback():torch.manual_seed(0)block = [128, 128]N, K = 250, 256 # N % 8 != 0 used to crash inside the CUTLASS kernel- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)Wq, scale = _block_quantize_weight(W, block)scale.block_size = block- X = torch.randn(4, K, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ X = torch.randn(4, K, device = dev, dtype = torch.bfloat16, requires_grad = True)out = FP8_fbgemm_block_linear.apply(X, Wq, scale)ref = _reference(X.detach(), Wq, scale, block)@@ -174,10 +175,10 @@ def test_non_square_block_uses_dequant_fallback():torch.manual_seed(0)block = [128, 64] # kernel only implements 128x128x128, used to crashN, K = 256, 256- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)Wq, scale = _block_quantize_weight(W, block)scale.block_size = block- X = torch.randn(64, K, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ X = torch.randn(64, K, device = dev, dtype = torch.bfloat16, requires_grad = True)out = FP8_fbgemm_block_linear.apply(X, Wq, scale)ref = _reference(X.detach(), Wq, scale, block)@@ -197,9 +198,9 @@ def test_inputs_the_kernel_rejects_use_dequant_fallback(kind):block = [128, 128]# strided_3d needs a shape the kernel rejects too, else it stays on the fast pathN, K = (250, 130) if kind == "strided_3d" else (256, 256)- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)Wq, scale = _block_quantize_weight(W, block)- X = torch.randn(8, K, device = "cuda", dtype = torch.bfloat16)+ X = torch.randn(8, K, device = dev, dtype = torch.bfloat16)if kind.startswith("per_tensor"):scale = scale.amax().clone()@@ -210,7 +211,7 @@ def test_inputs_the_kernel_rejects_use_dequant_fallback(kind):if kind == "bf16_scale":scale = scale.to(torch.bfloat16)else:- X = torch.randn(2, 4, K * 2, device = "cuda", dtype = torch.bfloat16)[..., ::2]+ X = torch.randn(2, 4, K * 2, device = dev, dtype = torch.bfloat16)[..., ::2]scale.block_size = blockref = _reference(X, Wq, scale, block)@@ -229,13 +230,13 @@ def test_transposed_weight_swaps_block_axes(block, N, K):from unsloth.kernels.fp8 import FP8_fbgemm_block_lineartorch.manual_seed(0)- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)Wq, scale = _block_quantize_weight(W, block)scale.block_size = blockWt = Wq.t()Wt.block_size = block- dY = torch.randn(8, N, device = "cuda", dtype = torch.bfloat16)+ dY = torch.randn(8, N, device = dev, dtype = torch.bfloat16)out = FP8_fbgemm_block_linear.apply(dY, Wt, scale)ref = (dY.float() @ _dequant(Wq, scale, block)).to(dY.dtype)assert out.shape == (8, K)diff --git a/tests/test_fp8_leftover_dequant_16bit.py b/tests/test_fp8_leftover_dequant_16bit.pyindex 68a0f9e6565..00856309ee8 100644--- a/tests/test_fp8_leftover_dequant_16bit.py+++ b/tests/test_fp8_leftover_dequant_16bit.py@@ -22,7 +22,7 @@ import tempfileimport pytestfrom real_accelerator import (- has_real_cuda,+ has_real_accelerator,) # tests/_shared, on sys.path via tests/conftest.pyimport torchfrom torch import nn@@ -220,19 +220,22 @@ def test_scale_grid_that_does_not_tile_is_refused():# has_real_cuda(): another test spoofs torch.cuda.is_available() process-wide.[email protected](not has_real_cuda(), reason = "needs CUDA")[email protected](not has_real_accelerator(), reason = "needs CUDA or XPU")def test_out_of_memory_on_the_device_is_finished_through_the_cpu(monkeypatch):from unsloth.models import loader_utils+ xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()+ cuda_available = torch.cuda.is_available()+ dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"model, tensors, expected = _build()- model = model.to("cuda")+ model = model.to(dev)real = loader_utils._fp8_scale_grid_dequant- calls = {"cuda": 0, "cpu": 0}+ calls = {dev: 0, "cpu": 0}def flaky(quantized, scale, out_dtype):calls[quantized.device.type] += 1- if quantized.device.type == "cuda":- raise torch.OutOfMemoryError("CUDA out of memory (simulated)")+ if quantized.device.type == dev:+ raise torch.OutOfMemoryError(f"{dev} out of memory (simulated)")return real(quantized, scale, out_dtype)monkeypatch.setattr(loader_utils, "_fp8_scale_grid_dequant", flaky)@@ -240,8 +243,8 @@ def test_out_of_memory_on_the_device_is_finished_through_the_cpu(monkeypatch):_write_checkpoint(d, tensors)done, skipped = loader_utils._dequantize_leftover_fp8_params(model, d, torch.bfloat16)assert done == 2- assert calls == {"cuda": 2, "cpu": 2}- assert model.experts.gate_up_proj.device.type == "cuda"+ assert calls == {dev: 2, "cpu": 2}+ assert model.experts.gate_up_proj.device.type == devassert model.experts.gate_up_proj.dtype == torch.bfloat16assert torch.equal(model.experts.gate_up_proj.detach().cpu(), expected["experts.gate_up_proj"])assert torch.equal(model.experts.down_proj.detach().cpu(), expected["experts.down_proj"])diff --git a/tests/test_fp8_rowwise_backend.py b/tests/test_fp8_rowwise_backend.pyindex 917897bd733..422bbe89913 100644--- a/tests/test_fp8_rowwise_backend.py+++ b/tests/test_fp8_rowwise_backend.py@@ -6,9 +6,13 @@ import pytesttorch = pytest.importorskip("torch")+cuda_available = torch.cuda.is_available()+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"+pytestmark = pytest.mark.skipif(- not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),- reason = "FP8 GEMMs need CUDA sm89+",+ not ((cuda_available and torch.cuda.get_device_capability() >= (8, 9)) or xpu_available),+ reason = "FP8 GEMMs need CUDA sm89+ or XPU",)@@ -20,13 +24,13 @@ def F():def _rowwise_weight(N, K):- w = torch.randn(N, K, device = "cuda") * 0.02+ w = torch.randn(N, K, device = dev) * 0.02scale = (w.abs().amax(1, keepdim = True) / 448).float()return (w / scale).to(torch.float8_e4m3fn), scaledef _backends(F):- device = torch.device("cuda", torch.cuda.current_device())+ device = torch.device(dev, 0)ones = lambda n: torch.ones(n, dtype = torch.float32, device = device)out = ["dequant"]if F._rowwise_gemm_works(@@ -42,9 +46,9 @@ def _backends(F):def test_backend_is_one_that_runs_here(F):- backend = F._fp8_rowwise_backend(torch.device("cuda", torch.cuda.current_device()))+ backend = F._fp8_rowwise_backend(torch.device(dev, 0))assert backend in ("fbgemm", "scaled_mm", "dequant")- X = torch.randn(64, 256, device = "cuda", dtype = torch.bfloat16)+ X = torch.randn(64, 256, device = dev, dtype = torch.bfloat16)w, s = _rowwise_weight(512, 256)y = F.fp8_linear(X, w, s)ref = X.float() @ (w.float() * s).t()@@ -58,16 +62,16 @@ def test_every_backend_matches_reference(F, backend):pytest.skip(f"{backend} is not usable on this GPU")torch.manual_seed(0)w, s = _rowwise_weight(768, 512)- X = torch.randn(2, 48, 512, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ X = torch.randn(2, 48, 512, device = dev, dtype = torch.bfloat16, requires_grad = True)# FbgemmFp8Linear stores its bias in float32.- bias = torch.randn(768, device = "cuda", dtype = torch.float32)+ bias = torch.randn(768, device = dev, dtype = torch.float32)y = F.FbgemmFp8Linear_matmul.apply(X, w, s, bias, backend)W = w.float() * sref = X.detach().float() @ W.t() + bias.float()assert y.shape == (2, 48, 768) and y.dtype == torch.bfloat16assert ((y.float() - ref).norm() / ref.norm()) < 0.05y.float().sum().backward()- ref_dx = torch.ones(2, 48, 768, device = "cuda") @ W+ ref_dx = torch.ones(2, 48, 768, device = dev) @ Wassert ((X.grad.float() - ref_dx).norm() / ref_dx.norm()) < 0.01# Decode-sized calls take the dequant branch on the scaled_mm backend.assert (@@ -80,10 +84,10 @@ def test_quantize_matches_fbgemm(F):if not hasattr(torch.ops.fbgemm, "quantize_fp8_per_row"):pytest.skip("FBGEMM not installed")torch.manual_seed(0)- x = torch.randn(256, 1024, device = "cuda", dtype = torch.bfloat16)+ x = torch.randn(256, 1024, device = dev, dtype = torch.bfloat16)x[3] *= 1000x[5] = 0- for scale_ub in (None, torch.tensor([30.0], device = "cuda")):+ for scale_ub in (None, torch.tensor([30.0], device = dev)):q_ref, s_ref = torch.ops.fbgemm.quantize_fp8_per_row(x, scale_ub = scale_ub)q, s = F._quantize_fp8_per_row(x, scale_ub)# FBGEMM's fast-math division can land 1 ulp lower, flipping exact rounding ties by one FP8 step.diff --git a/tests/test_fp8_to_nf4_load.py b/tests/test_fp8_to_nf4_load.pyindex 09a526b640c..da324ae0a82 100644--- a/tests/test_fp8_to_nf4_load.py+++ b/tests/test_fp8_to_nf4_load.py@@ -33,8 +33,12 @@ _BLOCK = 128_TOKENIZER_REPO = "trl-internal-testing/tiny-Qwen3ForCausalLM"_KEEP_16BIT = "model.layers.1.self_attn.o_proj"-needs_cuda = pytest.mark.skipif(- not torch.cuda.is_available(), reason = "bitsandbytes 4bit needs a GPU"+cuda_available = torch.cuda.is_available()+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"++needs_gpu = pytest.mark.skipif(+ not (cuda_available or xpu_available), reason = "bitsandbytes 4bit needs a GPU")@@ -51,8 +55,8 @@ needs_feature = pytest.mark.skipif(reason = f"fp8 -> 4bit loading unavailable: {_feature_unavailable()}",)needs_fp8_gpu = pytest.mark.skipif(- not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),- reason = "fp8 checkpoints load natively only on sm89+",+ not ((cuda_available and torch.cuda.get_device_capability() >= (8, 9)) or xpu_available),+ reason = "fp8 checkpoints load natively only on sm89+ or XPU",)@@ -239,10 +243,10 @@ def _free(*models):import gcgc.collect()- torch.cuda.empty_cache()+ getattr(torch, dev, torch.cuda).empty_cache()-@needs_cuda+@needs_gpu@needs_featuredef test_dense_fp8_loads_nf4_bit_identical_to_dequantized_bf16(dense_pair, capsys):fp8_dir, deq_dir = dense_pair@@ -260,7 +264,7 @@ def test_dense_fp8_loads_nf4_bit_identical_to_dequantized_bf16(dense_pair, capsy_assert_same(ours, theirs)-@needs_cuda+@needs_gpu@needs_featuredef test_moe_fp8_loads_nf4_experts_bit_identical(moe_pair):fp8_dir, deq_dir = moe_pair@@ -281,7 +285,7 @@ def test_moe_fp8_loads_nf4_experts_bit_identical(moe_pair):_assert_same(ours, theirs)-@needs_cuda+@needs_gpu@needs_featuredef test_checkpoint_16bit_modules_stay_16bit(dense_pair):from safetensors.torch import load_file@@ -322,7 +326,7 @@ def test_default_load_in_4bit_and_kill_switch_keep_fp8(dense_pair, monkeypatch):_free(model)-@needs_cuda+@needs_gpu@needs_featuredef test_fp8_keys_the_model_does_not_have_are_ignored(dense_pair, tmp_path):"""MTP layers ship in fp8 but transformers drops them as unexpected (DeepSeek-V3, GLM-4 MoE,@@ -347,7 +351,7 @@ def test_fp8_keys_the_model_does_not_have_are_ignored(dense_pair, tmp_path):_assert_same(ours, theirs)-@needs_cuda+@needs_gpu@needs_featuredef test_quantize_16bit_switch_matches_plain_4bit_skip_list(dense_pair, monkeypatch):fp8_dir, deq_dir = dense_pair@@ -370,7 +374,7 @@ def _zoo_reads_fp8_block_size_from_disk():return hasattr(saving_utils, "_fp8_block_size_on_disk")-@needs_cuda+@needs_gpu@needs_feature@pytest.mark.skipif(not _zoo_reads_fp8_block_size_from_disk(),diff --git a/tests/test_fp8_transposed_weight.py b/tests/test_fp8_transposed_weight.pyindex 54efd358738..9ada061c2b4 100644--- a/tests/test_fp8_transposed_weight.py+++ b/tests/test_fp8_transposed_weight.py@@ -7,7 +7,11 @@ import pytesttorch = pytest.importorskip("torch")-pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")+cuda_available = torch.cuda.is_available()+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"++pytestmark = pytest.mark.skipif(not (cuda_available or xpu_available), reason = "needs CUDA or XPU")@pytest.fixture(scope = "module")@@ -18,7 +22,7 @@ def F():def _rowwise(n):- w = torch.randn(n, n, device = "cuda") * torch.linspace(0.01, 1.0, n, device = "cuda")[:, None]+ w = torch.randn(n, n, device = dev) * torch.linspace(0.01, 1.0, n, device = dev)[:, None]s = (w.abs().amax(1, keepdim = True) / 448).float()q = (w / s).to(torch.float8_e4m3fn)return q, s, q.float() * s@@ -27,18 +31,18 @@ def _rowwise(n):def _block(n, bs):torch.manual_seed(0)p = -(-n // bs)- s = torch.rand(p, p, device = "cuda") * 4 + 0.1+ s = torch.rand(p, p, device = dev) * 4 + 0.1full = s.repeat_interleave(bs, 0)[:n].repeat_interleave(bs, 1)[:, :n]- q = (torch.randn(n, n, device = "cuda") * 0.02 / full).to(torch.float8_e4m3fn)+ q = (torch.randn(n, n, device = dev) * 0.02 / full).to(torch.float8_e4m3fn)s.block_size = [bs, bs]return q, s, q.float() * fulldef _block_rect(n, k, bs):torch.manual_seed(0)- s = torch.rand(-(-n // bs), -(-k // bs), device = "cuda") * 4 + 0.1+ s = torch.rand(-(-n // bs), -(-k // bs), device = dev) * 4 + 0.1full = s.repeat_interleave(bs, 0)[:n].repeat_interleave(bs, 1)[:, :k]- q = (torch.randn(n, k, device = "cuda") * 0.02 / full).to(torch.float8_e4m3fn)+ q = (torch.randn(n, k, device = dev) * 0.02 / full).to(torch.float8_e4m3fn)s.block_size = [bs, bs]return q, s, q.float() * full@@ -62,7 +66,7 @@ def test_fp8_linear_on_transposed_view(F, kind):q, s, ref = _block_rect(256, 384, 128)else:q, s, ref = _block(256, 128 if kind == "block128" else 64)- X = torch.randn(64, q.shape[0], device = "cuda", dtype = torch.bfloat16)+ X = torch.randn(64, q.shape[0], device = dev, dtype = torch.bfloat16)y = F.fp8_linear(X, q.t(), s)expect = X.float() @ refassert ((y.float() - expect).norm() / expect.norm()) < 0.05diff --git a/tests/test_fp8linear_high_precision_weight.py b/tests/test_fp8linear_high_precision_weight.pyindex 5d331fc221a..e9150f16f66 100644--- a/tests/test_fp8linear_high_precision_weight.py+++ b/tests/test_fp8linear_high_precision_weight.py@@ -6,7 +6,11 @@ import pytestimport torchimport torch.nn.functional as F-pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")+cuda_available = torch.cuda.is_available()+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"++pytestmark = pytest.mark.skipif(not (cuda_available or xpu_available), reason = "needs CUDA or XPU")@pytest.mark.parametrize(@@ -18,17 +22,15 @@ def test_bf16_weight_in_fp8linear_runs_plain_linear(bias):FP8Linear = fp8.FP8Linearif FP8Linear is None:pytest.skip("this transformers has no FP8Linear")- layer = FP8Linear(256, 128, block_size = (128, 128)).cuda()+ layer = FP8Linear(256, 128, block_size = (128, 128)).to(dev)torch.manual_seed(0)layer.weight = torch.nn.Parameter(- torch.randn(128, 256, device = "cuda", dtype = torch.bfloat16) * 0.05, requires_grad = False+ torch.randn(128, 256, device = dev, dtype = torch.bfloat16) * 0.05, requires_grad = False)layer.bias = (- torch.nn.Parameter(torch.randn(128, device = "cuda", dtype = bias))- if bias is not None- else None+ torch.nn.Parameter(torch.randn(128, device = dev, dtype = bias)) if bias is not None else None)- x = torch.randn(4, 7, 256, device = "cuda", dtype = torch.bfloat16)+ x = torch.randn(4, 7, 256, device = dev, dtype = torch.bfloat16)out = layer(x)assert out.dtype == torch.bfloat16ref = F.linear(x, layer.weight, None if layer.bias is None else layer.bias.to(x.dtype))diff --git a/tests/test_fp8linear_module_block_size.py b/tests/test_fp8linear_module_block_size.pyindex 58016fc7cff..b722bbc2800 100644--- a/tests/test_fp8linear_module_block_size.py+++ b/tests/test_fp8linear_module_block_size.py@@ -5,9 +5,13 @@import pytestimport torch+cuda_available = torch.cuda.is_available()+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"+pytestmark = pytest.mark.skipif(- not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),- reason = "block-FP8 kernels need CUDA sm89+",+ not ((cuda_available and torch.cuda.get_device_capability() >= (8, 9)) or xpu_available),+ reason = "block-FP8 kernels need CUDA sm89+ or XPU",)@@ -24,12 +28,12 @@ def _make_layer(out_features, in_features, block, scale_fmt):layer = fp8.FP8Linear(in_features, out_features, block_size = block, **extra)except TypeError:pytest.skip("this transformers FP8Linear takes no scale_fmt")- layer = layer.cuda()+ layer = layer.to(dev)torch.manual_seed(0)- weight = (torch.randn(out_features, in_features, device = "cuda") * 0.5).to(torch.float8_e4m3fn)+ weight = (torch.randn(out_features, in_features, device = dev) * 0.5).to(torch.float8_e4m3fn)# Power-of-two scales so float and ue8m0 formats hold the same values.grid = (-(-out_features // block[0]), -(-in_features // block[1]))- scale = torch.exp2(torch.randint(-9, -4, grid, device = "cuda").float())+ scale = torch.exp2(torch.randint(-9, -4, grid, device = dev).float())layer.weight = torch.nn.Parameter(weight, requires_grad = False)layer.weight_scale_inv = torch.nn.Parameter(scale.to(layer.weight_scale_inv.dtype), requires_grad = False@@ -53,7 +57,7 @@ def test_fp8linear_uses_module_block_size(block, scale_fmt):out_features, in_features = 512, 1024layer, weight, scale = _make_layer(out_features, in_features, block, scale_fmt)torch.manual_seed(1)- X = torch.randn(3, 64, in_features, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ X = torch.randn(3, 64, in_features, device = dev, dtype = torch.bfloat16, requires_grad = True)out = layer(X)assert out.shape == (3, 64, out_features) and out.dtype == torch.bfloat16diff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.pyindex e22e282e1b7..4612ef3d0e0 100644--- a/tests/test_modelopt_fp8_loading.py+++ b/tests/test_modelopt_fp8_loading.py@@ -12,7 +12,7 @@ from types import SimpleNamespaceimport pytestimport torchimport torch.nn as nn-from real_accelerator import has_real_cuda+from real_accelerator import has_real_cuda, has_real_acceleratorimport unsloth # noqa: F401from unsloth.models.modelopt_fp8 import (@@ -25,6 +25,9 @@ from unsloth.models.modelopt_fp8 import ()from unsloth.models.loader_utils import check_and_disable_bitsandbytes_loading+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()+dev = "cuda" if has_real_cuda() else "xpu" if xpu_available else "cpu"+def _sarvam_quant(**overrides):quant = {@@ -435,9 +438,9 @@ def _write_tiny_modelopt_llama(path):def test_tiny_modelopt_llama_round_trip(tmp_path, dequantize):from transformers import AutoConfig, AutoModelForCausalLM, FineGrainedFP8Config- if torch.cuda.get_device_capability()[0] < 9 and not dequantize:+ if dev == "cuda" and torch.cuda.get_device_capability()[0] < 9 and not dequantize:pytest.skip("fp8 matmul needs sm_89+")- reference = _write_tiny_modelopt_llama(str(tmp_path)).cuda()+ reference = _write_tiny_modelopt_llama(str(tmp_path)).to(dev)config = AutoConfig.from_pretrained(str(tmp_path))plan = arm_modelopt_fp8_loading(config, verbose = False)kwargs = {}@@ -445,7 +448,7 @@ def test_tiny_modelopt_llama_round_trip(tmp_path, dequantize):extra = {"dequantize": True} if dequantize else {}kwargs["quantization_config"] = FineGrainedFP8Config.from_dict(dict(plan), **extra)model = AutoModelForCausalLM.from_pretrained(- str(tmp_path), config = config, dtype = torch.bfloat16, device_map = "cuda", **kwargs+ str(tmp_path), config = config, dtype = torch.bfloat16, device_map = dev, **kwargs)q_proj = model.model.layers[0].self_attn.q_projif dequantize:@@ -454,7 +457,7 @@ def test_tiny_modelopt_llama_round_trip(tmp_path, dequantize):else:assert q_proj.weight.dtype == torch.float8_e4m3fnassert float(q_proj.activation_scale) == pytest.approx(0.02)- x = torch.randint(0, 512, (2, 32), device = "cuda")+ x = torch.randint(0, 512, (2, 32), device = dev)with torch.no_grad():got = model(x).logits.float()want = reference(x).logits.float()@@ -479,7 +482,7 @@ def test_rewrite_follows_who_loads_the_weights():assert "AUTO_QUANTIZATION_CONFIG_MAPPING[quant_method]" not in vision_source[email protected](not has_real_cuda(), reason = "FastModel loads need an accelerator")[email protected](not has_real_accelerator(), reason = "FastModel loads need an accelerator")def test_a_declined_modelopt_format_still_refuses_to_load_in_process(tmp_path):"""Subprocess: FastModel compiles and rebinds the Llama classes process-wide before it refuses."""import subprocess@@ -515,7 +518,9 @@ else:@needs_per_tensor_fp8[email protected](not has_real_cuda(), reason = "FastLanguageModel loads need an accelerator")+ not has_real_accelerator(), reason = "FastLanguageModel loads need an accelerator"+)def test_fast_llama_checks_fp8_hardware_on_the_rewritten_config(tmp_path):"""Subprocess: FastLanguageModel patches the Llama classes process-wide."""import subprocess@@ -646,7 +651,7 @@ def test_tiny_modelopt_llama_loads_a_classification_head(tmp_path):from transformers import AutoConfig, AutoModelForSequenceClassificationfrom unsloth.models.modelopt_fp8 import keep_task_heads_unquantized- if torch.cuda.get_device_capability()[0] < 9:+ if dev == "cuda" and torch.cuda.get_device_capability()[0] < 9:pytest.skip("fp8 matmul needs sm_89+")_write_tiny_modelopt_llama(str(tmp_path))config = AutoConfig.from_pretrained(str(tmp_path), num_labels = 2, pad_token_id = 0)@@ -655,13 +660,13 @@ def test_tiny_modelopt_llama_loads_a_classification_head(tmp_path):kwargs = {}pop_modelopt_key_mapping(config, kwargs)model = AutoModelForSequenceClassification.from_pretrained(- str(tmp_path), config = config, dtype = torch.bfloat16, device_map = "cuda", **kwargs+ str(tmp_path), config = config, dtype = torch.bfloat16, device_map = dev, **kwargs)assert type(model.score) is nn.Linearassert model.score.weight.dtype == torch.bfloat16assert model.model.layers[0].self_attn.q_proj.weight.dtype == torch.float8_e4m3fn- x = torch.randint(1, 512, (2, 16), device = "cuda")- out = model(x, labels = torch.tensor([0, 1], device = "cuda"))+ x = torch.randint(1, 512, (2, 16), device = dev)+ out = model(x, labels = torch.tensor([0, 1], device = dev))out.loss.backward()assert torch.isfinite(out.loss)assert model.score.weight.grad is not None and model.score.weight.grad.abs().sum() > 0@@ -684,7 +689,7 @@ def test_both_loaders_hand_the_planner_the_rewritten_plan():@needs_per_tensor_fp8@pytest.mark.skipif(- not has_real_cuda(), reason = "transformers dequantizes fp8 to bf16 without an accelerator"+ not has_real_accelerator(), reason = "transformers dequantizes fp8 to bf16 without an accelerator")def test_the_planner_sizes_a_modelopt_checkpoint_from_the_rewritten_plan(tmp_path):from transformers import AutoConfig, LlamaConfig@@ -846,7 +851,7 @@ def test_merged_save_detects_a_rewritten_modelopt_checkpoint_as_fp8(tmp_path, mo@needs_per_tensor_fp8[email protected](not has_real_cuda(), reason = "FastLanguageModel loads need an accelerator")[email protected](not has_real_cuda(), reason = "FastLanguageModel loads need CUDA")def test_merged_16bit_save_of_a_modelopt_lora_reloads_without_unsloth(tmp_path):"""Subprocess: FastLanguageModel patches the Llama classes process-wide."""import subprocess@@ -855,7 +860,7 @@ def test_merged_16bit_save_of_a_modelopt_lora_reloads_without_unsloth(tmp_path):from tokenizers import Tokenizer, models, pre_tokenizersfrom transformers import PreTrainedTokenizerFast- if torch.cuda.get_device_capability()[0] < 9:+ if dev == "cuda" and torch.cuda.get_device_capability()[0] < 9:pytest.skip("fp8 matmul needs sm_89+")ckpt, merged = tmp_path / "ckpt", tmp_path / "merged"ckpt.mkdir()@@ -876,7 +881,7 @@ with torch.no_grad():for name, p in model.named_parameters():if "lora_B" in name:p.normal_(0, 0.02)-x = torch.randint(0, 512, (2, 32), device = "cuda")+x = torch.randint(0, 512, (2, 32), device = {dev!r})model.eval()with torch.no_grad():lora = model.base_model.model.model.layers[0].self_attn.q_proj@@ -893,11 +898,11 @@ model.save_pretrained_merged({str(merged)!r}, tok, save_method = "merged_16bit")import sys, torchfrom transformers import AutoModelForCausalLMx, want, expected = torch.load({str(tmp_path / "want.pt")!r})-model = AutoModelForCausalLM.from_pretrained({str(merged)!r}, dtype = torch.bfloat16).cuda()+model = AutoModelForCausalLM.from_pretrained({str(merged)!r}, dtype = torch.bfloat16).to({dev!r})assert "unsloth" not in sys.modulesq = model.model.layers[0].self_attn.q_proj.weightwith torch.no_grad():- got = model(x.cuda()).logits.float().cpu()+ got = model(x.to({dev!r})).logits.float().cpu()w_rel = float((q.float().cpu() - expected).norm() / expected.norm())print("CHECK", q.dtype, w_rel, float((got - want).norm() / want.norm()))"""diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 5da8c40d4a9..c4a28ee385f 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -6044,10 +6044,10 @@ patch_hf_quantizer()def verify_fp8_support_if_applicable(model_config):quant_method = get_quant_type(model_config)- if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE != "cuda":+ if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE not in ("cuda", "xpu"):# In case of block quantized, we allow L4 because we fall back to torchao kernels.raise ValueError(- f"Unsloth: FP8 quantization is only supported on CUDA GPUs. You are using {DEVICE_TYPE}."+ f"Unsloth: FP8 quantization is only supported on CUDA and XPU GPUs. You are using {DEVICE_TYPE}.")if DEVICE_TYPE == "cuda":