Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

[XPU] Enable FP8 training on XPU (#12535)

JoshuaL3000cb67888d1b9f
diff --git a/tests/test_compressed_tensors_fp8_kernel_route.py b/tests/test_compressed_tensors_fp8_kernel_route.py
index 23f9870f3fd..bc1dc27fcb1 100644
--- a/tests/test_compressed_tensors_fp8_kernel_route.py
+++ b/tests/test_compressed_tensors_fp8_kernel_route.py
@@ -19,8 +19,12 @@ import types
import pytest
import torch
+from real_accelerator import has_real_cuda
-pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
+dev = "cuda" if has_real_cuda() else "xpu" if xpu_available else "cpu"
+
+pytestmark = pytest.mark.skipif(not (has_real_cuda() or xpu_available), reason = "needs CUDA or XPU")
ct_quant = pytest.importorskip("compressed_tensors.quantization")
@@ -63,8 +67,8 @@ def _ct_model(
weight_type = "float",
):
torch.manual_seed(0)
- lin = torch.nn.Linear(i, o, bias = bias, device = "cuda", dtype = torch.bfloat16)
- Wq, s, ref = _quantize(torch.randn(o, i, device = "cuda") * 0.02, strategy, block)
+ lin = torch.nn.Linear(i, o, bias = bias, device = dev, dtype = torch.bfloat16)
+ Wq, s, ref = _quantize(torch.randn(o, i, device = dev) * 0.02, strategy, block)
lin.weight = torch.nn.Parameter(Wq, requires_grad = False)
lin.weight_scale = torch.nn.Parameter(s, requires_grad = False)
kwargs = dict(num_bits = 8, type = weight_type, strategy = strategy, symmetric = True, dynamic = False)
@@ -92,7 +96,7 @@ def _ct_model(
def test_weight_dequant_transposed_row_scale(shape):
from unsloth.kernels.fp8 import weight_dequant
- Wq, s, ref = _quantize(torch.randn(*shape, device = "cuda") * 0.02, "channel")
+ Wq, s, ref = _quantize(torch.randn(*shape, device = dev) * 0.02, "channel")
torch.testing.assert_close(weight_dequant(Wq, s, torch.float32), ref)
torch.testing.assert_close(weight_dequant(Wq.t(), s, torch.float32), ref.t())
@@ -116,11 +120,11 @@ def test_fp8_modules_route_to_unsloth_kernels(strategy, shape, block, bias, monk
assert _route_compressed_tensors_fp8_to_unsloth(model) == 1
lin = model.lin
assert lin.weight.dtype == torch.float8_e4m3fn
- X = torch.randn(2, 5, shape[1], device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ X = torch.randn(2, 5, shape[1], device = dev, dtype = torch.bfloat16, requires_grad = True)
y = lin(X)
y_ref = X.float() @ ref.t() + (lin.bias.float() if bias else 0)
(dX,) = torch.autograd.grad(y.float().sum(), X)
- dX_ref = torch.ones(2, 5, shape[0], device = "cuda") @ ref
+ dX_ref = torch.ones(2, 5, shape[0], device = dev) @ ref
assert float((y.float() - y_ref).norm() / y_ref.norm()) < 0.05
assert float((dX.float() - dX_ref).norm() / dX_ref.norm()) < 0.01
@@ -185,9 +189,7 @@ def test_non_fp8_or_unsupported_modules_are_left_alone(monkeypatch):
model.lin.weight = torch.nn.Parameter(model.lin.weight.to(torch.bfloat16), requires_grad = False)
assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
model, _ = _ct_model(256, 256, "channel")
- model.lin.weight_scale = torch.nn.Parameter(
- torch.ones(3, 1, device = "cuda"), requires_grad = False
- )
+ model.lin.weight_scale = torch.nn.Parameter(torch.ones(3, 1, device = dev), requires_grad = False)
assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
model, _ = _ct_model(256, 256, "channel")
monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")
@@ -225,7 +227,7 @@ def test_single_token_fast_path_adds_the_bias_once(strategy, block, monkeypatch)
lin = model.lin
with torch.no_grad():
lin.bias.fill_(1.0)
- X = torch.randn(1, 1, 256, device = "cuda", dtype = torch.bfloat16)
+ X = torch.randn(1, 1, 256, device = dev, dtype = torch.bfloat16)
y = fast_linear_forward(lin, X)
y_ref = X.float() @ ref.t() + 1.0
assert float((y.float() - y_ref).norm() / y_ref.norm()) < 0.05
@@ -241,6 +243,7 @@ def test_a_model_mixing_fp8_and_other_schemes_is_not_partly_routed():
assert not hasattr(model.lin, "_unsloth_compressed_tensors_fp8")
[email protected](not has_real_cuda(), reason = "Decode GEMV needs a CUDA GPU")
@pytest.mark.parametrize(
"shape", [(2048, 2048), (256, 2048), (11008, 2048), (2048, 11008), (100, 300), (33, 7)]
)
@@ -252,7 +255,7 @@ def test_decode_gemv_matches_the_dequantized_matmul(shape, rows):
pytest.skip("triton cannot read float8_e4m3fn on this GPU")
model, ref = _ct_model(*shape, "channel")
W, s = model.lin.weight, model.lin.weight_scale
- X = torch.randn(*rows, shape[1], device = "cuda", dtype = torch.bfloat16)
+ X = torch.randn(*rows, shape[1], device = dev, dtype = torch.bfloat16)
with torch.no_grad():
assert can_use_fp8_rowwise_gemv(X, W, s)
y = fp8_rowwise_gemv(X, W, s)
@@ -263,17 +266,18 @@ def test_decode_gemv_matches_the_dequantized_matmul(shape, rows):
assert torch.equal(y, y_again)
[email protected](not has_real_cuda(), reason = "Decode GEMV needs a CUDA GPU")
def test_decode_gemv_refuses_what_it_cannot_compute():
from unsloth.kernels.fp8 import can_use_fp8_rowwise_gemv
model, _ = _ct_model(2048, 16384, "channel")
W, s = model.lin.weight, model.lin.weight_scale
- X = torch.randn(1, 16384, device = "cuda", dtype = torch.bfloat16)
+ X = torch.randn(1, 16384, device = dev, dtype = torch.bfloat16)
with torch.no_grad():
# 128x128 block grid here has 2048 == N elements.
- assert not can_use_fp8_rowwise_gemv(X, W, torch.ones(16, 128, device = "cuda"))
+ assert not can_use_fp8_rowwise_gemv(X, W, torch.ones(16, 128, device = dev))
assert not can_use_fp8_rowwise_gemv(
- torch.randn(2, 16384, device = "cuda", dtype = torch.bfloat16), W, s
+ torch.randn(2, 16384, device = dev, dtype = torch.bfloat16), W, s
)
assert not can_use_fp8_rowwise_gemv(X.float(), W, s)
assert not can_use_fp8_rowwise_gemv(X[:, :2048], W.t(), s)
@@ -282,7 +286,7 @@ def test_decode_gemv_refuses_what_it_cannot_compute():
def _fp8_kernel_unsupported_here():
from unsloth.kernels.fp8 import _fp8_kernel_unsupported
- return _fp8_kernel_unsupported(torch.empty(1, device = "cuda", dtype = torch.float8_e4m3fn))
+ return _fp8_kernel_unsupported(torch.empty(1, device = dev, dtype = torch.float8_e4m3fn))
def test_fp8_weights_off_the_gpu_keep_the_compressed_tensors_path():
@@ -300,13 +304,13 @@ def test_compiled_block_fp8_linear_passes_the_input_gradient():
torch._dynamo.reset()
torch.manual_seed(0)
- W = (torch.randn(256, 256, device = "cuda") * 0.05).to(torch.float8_e4m3fn)
- s = torch.rand(2, 2, device = "cuda") * 0.01 + 0.001
+ W = (torch.randn(256, 256, device = dev) * 0.05).to(torch.float8_e4m3fn)
+ s = torch.rand(2, 2, device = dev) * 0.01 + 0.001
W.block_size = s.block_size = [128, 128]
ref = W.float() * s.repeat_interleave(128, 0).repeat_interleave(128, 1)
- X = torch.randn(2, 5, 256, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ X = torch.randn(2, 5, 256, device = dev, dtype = torch.bfloat16, requires_grad = True)
(dX,) = torch.autograd.grad(fp8_linear(X, W, s).float().sum(), X)
- dX_ref = torch.ones(2, 5, 256, device = "cuda") @ ref
+ dX_ref = torch.ones(2, 5, 256, device = dev) @ ref
assert float((dX.float() - dX_ref).norm() / dX_ref.norm()) < 0.01
@@ -315,9 +319,9 @@ def test_rowwise_dequant_fallback_follows_the_input_dtype(dtype, monkeypatch):
from unsloth.kernels import fp8
monkeypatch.setattr(fp8, "_has_fbgemm_rowwise", lambda: False)
- W = (torch.randn(256, 512, device = "cuda") * 0.05).to(torch.float8_e4m3fn)
- s = torch.rand(256, 1, device = "cuda") * 0.01 + 0.001
- X = torch.randn(3, 512, device = "cuda", dtype = dtype, requires_grad = True)
+ W = (torch.randn(256, 512, device = dev) * 0.05).to(torch.float8_e4m3fn)
+ s = torch.rand(256, 1, device = dev) * 0.01 + 0.001
+ X = torch.randn(3, 512, device = dev, dtype = dtype, requires_grad = True)
y = fp8.FbgemmFp8Linear_matmul.apply(X, W, s, None)
(dX,) = torch.autograd.grad(y.float().sum(), X)
ref = W.float() * s
diff --git a/tests/test_fp8_compile_fullgraph.py b/tests/test_fp8_compile_fullgraph.py
index c3f98d73529..05d124d5fed 100644
--- a/tests/test_fp8_compile_fullgraph.py
+++ b/tests/test_fp8_compile_fullgraph.py
@@ -6,9 +6,13 @@ import pytest
torch = pytest.importorskip("torch")
+cuda_available = torch.cuda.is_available()
+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"
+
pytestmark = pytest.mark.skipif(
- not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),
- reason = "block-FP8 Triton kernels need CUDA sm89+",
+ not ((cuda_available and torch.cuda.get_device_capability() >= (8, 9)) or xpu_available),
+ reason = "block-FP8 Triton kernels need CUDA sm89+ or XPU",
)
@@ -25,9 +29,9 @@ def test_block_fp8_linear_compiles_fullgraph_and_matches_eager():
# Several weight shapes (the second triggers automatic dynamic shapes); K = 200 takes the dequant fallback.
for N, K in [(512, 256), (256, 256), (256, 768), (768, 256), (512, 200)]:
torch.manual_seed(0)
- X = torch.randn(256, K, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
- W = (torch.randn(N, K, device = "cuda") * 0.02).to(torch.float8_e4m3fn)
- s = torch.rand(-(-N // 128), -(-K // 128), device = "cuda") + 0.5
+ X = torch.randn(256, K, device = dev, dtype = torch.bfloat16, requires_grad = True)
+ W = (torch.randn(N, K, device = dev) * 0.02).to(torch.float8_e4m3fn)
+ s = torch.rand(-(-N // 128), -(-K // 128), device = dev) + 0.5
s.block_size = [128, 128]
compiled(X, W, s).backward()
diff --git a/tests/test_fp8_device_context.py b/tests/test_fp8_device_context.py
index 7e1569a2191..2eb99376ce0 100644
--- a/tests/test_fp8_device_context.py
+++ b/tests/test_fp8_device_context.py
@@ -169,24 +169,27 @@ def test_fp8_triton_launches_enter_tensor_device_context() -> None:
assert not (expected_launches & visitor.unguarded_launches)
-def _require_two_cuda_devices():
+def _require_two_accelerator_devices():
torch = pytest.importorskip("torch")
pytest.importorskip("triton")
- if not torch.cuda.is_available() or torch.cuda.device_count() < 2:
- pytest.skip("requires at least two CUDA devices")
- return torch
+ if torch.cuda.is_available() and torch.cuda.device_count() >= 2:
+ return "cuda", torch
+ if hasattr(torch, "xpu") and torch.xpu.is_available() and torch.xpu.device_count() >= 2:
+ return "xpu", torch
+ pytest.skip("requires at least two CUDA or XPU devices")
def test_weight_dequant_block_runs_on_tensor_device_when_current_device_differs() -> None:
- torch = _require_two_cuda_devices()
+ device, torch = _require_two_accelerator_devices()
+ accel = getattr(torch, device)
from unsloth.kernels.fp8 import weight_dequant_block
- previous_device = torch.cuda.current_device()
+ previous_device = accel.current_device()
try:
- torch.cuda.set_device(0)
- x = torch.arange(256 * 256, device = "cuda:1", dtype = torch.float32).reshape(256, 256)
- scales = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device = "cuda:1", dtype = torch.float32)
+ accel.set_device(0)
+ x = torch.arange(256 * 256, device = f"{device}:1", dtype = torch.float32).reshape(256, 256)
+ scales = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device = f"{device}:1", dtype = torch.float32)
actual = weight_dequant_block(x, scales, block_size = 128, dtype = torch.float32)
@@ -194,51 +197,54 @@ def test_weight_dequant_block_runs_on_tensor_device_when_current_device_differs(
expected = x * expanded_scales
assert actual.device == x.device
- assert torch.cuda.current_device() == 0
+ assert accel.current_device() == 0
torch.testing.assert_close(actual, expected)
finally:
- torch.cuda.set_device(previous_device)
+ accel.set_device(previous_device)
def test_act_quant_runs_on_tensor_device_when_current_device_differs() -> None:
- torch = _require_two_cuda_devices()
+ device, torch = _require_two_accelerator_devices()
+ accel = getattr(torch, device)
if not hasattr(torch, "float8_e4m3fn"):
pytest.skip("requires torch.float8_e4m3fn")
- if torch.cuda.get_device_capability(1)[0] < 9:
+ if device == "cuda" and torch.cuda.get_device_capability(1)[0] < 9:
pytest.skip("requires FP8-capable CUDA hardware")
from unsloth.kernels.fp8 import act_quant
- previous_device = torch.cuda.current_device()
+ previous_device = accel.current_device()
try:
- torch.cuda.set_device(0)
- x = torch.arange(256, device = "cuda:1", dtype = torch.float32).reshape(2, 128)
+ accel.set_device(0)
+ x = torch.arange(256, device = f"{device}:1", dtype = torch.float32).reshape(2, 128)
y, scales = act_quant(x, block_size = 128)
assert y.device == x.device
assert scales.device == x.device
- assert torch.cuda.current_device() == 0
+ assert accel.current_device() == 0
finally:
- torch.cuda.set_device(previous_device)
+ accel.set_device(previous_device)
def test_w8a8_block_fp8_matmul_triton_runs_on_tensor_device_when_current_device_differs() -> None:
- torch = _require_two_cuda_devices()
+ device, torch = _require_two_accelerator_devices()
+ accel = getattr(torch, device)
if not hasattr(torch, "float8_e4m3fn"):
pytest.skip("requires torch.float8_e4m3fn")
- if torch.cuda.get_device_capability(1)[0] < 9:
+ if device == "cuda" and torch.cuda.get_device_capability(1)[0] < 9:
pytest.skip("requires FP8-capable CUDA hardware")
from unsloth.kernels.fp8 import w8a8_block_fp8_matmul_triton
- previous_device = torch.cuda.current_device()
+ previous_device = accel.current_device()
try:
- torch.cuda.set_device(0)
- A = torch.ones((128, 128), device = "cuda:1", dtype = torch.float32).to(torch.float8_e4m3fn)
- B = torch.ones((128, 128), device = "cuda:1", dtype = torch.float32).to(torch.float8_e4m3fn)
- As = torch.ones((128, 1), device = "cuda:1", dtype = torch.float32)
- Bs = torch.ones((1, 1), device = "cuda:1", dtype = torch.float32)
+ accel.set_device(0)
+ dev1 = f"{device}:1"
+ A = torch.ones((128, 128), device = dev1, dtype = torch.float32).to(torch.float8_e4m3fn)
+ B = torch.ones((128, 128), device = dev1, dtype = torch.float32).to(torch.float8_e4m3fn)
+ As = torch.ones((128, 1), device = dev1, dtype = torch.float32)
+ Bs = torch.ones((1, 1), device = dev1, dtype = torch.float32)
actual = w8a8_block_fp8_matmul_triton(
A,
@@ -249,9 +255,9 @@ def test_w8a8_block_fp8_matmul_triton_runs_on_tensor_device_when_current_device_
output_dtype = torch.float32,
)
- expected = torch.full((128, 128), 128.0, device = "cuda:1", dtype = torch.float32)
+ expected = torch.full((128, 128), 128.0, device = dev1, dtype = torch.float32)
assert actual.device == A.device
- assert torch.cuda.current_device() == 0
+ assert accel.current_device() == 0
torch.testing.assert_close(actual, expected)
finally:
- torch.cuda.set_device(previous_device)
+ accel.set_device(previous_device)
diff --git a/tests/test_fp8_device_gate.py b/tests/test_fp8_device_gate.py
new file mode 100644
index 00000000000..38a6eee7f7a
--- /dev/null
+++ b/tests/test_fp8_device_gate.py
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# verify_fp8_support_if_applicable admits FP8 checkpoints on CUDA and XPU only.
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import unsloth # noqa: F401
+from unsloth.models import _utils
+
+
+def _config(quant_method):
+ return SimpleNamespace(quantization_config = {"quant_method": quant_method})
+
+
[email protected]("quant_method", ["fp8", "fbgemm_fp8"])
+def test_xpu_is_admitted(monkeypatch, quant_method):
+ monkeypatch.setattr(_utils, "DEVICE_TYPE", "xpu")
+ _utils.verify_fp8_support_if_applicable(_config(quant_method))
+
+
[email protected]("device_type", ["hip", "mps", "npu", "cpu"])
[email protected]("quant_method", ["fp8", "fbgemm_fp8"])
+def test_other_devices_are_still_refused(monkeypatch, device_type, quant_method):
+ monkeypatch.setattr(_utils, "DEVICE_TYPE", device_type)
+ with pytest.raises(ValueError, match = "FP8 quantization is only supported on"):
+ _utils.verify_fp8_support_if_applicable(_config(quant_method))
+
+
[email protected]("device_type", ["xpu", "hip", "mps", "cpu"])
+def test_non_fp8_checkpoints_load_anywhere(monkeypatch, device_type):
+ monkeypatch.setattr(_utils, "DEVICE_TYPE", device_type)
+ _utils.verify_fp8_support_if_applicable(_config("bitsandbytes"))
+ _utils.verify_fp8_support_if_applicable(SimpleNamespace())
+
+
+ "capability, quant_method, refused",
+ [
+ ((8, 0), "fp8", True),
+ ((8, 9), "fp8", False),
+ ((8, 9), "fbgemm_fp8", True),
+ ((9, 0), "fbgemm_fp8", False),
+ ],
+)
+def test_cuda_capability_rules_are_unchanged(monkeypatch, capability, quant_method, refused):
+ monkeypatch.setattr(_utils, "DEVICE_TYPE", "cuda")
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: capability)
+ monkeypatch.setattr(torch.cuda, "get_device_name", lambda *a: "spoofed")
+ if refused:
+ with pytest.raises(ValueError):
+ _utils.verify_fp8_support_if_applicable(_config(quant_method))
+ else:
+ _utils.verify_fp8_support_if_applicable(_config(quant_method))
diff --git a/tests/test_fp8_fbgemm_block_shapes.py b/tests/test_fp8_fbgemm_block_shapes.py
index 9a8804e2da4..2df3731f158 100644
--- a/tests/test_fp8_fbgemm_block_shapes.py
+++ b/tests/test_fp8_fbgemm_block_shapes.py
@@ -16,10 +16,11 @@ import pytest
import torch
cuda_available = torch.cuda.is_available()
+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"
-
-# Only the kernel battery needs fbgemm; the fallback tests below never reach it.
-pytestmark = pytest.mark.skipif(not cuda_available, reason = "needs CUDA")
+# Only the kernel battery needs fbgemm (CUDA-only); the fallback tests below never reach it.
+pytestmark = pytest.mark.skipif(not (cuda_available or xpu_available), reason = "needs CUDA or XPU")
def skip_without_fbgemm():
@@ -119,10 +120,10 @@ def test_output_tile_grid_battery_matches_reference():
(100, 136, 272),
(64, 8, 16),
]:
- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)
+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)
Wq, scale = _block_quantize_weight(W, block)
scale.block_size = block
- X = torch.randn(M, K, device = "cuda", dtype = torch.bfloat16)
+ X = torch.randn(M, K, device = dev, dtype = torch.bfloat16)
out = FP8_fbgemm_block_linear.apply(X, Wq, scale)
ref = _reference(X, Wq, scale, block)
@@ -136,10 +137,10 @@ def test_odd_k_uses_dequant_fallback():
torch.manual_seed(0)
block = [128, 128]
N, K = 320, 130 # K % 16 != 0 used to crash inside the CUTLASS kernel
- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)
+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)
Wq, scale = _block_quantize_weight(W, block)
scale.block_size = block
- X = torch.randn(4, K, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ X = torch.randn(4, K, device = dev, dtype = torch.bfloat16, requires_grad = True)
out = FP8_fbgemm_block_linear.apply(X, Wq, scale)
assert torch.isfinite(out).all()
@@ -156,10 +157,10 @@ def test_odd_n_uses_dequant_fallback():
torch.manual_seed(0)
block = [128, 128]
N, K = 250, 256 # N % 8 != 0 used to crash inside the CUTLASS kernel
- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)
+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)
Wq, scale = _block_quantize_weight(W, block)
scale.block_size = block
- X = torch.randn(4, K, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ X = torch.randn(4, K, device = dev, dtype = torch.bfloat16, requires_grad = True)
out = FP8_fbgemm_block_linear.apply(X, Wq, scale)
ref = _reference(X.detach(), Wq, scale, block)
@@ -174,10 +175,10 @@ def test_non_square_block_uses_dequant_fallback():
torch.manual_seed(0)
block = [128, 64] # kernel only implements 128x128x128, used to crash
N, K = 256, 256
- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)
+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)
Wq, scale = _block_quantize_weight(W, block)
scale.block_size = block
- X = torch.randn(64, K, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ X = torch.randn(64, K, device = dev, dtype = torch.bfloat16, requires_grad = True)
out = FP8_fbgemm_block_linear.apply(X, Wq, scale)
ref = _reference(X.detach(), Wq, scale, block)
@@ -197,9 +198,9 @@ def test_inputs_the_kernel_rejects_use_dequant_fallback(kind):
block = [128, 128]
# strided_3d needs a shape the kernel rejects too, else it stays on the fast path
N, K = (250, 130) if kind == "strided_3d" else (256, 256)
- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)
+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)
Wq, scale = _block_quantize_weight(W, block)
- X = torch.randn(8, K, device = "cuda", dtype = torch.bfloat16)
+ X = torch.randn(8, K, device = dev, dtype = torch.bfloat16)
if kind.startswith("per_tensor"):
scale = scale.amax().clone()
@@ -210,7 +211,7 @@ def test_inputs_the_kernel_rejects_use_dequant_fallback(kind):
if kind == "bf16_scale":
scale = scale.to(torch.bfloat16)
else:
- X = torch.randn(2, 4, K * 2, device = "cuda", dtype = torch.bfloat16)[..., ::2]
+ X = torch.randn(2, 4, K * 2, device = dev, dtype = torch.bfloat16)[..., ::2]
scale.block_size = block
ref = _reference(X, Wq, scale, block)
@@ -229,13 +230,13 @@ def test_transposed_weight_swaps_block_axes(block, N, K):
from unsloth.kernels.fp8 import FP8_fbgemm_block_linear
torch.manual_seed(0)
- W = torch.randn(N, K, device = "cuda", dtype = torch.bfloat16)
+ W = torch.randn(N, K, device = dev, dtype = torch.bfloat16)
Wq, scale = _block_quantize_weight(W, block)
scale.block_size = block
Wt = Wq.t()
Wt.block_size = block
- dY = torch.randn(8, N, device = "cuda", dtype = torch.bfloat16)
+ dY = torch.randn(8, N, device = dev, dtype = torch.bfloat16)
out = FP8_fbgemm_block_linear.apply(dY, Wt, scale)
ref = (dY.float() @ _dequant(Wq, scale, block)).to(dY.dtype)
assert out.shape == (8, K)
diff --git a/tests/test_fp8_leftover_dequant_16bit.py b/tests/test_fp8_leftover_dequant_16bit.py
index 68a0f9e6565..00856309ee8 100644
--- a/tests/test_fp8_leftover_dequant_16bit.py
+++ b/tests/test_fp8_leftover_dequant_16bit.py
@@ -22,7 +22,7 @@ import tempfile
import pytest
from real_accelerator import (
- has_real_cuda,
+ has_real_accelerator,
) # tests/_shared, on sys.path via tests/conftest.py
import torch
from torch import nn
@@ -220,19 +220,22 @@ def test_scale_grid_that_does_not_tile_is_refused():
# has_real_cuda(): another test spoofs torch.cuda.is_available() process-wide.
[email protected](not has_real_cuda(), reason = "needs CUDA")
[email protected](not has_real_accelerator(), reason = "needs CUDA or XPU")
def test_out_of_memory_on_the_device_is_finished_through_the_cpu(monkeypatch):
from unsloth.models import loader_utils
+ xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
+ cuda_available = torch.cuda.is_available()
+ dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"
model, tensors, expected = _build()
- model = model.to("cuda")
+ model = model.to(dev)
real = loader_utils._fp8_scale_grid_dequant
- calls = {"cuda": 0, "cpu": 0}
+ calls = {dev: 0, "cpu": 0}
def flaky(quantized, scale, out_dtype):
calls[quantized.device.type] += 1
- if quantized.device.type == "cuda":
- raise torch.OutOfMemoryError("CUDA out of memory (simulated)")
+ if quantized.device.type == dev:
+ raise torch.OutOfMemoryError(f"{dev} out of memory (simulated)")
return real(quantized, scale, out_dtype)
monkeypatch.setattr(loader_utils, "_fp8_scale_grid_dequant", flaky)
@@ -240,8 +243,8 @@ def test_out_of_memory_on_the_device_is_finished_through_the_cpu(monkeypatch):
_write_checkpoint(d, tensors)
done, skipped = loader_utils._dequantize_leftover_fp8_params(model, d, torch.bfloat16)
assert done == 2
- assert calls == {"cuda": 2, "cpu": 2}
- assert model.experts.gate_up_proj.device.type == "cuda"
+ assert calls == {dev: 2, "cpu": 2}
+ assert model.experts.gate_up_proj.device.type == dev
assert model.experts.gate_up_proj.dtype == torch.bfloat16
assert torch.equal(model.experts.gate_up_proj.detach().cpu(), expected["experts.gate_up_proj"])
assert torch.equal(model.experts.down_proj.detach().cpu(), expected["experts.down_proj"])
diff --git a/tests/test_fp8_rowwise_backend.py b/tests/test_fp8_rowwise_backend.py
index 917897bd733..422bbe89913 100644
--- a/tests/test_fp8_rowwise_backend.py
+++ b/tests/test_fp8_rowwise_backend.py
@@ -6,9 +6,13 @@ import pytest
torch = pytest.importorskip("torch")
+cuda_available = torch.cuda.is_available()
+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"
+
pytestmark = pytest.mark.skipif(
- not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),
- reason = "FP8 GEMMs need CUDA sm89+",
+ not ((cuda_available and torch.cuda.get_device_capability() >= (8, 9)) or xpu_available),
+ reason = "FP8 GEMMs need CUDA sm89+ or XPU",
)
@@ -20,13 +24,13 @@ def F():
def _rowwise_weight(N, K):
- w = torch.randn(N, K, device = "cuda") * 0.02
+ w = torch.randn(N, K, device = dev) * 0.02
scale = (w.abs().amax(1, keepdim = True) / 448).float()
return (w / scale).to(torch.float8_e4m3fn), scale
def _backends(F):
- device = torch.device("cuda", torch.cuda.current_device())
+ device = torch.device(dev, 0)
ones = lambda n: torch.ones(n, dtype = torch.float32, device = device)
out = ["dequant"]
if F._rowwise_gemm_works(
@@ -42,9 +46,9 @@ def _backends(F):
def test_backend_is_one_that_runs_here(F):
- backend = F._fp8_rowwise_backend(torch.device("cuda", torch.cuda.current_device()))
+ backend = F._fp8_rowwise_backend(torch.device(dev, 0))
assert backend in ("fbgemm", "scaled_mm", "dequant")
- X = torch.randn(64, 256, device = "cuda", dtype = torch.bfloat16)
+ X = torch.randn(64, 256, device = dev, dtype = torch.bfloat16)
w, s = _rowwise_weight(512, 256)
y = F.fp8_linear(X, w, s)
ref = X.float() @ (w.float() * s).t()
@@ -58,16 +62,16 @@ def test_every_backend_matches_reference(F, backend):
pytest.skip(f"{backend} is not usable on this GPU")
torch.manual_seed(0)
w, s = _rowwise_weight(768, 512)
- X = torch.randn(2, 48, 512, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ X = torch.randn(2, 48, 512, device = dev, dtype = torch.bfloat16, requires_grad = True)
# FbgemmFp8Linear stores its bias in float32.
- bias = torch.randn(768, device = "cuda", dtype = torch.float32)
+ bias = torch.randn(768, device = dev, dtype = torch.float32)
y = F.FbgemmFp8Linear_matmul.apply(X, w, s, bias, backend)
W = w.float() * s
ref = X.detach().float() @ W.t() + bias.float()
assert y.shape == (2, 48, 768) and y.dtype == torch.bfloat16
assert ((y.float() - ref).norm() / ref.norm()) < 0.05
y.float().sum().backward()
- ref_dx = torch.ones(2, 48, 768, device = "cuda") @ W
+ ref_dx = torch.ones(2, 48, 768, device = dev) @ W
assert ((X.grad.float() - ref_dx).norm() / ref_dx.norm()) < 0.01
# Decode-sized calls take the dequant branch on the scaled_mm backend.
assert (
@@ -80,10 +84,10 @@ def test_quantize_matches_fbgemm(F):
if not hasattr(torch.ops.fbgemm, "quantize_fp8_per_row"):
pytest.skip("FBGEMM not installed")
torch.manual_seed(0)
- x = torch.randn(256, 1024, device = "cuda", dtype = torch.bfloat16)
+ x = torch.randn(256, 1024, device = dev, dtype = torch.bfloat16)
x[3] *= 1000
x[5] = 0
- for scale_ub in (None, torch.tensor([30.0], device = "cuda")):
+ for scale_ub in (None, torch.tensor([30.0], device = dev)):
q_ref, s_ref = torch.ops.fbgemm.quantize_fp8_per_row(x, scale_ub = scale_ub)
q, s = F._quantize_fp8_per_row(x, scale_ub)
# FBGEMM's fast-math division can land 1 ulp lower, flipping exact rounding ties by one FP8 step.
diff --git a/tests/test_fp8_to_nf4_load.py b/tests/test_fp8_to_nf4_load.py
index 09a526b640c..da324ae0a82 100644
--- a/tests/test_fp8_to_nf4_load.py
+++ b/tests/test_fp8_to_nf4_load.py
@@ -33,8 +33,12 @@ _BLOCK = 128
_TOKENIZER_REPO = "trl-internal-testing/tiny-Qwen3ForCausalLM"
_KEEP_16BIT = "model.layers.1.self_attn.o_proj"
-needs_cuda = pytest.mark.skipif(
- not torch.cuda.is_available(), reason = "bitsandbytes 4bit needs a GPU"
+cuda_available = torch.cuda.is_available()
+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"
+
+needs_gpu = pytest.mark.skipif(
+ not (cuda_available or xpu_available), reason = "bitsandbytes 4bit needs a GPU"
)
@@ -51,8 +55,8 @@ needs_feature = pytest.mark.skipif(
reason = f"fp8 -> 4bit loading unavailable: {_feature_unavailable()}",
)
needs_fp8_gpu = pytest.mark.skipif(
- not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),
- reason = "fp8 checkpoints load natively only on sm89+",
+ not ((cuda_available and torch.cuda.get_device_capability() >= (8, 9)) or xpu_available),
+ reason = "fp8 checkpoints load natively only on sm89+ or XPU",
)
@@ -239,10 +243,10 @@ def _free(*models):
import gc
gc.collect()
- torch.cuda.empty_cache()
+ getattr(torch, dev, torch.cuda).empty_cache()
-@needs_cuda
+@needs_gpu
@needs_feature
def test_dense_fp8_loads_nf4_bit_identical_to_dequantized_bf16(dense_pair, capsys):
fp8_dir, deq_dir = dense_pair
@@ -260,7 +264,7 @@ def test_dense_fp8_loads_nf4_bit_identical_to_dequantized_bf16(dense_pair, capsy
_assert_same(ours, theirs)
-@needs_cuda
+@needs_gpu
@needs_feature
def test_moe_fp8_loads_nf4_experts_bit_identical(moe_pair):
fp8_dir, deq_dir = moe_pair
@@ -281,7 +285,7 @@ def test_moe_fp8_loads_nf4_experts_bit_identical(moe_pair):
_assert_same(ours, theirs)
-@needs_cuda
+@needs_gpu
@needs_feature
def test_checkpoint_16bit_modules_stay_16bit(dense_pair):
from safetensors.torch import load_file
@@ -322,7 +326,7 @@ def test_default_load_in_4bit_and_kill_switch_keep_fp8(dense_pair, monkeypatch):
_free(model)
-@needs_cuda
+@needs_gpu
@needs_feature
def test_fp8_keys_the_model_does_not_have_are_ignored(dense_pair, tmp_path):
"""MTP layers ship in fp8 but transformers drops them as unexpected (DeepSeek-V3, GLM-4 MoE,
@@ -347,7 +351,7 @@ def test_fp8_keys_the_model_does_not_have_are_ignored(dense_pair, tmp_path):
_assert_same(ours, theirs)
-@needs_cuda
+@needs_gpu
@needs_feature
def test_quantize_16bit_switch_matches_plain_4bit_skip_list(dense_pair, monkeypatch):
fp8_dir, deq_dir = dense_pair
@@ -370,7 +374,7 @@ def _zoo_reads_fp8_block_size_from_disk():
return hasattr(saving_utils, "_fp8_block_size_on_disk")
-@needs_cuda
+@needs_gpu
@needs_feature
@pytest.mark.skipif(
not _zoo_reads_fp8_block_size_from_disk(),
diff --git a/tests/test_fp8_transposed_weight.py b/tests/test_fp8_transposed_weight.py
index 54efd358738..9ada061c2b4 100644
--- a/tests/test_fp8_transposed_weight.py
+++ b/tests/test_fp8_transposed_weight.py
@@ -7,7 +7,11 @@ import pytest
torch = pytest.importorskip("torch")
-pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
+cuda_available = torch.cuda.is_available()
+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"
+
+pytestmark = pytest.mark.skipif(not (cuda_available or xpu_available), reason = "needs CUDA or XPU")
@pytest.fixture(scope = "module")
@@ -18,7 +22,7 @@ def F():
def _rowwise(n):
- w = torch.randn(n, n, device = "cuda") * torch.linspace(0.01, 1.0, n, device = "cuda")[:, None]
+ w = torch.randn(n, n, device = dev) * torch.linspace(0.01, 1.0, n, device = dev)[:, None]
s = (w.abs().amax(1, keepdim = True) / 448).float()
q = (w / s).to(torch.float8_e4m3fn)
return q, s, q.float() * s
@@ -27,18 +31,18 @@ def _rowwise(n):
def _block(n, bs):
torch.manual_seed(0)
p = -(-n // bs)
- s = torch.rand(p, p, device = "cuda") * 4 + 0.1
+ s = torch.rand(p, p, device = dev) * 4 + 0.1
full = s.repeat_interleave(bs, 0)[:n].repeat_interleave(bs, 1)[:, :n]
- q = (torch.randn(n, n, device = "cuda") * 0.02 / full).to(torch.float8_e4m3fn)
+ q = (torch.randn(n, n, device = dev) * 0.02 / full).to(torch.float8_e4m3fn)
s.block_size = [bs, bs]
return q, s, q.float() * full
def _block_rect(n, k, bs):
torch.manual_seed(0)
- s = torch.rand(-(-n // bs), -(-k // bs), device = "cuda") * 4 + 0.1
+ s = torch.rand(-(-n // bs), -(-k // bs), device = dev) * 4 + 0.1
full = s.repeat_interleave(bs, 0)[:n].repeat_interleave(bs, 1)[:, :k]
- q = (torch.randn(n, k, device = "cuda") * 0.02 / full).to(torch.float8_e4m3fn)
+ q = (torch.randn(n, k, device = dev) * 0.02 / full).to(torch.float8_e4m3fn)
s.block_size = [bs, bs]
return q, s, q.float() * full
@@ -62,7 +66,7 @@ def test_fp8_linear_on_transposed_view(F, kind):
q, s, ref = _block_rect(256, 384, 128)
else:
q, s, ref = _block(256, 128 if kind == "block128" else 64)
- X = torch.randn(64, q.shape[0], device = "cuda", dtype = torch.bfloat16)
+ X = torch.randn(64, q.shape[0], device = dev, dtype = torch.bfloat16)
y = F.fp8_linear(X, q.t(), s)
expect = X.float() @ ref
assert ((y.float() - expect).norm() / expect.norm()) < 0.05
diff --git a/tests/test_fp8linear_high_precision_weight.py b/tests/test_fp8linear_high_precision_weight.py
index 5d331fc221a..e9150f16f66 100644
--- a/tests/test_fp8linear_high_precision_weight.py
+++ b/tests/test_fp8linear_high_precision_weight.py
@@ -6,7 +6,11 @@ import pytest
import torch
import torch.nn.functional as F
-pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
+cuda_available = torch.cuda.is_available()
+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"
+
+pytestmark = pytest.mark.skipif(not (cuda_available or xpu_available), reason = "needs CUDA or XPU")
@pytest.mark.parametrize(
@@ -18,17 +22,15 @@ def test_bf16_weight_in_fp8linear_runs_plain_linear(bias):
FP8Linear = fp8.FP8Linear
if FP8Linear is None:
pytest.skip("this transformers has no FP8Linear")
- layer = FP8Linear(256, 128, block_size = (128, 128)).cuda()
+ layer = FP8Linear(256, 128, block_size = (128, 128)).to(dev)
torch.manual_seed(0)
layer.weight = torch.nn.Parameter(
- torch.randn(128, 256, device = "cuda", dtype = torch.bfloat16) * 0.05, requires_grad = False
+ torch.randn(128, 256, device = dev, dtype = torch.bfloat16) * 0.05, requires_grad = False
)
layer.bias = (
- torch.nn.Parameter(torch.randn(128, device = "cuda", dtype = bias))
- if bias is not None
- else None
+ torch.nn.Parameter(torch.randn(128, device = dev, dtype = bias)) if bias is not None else None
)
- x = torch.randn(4, 7, 256, device = "cuda", dtype = torch.bfloat16)
+ x = torch.randn(4, 7, 256, device = dev, dtype = torch.bfloat16)
out = layer(x)
assert out.dtype == torch.bfloat16
ref = F.linear(x, layer.weight, None if layer.bias is None else layer.bias.to(x.dtype))
diff --git a/tests/test_fp8linear_module_block_size.py b/tests/test_fp8linear_module_block_size.py
index 58016fc7cff..b722bbc2800 100644
--- a/tests/test_fp8linear_module_block_size.py
+++ b/tests/test_fp8linear_module_block_size.py
@@ -5,9 +5,13 @@
import pytest
import torch
+cuda_available = torch.cuda.is_available()
+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
+dev = "cuda" if cuda_available else "xpu" if xpu_available else "cpu"
+
pytestmark = pytest.mark.skipif(
- not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),
- reason = "block-FP8 kernels need CUDA sm89+",
+ not ((cuda_available and torch.cuda.get_device_capability() >= (8, 9)) or xpu_available),
+ reason = "block-FP8 kernels need CUDA sm89+ or XPU",
)
@@ -24,12 +28,12 @@ def _make_layer(out_features, in_features, block, scale_fmt):
layer = fp8.FP8Linear(in_features, out_features, block_size = block, **extra)
except TypeError:
pytest.skip("this transformers FP8Linear takes no scale_fmt")
- layer = layer.cuda()
+ layer = layer.to(dev)
torch.manual_seed(0)
- weight = (torch.randn(out_features, in_features, device = "cuda") * 0.5).to(torch.float8_e4m3fn)
+ weight = (torch.randn(out_features, in_features, device = dev) * 0.5).to(torch.float8_e4m3fn)
# Power-of-two scales so float and ue8m0 formats hold the same values.
grid = (-(-out_features // block[0]), -(-in_features // block[1]))
- scale = torch.exp2(torch.randint(-9, -4, grid, device = "cuda").float())
+ scale = torch.exp2(torch.randint(-9, -4, grid, device = dev).float())
layer.weight = torch.nn.Parameter(weight, requires_grad = False)
layer.weight_scale_inv = torch.nn.Parameter(
scale.to(layer.weight_scale_inv.dtype), requires_grad = False
@@ -53,7 +57,7 @@ def test_fp8linear_uses_module_block_size(block, scale_fmt):
out_features, in_features = 512, 1024
layer, weight, scale = _make_layer(out_features, in_features, block, scale_fmt)
torch.manual_seed(1)
- X = torch.randn(3, 64, in_features, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ X = torch.randn(3, 64, in_features, device = dev, dtype = torch.bfloat16, requires_grad = True)
out = layer(X)
assert out.shape == (3, 64, out_features) and out.dtype == torch.bfloat16
diff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.py
index e22e282e1b7..4612ef3d0e0 100644
--- a/tests/test_modelopt_fp8_loading.py
+++ b/tests/test_modelopt_fp8_loading.py
@@ -12,7 +12,7 @@ from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
-from real_accelerator import has_real_cuda
+from real_accelerator import has_real_cuda, has_real_accelerator
import unsloth # noqa: F401
from unsloth.models.modelopt_fp8 import (
@@ -25,6 +25,9 @@ from unsloth.models.modelopt_fp8 import (
)
from unsloth.models.loader_utils import check_and_disable_bitsandbytes_loading
+xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
+dev = "cuda" if has_real_cuda() else "xpu" if xpu_available else "cpu"
+
def _sarvam_quant(**overrides):
quant = {
@@ -435,9 +438,9 @@ def _write_tiny_modelopt_llama(path):
def test_tiny_modelopt_llama_round_trip(tmp_path, dequantize):
from transformers import AutoConfig, AutoModelForCausalLM, FineGrainedFP8Config
- if torch.cuda.get_device_capability()[0] < 9 and not dequantize:
+ if dev == "cuda" and torch.cuda.get_device_capability()[0] < 9 and not dequantize:
pytest.skip("fp8 matmul needs sm_89+")
- reference = _write_tiny_modelopt_llama(str(tmp_path)).cuda()
+ reference = _write_tiny_modelopt_llama(str(tmp_path)).to(dev)
config = AutoConfig.from_pretrained(str(tmp_path))
plan = arm_modelopt_fp8_loading(config, verbose = False)
kwargs = {}
@@ -445,7 +448,7 @@ def test_tiny_modelopt_llama_round_trip(tmp_path, dequantize):
extra = {"dequantize": True} if dequantize else {}
kwargs["quantization_config"] = FineGrainedFP8Config.from_dict(dict(plan), **extra)
model = AutoModelForCausalLM.from_pretrained(
- str(tmp_path), config = config, dtype = torch.bfloat16, device_map = "cuda", **kwargs
+ str(tmp_path), config = config, dtype = torch.bfloat16, device_map = dev, **kwargs
)
q_proj = model.model.layers[0].self_attn.q_proj
if dequantize:
@@ -454,7 +457,7 @@ def test_tiny_modelopt_llama_round_trip(tmp_path, dequantize):
else:
assert q_proj.weight.dtype == torch.float8_e4m3fn
assert float(q_proj.activation_scale) == pytest.approx(0.02)
- x = torch.randint(0, 512, (2, 32), device = "cuda")
+ x = torch.randint(0, 512, (2, 32), device = dev)
with torch.no_grad():
got = model(x).logits.float()
want = reference(x).logits.float()
@@ -479,7 +482,7 @@ def test_rewrite_follows_who_loads_the_weights():
assert "AUTO_QUANTIZATION_CONFIG_MAPPING[quant_method]" not in vision_source
[email protected](not has_real_cuda(), reason = "FastModel loads need an accelerator")
[email protected](not has_real_accelerator(), reason = "FastModel loads need an accelerator")
def test_a_declined_modelopt_format_still_refuses_to_load_in_process(tmp_path):
"""Subprocess: FastModel compiles and rebinds the Llama classes process-wide before it refuses."""
import subprocess
@@ -515,7 +518,9 @@ else:
@needs_per_tensor_fp8
[email protected](not has_real_cuda(), reason = "FastLanguageModel loads need an accelerator")
+ not has_real_accelerator(), reason = "FastLanguageModel loads need an accelerator"
+)
def test_fast_llama_checks_fp8_hardware_on_the_rewritten_config(tmp_path):
"""Subprocess: FastLanguageModel patches the Llama classes process-wide."""
import subprocess
@@ -646,7 +651,7 @@ def test_tiny_modelopt_llama_loads_a_classification_head(tmp_path):
from transformers import AutoConfig, AutoModelForSequenceClassification
from unsloth.models.modelopt_fp8 import keep_task_heads_unquantized
- if torch.cuda.get_device_capability()[0] < 9:
+ if dev == "cuda" and torch.cuda.get_device_capability()[0] < 9:
pytest.skip("fp8 matmul needs sm_89+")
_write_tiny_modelopt_llama(str(tmp_path))
config = AutoConfig.from_pretrained(str(tmp_path), num_labels = 2, pad_token_id = 0)
@@ -655,13 +660,13 @@ def test_tiny_modelopt_llama_loads_a_classification_head(tmp_path):
kwargs = {}
pop_modelopt_key_mapping(config, kwargs)
model = AutoModelForSequenceClassification.from_pretrained(
- str(tmp_path), config = config, dtype = torch.bfloat16, device_map = "cuda", **kwargs
+ str(tmp_path), config = config, dtype = torch.bfloat16, device_map = dev, **kwargs
)
assert type(model.score) is nn.Linear
assert model.score.weight.dtype == torch.bfloat16
assert model.model.layers[0].self_attn.q_proj.weight.dtype == torch.float8_e4m3fn
- x = torch.randint(1, 512, (2, 16), device = "cuda")
- out = model(x, labels = torch.tensor([0, 1], device = "cuda"))
+ x = torch.randint(1, 512, (2, 16), device = dev)
+ out = model(x, labels = torch.tensor([0, 1], device = dev))
out.loss.backward()
assert torch.isfinite(out.loss)
assert model.score.weight.grad is not None and model.score.weight.grad.abs().sum() > 0
@@ -684,7 +689,7 @@ def test_both_loaders_hand_the_planner_the_rewritten_plan():
@needs_per_tensor_fp8
@pytest.mark.skipif(
- not has_real_cuda(), reason = "transformers dequantizes fp8 to bf16 without an accelerator"
+ not has_real_accelerator(), reason = "transformers dequantizes fp8 to bf16 without an accelerator"
)
def test_the_planner_sizes_a_modelopt_checkpoint_from_the_rewritten_plan(tmp_path):
from transformers import AutoConfig, LlamaConfig
@@ -846,7 +851,7 @@ def test_merged_save_detects_a_rewritten_modelopt_checkpoint_as_fp8(tmp_path, mo
@needs_per_tensor_fp8
[email protected](not has_real_cuda(), reason = "FastLanguageModel loads need an accelerator")
[email protected](not has_real_cuda(), reason = "FastLanguageModel loads need CUDA")
def test_merged_16bit_save_of_a_modelopt_lora_reloads_without_unsloth(tmp_path):
"""Subprocess: FastLanguageModel patches the Llama classes process-wide."""
import subprocess
@@ -855,7 +860,7 @@ def test_merged_16bit_save_of_a_modelopt_lora_reloads_without_unsloth(tmp_path):
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import PreTrainedTokenizerFast
- if torch.cuda.get_device_capability()[0] < 9:
+ if dev == "cuda" and torch.cuda.get_device_capability()[0] < 9:
pytest.skip("fp8 matmul needs sm_89+")
ckpt, merged = tmp_path / "ckpt", tmp_path / "merged"
ckpt.mkdir()
@@ -876,7 +881,7 @@ with torch.no_grad():
for name, p in model.named_parameters():
if "lora_B" in name:
p.normal_(0, 0.02)
-x = torch.randint(0, 512, (2, 32), device = "cuda")
+x = torch.randint(0, 512, (2, 32), device = {dev!r})
model.eval()
with torch.no_grad():
lora = model.base_model.model.model.layers[0].self_attn.q_proj
@@ -893,11 +898,11 @@ model.save_pretrained_merged({str(merged)!r}, tok, save_method = "merged_16bit")
import sys, torch
from transformers import AutoModelForCausalLM
x, want, expected = torch.load({str(tmp_path / "want.pt")!r})
-model = AutoModelForCausalLM.from_pretrained({str(merged)!r}, dtype = torch.bfloat16).cuda()
+model = AutoModelForCausalLM.from_pretrained({str(merged)!r}, dtype = torch.bfloat16).to({dev!r})
assert "unsloth" not in sys.modules
q = model.model.layers[0].self_attn.q_proj.weight
with torch.no_grad():
- got = model(x.cuda()).logits.float().cpu()
+ got = model(x.to({dev!r})).logits.float().cpu()
w_rel = float((q.float().cpu() - expected).norm() / expected.norm())
print("CHECK", q.dtype, w_rel, float((got - want).norm() / want.norm()))
"""
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 5da8c40d4a9..c4a28ee385f 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -6044,10 +6044,10 @@ patch_hf_quantizer()
def verify_fp8_support_if_applicable(model_config):
quant_method = get_quant_type(model_config)
- if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE != "cuda":
+ if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE not in ("cuda", "xpu"):
# In case of block quantized, we allow L4 because we fall back to torchao kernels.
raise ValueError(
- f"Unsloth: FP8 quantization is only supported on CUDA GPUs. You are using {DEVICE_TYPE}."
+ f"Unsloth: FP8 quantization is only supported on CUDA and XPU GPUs. You are using {DEVICE_TYPE}."
)
if DEVICE_TYPE == "cuda":