Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

feat(device_type): add Ascend NPU device detection (#10686)

li-lizhec78fc38bab86
diff --git a/tests/test_device_helpers.py b/tests/test_device_helpers.py
index 78bf40c505c..c57d9030d09 100644
--- a/tests/test_device_helpers.py
+++ b/tests/test_device_helpers.py
@@ -3,6 +3,7 @@ import sys
import types
from pathlib import Path
+import pytest
from packaging.version import Version
@@ -201,3 +202,135 @@ def test_model_call_sites_use_shared_cache_dispatch():
assert "clean_gpu_cache()" in gemma2_source
assert "torch.cuda.empty_cache()" not in granite_source
assert "clean_gpu_cache()" in granite_source
+
+
+NPU_PROPERTIES = types.SimpleNamespace(
+ name = "Ascend910B2",
+ total_memory = 60 * 1024**3,
+ # major/minor are std::optional on real hardware, so None. The npu arm must not read them.
+ major = None,
+ minor = None,
+)
+
+
+def _npu_backend(
+ *,
+ available = True,
+ device_count = 4,
+ properties = NPU_PROPERTIES,
+ calls = None,
+):
+ def _is_available():
+ if available == "raise":
+ raise RuntimeError("npu driver not found")
+ return available
+
+ return types.SimpleNamespace(
+ is_available = _is_available,
+ device_count = lambda: device_count,
+ get_device_properties = lambda _index: properties,
+ empty_cache = lambda: (calls if calls is not None else []).append("empty_cache"),
+ current_device = lambda: 0,
+ )
+
+
+def test_npu_detected_with_count_and_stats(monkeypatch):
+ calls = []
+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)
+ torch.npu = _npu_backend(calls = calls)
+
+ device_type = _load_device_type(monkeypatch, torch)
+
+ assert device_type.DEVICE_TYPE == "npu"
+ assert device_type.DEVICE_TYPE_TORCH == "npu"
+ assert device_type.DEVICE_COUNT == 4
+ assert device_type._DEVICE_MODULE is torch.npu
+
+ name, snippet, max_memory = device_type.get_device_stats()
+ assert (name, snippet, max_memory) == ("Ascend910B2. ", "Ascend NPU.", 60.0)
+
+ device_type.clean_gpu_cache()
+ assert calls == ["empty_cache"]
+ assert device_type.get_current_device() == 0
+
+
+def test_npu_blank_name_falls_back(monkeypatch):
+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)
+ torch.npu = _npu_backend(
+ properties = types.SimpleNamespace(name = "", total_memory = 60 * 1024**3),
+ )
+
+ device_type = _load_device_type(monkeypatch, torch)
+ name, snippet, _ = device_type.get_device_stats()
+
+ assert name == "Ascend NPU Device. "
+ assert snippet == "Ascend NPU."
+
+
+def test_cuda_wins_over_npu(monkeypatch):
+ torch = _fake_torch(properties = CUDA_PROPERTIES)
+ torch.npu = _npu_backend()
+
+ device_type = _load_device_type(monkeypatch, torch)
+
+ assert device_type.DEVICE_TYPE == "cuda"
+
+
+def test_xpu_wins_over_npu(monkeypatch):
+ xpu_backend = types.SimpleNamespace(
+ is_available = lambda: True,
+ device_count = lambda: 2,
+ empty_cache = lambda: None,
+ current_device = lambda: 0,
+ get_device_properties = lambda _index: types.SimpleNamespace(
+ name = "Intel Arc",
+ total_memory = 8 * 1024**3,
+ ),
+ )
+ torch = _fake_torch(
+ properties = CUDA_PROPERTIES,
+ xpu_backend = xpu_backend,
+ cuda_available = False,
+ )
+ torch.npu = _npu_backend()
+
+ device_type = _load_device_type(monkeypatch, torch)
+
+ assert device_type.DEVICE_TYPE == "xpu"
+ assert device_type.DEVICE_COUNT == 2
+
+
+def test_npu_probe_survives_raising_is_available(monkeypatch):
+ # Must stay a clean NotImplementedError, not a RuntimeError escaping `import unsloth`.
+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)
+ torch.npu = _npu_backend(available = "raise")
+
+ with pytest.raises(NotImplementedError):
+ _load_device_type(monkeypatch, torch)
+
+
+def test_npu_unavailable_is_not_selected(monkeypatch):
+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)
+ torch.npu = _npu_backend(available = False)
+
+ with pytest.raises(NotImplementedError):
+ _load_device_type(monkeypatch, torch)
+
+
+def test_unsupported_accelerator_is_named_in_the_error(monkeypatch):
+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)
+ torch.accelerator = types.SimpleNamespace(
+ is_available = lambda: True,
+ current_accelerator = lambda: "mtia",
+ )
+
+ with pytest.raises(NotImplementedError, match = "does not currently work on mtia"):
+ _load_device_type(monkeypatch, torch)
+
+
+def test_error_without_torch_accelerator_has_no_device_name(monkeypatch):
+ # torch < 2.6 has no torch.accelerator, so there is no name to report.
+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)
+
+ with pytest.raises(NotImplementedError, match = "does not currently work on this device"):
+ _load_device_type(monkeypatch, torch)
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index f20543e1790..6d983cf32e8 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -1552,9 +1552,10 @@ else:
from ._gpu_init import __version__
def get_gpu_memory_stats():
- """Return CUDA/ROCm/XPU device stats, peak memory, and total memory in GiB."""
+ """Return CUDA/ROCm/XPU/NPU device stats, peak memory, and total memory in GiB."""
try:
import torch
+
if hasattr(torch, "xpu") and torch.xpu.is_available():
props = torch.xpu.get_device_properties(0)
peak = (
@@ -1569,19 +1570,31 @@ else:
peak = torch.cuda.max_memory_reserved()
total = getattr(props, "total_memory", 0)
return props, _bytes_to_gb(peak), _bytes_to_gb(total) or 1.0
+ # Last, so no existing device changes branch. npu fell through to a fake 1 GiB.
+ if hasattr(torch, "npu") and torch.npu.is_available():
+ props = torch.npu.get_device_properties(0)
+ peak = (
+ torch.npu.max_memory_reserved()
+ if hasattr(torch.npu, "max_memory_reserved")
+ else torch.npu.max_memory_allocated()
+ )
+ total = getattr(props, "total_memory", 0)
+ return props, _bytes_to_gb(peak), _bytes_to_gb(total) or 1.0
except Exception:
pass
stats = _UnslothDeviceStats("Unknown GPU", 0)
return stats, 0.0, 1.0
def clear_gpu_memory():
- """Clear cached GPU memory on CUDA, ROCm, or XPU when available."""
+ """Clear cached GPU memory on CUDA, ROCm, XPU, or NPU when available."""
try:
import torch
if hasattr(torch, "xpu") and torch.xpu.is_available():
torch.xpu.empty_cache()
elif hasattr(torch, "cuda") and torch.cuda.is_available():
torch.cuda.empty_cache()
+ elif hasattr(torch, "npu") and torch.npu.is_available():
+ torch.npu.empty_cache()
except Exception:
pass
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 8fd16941e9a..962494728bb 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -387,6 +387,9 @@ elif DEVICE_TYPE == "xpu":
# torch.xpu.is_bf16_supported() does not have including_emulation set SUPPORTS_BFLOAT16 as
# torch.xpu.is_bf16_supported()
SUPPORTS_BFLOAT16 = torch.xpu.is_bf16_supported()
+elif DEVICE_TYPE == "npu":
+ # No arm left the name unbound, so consumers fell back to their own False.
+ SUPPORTS_BFLOAT16 = torch.npu.is_bf16_supported()
# For Gradio HF Spaces?
# if "SPACE_AUTHOR_NAME" not in os.environ and "SPACE_REPO_NAME" not in os.environ:
diff --git a/unsloth/device_type.py b/unsloth/device_type.py
index 37f71324289..cbf4cf8265c 100644
--- a/unsloth/device_type.py
+++ b/unsloth/device_type.py
@@ -11,6 +11,7 @@
__all__ = [
"is_hip",
+ "npu_is_available",
"get_device_type",
"DEVICE_TYPE",
"DEVICE_TYPE_TORCH",
@@ -27,6 +28,7 @@ __all__ = [
]
import functools
+import importlib.util
import inspect
import os
import re
@@ -55,6 +57,32 @@ def is_hip():
return bool(getattr(getattr(torch, "version", None), "hip", None))
+def npu_is_available():
+ """True only when torch.npu is present AND usable.
+
+ Only torch_npu >= 2.5.1 autoloads the namespace, and importing it without a driver
+ raises, so an unguarded probe would break `import unsloth` on CUDA, ROCm and XPU too.
+ """
+ if _IS_MLX:
+ return False
+ npu = getattr(torch, "npu", None)
+ if npu is None:
+ if importlib.util.find_spec("torch_npu") is None:
+ return False
+ try:
+ import torch_npu # noqa: F401
+ except Exception:
+ return False
+ npu = getattr(torch, "npu", None)
+ if npu is None:
+ return False
+ try:
+ return bool(npu.is_available())
+ except Exception:
+ return False
+
+
@functools.cache
def get_device_type():
# MLX first: torch is never imported on the MLX runtime, so claiming "cuda" here would NameError in
@@ -71,17 +99,27 @@ def get_device_type():
return "cuda"
elif hasattr(torch, "xpu") and torch.xpu.is_available():
return "xpu"
+ # After xpu: a host exposing both keeps selecting xpu, as it did before NPU.
+ elif npu_is_available():
+ return "npu"
+ accelerator = None
if hasattr(torch, "accelerator"):
if not torch.accelerator.is_available():
raise NotImplementedError("Unsloth cannot find any torch accelerator? You need a GPU.")
accelerator = str(torch.accelerator.current_accelerator())
- if accelerator in ("cuda", "xpu", "hip"):
+ # Listed, not returned: torch.npu is unusable here, so it only defers the AttributeError.
+ if accelerator in ("cuda", "xpu", "hip", "npu"):
raise RuntimeError(
- f"Unsloth: Weirdly `torch.cuda.is_available()`, `torch.xpu.is_available()` and `is_hip` all failed.\n"
+ f"Unsloth: Weirdly `torch.cuda.is_available()`, `torch.xpu.is_available()`, `torch.npu.is_available()` and `is_hip` all failed.\n"
f"But `torch.accelerator.current_accelerator()` works with it being = `{accelerator}`\n"
f"Please reinstall torch - it's most likely broken :("
)
- raise NotImplementedError("Unsloth currently only works on NVIDIA, AMD and Intel GPUs.")
+ # torch.accelerator only exists from torch 2.6, so below that there is no name.
+ raise NotImplementedError(
+ f"Unsloth does not currently work on {accelerator}."
+ if accelerator
+ else "Unsloth does not currently work on this device."
+ )
DEVICE_TYPE: str = get_device_type()
@@ -99,6 +137,8 @@ def get_device_count():
return torch.cuda.device_count()
elif DEVICE_TYPE == "xpu":
return torch.xpu.device_count()
+ elif DEVICE_TYPE == "npu":
+ return torch.npu.device_count()
else:
return 1
@@ -244,6 +284,14 @@ def get_device_stats() -> tuple[str, str, float]:
elif DEVICE_TYPE == "xpu":
name = gpu_stats.name + ". " if gpu_stats.name else "Intel XPU Device. "
snippet = f"Intel Toolkit: {torch.version.xpu}."
+ elif DEVICE_TYPE == "npu":
+ name = gpu_stats.name + ". " if gpu_stats.name else "NPU Device. "
+ # Report the toolkit like the cuda/xpu arms, not the name already in `name`.
+ try:
+ import torch_npu
+ snippet = f"NPU. torch_npu: {torch_npu.__version__}."
+ except Exception:
+ snippet = "NPU."
else:
name = gpu_stats.name + ". " if gpu_stats.name else "NVIDIA GPU Device. "
snippet = f"CUDA: {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit: {torch.version.cuda}."
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 157b301a7e8..12eeb3943e0 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -39,6 +39,10 @@ from unsloth_zoo.utils import Version
if DEVICE_TYPE == "xpu" and Version(torch.__version__) < Version("2.6.0"):
raise RuntimeError("Intel xpu currently supports unsloth with torch.version >= 2.6.0")
+# torch.amp.custom_fwd(device_type=) is 2.4+; say so here, not as an AttributeError mid-import.
+if DEVICE_TYPE == "npu" and Version(torch.__version__) < Version("2.4.0"):
+ raise RuntimeError("NPUs currently support unsloth with torch.version >= 2.4.0")
+
if Version(torch.__version__) < Version("2.4.0"):
torch_amp_custom_fwd = torch.cuda.amp.custom_fwd
torch_amp_custom_bwd = torch.cuda.amp.custom_bwd
@@ -52,6 +56,10 @@ else:
if DEVICE_TYPE == "xpu":
torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "xpu")
torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "xpu")
+elif DEVICE_TYPE == "npu":
+ # A mismatched device_type makes these inert, not loud (pytorch#165730): "cuda" drops autocast.
+ torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "npu")
+ torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "npu")
# tl.math.tanh is now libdevice.tanh.
@@ -171,6 +179,9 @@ if DEVICE_COUNT > 1:
torch_gpu_device = torch.cuda.device
elif DEVICE_TYPE == "xpu":
torch_gpu_device = torch.xpu.device
+ elif DEVICE_TYPE == "npu":
+ # Unbound without this, and rope_embedding imports it at module scope: ImportError.
+ torch_gpu_device = torch.npu.device
else:
from contextlib import nullcontext
def torch_gpu_device(device):
@@ -224,7 +235,8 @@ if DEVICE_TYPE == "xpu":
XPU_STREAMS = ()
WEIGHT_BUFFERS = []
ABSMAX_BUFFERS = []
-elif DEVICE_TYPE == "mlx":
+elif DEVICE_TYPE in ("mlx", "npu"):
+ # npu joins mlx: the else arm reads CUDA raw streams a NPU build has no runtime for.
CUDA_STREAMS = ()
XPU_STREAMS = ()
WEIGHT_BUFFERS = []
@@ -274,9 +286,13 @@ else:
cgemm_4bit_inference_naive_bf16 = bnb_functional.lib.cgemm_4bit_inference_naive_bf16
-torch_device_stream = (
- torch.xpu.current_stream if DEVICE_TYPE == "xpu" else torch.cuda.current_stream
-)
+if DEVICE_TYPE == "xpu":
+ torch_device_stream = torch.xpu.current_stream
+elif DEVICE_TYPE == "npu":
+ # RoPE synchronizes on this in its first forward, which would otherwise reach CUDA.
+ torch_device_stream = torch.npu.current_stream
+else:
+ torch_device_stream = torch.cuda.current_stream
torch_mm = torch.mm
torch_mv = torch.mv
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 045ed54b998..b0b601ecdf2 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -1907,6 +1907,13 @@ elif DEVICE_TYPE == "xpu":
else:
torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "xpu")
torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "xpu")
+elif DEVICE_TYPE == "npu":
+ # Named, not left to the else: same 2.4-only API, but this says why it failed.
+ if Version(torch_version) < Version("2.4.0"):
+ raise RuntimeError("torch.npu currently only supports torch.version >= 2.4.0")
+ else:
+ torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "npu")
+ torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "npu")
else:
# Exhaustive because both names are in __all__: an unbound branch (mlx) breaks `import *`.
torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = DEVICE_TYPE_TORCH)
@@ -2032,6 +2039,9 @@ elif DEVICE_TYPE == "hip":
HAS_FLASH_ATTENTION = False
elif DEVICE_TYPE == "xpu":
SUPPORTS_BFLOAT16 = True
+elif DEVICE_TYPE == "npu":
+ # Ask torch_npu: the module-level False silently rewrote an explicit bfloat16 to float16.
+ SUPPORTS_BFLOAT16 = torch.npu.is_bf16_supported()
try:
_xformers_logger = logging.getLogger("xformers")
@@ -2448,11 +2458,13 @@ def get_statistics(local_files_only = False):
disabled = True
_get_statistics(None)
_get_statistics("repeat", force_download = False)
- total_memory = (
- torch.xpu.get_device_properties(0).total_memory
- if DEVICE_TYPE == "xpu"
- else torch.cuda.get_device_properties(0).total_memory
- )
+ if DEVICE_TYPE == "xpu":
+ total_memory = torch.xpu.get_device_properties(0).total_memory
+ elif DEVICE_TYPE == "npu":
+ # from_pretrained always calls this; a NPU build cannot answer the else arm.
+ total_memory = torch.npu.get_device_properties(0).total_memory
+ else:
+ total_memory = torch.cuda.get_device_properties(0).total_memory
vram = total_memory / 1024 / 1024 / 1024
if vram <= 8:
vram = 8
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 8f14d073593..45db9ef403c 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -105,6 +105,8 @@ def prepare_device_map():
torch.cuda.set_device(local_rank)
elif DEVICE_TYPE_TORCH == "xpu" and hasattr(torch, "xpu"):
torch.xpu.set_device(local_rank)
+ elif DEVICE_TYPE_TORCH == "npu" and hasattr(torch, "npu"):
+ torch.npu.set_device(local_rank)
except Exception:
pass
return device_map, True