Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
feat(device_type): add Ascend NPU device detection (#10686)
li-lizhec78fc38bab86
diff --git a/tests/test_device_helpers.py b/tests/test_device_helpers.pyindex 78bf40c505c..c57d9030d09 100644--- a/tests/test_device_helpers.py+++ b/tests/test_device_helpers.py@@ -3,6 +3,7 @@ import sysimport typesfrom pathlib import Path+import pytestfrom packaging.version import Version@@ -201,3 +202,135 @@ def test_model_call_sites_use_shared_cache_dispatch():assert "clean_gpu_cache()" in gemma2_sourceassert "torch.cuda.empty_cache()" not in granite_sourceassert "clean_gpu_cache()" in granite_source+++NPU_PROPERTIES = types.SimpleNamespace(+ name = "Ascend910B2",+ total_memory = 60 * 1024**3,+ # major/minor are std::optional on real hardware, so None. The npu arm must not read them.+ major = None,+ minor = None,+)+++def _npu_backend(+ *,+ available = True,+ device_count = 4,+ properties = NPU_PROPERTIES,+ calls = None,+):+ def _is_available():+ if available == "raise":+ raise RuntimeError("npu driver not found")+ return available++ return types.SimpleNamespace(+ is_available = _is_available,+ device_count = lambda: device_count,+ get_device_properties = lambda _index: properties,+ empty_cache = lambda: (calls if calls is not None else []).append("empty_cache"),+ current_device = lambda: 0,+ )+++def test_npu_detected_with_count_and_stats(monkeypatch):+ calls = []+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)+ torch.npu = _npu_backend(calls = calls)++ device_type = _load_device_type(monkeypatch, torch)++ assert device_type.DEVICE_TYPE == "npu"+ assert device_type.DEVICE_TYPE_TORCH == "npu"+ assert device_type.DEVICE_COUNT == 4+ assert device_type._DEVICE_MODULE is torch.npu++ name, snippet, max_memory = device_type.get_device_stats()+ assert (name, snippet, max_memory) == ("Ascend910B2. ", "Ascend NPU.", 60.0)++ device_type.clean_gpu_cache()+ assert calls == ["empty_cache"]+ assert device_type.get_current_device() == 0+++def test_npu_blank_name_falls_back(monkeypatch):+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)+ torch.npu = _npu_backend(+ properties = types.SimpleNamespace(name = "", total_memory = 60 * 1024**3),+ )++ device_type = _load_device_type(monkeypatch, torch)+ name, snippet, _ = device_type.get_device_stats()++ assert name == "Ascend NPU Device. "+ assert snippet == "Ascend NPU."+++def test_cuda_wins_over_npu(monkeypatch):+ torch = _fake_torch(properties = CUDA_PROPERTIES)+ torch.npu = _npu_backend()++ device_type = _load_device_type(monkeypatch, torch)++ assert device_type.DEVICE_TYPE == "cuda"+++def test_xpu_wins_over_npu(monkeypatch):+ xpu_backend = types.SimpleNamespace(+ is_available = lambda: True,+ device_count = lambda: 2,+ empty_cache = lambda: None,+ current_device = lambda: 0,+ get_device_properties = lambda _index: types.SimpleNamespace(+ name = "Intel Arc",+ total_memory = 8 * 1024**3,+ ),+ )+ torch = _fake_torch(+ properties = CUDA_PROPERTIES,+ xpu_backend = xpu_backend,+ cuda_available = False,+ )+ torch.npu = _npu_backend()++ device_type = _load_device_type(monkeypatch, torch)++ assert device_type.DEVICE_TYPE == "xpu"+ assert device_type.DEVICE_COUNT == 2+++def test_npu_probe_survives_raising_is_available(monkeypatch):+ # Must stay a clean NotImplementedError, not a RuntimeError escaping `import unsloth`.+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)+ torch.npu = _npu_backend(available = "raise")++ with pytest.raises(NotImplementedError):+ _load_device_type(monkeypatch, torch)+++def test_npu_unavailable_is_not_selected(monkeypatch):+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)+ torch.npu = _npu_backend(available = False)++ with pytest.raises(NotImplementedError):+ _load_device_type(monkeypatch, torch)+++def test_unsupported_accelerator_is_named_in_the_error(monkeypatch):+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)+ torch.accelerator = types.SimpleNamespace(+ is_available = lambda: True,+ current_accelerator = lambda: "mtia",+ )++ with pytest.raises(NotImplementedError, match = "does not currently work on mtia"):+ _load_device_type(monkeypatch, torch)+++def test_error_without_torch_accelerator_has_no_device_name(monkeypatch):+ # torch < 2.6 has no torch.accelerator, so there is no name to report.+ torch = _fake_torch(properties = CUDA_PROPERTIES, cuda_available = False)++ with pytest.raises(NotImplementedError, match = "does not currently work on this device"):+ _load_device_type(monkeypatch, torch)diff --git a/unsloth/__init__.py b/unsloth/__init__.pyindex f20543e1790..6d983cf32e8 100644--- a/unsloth/__init__.py+++ b/unsloth/__init__.py@@ -1552,9 +1552,10 @@ else:from ._gpu_init import __version__def get_gpu_memory_stats():- """Return CUDA/ROCm/XPU device stats, peak memory, and total memory in GiB."""+ """Return CUDA/ROCm/XPU/NPU device stats, peak memory, and total memory in GiB."""try:import torch+if hasattr(torch, "xpu") and torch.xpu.is_available():props = torch.xpu.get_device_properties(0)peak = (@@ -1569,19 +1570,31 @@ else:peak = torch.cuda.max_memory_reserved()total = getattr(props, "total_memory", 0)return props, _bytes_to_gb(peak), _bytes_to_gb(total) or 1.0+ # Last, so no existing device changes branch. npu fell through to a fake 1 GiB.+ if hasattr(torch, "npu") and torch.npu.is_available():+ props = torch.npu.get_device_properties(0)+ peak = (+ torch.npu.max_memory_reserved()+ if hasattr(torch.npu, "max_memory_reserved")+ else torch.npu.max_memory_allocated()+ )+ total = getattr(props, "total_memory", 0)+ return props, _bytes_to_gb(peak), _bytes_to_gb(total) or 1.0except Exception:passstats = _UnslothDeviceStats("Unknown GPU", 0)return stats, 0.0, 1.0def clear_gpu_memory():- """Clear cached GPU memory on CUDA, ROCm, or XPU when available."""+ """Clear cached GPU memory on CUDA, ROCm, XPU, or NPU when available."""try:import torchif hasattr(torch, "xpu") and torch.xpu.is_available():torch.xpu.empty_cache()elif hasattr(torch, "cuda") and torch.cuda.is_available():torch.cuda.empty_cache()+ elif hasattr(torch, "npu") and torch.npu.is_available():+ torch.npu.empty_cache()except Exception:passdiff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 8fd16941e9a..962494728bb 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -387,6 +387,9 @@ elif DEVICE_TYPE == "xpu":# torch.xpu.is_bf16_supported() does not have including_emulation set SUPPORTS_BFLOAT16 as# torch.xpu.is_bf16_supported()SUPPORTS_BFLOAT16 = torch.xpu.is_bf16_supported()+elif DEVICE_TYPE == "npu":+ # No arm left the name unbound, so consumers fell back to their own False.+ SUPPORTS_BFLOAT16 = torch.npu.is_bf16_supported()# For Gradio HF Spaces?# if "SPACE_AUTHOR_NAME" not in os.environ and "SPACE_REPO_NAME" not in os.environ:diff --git a/unsloth/device_type.py b/unsloth/device_type.pyindex 37f71324289..cbf4cf8265c 100644--- a/unsloth/device_type.py+++ b/unsloth/device_type.py@@ -11,6 +11,7 @@__all__ = ["is_hip",+ "npu_is_available","get_device_type","DEVICE_TYPE","DEVICE_TYPE_TORCH",@@ -27,6 +28,7 @@ __all__ = []import functools+import importlib.utilimport inspectimport osimport re@@ -55,6 +57,32 @@ def is_hip():return bool(getattr(getattr(torch, "version", None), "hip", None))+def npu_is_available():+ """True only when torch.npu is present AND usable.++ Only torch_npu >= 2.5.1 autoloads the namespace, and importing it without a driver+ raises, so an unguarded probe would break `import unsloth` on CUDA, ROCm and XPU too.+ """+ if _IS_MLX:+ return False+ npu = getattr(torch, "npu", None)+ if npu is None:+ if importlib.util.find_spec("torch_npu") is None:+ return False+ try:+ import torch_npu # noqa: F401+ except Exception:+ return False+ npu = getattr(torch, "npu", None)+ if npu is None:+ return False+ try:+ return bool(npu.is_available())+ except Exception:+ return False++@functools.cachedef get_device_type():# MLX first: torch is never imported on the MLX runtime, so claiming "cuda" here would NameError in@@ -71,17 +99,27 @@ def get_device_type():return "cuda"elif hasattr(torch, "xpu") and torch.xpu.is_available():return "xpu"+ # After xpu: a host exposing both keeps selecting xpu, as it did before NPU.+ elif npu_is_available():+ return "npu"+ accelerator = Noneif hasattr(torch, "accelerator"):if not torch.accelerator.is_available():raise NotImplementedError("Unsloth cannot find any torch accelerator? You need a GPU.")accelerator = str(torch.accelerator.current_accelerator())- if accelerator in ("cuda", "xpu", "hip"):+ # Listed, not returned: torch.npu is unusable here, so it only defers the AttributeError.+ if accelerator in ("cuda", "xpu", "hip", "npu"):raise RuntimeError(- f"Unsloth: Weirdly `torch.cuda.is_available()`, `torch.xpu.is_available()` and `is_hip` all failed.\n"+ f"Unsloth: Weirdly `torch.cuda.is_available()`, `torch.xpu.is_available()`, `torch.npu.is_available()` and `is_hip` all failed.\n"f"But `torch.accelerator.current_accelerator()` works with it being = `{accelerator}`\n"f"Please reinstall torch - it's most likely broken :(")- raise NotImplementedError("Unsloth currently only works on NVIDIA, AMD and Intel GPUs.")+ # torch.accelerator only exists from torch 2.6, so below that there is no name.+ raise NotImplementedError(+ f"Unsloth does not currently work on {accelerator}."+ if accelerator+ else "Unsloth does not currently work on this device."+ )DEVICE_TYPE: str = get_device_type()@@ -99,6 +137,8 @@ def get_device_count():return torch.cuda.device_count()elif DEVICE_TYPE == "xpu":return torch.xpu.device_count()+ elif DEVICE_TYPE == "npu":+ return torch.npu.device_count()else:return 1@@ -244,6 +284,14 @@ def get_device_stats() -> tuple[str, str, float]:elif DEVICE_TYPE == "xpu":name = gpu_stats.name + ". " if gpu_stats.name else "Intel XPU Device. "snippet = f"Intel Toolkit: {torch.version.xpu}."+ elif DEVICE_TYPE == "npu":+ name = gpu_stats.name + ". " if gpu_stats.name else "NPU Device. "+ # Report the toolkit like the cuda/xpu arms, not the name already in `name`.+ try:+ import torch_npu+ snippet = f"NPU. torch_npu: {torch_npu.__version__}."+ except Exception:+ snippet = "NPU."else:name = gpu_stats.name + ". " if gpu_stats.name else "NVIDIA GPU Device. "snippet = f"CUDA: {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit: {torch.version.cuda}."diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 157b301a7e8..12eeb3943e0 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -39,6 +39,10 @@ from unsloth_zoo.utils import Versionif DEVICE_TYPE == "xpu" and Version(torch.__version__) < Version("2.6.0"):raise RuntimeError("Intel xpu currently supports unsloth with torch.version >= 2.6.0")+# torch.amp.custom_fwd(device_type=) is 2.4+; say so here, not as an AttributeError mid-import.+if DEVICE_TYPE == "npu" and Version(torch.__version__) < Version("2.4.0"):+ raise RuntimeError("NPUs currently support unsloth with torch.version >= 2.4.0")+if Version(torch.__version__) < Version("2.4.0"):torch_amp_custom_fwd = torch.cuda.amp.custom_fwdtorch_amp_custom_bwd = torch.cuda.amp.custom_bwd@@ -52,6 +56,10 @@ else:if DEVICE_TYPE == "xpu":torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "xpu")torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "xpu")+elif DEVICE_TYPE == "npu":+ # A mismatched device_type makes these inert, not loud (pytorch#165730): "cuda" drops autocast.+ torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "npu")+ torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "npu")# tl.math.tanh is now libdevice.tanh.@@ -171,6 +179,9 @@ if DEVICE_COUNT > 1:torch_gpu_device = torch.cuda.deviceelif DEVICE_TYPE == "xpu":torch_gpu_device = torch.xpu.device+ elif DEVICE_TYPE == "npu":+ # Unbound without this, and rope_embedding imports it at module scope: ImportError.+ torch_gpu_device = torch.npu.deviceelse:from contextlib import nullcontextdef torch_gpu_device(device):@@ -224,7 +235,8 @@ if DEVICE_TYPE == "xpu":XPU_STREAMS = ()WEIGHT_BUFFERS = []ABSMAX_BUFFERS = []-elif DEVICE_TYPE == "mlx":+elif DEVICE_TYPE in ("mlx", "npu"):+ # npu joins mlx: the else arm reads CUDA raw streams a NPU build has no runtime for.CUDA_STREAMS = ()XPU_STREAMS = ()WEIGHT_BUFFERS = []@@ -274,9 +286,13 @@ else:cgemm_4bit_inference_naive_bf16 = bnb_functional.lib.cgemm_4bit_inference_naive_bf16-torch_device_stream = (- torch.xpu.current_stream if DEVICE_TYPE == "xpu" else torch.cuda.current_stream-)+if DEVICE_TYPE == "xpu":+ torch_device_stream = torch.xpu.current_stream+elif DEVICE_TYPE == "npu":+ # RoPE synchronizes on this in its first forward, which would otherwise reach CUDA.+ torch_device_stream = torch.npu.current_stream+else:+ torch_device_stream = torch.cuda.current_streamtorch_mm = torch.mmtorch_mv = torch.mvdiff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 045ed54b998..b0b601ecdf2 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -1907,6 +1907,13 @@ elif DEVICE_TYPE == "xpu":else:torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "xpu")torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "xpu")+elif DEVICE_TYPE == "npu":+ # Named, not left to the else: same 2.4-only API, but this says why it failed.+ if Version(torch_version) < Version("2.4.0"):+ raise RuntimeError("torch.npu currently only supports torch.version >= 2.4.0")+ else:+ torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "npu")+ torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "npu")else:# Exhaustive because both names are in __all__: an unbound branch (mlx) breaks `import *`.torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = DEVICE_TYPE_TORCH)@@ -2032,6 +2039,9 @@ elif DEVICE_TYPE == "hip":HAS_FLASH_ATTENTION = Falseelif DEVICE_TYPE == "xpu":SUPPORTS_BFLOAT16 = True+elif DEVICE_TYPE == "npu":+ # Ask torch_npu: the module-level False silently rewrote an explicit bfloat16 to float16.+ SUPPORTS_BFLOAT16 = torch.npu.is_bf16_supported()try:_xformers_logger = logging.getLogger("xformers")@@ -2448,11 +2458,13 @@ def get_statistics(local_files_only = False):disabled = True_get_statistics(None)_get_statistics("repeat", force_download = False)- total_memory = (- torch.xpu.get_device_properties(0).total_memory- if DEVICE_TYPE == "xpu"- else torch.cuda.get_device_properties(0).total_memory- )+ if DEVICE_TYPE == "xpu":+ total_memory = torch.xpu.get_device_properties(0).total_memory+ elif DEVICE_TYPE == "npu":+ # from_pretrained always calls this; a NPU build cannot answer the else arm.+ total_memory = torch.npu.get_device_properties(0).total_memory+ else:+ total_memory = torch.cuda.get_device_properties(0).total_memoryvram = total_memory / 1024 / 1024 / 1024if vram <= 8:vram = 8diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 8f14d073593..45db9ef403c 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -105,6 +105,8 @@ def prepare_device_map():torch.cuda.set_device(local_rank)elif DEVICE_TYPE_TORCH == "xpu" and hasattr(torch, "xpu"):torch.xpu.set_device(local_rank)+ elif DEVICE_TYPE_TORCH == "npu" and hasattr(torch, "npu"):+ torch.npu.set_device(local_rank)except Exception:passreturn device_map, True