Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix training with accelerate 1.15 on torch without a distributed backend (AMD Windows ROCm) (#12162)
Daniel Haneaab494673c5
diff --git a/tests/test_torch_without_distributed_backend.py b/tests/test_torch_without_distributed_backend.pynew file mode 100644index 00000000000..0d15942dd1d--- /dev/null+++ b/tests/test_torch_without_distributed_backend.py@@ -0,0 +1,117 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""accelerate 1.15.0's `model_has_dtensor` on a torch without `torch._C._distributed_c10d`+(AMD's Windows ROCm torch, huggingface/accelerate#4249), simulated on any torch."""++from __future__ import annotations++import importlib.abc+import sys++import pytest++import unsloth # noqa: F401+from unsloth import import_fixes++torch = pytest.importorskip("torch")++_C10D = "torch._C._distributed_c10d"+++def _missing_c10d():+ return ModuleNotFoundError(+ f"No module named '{_C10D}'; 'torch._C' is not a package", name = _C10D+ )+++class _Raises(importlib.abc.MetaPathFinder):+ def __init__(self, prefix, error):+ self.prefix, self.error = prefix, error++ def find_spec(+ self,+ name,+ path = None,+ target = None,+ ):+ if name == self.prefix or name.startswith(self.prefix + "."):+ raise self.error+ return None+++def no_backend(monkeypatch):+ monkeypatch.setattr(torch.distributed, "is_available", lambda: False)+++def accelerate_dtensor(monkeypatch):+ other = pytest.importorskip("accelerate.utils.other")+ import accelerate.accelerator+ import accelerate.utils++ original = getattr(other.model_has_dtensor, "__wrapped__", other.model_has_dtensor)+ for module in (other, accelerate.utils, accelerate.accelerator):+ monkeypatch.setattr(module, "model_has_dtensor", original)+ # What `from torch.distributed.tensor import DTensor` does on a torch without the backend.+ for name in [+ n+ for n in sys.modules+ if n.startswith("torch.distributed.tensor") or n.startswith("torch.distributed._tensor")+ ]:+ monkeypatch.delitem(sys.modules, name)+ for attr in ("tensor", "_tensor"):+ monkeypatch.delattr(torch.distributed, attr, raising = False)+ monkeypatch.setattr(+ sys,+ "meta_path",+ [+ _Raises("torch.distributed.tensor", _missing_c10d()),+ _Raises("torch.distributed._tensor", _missing_c10d()),+ *sys.meta_path,+ ],+ )+ return original+++def test_accelerate_dtensor_check_without_backend(accelerate_dtensor, no_backend):+ import accelerate.accelerator+ import accelerate.utils+ import accelerate.utils.other as other++ model = torch.nn.Linear(2, 2)+ try:+ unpatched = accelerate_dtensor(model)+ except ImportError:+ unpatched = "raises"+ assert import_fixes.fix_accelerate_dtensor_check_without_torch_distributed() is True+ assert other.model_has_dtensor(model) is False+ assert accelerate.utils.model_has_dtensor is other.model_has_dtensor+ assert accelerate.accelerator.model_has_dtensor is other.model_has_dtensor+ assert other.model_has_dtensor.__wrapped__ is accelerate_dtensor+ assert import_fixes.fix_accelerate_dtensor_check_without_torch_distributed() is True+ assert other.model_has_dtensor.__wrapped__ is accelerate_dtensor+ # accelerate releases before huggingface/accelerate#4250 raise here; later ones return False.+ assert unpatched in ("raises", False)+++def test_accelerate_untouched_with_backend(accelerate_dtensor, monkeypatch):+ import accelerate.utils.other as other++ monkeypatch.setattr(torch.distributed, "is_available", lambda: True)++ assert import_fixes.fix_accelerate_dtensor_check_without_torch_distributed() is False+ assert other.model_has_dtensor is accelerate_dtensor+++def test_other_import_errors_still_raise(monkeypatch, no_backend):+ other = pytest.importorskip("accelerate.utils.other")++ def broken(model):+ raise ModuleNotFoundError("No module named 'numpy'", name = "numpy")++ monkeypatch.setattr(other, "model_has_dtensor", broken)+ assert import_fixes.fix_accelerate_dtensor_check_without_torch_distributed() is True+ with pytest.raises(ModuleNotFoundError, match = "numpy"):+ other.model_has_dtensor(torch.nn.Linear(2, 2))diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex e2c344c43db..32a159c5955 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -290,6 +290,7 @@ from .import_fixes import (patch_torchcodec_audio_decoder,disable_torchcodec_if_broken,disable_broken_wandb,+ fix_accelerate_dtensor_check_without_torch_distributed,fix_trl_vllm_ascend,fix_peft_transformers_tensor_parallel_import_compat,fix_peft_transformers_weight_conversion_import,@@ -372,6 +373,8 @@ patch_vllm_for_notebooks()patch_torchcodec_audio_decoder()disable_torchcodec_if_broken()disable_broken_wandb()+# After unsloth_zoo, whose ROCm torchao loader must be in place before accelerate is imported.+fix_accelerate_dtensor_check_without_torch_distributed()# Must run before patch_peft_weight_converter_compatibility: it stubs the transformers v5# submodules peft 0.19.x imports, so the next patch can wrap build_peft_weight_mapping instead of# being swallowed by its ImportError.@@ -422,6 +425,7 @@ del patch_torchcodec_audio_decoderdel disable_torchcodec_if_brokendel disable_torchaudio_if_cuda_mismatcheddel disable_broken_wandb+del fix_accelerate_dtensor_check_without_torch_distributeddel fix_peft_transformers_tensor_parallel_import_compatdel fix_peft_transformers_weight_conversion_importdel patch_peft_weight_converter_compatibilitydiff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 9b5adcadc15..a08d93fd317 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -6846,6 +6846,58 @@ def disable_torchaudio_if_cuda_mismatched():sys.modules["torchaudio"] = None+def _torch_distributed_unavailable():+ """True when torch has no `torch._C._distributed_c10d` (AMD's Windows ROCm wheels, torch 2.11)."""+ try:+ import torch.distributed as dist+ return not dist.is_available()+ except Exception:+ return False+++def _is_missing_torch_distributed(exc):+ name = getattr(exc, "name", None) or ""+ return name.startswith("torch._C._distributed") or "torch._C._distributed" in str(exc)+++def fix_accelerate_dtensor_check_without_torch_distributed():+ """Backport huggingface/accelerate#4250: accelerate 1.15.0's `prepare_model` calls `model_has_dtensor`,+ which imports `torch.distributed.tensor` and kills every Trainer on a torch without a distributed+ backend (huggingface/accelerate#4249). No DTensor can exist there, so the answer is False.+ """+ if not _torch_distributed_unavailable():+ return False+ if importlib.util.find_spec("accelerate") is None:+ return False+ try:+ import accelerate.utils.other as acc_other+ except Exception:+ return False+ original = getattr(acc_other, "model_has_dtensor", None)+ if original is None:+ return False+ if getattr(original, "__unsloth_patched__", False):+ return True++ @functools.wraps(original)+ def model_has_dtensor(model):+ try:+ return original(model)+ except ImportError as exc:+ if _is_missing_torch_distributed(exc):+ return False+ raise++ model_has_dtensor.__unsloth_patched__ = True+ acc_other.model_has_dtensor = model_has_dtensor+ # Both re-export the function by name at import time.+ for module_name in ("accelerate.utils", "accelerate.accelerator"):+ module = sys.modules.get(module_name)+ if getattr(module, "model_has_dtensor", None) is original:+ module.model_has_dtensor = model_has_dtensor+ return True++def disable_broken_wandb():"""Disable wandb if it's installed but cannot actually import.