Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix training with accelerate 1.15 on torch without a distributed backend (AMD Windows ROCm) (#12162)

Daniel Haneaab494673c5
diff --git a/tests/test_torch_without_distributed_backend.py b/tests/test_torch_without_distributed_backend.py
new file mode 100644
index 00000000000..0d15942dd1d
--- /dev/null
+++ b/tests/test_torch_without_distributed_backend.py
@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""accelerate 1.15.0's `model_has_dtensor` on a torch without `torch._C._distributed_c10d`
+(AMD's Windows ROCm torch, huggingface/accelerate#4249), simulated on any torch."""
+
+from __future__ import annotations
+
+import importlib.abc
+import sys
+
+import pytest
+
+import unsloth # noqa: F401
+from unsloth import import_fixes
+
+torch = pytest.importorskip("torch")
+
+_C10D = "torch._C._distributed_c10d"
+
+
+def _missing_c10d():
+ return ModuleNotFoundError(
+ f"No module named '{_C10D}'; 'torch._C' is not a package", name = _C10D
+ )
+
+
+class _Raises(importlib.abc.MetaPathFinder):
+ def __init__(self, prefix, error):
+ self.prefix, self.error = prefix, error
+
+ def find_spec(
+ self,
+ name,
+ path = None,
+ target = None,
+ ):
+ if name == self.prefix or name.startswith(self.prefix + "."):
+ raise self.error
+ return None
+
+
+def no_backend(monkeypatch):
+ monkeypatch.setattr(torch.distributed, "is_available", lambda: False)
+
+
+def accelerate_dtensor(monkeypatch):
+ other = pytest.importorskip("accelerate.utils.other")
+ import accelerate.accelerator
+ import accelerate.utils
+
+ original = getattr(other.model_has_dtensor, "__wrapped__", other.model_has_dtensor)
+ for module in (other, accelerate.utils, accelerate.accelerator):
+ monkeypatch.setattr(module, "model_has_dtensor", original)
+ # What `from torch.distributed.tensor import DTensor` does on a torch without the backend.
+ for name in [
+ n
+ for n in sys.modules
+ if n.startswith("torch.distributed.tensor") or n.startswith("torch.distributed._tensor")
+ ]:
+ monkeypatch.delitem(sys.modules, name)
+ for attr in ("tensor", "_tensor"):
+ monkeypatch.delattr(torch.distributed, attr, raising = False)
+ monkeypatch.setattr(
+ sys,
+ "meta_path",
+ [
+ _Raises("torch.distributed.tensor", _missing_c10d()),
+ _Raises("torch.distributed._tensor", _missing_c10d()),
+ *sys.meta_path,
+ ],
+ )
+ return original
+
+
+def test_accelerate_dtensor_check_without_backend(accelerate_dtensor, no_backend):
+ import accelerate.accelerator
+ import accelerate.utils
+ import accelerate.utils.other as other
+
+ model = torch.nn.Linear(2, 2)
+ try:
+ unpatched = accelerate_dtensor(model)
+ except ImportError:
+ unpatched = "raises"
+ assert import_fixes.fix_accelerate_dtensor_check_without_torch_distributed() is True
+ assert other.model_has_dtensor(model) is False
+ assert accelerate.utils.model_has_dtensor is other.model_has_dtensor
+ assert accelerate.accelerator.model_has_dtensor is other.model_has_dtensor
+ assert other.model_has_dtensor.__wrapped__ is accelerate_dtensor
+ assert import_fixes.fix_accelerate_dtensor_check_without_torch_distributed() is True
+ assert other.model_has_dtensor.__wrapped__ is accelerate_dtensor
+ # accelerate releases before huggingface/accelerate#4250 raise here; later ones return False.
+ assert unpatched in ("raises", False)
+
+
+def test_accelerate_untouched_with_backend(accelerate_dtensor, monkeypatch):
+ import accelerate.utils.other as other
+
+ monkeypatch.setattr(torch.distributed, "is_available", lambda: True)
+
+ assert import_fixes.fix_accelerate_dtensor_check_without_torch_distributed() is False
+ assert other.model_has_dtensor is accelerate_dtensor
+
+
+def test_other_import_errors_still_raise(monkeypatch, no_backend):
+ other = pytest.importorskip("accelerate.utils.other")
+
+ def broken(model):
+ raise ModuleNotFoundError("No module named 'numpy'", name = "numpy")
+
+ monkeypatch.setattr(other, "model_has_dtensor", broken)
+ assert import_fixes.fix_accelerate_dtensor_check_without_torch_distributed() is True
+ with pytest.raises(ModuleNotFoundError, match = "numpy"):
+ other.model_has_dtensor(torch.nn.Linear(2, 2))
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index e2c344c43db..32a159c5955 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -290,6 +290,7 @@ from .import_fixes import (
patch_torchcodec_audio_decoder,
disable_torchcodec_if_broken,
disable_broken_wandb,
+ fix_accelerate_dtensor_check_without_torch_distributed,
fix_trl_vllm_ascend,
fix_peft_transformers_tensor_parallel_import_compat,
fix_peft_transformers_weight_conversion_import,
@@ -372,6 +373,8 @@ patch_vllm_for_notebooks()
patch_torchcodec_audio_decoder()
disable_torchcodec_if_broken()
disable_broken_wandb()
+# After unsloth_zoo, whose ROCm torchao loader must be in place before accelerate is imported.
+fix_accelerate_dtensor_check_without_torch_distributed()
# Must run before patch_peft_weight_converter_compatibility: it stubs the transformers v5
# submodules peft 0.19.x imports, so the next patch can wrap build_peft_weight_mapping instead of
# being swallowed by its ImportError.
@@ -422,6 +425,7 @@ del patch_torchcodec_audio_decoder
del disable_torchcodec_if_broken
del disable_torchaudio_if_cuda_mismatched
del disable_broken_wandb
+del fix_accelerate_dtensor_check_without_torch_distributed
del fix_peft_transformers_tensor_parallel_import_compat
del fix_peft_transformers_weight_conversion_import
del patch_peft_weight_converter_compatibility
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 9b5adcadc15..a08d93fd317 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -6846,6 +6846,58 @@ def disable_torchaudio_if_cuda_mismatched():
sys.modules["torchaudio"] = None
+def _torch_distributed_unavailable():
+ """True when torch has no `torch._C._distributed_c10d` (AMD's Windows ROCm wheels, torch 2.11)."""
+ try:
+ import torch.distributed as dist
+ return not dist.is_available()
+ except Exception:
+ return False
+
+
+def _is_missing_torch_distributed(exc):
+ name = getattr(exc, "name", None) or ""
+ return name.startswith("torch._C._distributed") or "torch._C._distributed" in str(exc)
+
+
+def fix_accelerate_dtensor_check_without_torch_distributed():
+ """Backport huggingface/accelerate#4250: accelerate 1.15.0's `prepare_model` calls `model_has_dtensor`,
+ which imports `torch.distributed.tensor` and kills every Trainer on a torch without a distributed
+ backend (huggingface/accelerate#4249). No DTensor can exist there, so the answer is False.
+ """
+ if not _torch_distributed_unavailable():
+ return False
+ if importlib.util.find_spec("accelerate") is None:
+ return False
+ try:
+ import accelerate.utils.other as acc_other
+ except Exception:
+ return False
+ original = getattr(acc_other, "model_has_dtensor", None)
+ if original is None:
+ return False
+ if getattr(original, "__unsloth_patched__", False):
+ return True
+
+ @functools.wraps(original)
+ def model_has_dtensor(model):
+ try:
+ return original(model)
+ except ImportError as exc:
+ if _is_missing_torch_distributed(exc):
+ return False
+ raise
+
+ model_has_dtensor.__unsloth_patched__ = True
+ acc_other.model_has_dtensor = model_has_dtensor
+ # Both re-export the function by name at import time.
+ for module_name in ("accelerate.utils", "accelerate.accelerator"):
+ module = sys.modules.get(module_name)
+ if getattr(module, "model_has_dtensor", None) is original:
+ module.model_has_dtensor = model_has_dtensor
+ return True
+
+
def disable_broken_wandb():
"""Disable wandb if it's installed but cannot actually import.