Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2: keep accelerate's FSDP2 for UNSLOTH_FSDP2=1 on XPU / NPU / other non-CUDA installs (#14760)

Daniel Han40ef4b9bd0de
diff --git a/tests/test_fsdp2_native_backend_table.py b/tests/test_fsdp2_native_backend_table.py
index bf28fe0754f..30dac192bea 100644
--- a/tests/test_fsdp2_native_backend_table.py
+++ b/tests/test_fsdp2_native_backend_table.py
@@ -94,6 +94,9 @@ def test_table_covers_every_accelerate_distributed_type(native):
def test_every_refuse_row_is_refused_at_both_entry_points(native, monkeypatch):
launch = sys.modules["unsloth.fsdp2_launch"]
+ # Explicit native: on a torch build without CUDA / ROCm (CPU CI) the generic flag routes a vendor
+ # device to accelerate's FSDP2 instead (non_cuda_install), which is not refused.
+ monkeypatch.setenv("UNSLOTH_FSDP2_BACKEND", "native")
rows = [r for r in launch.NATIVE_BACKEND_TABLE if r["action"] == "refuse"]
assert len(rows) >= 10
for row in rows:
diff --git a/tests/test_fsdp2_native_trainer_args_gloo.py b/tests/test_fsdp2_native_trainer_args_gloo.py
index 7ec54c5460c..cfbdf3a2aee 100644
--- a/tests/test_fsdp2_native_trainer_args_gloo.py
+++ b/tests/test_fsdp2_native_trainer_args_gloo.py
@@ -7,7 +7,9 @@
* an optimizer passed positionally (Trainer's own order) is refused like a keyword one;
* `model_init` (and with it hyperparameter_search) is refused instead of training unsharded;
* `save_on_each_node=True` across nodes is refused (only global rank 0 holds the gathered state);
-* a resume of a non-default active PEFT adapter reads `<ckpt>/<adapter>/`, never the root adapter.
+* a resume of a non-default active PEFT adapter reads `<ckpt>/<adapter>/`, never the root adapter;
+* UNSLOTH_FSDP2=1 on an XPU install with TrainingArguments(use_cpu = True) still shards natively
+ (accelerate runs a CPU Trainer as MULTI_CPU, which drops its FSDP plugin).
Plus the launcher env: an explicit native backend turns accelerate's FSDP launch env off."""
import multiprocessing
@@ -182,6 +184,18 @@ def _worker(rank, world, port, out, case, tmp, env):
res["optimizer_keys"] = sorted(state)
except RuntimeError as exc:
res["error"] = str(exc)
+ elif case == "xpu_install_use_cpu":
+ # An XPU torch build (no CUDA / ROCm): the launch step routes UNSLOTH_FSDP2=1 to
+ # accelerate's FSDP2, then the use_cpu Trainer takes native FSDP2 back.
+ torch.version.cuda = None
+ torch.version.hip = None
+ torch.version.xpu = "20250101"
+ launch = sys.modules["unsloth.fsdp2_launch"]
+ res["routed"] = launch.apply_unsloth_fsdp2_launch_env()
+ trainer = Trainer(model = _lora(1), args = _targs(os.path.join(tmp, "out")))
+ res["sharded"] = native.is_native_fsdp2(trainer.model)
+ res["accelerate_fsdp"] = os.environ.get("ACCELERATE_USE_FSDP")
+ res["distributed_type"] = str(trainer.accelerator.distributed_type)
elif case == "named_adapter":
cfg = native.FSDP2Config(param_dtype = torch.float32)
ckpt = os.path.join(tmp, "ckpt")
@@ -269,6 +283,15 @@ def test_resume_reads_named_adapter_from_its_subdirectory(tmp_path):
assert r["n_other"] > 0 and r["other_diff"] == 0.0, (rank, r)
+def test_xpu_install_cpu_trainer_shards_natively(tmp_path):
+ pytest.importorskip("transformers")
+ pytest.importorskip("peft")
+ for rank, r in _run("xpu_install_use_cpu", tmp_path).items():
+ assert r["routed"] is True, (rank, r)
+ assert r["sharded"] is True and r["accelerate_fsdp"] is None, (rank, r)
+ assert "MULTI_CPU" in r["distributed_type"], (rank, r)
+
+
# ---- launcher env (no process group) ----------------------------------------------------------
diff --git a/tests/test_fsdp2_non_cuda_accelerate_route.py b/tests/test_fsdp2_non_cuda_accelerate_route.py
new file mode 100644
index 00000000000..c0935683495
--- /dev/null
+++ b/tests/test_fsdp2_non_cuda_accelerate_route.py
@@ -0,0 +1,209 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_FSDP2=1 on an XPU / NPU / ... install (no CUDA / ROCm torch build) keeps accelerate's FSDP2
+wrap, as it did before the native backend existed, instead of being refused: native FSDP2 runs on CUDA /
+ROCm or the CPU only. The decision is made from the torch build and the installed vendor packages (no
+torch import), so the launch step and Trainer setup agree. An explicit UNSLOTH_FSDP2_BACKEND=native is
+still native (and refused at Trainer setup when that device is present)."""
+
+import importlib.util
+import os
+import sys
+import types
+
+import pytest
+import torch
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_ENV_AT_IMPORT = dict(os.environ)
+
+_CLEAN = (
+ "ACCELERATE_USE_DEEPSPEED",
+ "ACCELERATE_USE_FSDP",
+ "ACCELERATE_USE_CPU",
+ "FSDP_VERSION",
+ "FSDP_AUTO_WRAP_POLICY",
+ "FSDP_STATE_DICT_TYPE",
+ "UNSLOTH_FSDP2_BACKEND",
+)
+
+
+def launch(monkeypatch):
+ # apply_unsloth_fsdp2_launch_env writes os.environ directly; monkeypatch.delenv records nothing for
+ # an absent key, so restore the whole environment or the FSDP launch env leaks into later tests.
+ saved = dict(os.environ)
+ yield _launch_module(monkeypatch)
+ os.environ.clear()
+ os.environ.update(saved)
+
+
+def _launch_module(monkeypatch):
+ for key in _CLEAN:
+ monkeypatch.delenv(key, raising = False)
+ for key, value in dict(UNSLOTH_FSDP2 = "1", WORLD_SIZE = "2", RANK = "0").items():
+ monkeypatch.setenv(key, value)
+ spec = importlib.util.spec_from_file_location(
+ "_fsdp2_launch_under_test", os.path.join(REPO, "unsloth", "fsdp2_launch.py")
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _build(
+ monkeypatch,
+ cuda = None,
+ hip = None,
+ xpu = None,
+):
+ monkeypatch.setattr(torch.version, "cuda", cuda, raising = False)
+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)
+ monkeypatch.setattr(torch.version, "xpu", xpu, raising = False)
+
+
+def test_xpu_build_routes_to_accelerate_fsdp2(launch, monkeypatch):
+ _build(monkeypatch, xpu = "20250101")
+ assert launch.unsloth_fsdp2_native_requested() is False
+ assert launch.apply_unsloth_fsdp2_launch_env() is True
+ assert os.environ["ACCELERATE_USE_FSDP"] == "true"
+ assert os.environ["FSDP_VERSION"] == "2"
+ assert launch.fsdp_version_from_env() == 2
+
+
+def test_vendor_package_on_a_non_cuda_build_routes_to_accelerate(launch, monkeypatch):
+ _build(monkeypatch)
+ real = importlib.util.find_spec
+ monkeypatch.setattr(
+ importlib.util,
+ "find_spec",
+ lambda name, *a, **k: object() if name == "torch_npu" else real(name, *a, **k),
+ )
+ assert launch.unsloth_fsdp2_native_requested() is False
+ assert launch.apply_unsloth_fsdp2_launch_env() is True
+
+
[email protected]("build", [dict(cuda = "12.8"), dict(hip = "6.3"), dict(cuda = "12.8", xpu = "1")])
+def test_cuda_and_rocm_builds_stay_native(launch, monkeypatch, build):
+ _build(monkeypatch, **build)
+ assert launch.unsloth_fsdp2_native_requested() is True
+ assert launch.apply_unsloth_fsdp2_launch_env() is False
+ assert "ACCELERATE_USE_FSDP" not in os.environ
+
+
+def test_explicit_native_backend_and_cpu_launch_stay_native(launch, monkeypatch):
+ _build(monkeypatch, xpu = "20250101")
+ monkeypatch.setenv("UNSLOTH_FSDP2_BACKEND", "native")
+ assert launch.unsloth_fsdp2_native_requested() is True
+ monkeypatch.delenv("UNSLOTH_FSDP2_BACKEND")
+ monkeypatch.setenv("ACCELERATE_USE_CPU", "true")
+ assert launch.unsloth_fsdp2_native_requested() is True
+
+
+def test_torch_build_is_read_without_importing_torch(launch, monkeypatch, tmp_path):
+ pkg = tmp_path / "torch"
+ pkg.mkdir()
+ (pkg / "version.py").write_text(
+ "from typing import Optional\n__version__ = '2.11.0+xpu'\n"
+ "cuda: Optional[str] = None\nhip: Optional[str] = None\nxpu: Optional[str] = '20250101'\n"
+ )
+ spec = importlib.machinery.ModuleSpec("torch", None, is_package = True)
+ spec.submodule_search_locations = [str(pkg)]
+ real = importlib.util.find_spec
+ monkeypatch.setattr(
+ importlib.util,
+ "find_spec",
+ lambda name, *a, **k: spec if name == "torch" else real(name, *a, **k),
+ )
+ monkeypatch.delitem(sys.modules, "torch")
+ try:
+ assert launch.unsloth_fsdp2_native_requested() is False
+ finally:
+ sys.modules["torch"] = torch
+
+
+def test_xpu_version_label_with_a_null_runtime_marker(launch, monkeypatch):
+ # A +xpu wheel can carry torch.version.xpu = None: the version label is the build marker.
+ _build(monkeypatch)
+ monkeypatch.setattr(torch.version, "__version__", "2.9.1+xpu", raising = False)
+ monkeypatch.setattr(torch, "__version__", "2.9.1+xpu")
+ assert launch.non_cuda_install() == "Intel XPU"
+ assert launch.unsloth_fsdp2_native_requested() is False
+ assert launch.apply_unsloth_fsdp2_launch_env() is True
+
+
+def test_xpu_version_label_read_without_importing_torch(launch, monkeypatch, tmp_path):
+ pkg = tmp_path / "torch"
+ pkg.mkdir()
+ (pkg / "version.py").write_text(
+ "from typing import Optional\n__version__ = '2.9.1+xpu'\n"
+ "cuda: Optional[str] = None\nhip: Optional[str] = None\nxpu: Optional[str] = None\n"
+ )
+ spec = importlib.machinery.ModuleSpec("torch", None, is_package = True)
+ spec.submodule_search_locations = [str(pkg)]
+ real = importlib.util.find_spec
+ monkeypatch.setattr(
+ importlib.util,
+ "find_spec",
+ lambda name, *a, **k: spec if name == "torch" else real(name, *a, **k),
+ )
+ monkeypatch.delitem(sys.modules, "torch")
+ try:
+ assert launch.non_cuda_install() == "Intel XPU"
+ finally:
+ sys.modules["torch"] = torch
+
+
+def test_cpu_wheel_label_is_not_xpu(launch, monkeypatch):
+ _build(monkeypatch)
+ monkeypatch.setattr(torch.version, "__version__", "2.9.1+cpu", raising = False)
+ monkeypatch.setattr(torch, "__version__", "2.9.1+cpu")
+ assert launch.non_cuda_install() is None
+ assert launch.unsloth_fsdp2_native_requested() is True
+
+
+def test_unsloth_fsdp2_native_on_xpu_stays_native_and_is_refused(launch, monkeypatch):
+ _build(monkeypatch, xpu = "20250101")
+ monkeypatch.setenv("UNSLOTH_FSDP2", "native")
+ assert launch.unsloth_fsdp2_native_requested() is True
+ assert launch.apply_unsloth_fsdp2_launch_env() is False
+ assert "ACCELERATE_USE_FSDP" not in os.environ
+ # Trainer setup with a live XPU device: refused with the accelerate alternative named.
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setattr(
+ torch, "xpu", types.SimpleNamespace(is_available = lambda: True), raising = False
+ )
+ with pytest.raises(RuntimeError, match = "UNSLOTH_FSDP2_BACKEND=accelerate"):
+ launch.refuse_native_backend_conflicts(None, uses_cpu = False)
+ # The same explicit request with a CPU Trainer runs natively on the CPU.
+ launch.refuse_native_backend_conflicts(None, uses_cpu = True)
+
+
[email protected]("flag", ["use_cpu", "no_cuda"])
+def test_cpu_trainer_on_xpu_install_takes_native_back(launch, monkeypatch, flag):
+ _build(monkeypatch, xpu = "20250101")
+ monkeypatch.setenv("FSDP_STATE_DICT_TYPE", "SHARDED_STATE_DICT")
+ assert launch.apply_unsloth_fsdp2_launch_env() is True
+ assert launch.reclaim_native_fsdp2_for_cpu_trainer(types.SimpleNamespace()) is False
+ assert os.environ["ACCELERATE_USE_FSDP"] == "true"
+ assert launch.reclaim_native_fsdp2_for_cpu_trainer(types.SimpleNamespace(**{flag: True}))
+ for key in ("ACCELERATE_USE_FSDP", "FSDP_VERSION", "FSDP_AUTO_WRAP_POLICY"):
+ assert key not in os.environ, key
+ assert os.environ["FSDP_STATE_DICT_TYPE"] == "SHARDED_STATE_DICT" # the user's value stays
+ assert launch.unsloth_fsdp2_native_requested() is True
+ assert launch.native_backend_conflicts(None, uses_cpu = True) == []
+
+
+def test_cpu_trainer_reclaim_needs_the_non_cuda_route(launch, monkeypatch):
+ _build(monkeypatch, cuda = "12.8")
+ monkeypatch.setenv("UNSLOTH_FSDP2_BACKEND", "accelerate")
+ assert launch.apply_unsloth_fsdp2_launch_env() is True
+ assert launch.reclaim_native_fsdp2_for_cpu_trainer(types.SimpleNamespace(use_cpu = True)) is False
+ assert os.environ["ACCELERATE_USE_FSDP"] == "true"
+
+
+def test_zz_launch_env_does_not_leak_out_of_this_module():
+ # Runs last in this file: every key the launch tests wrote is back to its value at import.
+ for key in _CLEAN + ("UNSLOTH_FSDP2", "WORLD_SIZE", "RANK"):
+ assert os.environ.get(key) == _ENV_AT_IMPORT.get(key), key
diff --git a/unsloth/fsdp2_launch.py b/unsloth/fsdp2_launch.py
index 8e1a9b489bc..e9f3a9567b6 100644
--- a/unsloth/fsdp2_launch.py
+++ b/unsloth/fsdp2_launch.py
@@ -3,13 +3,17 @@
"""FSDP launch detection from the launcher env only (no torch): identical on every rank."""
+import ast
import importlib.util
import os
+import re
import sys
__all__ = [
"unsloth_fsdp2_requested",
"unsloth_fsdp2_native_requested",
+ "non_cuda_install",
+ "reclaim_native_fsdp2_for_cpu_trainer",
"fsdp_version_from_env",
"apply_unsloth_fsdp2_launch_env",
"NATIVE_BACKEND_TABLE",
@@ -58,9 +62,124 @@ def unsloth_fsdp2_native_requested():
)
if backend == "" and _env_flag("ACCELERATE_USE_FSDP"):
return False
+ if _non_cuda_accelerate_route() is not None:
+ # Native FSDP2 is CUDA / ROCm / CPU only: an XPU / NPU / ... install keeps accelerate's FSDP2
+ # (pre-native behaviour); an explicit native pick stays native and is refused there.
+ return False
return True
+# {key: value before} of the accelerate env exported only because this is a non-CUDA install, so a
+# CPU Trainer can be handed back to native FSDP2.
+_NON_CUDA_EXPORT = None
+_CPU_TRAINER = False
+
+
+def _non_cuda_accelerate_route():
+ """The non_cuda_install() label when the generic UNSLOTH_FSDP2=1 request (no explicit native pick,
+ no `accelerate launch` FSDP config) goes to accelerate's FSDP2, else None."""
+ if not _env_flag("UNSLOTH_FSDP2") or _env_value("UNSLOTH_FSDP2_BACKEND") != "":
+ return None
+ if _env_flag("ACCELERATE_USE_FSDP"):
+ return None
+ return non_cuda_install()
+
+
+def _trainer_args_use_cpu(training_args):
+ return bool(
+ getattr(training_args, "use_cpu", False) or getattr(training_args, "no_cuda", False)
+ )
+
+
+def reclaim_native_fsdp2_for_cpu_trainer(training_args):
+ """TrainingArguments(use_cpu = True) (or `no_cuda`) on a non-CUDA install whose launch step routed
+ UNSLOTH_FSDP2=1 to accelerate's FSDP2: accelerate runs that Trainer as MULTI_CPU, which never builds
+ its FSDP plugin (accelerate/state.py AcceleratorState.__init__), so the sharding would be lost. Undo
+ the exported env before the Accelerator exists and let native FSDP2 shard on the gloo CPU mesh, as
+ it does for the same Trainer on a CUDA install. True when native FSDP2 now applies."""
+ global _NON_CUDA_EXPORT, _CPU_TRAINER
+ if _NON_CUDA_EXPORT is None or not _trainer_args_use_cpu(training_args):
+ return False
+ for key, value in _NON_CUDA_EXPORT.items():
+ if value is None:
+ os.environ.pop(key, None)
+ else:
+ os.environ[key] = value
+ _NON_CUDA_EXPORT = None
+ _CPU_TRAINER = True
+ if str(os.environ.get("RANK", "0")).strip() in ("", "0"):
+ print(
+ "Unsloth: UNSLOTH_FSDP2=1 with TrainingArguments(use_cpu = True): native FSDP2 shards the "
+ "model on the CPU (accelerate's FSDP2 does not run on a CPU launch)."
+ )
+ return unsloth_fsdp2_native_requested()
+
+
+def _torch_build():
+ """{"cuda", "hip", "xpu"}: the torch build's `torch.version` strings (None when absent), read from
+ torch/version.py without importing torch (the launch step runs before torch is imported), or None
+ when torch is not found. Same answer at the launch step and at Trainer setup."""
+ torch = sys.modules.get("torch")
+ version = getattr(torch, "version", None) if torch is not None else None
+ if version is not None:
+ build = {name: getattr(version, name, None) for name in ("cuda", "hip", "xpu")}
+ label = getattr(version, "__version__", None) or getattr(torch, "__version__", None)
+ return _xpu_from_label(build, label)
+ try:
+ spec = importlib.util.find_spec("torch")
+ except (ImportError, ValueError):
+ return None
+ for location in getattr(spec, "submodule_search_locations", None) or ():
+ try:
+ with open(os.path.join(location, "version.py"), encoding = "utf-8") as f:
+ source = f.read()
+ except OSError:
+ continue
+ build = {}
+ for name in ("cuda", "hip", "xpu", "__version__"):
+ match = re.search(rf"^{name}\s*(?::[^=\n]*)?=\s*(.+?)\s*$", source, re.M)
+ try:
+ build[name] = ast.literal_eval(match.group(1)) if match else None
+ except (ValueError, SyntaxError):
+ build[name] = None
+ return _xpu_from_label(build, build.pop("__version__"))
+ return None
+
+
+def _xpu_from_label(build, label):
+ """A `+xpu` wheel can ship torch.version.xpu = None (and torch < 2.7 has no xpu attribute at all):
+ the version label is then the only build marker, read the same way with and without torch."""
+ if not build.get("xpu") and "+xpu" in str(label or "").lower():
+ build["xpu"] = str(label)
+ return build
+
+
+def non_cuda_install():
+ """The label of the non-CUDA accelerator accelerate would pick on this install (a
+ NATIVE_BACKEND_TABLE device row), from the torch build and the installed vendor packages only, so
+ every rank and every call agrees; None on a CUDA / ROCm torch build, for a CPU launch
+ (ACCELERATE_USE_CPU), or when nothing matches."""
+ if _CPU_TRAINER or _env_flag("ACCELERATE_USE_CPU"):
+ return None
+ build = _torch_build()
+ if not build or build.get("cuda") or build.get("hip"):
+ return None
+ for row in NATIVE_BACKEND_TABLE:
+ if row["action"] != "refuse" or "device" not in row:
+ continue
+ package, attr, _ = row["device"]
+ if package is None:
+ if build.get(attr):
+ return row["label"]
+ continue
+ try:
+ if importlib.util.find_spec(package.split(".")[0]) is not None:
+ return row["label"]
+ except (ImportError, ValueError):
+ continue
+ return None
+
+
def fsdp_version_from_env():
"""0 for no FSDP, else 1 or 2. accelerate builds FSDP1 when ACCELERATE_USE_FSDP names no
version; FSDP_VERSION="0" means "not FSDP"."""
@@ -299,12 +418,28 @@ def refuse_native_backend_conflicts(training_args = None, uses_cpu = None):
found = native_backend_conflicts(training_args, uses_cpu)
if found:
what = "; ".join(f"{label} ({', '.join(reasons)})" for _, label, reasons in found)
+ devices = {row["type"] for row in NATIVE_BACKEND_TABLE if "device" in row}
+ hint = ""
+ if any(kind in devices for kind, _, _ in found):
+ hint = (
+ " For accelerate's FSDP2 on this device, set UNSLOTH_FSDP2=1 without a native pick "
+ "(or UNSLOTH_FSDP2_BACKEND=accelerate)."
+ )
raise RuntimeError(
"Unsloth: UNSLOTH_FSDP2=1 shards the model with native FSDP2 (CUDA / ROCm, or the CPU), "
- f"which cannot run under {what}. Drop that backend, or unset UNSLOTH_FSDP2."
+ f"which cannot run under {what}. Drop that backend, or unset UNSLOTH_FSDP2.{hint}"
)
+_LAUNCH_EXPORTS = (
+ "ACCELERATE_USE_FSDP",
+ "FSDP_VERSION",
+ "FSDP_AUTO_WRAP_POLICY",
+ "FSDP_RESHARD_AFTER_FORWARD",
+ "FSDP_STATE_DICT_TYPE",
+)
+
+
def apply_unsloth_fsdp2_launch_env():
"""UNSLOTH_FSDP2=1 with WORLD_SIZE > 1: export what `accelerate launch` exports for an FSDP2
config (explicit FSDP_* values win). Must run before transformers / accelerate read the env."""
@@ -332,10 +467,18 @@ def apply_unsloth_fsdp2_launch_env():
def _export_accelerate_fsdp2_env():
"""What `accelerate launch` exports for an FSDP2 config (explicit FSDP_* values win)."""
+ global _NON_CUDA_EXPORT
if _env_flag("ACCELERATE_USE_DEEPSPEED"):
raise RuntimeError(
"Unsloth: UNSLOTH_FSDP2=1 was set on a DeepSpeed launch. Unset one of them."
)
+ device = _non_cuda_accelerate_route()
+ if device is not None and str(os.environ.get("RANK", "0")).strip() in ("", "0"):
+ print(
+ f"Unsloth: UNSLOTH_FSDP2=1 with {device} (not a CUDA / ROCm torch build): native FSDP2 "
+ "runs on CUDA / ROCm or the CPU, so accelerate's FSDP2 shards the model."
+ )
+ before = {key: os.environ.get(key) for key in _LAUNCH_EXPORTS}
version = str(os.environ.get("FSDP_VERSION", "")).strip()
if version not in ("", "2"):
raise ValueError(
@@ -358,6 +501,10 @@ def _export_accelerate_fsdp2_env():
"false" if reshard.strip().lower() in _FALSY else "true",
)
os.environ.setdefault("FSDP_STATE_DICT_TYPE", "FULL_STATE_DICT")
+ if device is not None:
+ _NON_CUDA_EXPORT = {
+ key: value for key, value in before.items() if os.environ.get(key) != value
+ }
return True
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 3bfdf35175a..bd4881b82b8 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -5836,10 +5836,19 @@ def patch_gradient_accumulation_fix(Trainer):
# sharded over dp only, tp-aware clipping (tp_native.tp_trainer_setup). Dict lookups otherwise.
_refuse_pipeline_parallel_for_trainer(model)
tensor_parallel = _tensor_parallel_trainer_pre(model)
- from ..fsdp2_launch import unsloth_fsdp2_native_requested
+ from .. import fsdp2_launch as _fsdp2_launch
# Env reads only; fsdp2_native (and torch.distributed.fsdp) is imported only under the flag.
- native_fsdp2 = unsloth_fsdp2_native_requested()
+ native_fsdp2 = _fsdp2_launch.unsloth_fsdp2_native_requested()
+ if not native_fsdp2 and _fsdp2_launch._NON_CUDA_EXPORT is not None:
+ # UNSLOTH_FSDP2=1 on a non-CUDA install went to accelerate's FSDP2 at import, but a
+ # use_cpu Trainer is MULTI_CPU to accelerate, which drops FSDP: shard natively instead.
+ from .fsdp2_native import native_fsdp2_trainer_init_arguments
+ native_fsdp2 = _fsdp2_launch.reclaim_native_fsdp2_for_cpu_trainer(
+ native_fsdp2_trainer_init_arguments(_original_trainer_init, args, kwargs)[
+ "args"
+ ]
+ )
if native_fsdp2:
from .fsdp2_native import (
native_fsdp2_trainer_init_arguments,