Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Native FSDP2: keep accelerate's FSDP2 for UNSLOTH_FSDP2=1 on XPU / NPU / other non-CUDA installs (#14760)
Daniel Han40ef4b9bd0de
diff --git a/tests/test_fsdp2_native_backend_table.py b/tests/test_fsdp2_native_backend_table.pyindex bf28fe0754f..30dac192bea 100644--- a/tests/test_fsdp2_native_backend_table.py+++ b/tests/test_fsdp2_native_backend_table.py@@ -94,6 +94,9 @@ def test_table_covers_every_accelerate_distributed_type(native):def test_every_refuse_row_is_refused_at_both_entry_points(native, monkeypatch):launch = sys.modules["unsloth.fsdp2_launch"]+ # Explicit native: on a torch build without CUDA / ROCm (CPU CI) the generic flag routes a vendor+ # device to accelerate's FSDP2 instead (non_cuda_install), which is not refused.+ monkeypatch.setenv("UNSLOTH_FSDP2_BACKEND", "native")rows = [r for r in launch.NATIVE_BACKEND_TABLE if r["action"] == "refuse"]assert len(rows) >= 10for row in rows:diff --git a/tests/test_fsdp2_native_trainer_args_gloo.py b/tests/test_fsdp2_native_trainer_args_gloo.pyindex 7ec54c5460c..cfbdf3a2aee 100644--- a/tests/test_fsdp2_native_trainer_args_gloo.py+++ b/tests/test_fsdp2_native_trainer_args_gloo.py@@ -7,7 +7,9 @@* an optimizer passed positionally (Trainer's own order) is refused like a keyword one;* `model_init` (and with it hyperparameter_search) is refused instead of training unsharded;* `save_on_each_node=True` across nodes is refused (only global rank 0 holds the gathered state);-* a resume of a non-default active PEFT adapter reads `<ckpt>/<adapter>/`, never the root adapter.+* a resume of a non-default active PEFT adapter reads `<ckpt>/<adapter>/`, never the root adapter;+* UNSLOTH_FSDP2=1 on an XPU install with TrainingArguments(use_cpu = True) still shards natively+ (accelerate runs a CPU Trainer as MULTI_CPU, which drops its FSDP plugin).Plus the launcher env: an explicit native backend turns accelerate's FSDP launch env off."""import multiprocessing@@ -182,6 +184,18 @@ def _worker(rank, world, port, out, case, tmp, env):res["optimizer_keys"] = sorted(state)except RuntimeError as exc:res["error"] = str(exc)+ elif case == "xpu_install_use_cpu":+ # An XPU torch build (no CUDA / ROCm): the launch step routes UNSLOTH_FSDP2=1 to+ # accelerate's FSDP2, then the use_cpu Trainer takes native FSDP2 back.+ torch.version.cuda = None+ torch.version.hip = None+ torch.version.xpu = "20250101"+ launch = sys.modules["unsloth.fsdp2_launch"]+ res["routed"] = launch.apply_unsloth_fsdp2_launch_env()+ trainer = Trainer(model = _lora(1), args = _targs(os.path.join(tmp, "out")))+ res["sharded"] = native.is_native_fsdp2(trainer.model)+ res["accelerate_fsdp"] = os.environ.get("ACCELERATE_USE_FSDP")+ res["distributed_type"] = str(trainer.accelerator.distributed_type)elif case == "named_adapter":cfg = native.FSDP2Config(param_dtype = torch.float32)ckpt = os.path.join(tmp, "ckpt")@@ -269,6 +283,15 @@ def test_resume_reads_named_adapter_from_its_subdirectory(tmp_path):assert r["n_other"] > 0 and r["other_diff"] == 0.0, (rank, r)+def test_xpu_install_cpu_trainer_shards_natively(tmp_path):+ pytest.importorskip("transformers")+ pytest.importorskip("peft")+ for rank, r in _run("xpu_install_use_cpu", tmp_path).items():+ assert r["routed"] is True, (rank, r)+ assert r["sharded"] is True and r["accelerate_fsdp"] is None, (rank, r)+ assert "MULTI_CPU" in r["distributed_type"], (rank, r)++# ---- launcher env (no process group) ----------------------------------------------------------diff --git a/tests/test_fsdp2_non_cuda_accelerate_route.py b/tests/test_fsdp2_non_cuda_accelerate_route.pynew file mode 100644index 00000000000..c0935683495--- /dev/null+++ b/tests/test_fsdp2_non_cuda_accelerate_route.py@@ -0,0 +1,209 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_FSDP2=1 on an XPU / NPU / ... install (no CUDA / ROCm torch build) keeps accelerate's FSDP2+wrap, as it did before the native backend existed, instead of being refused: native FSDP2 runs on CUDA /+ROCm or the CPU only. The decision is made from the torch build and the installed vendor packages (no+torch import), so the launch step and Trainer setup agree. An explicit UNSLOTH_FSDP2_BACKEND=native is+still native (and refused at Trainer setup when that device is present)."""++import importlib.util+import os+import sys+import types++import pytest+import torch++REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+_ENV_AT_IMPORT = dict(os.environ)++_CLEAN = (+ "ACCELERATE_USE_DEEPSPEED",+ "ACCELERATE_USE_FSDP",+ "ACCELERATE_USE_CPU",+ "FSDP_VERSION",+ "FSDP_AUTO_WRAP_POLICY",+ "FSDP_STATE_DICT_TYPE",+ "UNSLOTH_FSDP2_BACKEND",+)+++def launch(monkeypatch):+ # apply_unsloth_fsdp2_launch_env writes os.environ directly; monkeypatch.delenv records nothing for+ # an absent key, so restore the whole environment or the FSDP launch env leaks into later tests.+ saved = dict(os.environ)+ yield _launch_module(monkeypatch)+ os.environ.clear()+ os.environ.update(saved)+++def _launch_module(monkeypatch):+ for key in _CLEAN:+ monkeypatch.delenv(key, raising = False)+ for key, value in dict(UNSLOTH_FSDP2 = "1", WORLD_SIZE = "2", RANK = "0").items():+ monkeypatch.setenv(key, value)+ spec = importlib.util.spec_from_file_location(+ "_fsdp2_launch_under_test", os.path.join(REPO, "unsloth", "fsdp2_launch.py")+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def _build(+ monkeypatch,+ cuda = None,+ hip = None,+ xpu = None,+):+ monkeypatch.setattr(torch.version, "cuda", cuda, raising = False)+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)+ monkeypatch.setattr(torch.version, "xpu", xpu, raising = False)+++def test_xpu_build_routes_to_accelerate_fsdp2(launch, monkeypatch):+ _build(monkeypatch, xpu = "20250101")+ assert launch.unsloth_fsdp2_native_requested() is False+ assert launch.apply_unsloth_fsdp2_launch_env() is True+ assert os.environ["ACCELERATE_USE_FSDP"] == "true"+ assert os.environ["FSDP_VERSION"] == "2"+ assert launch.fsdp_version_from_env() == 2+++def test_vendor_package_on_a_non_cuda_build_routes_to_accelerate(launch, monkeypatch):+ _build(monkeypatch)+ real = importlib.util.find_spec+ monkeypatch.setattr(+ importlib.util,+ "find_spec",+ lambda name, *a, **k: object() if name == "torch_npu" else real(name, *a, **k),+ )+ assert launch.unsloth_fsdp2_native_requested() is False+ assert launch.apply_unsloth_fsdp2_launch_env() is True++[email protected]("build", [dict(cuda = "12.8"), dict(hip = "6.3"), dict(cuda = "12.8", xpu = "1")])+def test_cuda_and_rocm_builds_stay_native(launch, monkeypatch, build):+ _build(monkeypatch, **build)+ assert launch.unsloth_fsdp2_native_requested() is True+ assert launch.apply_unsloth_fsdp2_launch_env() is False+ assert "ACCELERATE_USE_FSDP" not in os.environ+++def test_explicit_native_backend_and_cpu_launch_stay_native(launch, monkeypatch):+ _build(monkeypatch, xpu = "20250101")+ monkeypatch.setenv("UNSLOTH_FSDP2_BACKEND", "native")+ assert launch.unsloth_fsdp2_native_requested() is True+ monkeypatch.delenv("UNSLOTH_FSDP2_BACKEND")+ monkeypatch.setenv("ACCELERATE_USE_CPU", "true")+ assert launch.unsloth_fsdp2_native_requested() is True+++def test_torch_build_is_read_without_importing_torch(launch, monkeypatch, tmp_path):+ pkg = tmp_path / "torch"+ pkg.mkdir()+ (pkg / "version.py").write_text(+ "from typing import Optional\n__version__ = '2.11.0+xpu'\n"+ "cuda: Optional[str] = None\nhip: Optional[str] = None\nxpu: Optional[str] = '20250101'\n"+ )+ spec = importlib.machinery.ModuleSpec("torch", None, is_package = True)+ spec.submodule_search_locations = [str(pkg)]+ real = importlib.util.find_spec+ monkeypatch.setattr(+ importlib.util,+ "find_spec",+ lambda name, *a, **k: spec if name == "torch" else real(name, *a, **k),+ )+ monkeypatch.delitem(sys.modules, "torch")+ try:+ assert launch.unsloth_fsdp2_native_requested() is False+ finally:+ sys.modules["torch"] = torch+++def test_xpu_version_label_with_a_null_runtime_marker(launch, monkeypatch):+ # A +xpu wheel can carry torch.version.xpu = None: the version label is the build marker.+ _build(monkeypatch)+ monkeypatch.setattr(torch.version, "__version__", "2.9.1+xpu", raising = False)+ monkeypatch.setattr(torch, "__version__", "2.9.1+xpu")+ assert launch.non_cuda_install() == "Intel XPU"+ assert launch.unsloth_fsdp2_native_requested() is False+ assert launch.apply_unsloth_fsdp2_launch_env() is True+++def test_xpu_version_label_read_without_importing_torch(launch, monkeypatch, tmp_path):+ pkg = tmp_path / "torch"+ pkg.mkdir()+ (pkg / "version.py").write_text(+ "from typing import Optional\n__version__ = '2.9.1+xpu'\n"+ "cuda: Optional[str] = None\nhip: Optional[str] = None\nxpu: Optional[str] = None\n"+ )+ spec = importlib.machinery.ModuleSpec("torch", None, is_package = True)+ spec.submodule_search_locations = [str(pkg)]+ real = importlib.util.find_spec+ monkeypatch.setattr(+ importlib.util,+ "find_spec",+ lambda name, *a, **k: spec if name == "torch" else real(name, *a, **k),+ )+ monkeypatch.delitem(sys.modules, "torch")+ try:+ assert launch.non_cuda_install() == "Intel XPU"+ finally:+ sys.modules["torch"] = torch+++def test_cpu_wheel_label_is_not_xpu(launch, monkeypatch):+ _build(monkeypatch)+ monkeypatch.setattr(torch.version, "__version__", "2.9.1+cpu", raising = False)+ monkeypatch.setattr(torch, "__version__", "2.9.1+cpu")+ assert launch.non_cuda_install() is None+ assert launch.unsloth_fsdp2_native_requested() is True+++def test_unsloth_fsdp2_native_on_xpu_stays_native_and_is_refused(launch, monkeypatch):+ _build(monkeypatch, xpu = "20250101")+ monkeypatch.setenv("UNSLOTH_FSDP2", "native")+ assert launch.unsloth_fsdp2_native_requested() is True+ assert launch.apply_unsloth_fsdp2_launch_env() is False+ assert "ACCELERATE_USE_FSDP" not in os.environ+ # Trainer setup with a live XPU device: refused with the accelerate alternative named.+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ monkeypatch.setattr(+ torch, "xpu", types.SimpleNamespace(is_available = lambda: True), raising = False+ )+ with pytest.raises(RuntimeError, match = "UNSLOTH_FSDP2_BACKEND=accelerate"):+ launch.refuse_native_backend_conflicts(None, uses_cpu = False)+ # The same explicit request with a CPU Trainer runs natively on the CPU.+ launch.refuse_native_backend_conflicts(None, uses_cpu = True)++[email protected]("flag", ["use_cpu", "no_cuda"])+def test_cpu_trainer_on_xpu_install_takes_native_back(launch, monkeypatch, flag):+ _build(monkeypatch, xpu = "20250101")+ monkeypatch.setenv("FSDP_STATE_DICT_TYPE", "SHARDED_STATE_DICT")+ assert launch.apply_unsloth_fsdp2_launch_env() is True+ assert launch.reclaim_native_fsdp2_for_cpu_trainer(types.SimpleNamespace()) is False+ assert os.environ["ACCELERATE_USE_FSDP"] == "true"+ assert launch.reclaim_native_fsdp2_for_cpu_trainer(types.SimpleNamespace(**{flag: True}))+ for key in ("ACCELERATE_USE_FSDP", "FSDP_VERSION", "FSDP_AUTO_WRAP_POLICY"):+ assert key not in os.environ, key+ assert os.environ["FSDP_STATE_DICT_TYPE"] == "SHARDED_STATE_DICT" # the user's value stays+ assert launch.unsloth_fsdp2_native_requested() is True+ assert launch.native_backend_conflicts(None, uses_cpu = True) == []+++def test_cpu_trainer_reclaim_needs_the_non_cuda_route(launch, monkeypatch):+ _build(monkeypatch, cuda = "12.8")+ monkeypatch.setenv("UNSLOTH_FSDP2_BACKEND", "accelerate")+ assert launch.apply_unsloth_fsdp2_launch_env() is True+ assert launch.reclaim_native_fsdp2_for_cpu_trainer(types.SimpleNamespace(use_cpu = True)) is False+ assert os.environ["ACCELERATE_USE_FSDP"] == "true"+++def test_zz_launch_env_does_not_leak_out_of_this_module():+ # Runs last in this file: every key the launch tests wrote is back to its value at import.+ for key in _CLEAN + ("UNSLOTH_FSDP2", "WORLD_SIZE", "RANK"):+ assert os.environ.get(key) == _ENV_AT_IMPORT.get(key), keydiff --git a/unsloth/fsdp2_launch.py b/unsloth/fsdp2_launch.pyindex 8e1a9b489bc..e9f3a9567b6 100644--- a/unsloth/fsdp2_launch.py+++ b/unsloth/fsdp2_launch.py@@ -3,13 +3,17 @@"""FSDP launch detection from the launcher env only (no torch): identical on every rank."""+import astimport importlib.utilimport os+import reimport sys__all__ = ["unsloth_fsdp2_requested","unsloth_fsdp2_native_requested",+ "non_cuda_install",+ "reclaim_native_fsdp2_for_cpu_trainer","fsdp_version_from_env","apply_unsloth_fsdp2_launch_env","NATIVE_BACKEND_TABLE",@@ -58,9 +62,124 @@ def unsloth_fsdp2_native_requested():)if backend == "" and _env_flag("ACCELERATE_USE_FSDP"):return False+ if _non_cuda_accelerate_route() is not None:+ # Native FSDP2 is CUDA / ROCm / CPU only: an XPU / NPU / ... install keeps accelerate's FSDP2+ # (pre-native behaviour); an explicit native pick stays native and is refused there.+ return Falsereturn True+# {key: value before} of the accelerate env exported only because this is a non-CUDA install, so a+# CPU Trainer can be handed back to native FSDP2.+_NON_CUDA_EXPORT = None+_CPU_TRAINER = False+++def _non_cuda_accelerate_route():+ """The non_cuda_install() label when the generic UNSLOTH_FSDP2=1 request (no explicit native pick,+ no `accelerate launch` FSDP config) goes to accelerate's FSDP2, else None."""+ if not _env_flag("UNSLOTH_FSDP2") or _env_value("UNSLOTH_FSDP2_BACKEND") != "":+ return None+ if _env_flag("ACCELERATE_USE_FSDP"):+ return None+ return non_cuda_install()+++def _trainer_args_use_cpu(training_args):+ return bool(+ getattr(training_args, "use_cpu", False) or getattr(training_args, "no_cuda", False)+ )+++def reclaim_native_fsdp2_for_cpu_trainer(training_args):+ """TrainingArguments(use_cpu = True) (or `no_cuda`) on a non-CUDA install whose launch step routed+ UNSLOTH_FSDP2=1 to accelerate's FSDP2: accelerate runs that Trainer as MULTI_CPU, which never builds+ its FSDP plugin (accelerate/state.py AcceleratorState.__init__), so the sharding would be lost. Undo+ the exported env before the Accelerator exists and let native FSDP2 shard on the gloo CPU mesh, as+ it does for the same Trainer on a CUDA install. True when native FSDP2 now applies."""+ global _NON_CUDA_EXPORT, _CPU_TRAINER+ if _NON_CUDA_EXPORT is None or not _trainer_args_use_cpu(training_args):+ return False+ for key, value in _NON_CUDA_EXPORT.items():+ if value is None:+ os.environ.pop(key, None)+ else:+ os.environ[key] = value+ _NON_CUDA_EXPORT = None+ _CPU_TRAINER = True+ if str(os.environ.get("RANK", "0")).strip() in ("", "0"):+ print(+ "Unsloth: UNSLOTH_FSDP2=1 with TrainingArguments(use_cpu = True): native FSDP2 shards the "+ "model on the CPU (accelerate's FSDP2 does not run on a CPU launch)."+ )+ return unsloth_fsdp2_native_requested()+++def _torch_build():+ """{"cuda", "hip", "xpu"}: the torch build's `torch.version` strings (None when absent), read from+ torch/version.py without importing torch (the launch step runs before torch is imported), or None+ when torch is not found. Same answer at the launch step and at Trainer setup."""+ torch = sys.modules.get("torch")+ version = getattr(torch, "version", None) if torch is not None else None+ if version is not None:+ build = {name: getattr(version, name, None) for name in ("cuda", "hip", "xpu")}+ label = getattr(version, "__version__", None) or getattr(torch, "__version__", None)+ return _xpu_from_label(build, label)+ try:+ spec = importlib.util.find_spec("torch")+ except (ImportError, ValueError):+ return None+ for location in getattr(spec, "submodule_search_locations", None) or ():+ try:+ with open(os.path.join(location, "version.py"), encoding = "utf-8") as f:+ source = f.read()+ except OSError:+ continue+ build = {}+ for name in ("cuda", "hip", "xpu", "__version__"):+ match = re.search(rf"^{name}\s*(?::[^=\n]*)?=\s*(.+?)\s*$", source, re.M)+ try:+ build[name] = ast.literal_eval(match.group(1)) if match else None+ except (ValueError, SyntaxError):+ build[name] = None+ return _xpu_from_label(build, build.pop("__version__"))+ return None+++def _xpu_from_label(build, label):+ """A `+xpu` wheel can ship torch.version.xpu = None (and torch < 2.7 has no xpu attribute at all):+ the version label is then the only build marker, read the same way with and without torch."""+ if not build.get("xpu") and "+xpu" in str(label or "").lower():+ build["xpu"] = str(label)+ return build+++def non_cuda_install():+ """The label of the non-CUDA accelerator accelerate would pick on this install (a+ NATIVE_BACKEND_TABLE device row), from the torch build and the installed vendor packages only, so+ every rank and every call agrees; None on a CUDA / ROCm torch build, for a CPU launch+ (ACCELERATE_USE_CPU), or when nothing matches."""+ if _CPU_TRAINER or _env_flag("ACCELERATE_USE_CPU"):+ return None+ build = _torch_build()+ if not build or build.get("cuda") or build.get("hip"):+ return None+ for row in NATIVE_BACKEND_TABLE:+ if row["action"] != "refuse" or "device" not in row:+ continue+ package, attr, _ = row["device"]+ if package is None:+ if build.get(attr):+ return row["label"]+ continue+ try:+ if importlib.util.find_spec(package.split(".")[0]) is not None:+ return row["label"]+ except (ImportError, ValueError):+ continue+ return None++def fsdp_version_from_env():"""0 for no FSDP, else 1 or 2. accelerate builds FSDP1 when ACCELERATE_USE_FSDP names noversion; FSDP_VERSION="0" means "not FSDP"."""@@ -299,12 +418,28 @@ def refuse_native_backend_conflicts(training_args = None, uses_cpu = None):found = native_backend_conflicts(training_args, uses_cpu)if found:what = "; ".join(f"{label} ({', '.join(reasons)})" for _, label, reasons in found)+ devices = {row["type"] for row in NATIVE_BACKEND_TABLE if "device" in row}+ hint = ""+ if any(kind in devices for kind, _, _ in found):+ hint = (+ " For accelerate's FSDP2 on this device, set UNSLOTH_FSDP2=1 without a native pick "+ "(or UNSLOTH_FSDP2_BACKEND=accelerate)."+ )raise RuntimeError("Unsloth: UNSLOTH_FSDP2=1 shards the model with native FSDP2 (CUDA / ROCm, or the CPU), "- f"which cannot run under {what}. Drop that backend, or unset UNSLOTH_FSDP2."+ f"which cannot run under {what}. Drop that backend, or unset UNSLOTH_FSDP2.{hint}")+_LAUNCH_EXPORTS = (+ "ACCELERATE_USE_FSDP",+ "FSDP_VERSION",+ "FSDP_AUTO_WRAP_POLICY",+ "FSDP_RESHARD_AFTER_FORWARD",+ "FSDP_STATE_DICT_TYPE",+)++def apply_unsloth_fsdp2_launch_env():"""UNSLOTH_FSDP2=1 with WORLD_SIZE > 1: export what `accelerate launch` exports for an FSDP2config (explicit FSDP_* values win). Must run before transformers / accelerate read the env."""@@ -332,10 +467,18 @@ def apply_unsloth_fsdp2_launch_env():def _export_accelerate_fsdp2_env():"""What `accelerate launch` exports for an FSDP2 config (explicit FSDP_* values win)."""+ global _NON_CUDA_EXPORTif _env_flag("ACCELERATE_USE_DEEPSPEED"):raise RuntimeError("Unsloth: UNSLOTH_FSDP2=1 was set on a DeepSpeed launch. Unset one of them.")+ device = _non_cuda_accelerate_route()+ if device is not None and str(os.environ.get("RANK", "0")).strip() in ("", "0"):+ print(+ f"Unsloth: UNSLOTH_FSDP2=1 with {device} (not a CUDA / ROCm torch build): native FSDP2 "+ "runs on CUDA / ROCm or the CPU, so accelerate's FSDP2 shards the model."+ )+ before = {key: os.environ.get(key) for key in _LAUNCH_EXPORTS}version = str(os.environ.get("FSDP_VERSION", "")).strip()if version not in ("", "2"):raise ValueError(@@ -358,6 +501,10 @@ def _export_accelerate_fsdp2_env():"false" if reshard.strip().lower() in _FALSY else "true",)os.environ.setdefault("FSDP_STATE_DICT_TYPE", "FULL_STATE_DICT")+ if device is not None:+ _NON_CUDA_EXPORT = {+ key: value for key, value in before.items() if os.environ.get(key) != value+ }return Truediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 3bfdf35175a..bd4881b82b8 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -5836,10 +5836,19 @@ def patch_gradient_accumulation_fix(Trainer):# sharded over dp only, tp-aware clipping (tp_native.tp_trainer_setup). Dict lookups otherwise._refuse_pipeline_parallel_for_trainer(model)tensor_parallel = _tensor_parallel_trainer_pre(model)- from ..fsdp2_launch import unsloth_fsdp2_native_requested+ from .. import fsdp2_launch as _fsdp2_launch# Env reads only; fsdp2_native (and torch.distributed.fsdp) is imported only under the flag.- native_fsdp2 = unsloth_fsdp2_native_requested()+ native_fsdp2 = _fsdp2_launch.unsloth_fsdp2_native_requested()+ if not native_fsdp2 and _fsdp2_launch._NON_CUDA_EXPORT is not None:+ # UNSLOTH_FSDP2=1 on a non-CUDA install went to accelerate's FSDP2 at import, but a+ # use_cpu Trainer is MULTI_CPU to accelerate, which drops FSDP: shard natively instead.+ from .fsdp2_native import native_fsdp2_trainer_init_arguments+ native_fsdp2 = _fsdp2_launch.reclaim_native_fsdp2_for_cpu_trainer(+ native_fsdp2_trainer_init_arguments(_original_trainer_init, args, kwargs)[+ "args"+ ]+ )if native_fsdp2:from .fsdp2_native import (native_fsdp2_trainer_init_arguments,