Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Run GRPO under FSDP2: no inference mode around FSDP, an optimizer that steps DTensors, and refuse vLLM with FSDP2 (#13156)

Daniel Hancc10f80cefee
diff --git a/tests/test_dia_generate_pad_token.py b/tests/test_dia_generate_pad_token.py
index b0808f50658..d65f560c4f4 100644
--- a/tests/test_dia_generate_pad_token.py
+++ b/tests/test_dia_generate_pad_token.py
@@ -49,6 +49,7 @@ def _captured_generate_kwargs(config, **generate_kwargs):
"FastBaseModel": SimpleNamespace(for_inference = lambda model: None),
"dtype_from_config": lambda config: "bfloat16",
"_get_dtype": lambda dtype: dtype,
+ "_get_inference_mode_context_manager": lambda model: nullcontext(),
"_unsloth_generate_accepts_kwarg": lambda model, name: False,
"NUM_LOGITS_TO_KEEP": {"DiaForConditionalGeneration": None},
"DEVICE_TYPE_TORCH": "cuda",
diff --git a/tests/test_fa2_fast_generate_bypass.py b/tests/test_fa2_fast_generate_bypass.py
index 1b3d4d69e60..67f3e25fede 100644
--- a/tests/test_fa2_fast_generate_bypass.py
+++ b/tests/test_fa2_fast_generate_bypass.py
@@ -211,6 +211,7 @@ def test_wrapper_dispatch_preserves_normalization_and_selects_expected_path():
"FastBaseModel": FakeFastBaseModel,
"dtype_from_config": lambda config: "bfloat16",
"_get_dtype": lambda dtype: dtype,
+ "_get_inference_mode_context_manager": lambda model: nullcontext(),
"_unsloth_generate_accepts_kwarg": lambda model, name: False,
"NUM_LOGITS_TO_KEEP": {architecture: None},
"DEVICE_TYPE_TORCH": "cuda",
@@ -298,6 +299,7 @@ def test_flash_attention_fallback_pins_a_dynamic_cache():
"FastBaseModel": SimpleNamespace(for_inference = lambda model: None),
"dtype_from_config": lambda config: "bfloat16",
"_get_dtype": lambda dtype: dtype,
+ "_get_inference_mode_context_manager": lambda model: nullcontext(),
"_unsloth_generate_accepts_kwarg": lambda model, name: False,
"NUM_LOGITS_TO_KEEP": {"Qwen3VLForConditionalGeneration": None},
"DEVICE_TYPE_TORCH": "cuda",
diff --git a/tests/test_fast_inference_refuses_fsdp2.py b/tests/test_fast_inference_refuses_fsdp2.py
new file mode 100644
index 00000000000..944714c5318
--- /dev/null
+++ b/tests/test_fast_inference_refuses_fsdp2.py
@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""unsloth#3551: `fast_inference = True` (vLLM) is refused under FSDP2, before vLLM loads."""
+
+from __future__ import annotations
+
+import ast
+import os
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from real_accelerator import has_real_cuda
+from unsloth.models import loader_utils
+from unsloth.models.loader_utils import fsdp2_requested, raise_if_fast_inference_under_fsdp2
+
+
+def no_launcher(monkeypatch):
+ monkeypatch.delenv("FSDP_VERSION", raising = False)
+ state = pytest.importorskip("accelerate.state")
+ monkeypatch.setattr(state.AcceleratorState, "_shared_state", {})
+ return monkeypatch
+
+
[email protected]("value, fsdp2", [("2", True), ("1", False), ("0", False)])
+def test_the_launcher_env_names_fsdp2(no_launcher, value, fsdp2):
+ no_launcher.setenv("FSDP_VERSION", value)
+ assert fsdp2_requested() is fsdp2
+
+
+def test_an_accelerator_with_an_fsdp2_plugin(no_launcher):
+ from accelerate.state import AcceleratorState
+ no_launcher.setattr(
+ AcceleratorState, "_shared_state", {"fsdp_plugin": SimpleNamespace(fsdp_version = 2)}
+ )
+ assert fsdp2_requested()
+
+
+def test_vllm_under_fsdp2_is_refused_with_the_way_out(no_launcher):
+ no_launcher.setenv("FSDP_VERSION", "2")
+ with pytest.raises(NotImplementedError, match = "fast_inference = False"):
+ raise_if_fast_inference_under_fsdp2(True)
+
+
[email protected]("fsdp_version, fast_inference", [("2", False), ("1", True), (None, True)])
+def test_everything_else_loads(no_launcher, fsdp_version, fast_inference):
+ if fsdp_version is not None:
+ no_launcher.setenv("FSDP_VERSION", fsdp_version)
+ raise_if_fast_inference_under_fsdp2(fast_inference)
+
+
+def test_both_loaders_refuse_after_the_dgx_spark_fallback():
+ """FastLanguageModel and FastModel: GB10 still falls back to native inference before the FSDP2 check."""
+ source = Path(loader_utils.__file__).with_name("loader.py").read_text(encoding = "utf-8")
+ tree = ast.parse(source)
+ checked = []
+ for cls in (n for n in tree.body if isinstance(n, ast.ClassDef)):
+ for fn in (
+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained"
+ ):
+ guards = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call)
+ and ast.unparse(n.func) == "raise_if_fast_inference_under_fsdp2"
+ ]
+ spark = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Constant)
+ and isinstance(n.value, str)
+ and "DGX Spark detected" in n.value
+ ]
+ if spark:
+ assert len(guards) == 1 and guards[0] > max(spark), (cls.name, guards, spark)
+ checked.append(cls.name)
+ assert {"FastLanguageModel", "FastModel"} <= set(checked), checked
+
+
+_TRAINER_PROBE = textwrap.dedent(
+ """
+ import os
+ from types import SimpleNamespace
+ import torch
+ from unsloth import FastLanguageModel # noqa: F401, patches the TRL trainers
+ from trl import GRPOConfig, GRPOTrainer
+
+ class FastInferenceModel(torch.nn.Module):
+ vllm_engine = object()
+ max_seq_length = 64
+
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(dtype = torch.bfloat16, torch_dtype = torch.bfloat16)
+ self.warnings_issued = {}
+
+ def for_training(self, *args, **kwargs):
+ pass
+
+ def build(**kwargs):
+ try:
+ args = GRPOConfig(output_dir = os.environ["PROBE_OUT"], **kwargs)
+ GRPOTrainer(model = FastInferenceModel(), reward_funcs = [], args = args)
+ except NotImplementedError as e:
+ return "REFUSED" if "FSDP2" in str(e) else "OTHER"
+ except Exception:
+ return "OTHER"
+ return "BUILT"
+
+ print("PROBE", build(fsdp = "full_shard", fsdp_config = {"fsdp_version": 2}),
+ build(fsdp = "full_shard", fsdp_config = {"fsdp_version": 1}), build(), build(fsdp = "full_shard"))
+ """
+)
+
+
+ not has_real_cuda(), reason = "the probe imports unsloth outside the conftest CPU spoof"
+)
+def test_trainer_args_fsdp2_is_refused_when_the_trainer_is_built(tmp_path):
+ """torchrun + TrainingArguments(fsdp = ..., fsdp_config = {"fsdp_version": 2}): nothing said FSDP2 at load time."""
+ env = {k: v for k, v in os.environ.items() if k not in ("FSDP_VERSION", "ACCELERATE_USE_FSDP")}
+ env.update(
+ UNSLOTH_COMPILE_LOCATION = str(tmp_path / "cache"),
+ PROBE_OUT = str(tmp_path / "out"),
+ HF_HUB_OFFLINE = "1",
+ )
+ run = subprocess.run(
+ [sys.executable, "-c", _TRAINER_PROBE], env = env, capture_output = True, text = True, timeout = 900
+ )
+ lines = [line for line in run.stdout.splitlines() if line.startswith("PROBE ")]
+ assert lines, f"probe failed:\n{run.stdout[-3000:]}\n{run.stderr[-3000:]}"
+ fsdp2, fsdp1, plain, unversioned = lines[-1].split()[1:]
+ assert fsdp2 == "REFUSED"
+ assert fsdp1 != "REFUSED" and plain != "REFUSED"
+ from transformers import __version__
+ from unsloth_zoo.utils import Version
+
+ # No version named: TrainingArguments picks FSDP2 from transformers 5.10.0, FSDP1 before.
+ assert (unversioned == "REFUSED") == (Version(__version__) >= Version("5.10.0"))
diff --git a/tests/test_fsdp2_default_optim.py b/tests/test_fsdp2_default_optim.py
new file mode 100644
index 00000000000..1888fdb5ed7
--- /dev/null
+++ b/tests/test_fsdp2_default_optim.py
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""unsloth#3551: Unsloth's adamw_8bit default cannot step FSDP2's DTensor parameters."""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+from real_accelerator import has_real_cuda
+
+pytest.importorskip("trl")
+# The probe imports unsloth in a fresh process, outside the conftest CPU spoof.
+pytestmark = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA GPU")
+
+_PROBE = textwrap.dedent(
+ """
+ import json, os
+ from unsloth import FastLanguageModel # noqa: F401, patches the TRL configs
+ from trl import GRPOConfig, SFTConfig
+
+ def optim(cls, env, **kwargs):
+ for name in ("FSDP_VERSION", "ACCELERATE_USE_FSDP"):
+ os.environ.pop(name, None)
+ os.environ.update(env)
+ value = cls(output_dir = os.environ["PROBE_OUT"], **kwargs).optim
+ return str(getattr(value, "value", value))
+
+ fsdp2_args = {"fsdp": "full_shard", "fsdp_config": {"fsdp_version": 2}}
+ fsdp2_json = os.environ["PROBE_OUT"] + "_cfg.json"
+ with open(fsdp2_json, "w", encoding = "utf-8") as f:
+ json.dump({"fsdp_version": 2}, f)
+ print("PROBE " + json.dumps({
+ "grpo_plain": optim(GRPOConfig, {}),
+ "sft_plain": optim(SFTConfig, {}),
+ "grpo_fsdp1": optim(GRPOConfig, {"ACCELERATE_USE_FSDP": "true", "FSDP_VERSION": "1"}),
+ "grpo_fsdp2": optim(GRPOConfig, {"ACCELERATE_USE_FSDP": "true", "FSDP_VERSION": "2"}),
+ "sft_fsdp2": optim(SFTConfig, {"ACCELERATE_USE_FSDP": "true", "FSDP_VERSION": "2"}),
+ "grpo_fsdp2_args": optim(GRPOConfig, {}, **fsdp2_args),
+ "grpo_fsdp2_json": optim(GRPOConfig, {}, fsdp = "full_shard", fsdp_config = fsdp2_json),
+ "grpo_fsdp2_explicit_torch": optim(GRPOConfig, {"FSDP_VERSION": "2"}, optim = "adamw_torch"),
+ "grpo_fsdp_unversioned": optim(GRPOConfig, {}, fsdp = "full_shard"),
+ }))
+ """
+)
+
+
[email protected](scope = "module")
+def probe(tmp_path_factory):
+ tmp = tmp_path_factory.mktemp("fsdp2_optim")
+ env = dict(os.environ)
+ env.update(UNSLOTH_COMPILE_LOCATION = str(tmp / "cache"), PROBE_OUT = str(tmp / "out"))
+ run = subprocess.run(
+ [sys.executable, "-c", _PROBE], env = env, capture_output = True, text = True, timeout = 900
+ )
+ lines = [line for line in run.stdout.splitlines() if line.startswith("PROBE ")]
+ assert lines, f"probe failed:\n{run.stdout[-3000:]}\n{run.stderr[-3000:]}"
+ return json.loads(lines[-1][len("PROBE ") :])
+
+
+def test_without_fsdp2_the_default_is_unchanged(probe):
+ assert probe["grpo_plain"] == "adamw_8bit"
+ assert probe["sft_plain"] == "adamw_8bit"
+ assert probe["grpo_fsdp1"] == "adamw_8bit"
+
+
[email protected]("case", ["grpo_fsdp2", "sft_fsdp2", "grpo_fsdp2_args", "grpo_fsdp2_json"])
+def test_fsdp2_gets_an_optimizer_that_steps_dtensors(probe, case):
+ assert probe[case] == "adamw_torch_fused"
+
+
+def test_fsdp_without_a_version_follows_the_transformers_default(probe):
+ """TrainingArguments picks FSDP2 when fsdp_config names no version from transformers 5.10.0, FSDP1 before."""
+ from transformers import __version__
+ from unsloth_zoo.utils import Version
+
+ fsdp2 = Version(__version__) >= Version("5.10.0")
+ assert probe["grpo_fsdp_unversioned"] == ("adamw_torch_fused" if fsdp2 else "adamw_8bit")
+
+
+def test_an_explicit_non_bnb_choice_is_kept(probe):
+ assert probe["grpo_fsdp2_explicit_torch"] == "adamw_torch"
diff --git a/tests/test_fsdp_generation_grad_mode.py b/tests/test_fsdp_generation_grad_mode.py
new file mode 100644
index 00000000000..04601c35bc0
--- /dev/null
+++ b/tests/test_fsdp_generation_grad_mode.py
@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""unsloth#3551: GRPO generation and logprob passes must not run FSDP under inference mode."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from unsloth.models._utils import _get_inference_mode_context_manager
+
+_FSDP_ENV = ("ACCELERATE_USE_FSDP", "FSDP_VERSION")
+
+
+def no_launcher(monkeypatch):
+ for name in _FSDP_ENV:
+ monkeypatch.delenv(name, raising = False)
+ state = pytest.importorskip("accelerate.state")
+ monkeypatch.setattr(state.AcceleratorState, "_shared_state", {})
+ return monkeypatch
+
+
+def _mode_inside(model):
+ with _get_inference_mode_context_manager(model):
+ return torch.is_inference_mode_enabled(), torch.is_grad_enabled()
+
+
+def test_single_process_keeps_inference_mode(no_launcher):
+ assert _mode_inside(torch.nn.Linear(2, 2)) == (True, False)
+
+
+ "name, value, inference",
+ [
+ ("ACCELERATE_USE_FSDP", "true", False),
+ ("ACCELERATE_USE_FSDP", "false", True),
+ ("FSDP_VERSION", "2", False),
+ ("FSDP_VERSION", "0", True),
+ ],
+)
+def test_the_accelerate_launcher_env_decides(no_launcher, name, value, inference):
+ no_launcher.setenv(name, value)
+ assert _mode_inside(torch.nn.Linear(2, 2)) == (inference, False)
+
+
+def test_trainer_args_fsdp_without_the_launcher(no_launcher):
+ """`TrainingArguments(fsdp=...)` reaches only the Accelerator, never the env."""
+ from accelerate.state import AcceleratorState
+ from accelerate.utils import DistributedType
+
+ no_launcher.setattr(
+ AcceleratorState, "_shared_state", {"distributed_type": DistributedType.FSDP}
+ )
+ assert _mode_inside(torch.nn.Linear(2, 2)) == (False, False)
+
+
+def test_torchao_still_gets_no_grad(no_launcher):
+ model = torch.nn.Linear(2, 2)
+ model.torchao_config = type("Cfg", (), {"qat_scheme": None})()
+ assert _mode_inside(model) == (False, False)
+
+
+def _fsdp2_generate_then_train(tmp_path, context):
+ import torch.distributed as dist
+
+ fully_shard = pytest.importorskip("torch.distributed.fsdp").fully_shard
+ dist.init_process_group("gloo", init_method = f"file://{tmp_path}/pg", rank = 0, world_size = 1)
+ try:
+ torch.manual_seed(0)
+ model = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.ReLU(), torch.nn.Linear(8, 2))
+ fully_shard(model[0], reshard_after_forward = False)
+ fully_shard(model, reshard_after_forward = False)
+ x = torch.randn(3, 8)
+ with context(model):
+ model(x)
+ model(x).sum().backward()
+ return model[0].weight.grad
+ finally:
+ dist.destroy_process_group()
+
+
+def test_fsdp2_trains_after_a_generation_pass(no_launcher, tmp_path):
+ no_launcher.setenv("ACCELERATE_USE_FSDP", "true")
+ grad = _fsdp2_generate_then_train(tmp_path, _get_inference_mode_context_manager)
+ assert grad is not None
+
+
+def test_fsdp2_under_inference_mode_is_the_reported_failure(no_launcher, tmp_path):
+ """Negative control: what the helper returned before unsloth#3551."""
+ with pytest.raises(RuntimeError, match = "[Ii]nference tensor"):
+ _fsdp2_generate_then_train(tmp_path, lambda model: torch.inference_mode())
diff --git a/tests/test_optimized_precision_conflicts.py b/tests/test_optimized_precision_conflicts.py
index d8a3f9c946e..0f747cc1038 100644
--- a/tests/test_optimized_precision_conflicts.py
+++ b/tests/test_optimized_precision_conflicts.py
@@ -85,6 +85,7 @@ def loader():
_get_dtype = lambda dtype: dtype,
_revision_for_tokenizer_repo = lambda *args: None,
_raise_if_modeling_ignores_config = lambda *args: None,
+ raise_if_fast_inference_under_fsdp2 = lambda fast_inference: None,
)
# Mistral-format checkpoints (#12144) are redirected before the precision check. None of
# these fixtures is one, so every helper that decides that answers False and the redirect
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index a881d9604cc..c8c564a1212 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -7232,8 +7232,22 @@ def _get_inference_mode_context_manager(model: torch.nn.Module):
torchao_config = getattr(model, "torchao_config", None)
if torchao_config is not None and torchao_config.qat_scheme is None:
return torch.no_grad()
- else:
- return torch.inference_mode()
+ # FSDP keeps the parameters it all-gathers for one forward and reuses them in the next, so a
+ # gather under inference mode poisons the training step: "Inference tensors cannot be saved
+ # for backward" (unsloth#3551). TRL generates under no_grad for the same reason.
+ import os, sys
+
+ if os.environ.get("ACCELERATE_USE_FSDP", "").strip().lower() in ("1", "true", "yes", "on"):
+ return torch.no_grad()
+ if os.environ.get("FSDP_VERSION", "").strip() not in ("", "0"):
+ return torch.no_grad()
+ accelerate_state = sys.modules.get("accelerate.state", None)
+ shared_state = getattr(
+ getattr(accelerate_state, "AcceleratorState", None), "_shared_state", None
+ )
+ if isinstance(shared_state, dict) and "FSDP" in str(shared_state.get("distributed_type", "")):
+ return torch.no_grad()
+ return torch.inference_mode()
def hf_login(token: Optional[str] = None) -> Optional[str]:
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index a2f770c3daa..8a190973690 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -60,6 +60,7 @@ from .loader_utils import (
_restore_load_scoped_env,
_resolve_checkpoint_tokenizer_name,
_is_offline_related_error,
+ raise_if_fast_inference_under_fsdp2,
)
import os, contextlib, sys
@@ -1101,6 +1102,7 @@ class FastLanguageModel(FastLlamaModel):
)
fast_inference = False
break
+ raise_if_fast_inference_under_fsdp2(fast_inference)
# bitsandbytes unusable (absent, or unstable on some AMD stacks). A capability check, so not gated on use_exact_model_name, which only suppresses repo-name remapping.
if not ALLOW_BITSANDBYTES:
@@ -2012,6 +2014,7 @@ class FastModel(FastBaseModel):
)
fast_inference = False
break
+ raise_if_fast_inference_under_fsdp2(fast_inference)
old_model_name = model_name
fp8_mode = None
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 600152e3aa1..95e57ff18fc 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -154,6 +154,29 @@ def fsdp_will_wrap():
return False
+def fsdp2_requested():
+ """True when this process runs under FSDP2: `accelerate launch` exports FSDP_VERSION=2, an
+ Accelerator built already records its fsdp_plugin."""
+ if os.environ.get("FSDP_VERSION", "").strip() == "2":
+ return True
+ try:
+ from accelerate.state import AcceleratorState
+ plugin = AcceleratorState._shared_state.get("fsdp_plugin", None)
+ return getattr(plugin, "fsdp_version", None) == 2
+ except Exception:
+ return False
+
+
+def raise_if_fast_inference_under_fsdp2(fast_inference):
+ """Unsupported combination: refuse it before vLLM loads instead of failing mid-run (unsloth#3551)."""
+ if fast_inference and fsdp2_requested():
+ raise NotImplementedError(
+ "Unsloth: `fast_inference = True` (vLLM) is not supported with FSDP2.\n"
+ "Load the model with `fast_inference = False` to train with FSDP2, "
+ "or launch without FSDP2 to use vLLM."
+ )
+
+
def prepare_device_map():
rank, world_size = _infer_distributed_ranks()
distributed = (world_size or 1) > 1 or (rank is not None and rank > 0)
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 90d185d823b..4643c88b30f 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -94,6 +94,9 @@ try:
except Exception:
transformers_version = Version("0.0.0")
+# TrainingArguments falls back to this FSDP version when fsdp_config names none (1 -> 2 in transformers 5.10.0).
+_FSDP_DEFAULT_VERSION = 2 if transformers_version >= Version("5.10.0") else 1
+
def vLLMSamplingParams(**kwargs):
from vllm import SamplingParams
@@ -3124,6 +3127,32 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += learning_rate_check
+ # bitsandbytes' 8-bit optimizers step plain tensors, but FSDP2 shards parameters as DTensors, so the
+ # first step dies with "got mixed torch.Tensor and DTensor" (unsloth#3551).
+ if "optim" in call_args:
+ extra_args += "_unsloth_fsdp2 = os.environ.get('FSDP_VERSION', '').strip() == '2'\n"
+ if "fsdp" in call_args and "fsdp_config" in call_args:
+ # TrainingArguments(fsdp = ..., fsdp_config = {"fsdp_version": 2} or a JSON path) without the launcher.
+ extra_args += (
+ "_unsloth_fsdp_cfg = fsdp_config\n"
+ "if fsdp and isinstance(_unsloth_fsdp_cfg, str) and os.path.isfile(_unsloth_fsdp_cfg):\n"
+ " try:\n"
+ " import json as _unsloth_json\n"
+ " with open(_unsloth_fsdp_cfg, encoding = 'utf-8') as _unsloth_f:\n"
+ " _unsloth_fsdp_cfg = _unsloth_json.load(_unsloth_f)\n"
+ " except Exception:\n"
+ " _unsloth_fsdp_cfg = None\n"
+ "if fsdp and _unsloth_fsdp_cfg is None:\n"
+ " _unsloth_fsdp_cfg = {}\n"
+ "if fsdp and isinstance(_unsloth_fsdp_cfg, dict):\n"
+ f" _unsloth_fsdp2 = _unsloth_fsdp2 or str(_unsloth_fsdp_cfg.get('fsdp_version', _unsloth_fsdp_cfg.get('version', {_FSDP_DEFAULT_VERSION}))) == '2'\n"
+ )
+ extra_args += (
+ "if _unsloth_fsdp2 and str(getattr(optim, 'value', optim)) == 'adamw_8bit':\n"
+ " print(\"Unsloth: FSDP2 shards parameters as DTensors, which bitsandbytes' 8-bit optimizers cannot step. Switching optim = 'adamw_8bit' to 'adamw_torch_fused'.\")\n"
+ " optim = 'adamw_torch_fused'\n"
+ )
+
# Fix num_train_epochs = None causing a TypeError in Trainer.__init__, which does `args.num_train_epochs > 0`.
if "num_train_epochs" in call_args:
num_train_epochs_check = (
@@ -3796,6 +3825,13 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
"\n"
+ " " * 8
+ "if hasattr(model, 'vllm_engine') and hasattr(args, 'use_vllm'):\n"
+ # TrainingArguments(fsdp = ..., fsdp_config = {"version": 2}) under torchrun: nothing at load time said FSDP2 (unsloth#3551).
+ + " " * 12
+ + "_unsloth_fsdp_cfg = getattr(args, 'fsdp_config', None)\n"
+ + " " * 12
+ + f"if getattr(args, 'fsdp', None) and isinstance(_unsloth_fsdp_cfg, dict) and str(_unsloth_fsdp_cfg.get('version', _unsloth_fsdp_cfg.get('fsdp_version', {_FSDP_DEFAULT_VERSION}))) == '2':\n"
+ + " " * 16
+ + "raise NotImplementedError('Unsloth: `fast_inference = True` (vLLM) is not supported with FSDP2.\\nLoad the model with `fast_inference = False` to train with FSDP2, or launch without FSDP2 to use vLLM.')\n"
+ " " * 12
+ "if (getattr(args, 'use_vllm', False) == False):\n"
+ " " * 16
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index a6ef9680182..ad43500772e 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -1803,7 +1803,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
if kwargs.get("past_key_values") is None:
kwargs["cache_implementation"] = "dynamic"
try:
- with torch.inference_mode(), autocaster:
+ with _get_inference_mode_context_manager(self), autocaster:
return self._old_generate(*args, **kwargs)
finally:
_clear_generation_caches(self)
@@ -1889,7 +1889,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
decode_scope = _CompileDecodeOnRepeat(self) if compile_decode else contextlib.nullcontext()
eager_scope = _EagerDecodeSteps(self) if _eager_decodes(self) else contextlib.nullcontext()
try:
- with decode_scope, eager_scope, torch.inference_mode(), autocaster:
+ with decode_scope, eager_scope, _get_inference_mode_context_manager(self), autocaster:
output = self._old_generate(*args, **kwargs)
finally:
_clear_generation_caches(self)