Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Run GRPO under FSDP2: no inference mode around FSDP, an optimizer that steps DTensors, and refuse vLLM with FSDP2 (#13156)
Daniel Hancc10f80cefee
diff --git a/tests/test_dia_generate_pad_token.py b/tests/test_dia_generate_pad_token.pyindex b0808f50658..d65f560c4f4 100644--- a/tests/test_dia_generate_pad_token.py+++ b/tests/test_dia_generate_pad_token.py@@ -49,6 +49,7 @@ def _captured_generate_kwargs(config, **generate_kwargs):"FastBaseModel": SimpleNamespace(for_inference = lambda model: None),"dtype_from_config": lambda config: "bfloat16","_get_dtype": lambda dtype: dtype,+ "_get_inference_mode_context_manager": lambda model: nullcontext(),"_unsloth_generate_accepts_kwarg": lambda model, name: False,"NUM_LOGITS_TO_KEEP": {"DiaForConditionalGeneration": None},"DEVICE_TYPE_TORCH": "cuda",diff --git a/tests/test_fa2_fast_generate_bypass.py b/tests/test_fa2_fast_generate_bypass.pyindex 1b3d4d69e60..67f3e25fede 100644--- a/tests/test_fa2_fast_generate_bypass.py+++ b/tests/test_fa2_fast_generate_bypass.py@@ -211,6 +211,7 @@ def test_wrapper_dispatch_preserves_normalization_and_selects_expected_path():"FastBaseModel": FakeFastBaseModel,"dtype_from_config": lambda config: "bfloat16","_get_dtype": lambda dtype: dtype,+ "_get_inference_mode_context_manager": lambda model: nullcontext(),"_unsloth_generate_accepts_kwarg": lambda model, name: False,"NUM_LOGITS_TO_KEEP": {architecture: None},"DEVICE_TYPE_TORCH": "cuda",@@ -298,6 +299,7 @@ def test_flash_attention_fallback_pins_a_dynamic_cache():"FastBaseModel": SimpleNamespace(for_inference = lambda model: None),"dtype_from_config": lambda config: "bfloat16","_get_dtype": lambda dtype: dtype,+ "_get_inference_mode_context_manager": lambda model: nullcontext(),"_unsloth_generate_accepts_kwarg": lambda model, name: False,"NUM_LOGITS_TO_KEEP": {"Qwen3VLForConditionalGeneration": None},"DEVICE_TYPE_TORCH": "cuda",diff --git a/tests/test_fast_inference_refuses_fsdp2.py b/tests/test_fast_inference_refuses_fsdp2.pynew file mode 100644index 00000000000..944714c5318--- /dev/null+++ b/tests/test_fast_inference_refuses_fsdp2.py@@ -0,0 +1,145 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""unsloth#3551: `fast_inference = True` (vLLM) is refused under FSDP2, before vLLM loads."""++from __future__ import annotations++import ast+import os+import subprocess+import sys+import textwrap+from pathlib import Path+from types import SimpleNamespace++import pytest++from real_accelerator import has_real_cuda+from unsloth.models import loader_utils+from unsloth.models.loader_utils import fsdp2_requested, raise_if_fast_inference_under_fsdp2+++def no_launcher(monkeypatch):+ monkeypatch.delenv("FSDP_VERSION", raising = False)+ state = pytest.importorskip("accelerate.state")+ monkeypatch.setattr(state.AcceleratorState, "_shared_state", {})+ return monkeypatch++[email protected]("value, fsdp2", [("2", True), ("1", False), ("0", False)])+def test_the_launcher_env_names_fsdp2(no_launcher, value, fsdp2):+ no_launcher.setenv("FSDP_VERSION", value)+ assert fsdp2_requested() is fsdp2+++def test_an_accelerator_with_an_fsdp2_plugin(no_launcher):+ from accelerate.state import AcceleratorState+ no_launcher.setattr(+ AcceleratorState, "_shared_state", {"fsdp_plugin": SimpleNamespace(fsdp_version = 2)}+ )+ assert fsdp2_requested()+++def test_vllm_under_fsdp2_is_refused_with_the_way_out(no_launcher):+ no_launcher.setenv("FSDP_VERSION", "2")+ with pytest.raises(NotImplementedError, match = "fast_inference = False"):+ raise_if_fast_inference_under_fsdp2(True)++[email protected]("fsdp_version, fast_inference", [("2", False), ("1", True), (None, True)])+def test_everything_else_loads(no_launcher, fsdp_version, fast_inference):+ if fsdp_version is not None:+ no_launcher.setenv("FSDP_VERSION", fsdp_version)+ raise_if_fast_inference_under_fsdp2(fast_inference)+++def test_both_loaders_refuse_after_the_dgx_spark_fallback():+ """FastLanguageModel and FastModel: GB10 still falls back to native inference before the FSDP2 check."""+ source = Path(loader_utils.__file__).with_name("loader.py").read_text(encoding = "utf-8")+ tree = ast.parse(source)+ checked = []+ for cls in (n for n in tree.body if isinstance(n, ast.ClassDef)):+ for fn in (+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained"+ ):+ guards = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Call)+ and ast.unparse(n.func) == "raise_if_fast_inference_under_fsdp2"+ ]+ spark = [+ n.lineno+ for n in ast.walk(fn)+ if isinstance(n, ast.Constant)+ and isinstance(n.value, str)+ and "DGX Spark detected" in n.value+ ]+ if spark:+ assert len(guards) == 1 and guards[0] > max(spark), (cls.name, guards, spark)+ checked.append(cls.name)+ assert {"FastLanguageModel", "FastModel"} <= set(checked), checked+++_TRAINER_PROBE = textwrap.dedent(+ """+ import os+ from types import SimpleNamespace+ import torch+ from unsloth import FastLanguageModel # noqa: F401, patches the TRL trainers+ from trl import GRPOConfig, GRPOTrainer++ class FastInferenceModel(torch.nn.Module):+ vllm_engine = object()+ max_seq_length = 64++ def __init__(self):+ super().__init__()+ self.config = SimpleNamespace(dtype = torch.bfloat16, torch_dtype = torch.bfloat16)+ self.warnings_issued = {}++ def for_training(self, *args, **kwargs):+ pass++ def build(**kwargs):+ try:+ args = GRPOConfig(output_dir = os.environ["PROBE_OUT"], **kwargs)+ GRPOTrainer(model = FastInferenceModel(), reward_funcs = [], args = args)+ except NotImplementedError as e:+ return "REFUSED" if "FSDP2" in str(e) else "OTHER"+ except Exception:+ return "OTHER"+ return "BUILT"++ print("PROBE", build(fsdp = "full_shard", fsdp_config = {"fsdp_version": 2}),+ build(fsdp = "full_shard", fsdp_config = {"fsdp_version": 1}), build(), build(fsdp = "full_shard"))+ """+)+++ not has_real_cuda(), reason = "the probe imports unsloth outside the conftest CPU spoof"+)+def test_trainer_args_fsdp2_is_refused_when_the_trainer_is_built(tmp_path):+ """torchrun + TrainingArguments(fsdp = ..., fsdp_config = {"fsdp_version": 2}): nothing said FSDP2 at load time."""+ env = {k: v for k, v in os.environ.items() if k not in ("FSDP_VERSION", "ACCELERATE_USE_FSDP")}+ env.update(+ UNSLOTH_COMPILE_LOCATION = str(tmp_path / "cache"),+ PROBE_OUT = str(tmp_path / "out"),+ HF_HUB_OFFLINE = "1",+ )+ run = subprocess.run(+ [sys.executable, "-c", _TRAINER_PROBE], env = env, capture_output = True, text = True, timeout = 900+ )+ lines = [line for line in run.stdout.splitlines() if line.startswith("PROBE ")]+ assert lines, f"probe failed:\n{run.stdout[-3000:]}\n{run.stderr[-3000:]}"+ fsdp2, fsdp1, plain, unversioned = lines[-1].split()[1:]+ assert fsdp2 == "REFUSED"+ assert fsdp1 != "REFUSED" and plain != "REFUSED"+ from transformers import __version__+ from unsloth_zoo.utils import Version++ # No version named: TrainingArguments picks FSDP2 from transformers 5.10.0, FSDP1 before.+ assert (unversioned == "REFUSED") == (Version(__version__) >= Version("5.10.0"))diff --git a/tests/test_fsdp2_default_optim.py b/tests/test_fsdp2_default_optim.pynew file mode 100644index 00000000000..1888fdb5ed7--- /dev/null+++ b/tests/test_fsdp2_default_optim.py@@ -0,0 +1,88 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""unsloth#3551: Unsloth's adamw_8bit default cannot step FSDP2's DTensor parameters."""++from __future__ import annotations++import json+import os+import subprocess+import sys+import textwrap++import pytest++from real_accelerator import has_real_cuda++pytest.importorskip("trl")+# The probe imports unsloth in a fresh process, outside the conftest CPU spoof.+pytestmark = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA GPU")++_PROBE = textwrap.dedent(+ """+ import json, os+ from unsloth import FastLanguageModel # noqa: F401, patches the TRL configs+ from trl import GRPOConfig, SFTConfig++ def optim(cls, env, **kwargs):+ for name in ("FSDP_VERSION", "ACCELERATE_USE_FSDP"):+ os.environ.pop(name, None)+ os.environ.update(env)+ value = cls(output_dir = os.environ["PROBE_OUT"], **kwargs).optim+ return str(getattr(value, "value", value))++ fsdp2_args = {"fsdp": "full_shard", "fsdp_config": {"fsdp_version": 2}}+ fsdp2_json = os.environ["PROBE_OUT"] + "_cfg.json"+ with open(fsdp2_json, "w", encoding = "utf-8") as f:+ json.dump({"fsdp_version": 2}, f)+ print("PROBE " + json.dumps({+ "grpo_plain": optim(GRPOConfig, {}),+ "sft_plain": optim(SFTConfig, {}),+ "grpo_fsdp1": optim(GRPOConfig, {"ACCELERATE_USE_FSDP": "true", "FSDP_VERSION": "1"}),+ "grpo_fsdp2": optim(GRPOConfig, {"ACCELERATE_USE_FSDP": "true", "FSDP_VERSION": "2"}),+ "sft_fsdp2": optim(SFTConfig, {"ACCELERATE_USE_FSDP": "true", "FSDP_VERSION": "2"}),+ "grpo_fsdp2_args": optim(GRPOConfig, {}, **fsdp2_args),+ "grpo_fsdp2_json": optim(GRPOConfig, {}, fsdp = "full_shard", fsdp_config = fsdp2_json),+ "grpo_fsdp2_explicit_torch": optim(GRPOConfig, {"FSDP_VERSION": "2"}, optim = "adamw_torch"),+ "grpo_fsdp_unversioned": optim(GRPOConfig, {}, fsdp = "full_shard"),+ }))+ """+)++[email protected](scope = "module")+def probe(tmp_path_factory):+ tmp = tmp_path_factory.mktemp("fsdp2_optim")+ env = dict(os.environ)+ env.update(UNSLOTH_COMPILE_LOCATION = str(tmp / "cache"), PROBE_OUT = str(tmp / "out"))+ run = subprocess.run(+ [sys.executable, "-c", _PROBE], env = env, capture_output = True, text = True, timeout = 900+ )+ lines = [line for line in run.stdout.splitlines() if line.startswith("PROBE ")]+ assert lines, f"probe failed:\n{run.stdout[-3000:]}\n{run.stderr[-3000:]}"+ return json.loads(lines[-1][len("PROBE ") :])+++def test_without_fsdp2_the_default_is_unchanged(probe):+ assert probe["grpo_plain"] == "adamw_8bit"+ assert probe["sft_plain"] == "adamw_8bit"+ assert probe["grpo_fsdp1"] == "adamw_8bit"++[email protected]("case", ["grpo_fsdp2", "sft_fsdp2", "grpo_fsdp2_args", "grpo_fsdp2_json"])+def test_fsdp2_gets_an_optimizer_that_steps_dtensors(probe, case):+ assert probe[case] == "adamw_torch_fused"+++def test_fsdp_without_a_version_follows_the_transformers_default(probe):+ """TrainingArguments picks FSDP2 when fsdp_config names no version from transformers 5.10.0, FSDP1 before."""+ from transformers import __version__+ from unsloth_zoo.utils import Version++ fsdp2 = Version(__version__) >= Version("5.10.0")+ assert probe["grpo_fsdp_unversioned"] == ("adamw_torch_fused" if fsdp2 else "adamw_8bit")+++def test_an_explicit_non_bnb_choice_is_kept(probe):+ assert probe["grpo_fsdp2_explicit_torch"] == "adamw_torch"diff --git a/tests/test_fsdp_generation_grad_mode.py b/tests/test_fsdp_generation_grad_mode.pynew file mode 100644index 00000000000..04601c35bc0--- /dev/null+++ b/tests/test_fsdp_generation_grad_mode.py@@ -0,0 +1,93 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""unsloth#3551: GRPO generation and logprob passes must not run FSDP under inference mode."""++from __future__ import annotations++import pytest+import torch++from unsloth.models._utils import _get_inference_mode_context_manager++_FSDP_ENV = ("ACCELERATE_USE_FSDP", "FSDP_VERSION")+++def no_launcher(monkeypatch):+ for name in _FSDP_ENV:+ monkeypatch.delenv(name, raising = False)+ state = pytest.importorskip("accelerate.state")+ monkeypatch.setattr(state.AcceleratorState, "_shared_state", {})+ return monkeypatch+++def _mode_inside(model):+ with _get_inference_mode_context_manager(model):+ return torch.is_inference_mode_enabled(), torch.is_grad_enabled()+++def test_single_process_keeps_inference_mode(no_launcher):+ assert _mode_inside(torch.nn.Linear(2, 2)) == (True, False)+++ "name, value, inference",+ [+ ("ACCELERATE_USE_FSDP", "true", False),+ ("ACCELERATE_USE_FSDP", "false", True),+ ("FSDP_VERSION", "2", False),+ ("FSDP_VERSION", "0", True),+ ],+)+def test_the_accelerate_launcher_env_decides(no_launcher, name, value, inference):+ no_launcher.setenv(name, value)+ assert _mode_inside(torch.nn.Linear(2, 2)) == (inference, False)+++def test_trainer_args_fsdp_without_the_launcher(no_launcher):+ """`TrainingArguments(fsdp=...)` reaches only the Accelerator, never the env."""+ from accelerate.state import AcceleratorState+ from accelerate.utils import DistributedType++ no_launcher.setattr(+ AcceleratorState, "_shared_state", {"distributed_type": DistributedType.FSDP}+ )+ assert _mode_inside(torch.nn.Linear(2, 2)) == (False, False)+++def test_torchao_still_gets_no_grad(no_launcher):+ model = torch.nn.Linear(2, 2)+ model.torchao_config = type("Cfg", (), {"qat_scheme": None})()+ assert _mode_inside(model) == (False, False)+++def _fsdp2_generate_then_train(tmp_path, context):+ import torch.distributed as dist++ fully_shard = pytest.importorskip("torch.distributed.fsdp").fully_shard+ dist.init_process_group("gloo", init_method = f"file://{tmp_path}/pg", rank = 0, world_size = 1)+ try:+ torch.manual_seed(0)+ model = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.ReLU(), torch.nn.Linear(8, 2))+ fully_shard(model[0], reshard_after_forward = False)+ fully_shard(model, reshard_after_forward = False)+ x = torch.randn(3, 8)+ with context(model):+ model(x)+ model(x).sum().backward()+ return model[0].weight.grad+ finally:+ dist.destroy_process_group()+++def test_fsdp2_trains_after_a_generation_pass(no_launcher, tmp_path):+ no_launcher.setenv("ACCELERATE_USE_FSDP", "true")+ grad = _fsdp2_generate_then_train(tmp_path, _get_inference_mode_context_manager)+ assert grad is not None+++def test_fsdp2_under_inference_mode_is_the_reported_failure(no_launcher, tmp_path):+ """Negative control: what the helper returned before unsloth#3551."""+ with pytest.raises(RuntimeError, match = "[Ii]nference tensor"):+ _fsdp2_generate_then_train(tmp_path, lambda model: torch.inference_mode())diff --git a/tests/test_optimized_precision_conflicts.py b/tests/test_optimized_precision_conflicts.pyindex d8a3f9c946e..0f747cc1038 100644--- a/tests/test_optimized_precision_conflicts.py+++ b/tests/test_optimized_precision_conflicts.py@@ -85,6 +85,7 @@ def loader():_get_dtype = lambda dtype: dtype,_revision_for_tokenizer_repo = lambda *args: None,_raise_if_modeling_ignores_config = lambda *args: None,+ raise_if_fast_inference_under_fsdp2 = lambda fast_inference: None,)# Mistral-format checkpoints (#12144) are redirected before the precision check. None of# these fixtures is one, so every helper that decides that answers False and the redirectdiff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex a881d9604cc..c8c564a1212 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -7232,8 +7232,22 @@ def _get_inference_mode_context_manager(model: torch.nn.Module):torchao_config = getattr(model, "torchao_config", None)if torchao_config is not None and torchao_config.qat_scheme is None:return torch.no_grad()- else:- return torch.inference_mode()+ # FSDP keeps the parameters it all-gathers for one forward and reuses them in the next, so a+ # gather under inference mode poisons the training step: "Inference tensors cannot be saved+ # for backward" (unsloth#3551). TRL generates under no_grad for the same reason.+ import os, sys++ if os.environ.get("ACCELERATE_USE_FSDP", "").strip().lower() in ("1", "true", "yes", "on"):+ return torch.no_grad()+ if os.environ.get("FSDP_VERSION", "").strip() not in ("", "0"):+ return torch.no_grad()+ accelerate_state = sys.modules.get("accelerate.state", None)+ shared_state = getattr(+ getattr(accelerate_state, "AcceleratorState", None), "_shared_state", None+ )+ if isinstance(shared_state, dict) and "FSDP" in str(shared_state.get("distributed_type", "")):+ return torch.no_grad()+ return torch.inference_mode()def hf_login(token: Optional[str] = None) -> Optional[str]:diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex a2f770c3daa..8a190973690 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -60,6 +60,7 @@ from .loader_utils import (_restore_load_scoped_env,_resolve_checkpoint_tokenizer_name,_is_offline_related_error,+ raise_if_fast_inference_under_fsdp2,)import os, contextlib, sys@@ -1101,6 +1102,7 @@ class FastLanguageModel(FastLlamaModel):)fast_inference = Falsebreak+ raise_if_fast_inference_under_fsdp2(fast_inference)# bitsandbytes unusable (absent, or unstable on some AMD stacks). A capability check, so not gated on use_exact_model_name, which only suppresses repo-name remapping.if not ALLOW_BITSANDBYTES:@@ -2012,6 +2014,7 @@ class FastModel(FastBaseModel):)fast_inference = Falsebreak+ raise_if_fast_inference_under_fsdp2(fast_inference)old_model_name = model_namefp8_mode = Nonediff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 600152e3aa1..95e57ff18fc 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -154,6 +154,29 @@ def fsdp_will_wrap():return False+def fsdp2_requested():+ """True when this process runs under FSDP2: `accelerate launch` exports FSDP_VERSION=2, an+ Accelerator built already records its fsdp_plugin."""+ if os.environ.get("FSDP_VERSION", "").strip() == "2":+ return True+ try:+ from accelerate.state import AcceleratorState+ plugin = AcceleratorState._shared_state.get("fsdp_plugin", None)+ return getattr(plugin, "fsdp_version", None) == 2+ except Exception:+ return False+++def raise_if_fast_inference_under_fsdp2(fast_inference):+ """Unsupported combination: refuse it before vLLM loads instead of failing mid-run (unsloth#3551)."""+ if fast_inference and fsdp2_requested():+ raise NotImplementedError(+ "Unsloth: `fast_inference = True` (vLLM) is not supported with FSDP2.\n"+ "Load the model with `fast_inference = False` to train with FSDP2, "+ "or launch without FSDP2 to use vLLM."+ )++def prepare_device_map():rank, world_size = _infer_distributed_ranks()distributed = (world_size or 1) > 1 or (rank is not None and rank > 0)diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 90d185d823b..4643c88b30f 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -94,6 +94,9 @@ try:except Exception:transformers_version = Version("0.0.0")+# TrainingArguments falls back to this FSDP version when fsdp_config names none (1 -> 2 in transformers 5.10.0).+_FSDP_DEFAULT_VERSION = 2 if transformers_version >= Version("5.10.0") else 1+def vLLMSamplingParams(**kwargs):from vllm import SamplingParams@@ -3124,6 +3127,32 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += learning_rate_check+ # bitsandbytes' 8-bit optimizers step plain tensors, but FSDP2 shards parameters as DTensors, so the+ # first step dies with "got mixed torch.Tensor and DTensor" (unsloth#3551).+ if "optim" in call_args:+ extra_args += "_unsloth_fsdp2 = os.environ.get('FSDP_VERSION', '').strip() == '2'\n"+ if "fsdp" in call_args and "fsdp_config" in call_args:+ # TrainingArguments(fsdp = ..., fsdp_config = {"fsdp_version": 2} or a JSON path) without the launcher.+ extra_args += (+ "_unsloth_fsdp_cfg = fsdp_config\n"+ "if fsdp and isinstance(_unsloth_fsdp_cfg, str) and os.path.isfile(_unsloth_fsdp_cfg):\n"+ " try:\n"+ " import json as _unsloth_json\n"+ " with open(_unsloth_fsdp_cfg, encoding = 'utf-8') as _unsloth_f:\n"+ " _unsloth_fsdp_cfg = _unsloth_json.load(_unsloth_f)\n"+ " except Exception:\n"+ " _unsloth_fsdp_cfg = None\n"+ "if fsdp and _unsloth_fsdp_cfg is None:\n"+ " _unsloth_fsdp_cfg = {}\n"+ "if fsdp and isinstance(_unsloth_fsdp_cfg, dict):\n"+ f" _unsloth_fsdp2 = _unsloth_fsdp2 or str(_unsloth_fsdp_cfg.get('fsdp_version', _unsloth_fsdp_cfg.get('version', {_FSDP_DEFAULT_VERSION}))) == '2'\n"+ )+ extra_args += (+ "if _unsloth_fsdp2 and str(getattr(optim, 'value', optim)) == 'adamw_8bit':\n"+ " print(\"Unsloth: FSDP2 shards parameters as DTensors, which bitsandbytes' 8-bit optimizers cannot step. Switching optim = 'adamw_8bit' to 'adamw_torch_fused'.\")\n"+ " optim = 'adamw_torch_fused'\n"+ )+# Fix num_train_epochs = None causing a TypeError in Trainer.__init__, which does `args.num_train_epochs > 0`.if "num_train_epochs" in call_args:num_train_epochs_check = (@@ -3796,6 +3825,13 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import"\n"+ " " * 8+ "if hasattr(model, 'vllm_engine') and hasattr(args, 'use_vllm'):\n"+ # TrainingArguments(fsdp = ..., fsdp_config = {"version": 2}) under torchrun: nothing at load time said FSDP2 (unsloth#3551).+ + " " * 12+ + "_unsloth_fsdp_cfg = getattr(args, 'fsdp_config', None)\n"+ + " " * 12+ + f"if getattr(args, 'fsdp', None) and isinstance(_unsloth_fsdp_cfg, dict) and str(_unsloth_fsdp_cfg.get('version', _unsloth_fsdp_cfg.get('fsdp_version', {_FSDP_DEFAULT_VERSION}))) == '2':\n"+ + " " * 16+ + "raise NotImplementedError('Unsloth: `fast_inference = True` (vLLM) is not supported with FSDP2.\\nLoad the model with `fast_inference = False` to train with FSDP2, or launch without FSDP2 to use vLLM.')\n"+ " " * 12+ "if (getattr(args, 'use_vllm', False) == False):\n"+ " " * 16diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex a6ef9680182..ad43500772e 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -1803,7 +1803,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):if kwargs.get("past_key_values") is None:kwargs["cache_implementation"] = "dynamic"try:- with torch.inference_mode(), autocaster:+ with _get_inference_mode_context_manager(self), autocaster:return self._old_generate(*args, **kwargs)finally:_clear_generation_caches(self)@@ -1889,7 +1889,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):decode_scope = _CompileDecodeOnRepeat(self) if compile_decode else contextlib.nullcontext()eager_scope = _EagerDecodeSteps(self) if _eager_decodes(self) else contextlib.nullcontext()try:- with decode_scope, eager_scope, torch.inference_mode(), autocaster:+ with decode_scope, eager_scope, _get_inference_mode_context_manager(self), autocaster:output = self._old_generate(*args, **kwargs)finally:_clear_generation_caches(self)