Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
DeepSpeed ZeRO-2 / ZeRO-3: stage detection at load, MoE and fused LoRA leaves, precision resync, reentrant checkpointing (#14394)
Daniel Hanb098d77e8107
diff --git a/tests/test_deepspeed_zero3.py b/tests/test_deepspeed_zero3.pyindex 71e558d2bfc..93053a89d13 100644--- a/tests/test_deepspeed_zero3.py+++ b/tests/test_deepspeed_zero3.py@@ -51,3 +51,374 @@ def test_without_zero3_the_device_map_is_untouched(zero3):zero3(False)for device_map in (loader_utils.DEFAULT_DEVICE_MAP, "sequential", {"": "cuda:1"}, None):assert loader_utils.deepspeed_zero3_device_map(device_map) is device_map+++_DS_ENV = (+ "ACCELERATE_USE_DEEPSPEED",+ "ACCELERATE_DEEPSPEED_ZERO_STAGE",+ "ACCELERATE_DEEPSPEED_CONFIG_FILE",+)+++def no_ds_env(monkeypatch):+ for key in _DS_ENV:+ monkeypatch.delenv(key, raising = False)+ import transformers.integrations.deepspeed as hf_ds++ monkeypatch.setattr(hf_ds, "_hf_deepspeed_config_weak_ref", None, raising = False)+ try:+ from accelerate.state import AcceleratorState+ monkeypatch.setitem(AcceleratorState._shared_state, "deepspeed_plugins", None)+ except Exception:+ pass+ return monkeypatch+++def test_zero_stage_unset_without_deepspeed(no_ds_env):+ assert loader_utils.deepspeed_zero_stage() is None+ assert not loader_utils.deepspeed_zero3_requested()++[email protected]("stage", ["2", "3"])+def test_zero_stage_from_accelerate_env(no_ds_env, stage):+ no_ds_env.setenv("ACCELERATE_USE_DEEPSPEED", "true")+ no_ds_env.setenv("ACCELERATE_DEEPSPEED_ZERO_STAGE", stage)+ assert loader_utils.deepspeed_zero_stage() == int(stage)+ assert loader_utils.deepspeed_zero3_requested() == (stage == "3")+++def test_zero_stage_from_accelerate_config_file(no_ds_env, tmp_path):+ import json++ path = tmp_path / "ds.json"+ path.write_text(json.dumps({"zero_optimization": {"stage": 3}}))+ no_ds_env.setenv("ACCELERATE_USE_DEEPSPEED", "true")+ no_ds_env.setenv("ACCELERATE_DEEPSPEED_CONFIG_FILE", str(path))+ assert loader_utils.deepspeed_zero3_requested()+ no_ds_env.delenv("ACCELERATE_USE_DEEPSPEED")+ assert loader_utils.deepspeed_zero_stage() is None+++def test_zero_stage_from_training_arguments_config(no_ds_env):+ import weakref+ import transformers.integrations.deepspeed as hf_ds++ class _Cfg:+ config = {"zero_optimization": {"stage": 3}}++ cfg = _Cfg()+ no_ds_env.setattr(hf_ds, "_hf_deepspeed_config_weak_ref", weakref.ref(cfg))+ assert loader_utils.deepspeed_zero3_requested()+++def _tiny_qwen3_moe():+ torch = pytest.importorskip("torch")+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM++ config = Qwen3MoeConfig(+ vocab_size = 64,+ hidden_size = 16,+ intermediate_size = 32,+ moe_intermediate_size = 8,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 8,+ num_experts = 4,+ num_experts_per_tok = 2,+ )+ with torch.device("meta"):+ return Qwen3MoeForCausalLM(config)+++def fake_set_z3_leaf(monkeypatch):+ import sys+ import types++ calls = []++ def set_z3_leaf_modules(+ model,+ classes,+ raise_if_not_found = True,+ ):+ calls.append(list(classes))+ out = []+ for module in model.modules():+ if isinstance(module, tuple(classes)):+ module._z3_leaf = True+ out.append(module)+ return out++ utils = types.ModuleType("deepspeed.utils")+ utils.set_z3_leaf_modules = set_z3_leaf_modules+ ds = types.ModuleType("deepspeed")+ ds.utils = utils+ monkeypatch.setitem(sys.modules, "deepspeed", ds)+ monkeypatch.setitem(sys.modules, "deepspeed.utils", utils)+ return calls+++def test_moe_blocks_become_zero3_leaves(no_ds_env, fake_set_z3_leaf):+ no_ds_env.setattr(loader_utils, "deepspeed_zero3_requested", lambda: True)+ model = _tiny_qwen3_moe()+ names = loader_utils.set_deepspeed_moe_leaf_modules(model)+ assert names == ("Qwen3MoeSparseMoeBlock",), names+ blocks = [m for m in model.modules() if type(m).__name__ == "Qwen3MoeSparseMoeBlock"]+ assert len(blocks) == 2 and all(getattr(m, "_z3_leaf", False) for m in blocks)+ assert sum(bool(getattr(m, "_z3_leaf", False)) for m in model.modules()) == 2+++def test_moe_leaf_marking_is_zero3_only(no_ds_env, fake_set_z3_leaf):+ no_ds_env.setattr(loader_utils, "deepspeed_zero3_requested", lambda: False)+ model = _tiny_qwen3_moe()+ assert loader_utils.set_deepspeed_moe_leaf_modules(model) == ()+ assert fake_set_z3_leaf == []+ assert not any(getattr(m, "_z3_leaf", False) for m in model.modules())+++def test_dense_model_has_no_moe_leaf(no_ds_env, fake_set_z3_leaf):+ torch = pytest.importorskip("torch")+ from transformers import LlamaConfig, LlamaForCausalLM++ no_ds_env.setattr(loader_utils, "deepspeed_zero3_requested", lambda: True)+ with torch.device("meta"):+ model = LlamaForCausalLM(+ LlamaConfig(+ vocab_size = 64,+ hidden_size = 16,+ intermediate_size = 32,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ )+ )+ assert loader_utils.set_deepspeed_moe_leaf_modules(model) == ()+ assert fake_set_z3_leaf == []+++def _filled_args(+ fp16,+ bf16,+ ds_fp16,+ ds_bf16,+ shared = True,+):+ import types++ config = {+ "fp16": {"enabled": ds_fp16},+ "bf16": {"enabled": ds_bf16},+ "zero_optimization": {"stage": 3},+ }+ plugin = types.SimpleNamespace(deepspeed_config = config)+ hf = types.SimpleNamespace(config = config if shared else dict(config))+ return types.SimpleNamespace(+ fp16 = fp16,+ bf16 = bf16,+ fp16_full_eval = False,+ bf16_full_eval = False,+ deepspeed_plugin = plugin,+ hf_deepspeed_config = hf,+ ), config+++def test_forced_float32_disables_the_filled_deepspeed_fp16():+ # T4 + Gemma3 / Qwen3-MoE: fp16 = True was filled in, then Unsloth switched to float32.+ args, config = _filled_args(fp16 = False, bf16 = False, ds_fp16 = True, ds_bf16 = False)+ assert loader_utils.sync_deepspeed_mixed_precision(args) == ["fp16"]+ assert config["fp16"]["enabled"] is False and config["bf16"]["enabled"] is False+ assert loader_utils.sync_deepspeed_mixed_precision(args) == []+++def test_resync_clears_queued_precision_mismatches_and_dtype():+ import inspect+ import types++ import torch++ from transformers.integrations import deepspeed as hf_ds++ assert 'self.mismatches.append(f"- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}")' in (+ inspect.getsource(hf_ds)+ )+ args, config = _filled_args(fp16 = False, bf16 = False, ds_fp16 = True, ds_bf16 = False)+ args.hf_deepspeed_config = types.SimpleNamespace(+ config = config,+ mismatches = [+ "- ds fp16.enabled=True vs hf fp16|fp16_full_eval=False",+ "- ds train_micro_batch_size_per_gpu=4 vs hf per_device_train_batch_size=2",+ ],+ _dtype = torch.float16,+ )+ assert loader_utils.sync_deepspeed_mixed_precision(args) == ["fp16"]+ assert args.hf_deepspeed_config.mismatches == [+ "- ds train_micro_batch_size_per_gpu=4 vs hf per_device_train_batch_size=2"+ ]+ assert args.hf_deepspeed_config._dtype is torch.float32+++def test_resync_dtype_follows_the_flags_when_sections_are_absent():+ import types++ import torch++ config = {"zero_optimization": {"stage": 3}}+ args = types.SimpleNamespace(+ fp16 = False,+ bf16 = True,+ fp16_full_eval = False,+ bf16_full_eval = False,+ deepspeed_plugin = types.SimpleNamespace(deepspeed_config = config),+ hf_deepspeed_config = types.SimpleNamespace(+ config = config, mismatches = [], _dtype = torch.float32+ ),+ )+ assert loader_utils.sync_deepspeed_mixed_precision(args) == []+ assert args.hf_deepspeed_config._dtype is torch.bfloat16+++def test_auto_and_absent_sections_are_left_to_transformers():+ import types++ config = {"fp16": {"enabled": "auto"}}+ args = types.SimpleNamespace(+ fp16 = False,+ bf16 = True,+ deepspeed_plugin = types.SimpleNamespace(deepspeed_config = config),+ hf_deepspeed_config = None,+ )+ assert loader_utils.sync_deepspeed_mixed_precision(args) == []+ assert config == {"fp16": {"enabled": "auto"}}+++def test_no_deepspeed_config_is_a_no_op():+ import types+ args = types.SimpleNamespace(fp16 = True, bf16 = False)+ assert loader_utils.sync_deepspeed_mixed_precision(args) == []+++def test_rl_trainer_source_resyncs_deepspeed_after_the_precision_switch():+ """The generated Unsloth*Trainer __init__ records the requested precision before its switches."""+ import inspect+ from unsloth.models import rl++ source = inspect.getsource(rl)+ first = source.index("_unsloth_precision_requested = tuple(")+ switch = source.index("Switching to float32 training since model cannot work with float16")+ sync = source.index("sync_deepspeed_mixed_precision(args)")+ assert first < switch < sync+++ "distributed, fsdp2, stage, reentrant",+ [+ (False, False, None, True), # one GPU+ (True, False, None, False), # DDP: non-reentrant (#3713)+ (True, True, None, True), # FSDP2+ (True, False, 3, True), # DeepSpeed ZeRO-3: non-reentrant recompute fails its checks+ (True, False, 2, True), # DeepSpeed ZeRO-2: no DDP reducer either+ ],+)+def test_vlm_gradient_checkpointing_reentrant(monkeypatch, distributed, fsdp2, stage, reentrant):+ monkeypatch.setattr(loader_utils, "is_distributed", lambda: distributed)+ monkeypatch.setattr(loader_utils, "fsdp2_requested", lambda: fsdp2)+ monkeypatch.setattr(loader_utils, "deepspeed_zero_stage", lambda: stage)+ assert loader_utils.vlm_gradient_checkpointing_reentrant() is reentrant+++def test_fused_lora_layers_become_zero3_leaves_only_under_zero3(monkeypatch):+ torch = pytest.importorskip("torch")+ layers = torch.nn.ModuleList([torch.nn.Linear(2, 2), torch.nn.Linear(2, 2)])+ monkeypatch.setattr(loader_utils, "deepspeed_zero3_requested", lambda: False)+ assert loader_utils.set_deepspeed_fused_lora_leaf_modules(layers) == 0+ assert not any(getattr(m, "_z3_leaf", False) for m in layers)+ monkeypatch.setattr(loader_utils, "deepspeed_zero3_requested", lambda: True)+ assert loader_utils.set_deepspeed_fused_lora_leaf_modules(layers) == 2+ assert all(m._z3_leaf for m in layers)+ try:+ from deepspeed.utils import z3_leaf_module+ except Exception:+ return+ # The flag DeepSpeed's hook installer reads (deepspeed/utils/z3_leaf_module.py).+ assert all(z3_leaf_module(m) for m in layers)+++def test_only_layers_with_installed_fused_kernels_are_leaves():+ import types++ torch = pytest.importorskip("torch")+ from unsloth.models import llama++ def fused_mlp(self, x):+ return x++ def layer(+ qkv = None,+ o = None,+ mlp = False,+ ):+ m = torch.nn.Module()+ m.self_attn = torch.nn.Module()+ m.self_attn.apply_qkv = qkv or (lambda *a: None)+ m.self_attn.apply_o = o or (lambda *a: None)+ m.mlp = torch.nn.Module()+ if mlp:+ m.mlp.forward = types.MethodType(fused_mlp, m.mlp)+ return m++ # Qwen2 with a q_proj-only adapter: nothing fused, nothing gathered as a whole block.+ plain = layer()+ qkv = layer(qkv = llama.apply_lora_qkv)+ o = layer(o = llama.apply_lora_o)+ mlp = layer(mlp = True)+ assert llama._fused_lora_layers([plain, qkv, o, mlp], fused_mlp) == [qkv, o, mlp]+++def test_trainer_hook_redoes_zero3_setup_for_a_model_loaded_before_deepspeed(+ no_ds_env, fake_set_z3_leaf, capsys+):+ """Loaded and PEFT-patched before TrainingArguments(deepspeed=...): the trainer hook marks the leaves."""+ import weakref+ import transformers.integrations.deepspeed as hf_ds++ model = _tiny_qwen3_moe()+ layers = list(model.model.layers)+ assert loader_utils.set_deepspeed_moe_leaf_modules(model) == ()+ assert loader_utils.set_deepspeed_fused_lora_leaf_modules(layers[:1]) == 0+ model._unsloth_vlm_gc_non_reentrant = True+ assert not any(getattr(m, "_z3_leaf", False) for m in model.modules())++ class _Cfg:+ config = {"zero_optimization": {"stage": 3}}++ cfg = _Cfg()+ no_ds_env.setattr(hf_ds, "_hf_deepspeed_config_weak_ref", weakref.ref(cfg))+ for _ in range(2):+ loader_utils.apply_deepspeed_zero3_model_setup(model)+ blocks = [m for m in model.modules() if type(m).__name__ == "Qwen3MoeSparseMoeBlock"]+ assert len(blocks) == 2 and all(m._z3_leaf for m in blocks)+ assert layers[0]._z3_leaf and not getattr(layers[1], "_z3_leaf", False)+ assert sum(bool(getattr(m, "_z3_leaf", False)) for m in model.modules()) == 3+ assert capsys.readouterr().out.count("non-reentrant gradient checkpointing") == 1+++def test_trainer_hook_is_a_no_op_without_zero3(no_ds_env, fake_set_z3_leaf):+ model = _tiny_qwen3_moe()+ loader_utils.set_deepspeed_fused_lora_leaf_modules(list(model.model.layers))+ loader_utils.apply_deepspeed_zero3_model_setup(model)+ loader_utils.apply_deepspeed_zero3_model_setup("not-a-model")+ assert fake_set_z3_leaf == []+ assert not any(getattr(m, "_z3_leaf", False) for m in model.modules())+++def test_rl_trainer_source_redoes_zero3_setup():+ import inspect+ from unsloth.models import rl++ source = inspect.getsource(rl)+ assert source.index("sync_deepspeed_mixed_precision(args)") < source.index(+ "apply_deepspeed_zero3_model_setup(model)"+ )diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex e63fa142401..36eafe10b31 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -60,6 +60,8 @@ except ImportError: # unsloth_zoo predating the helper (unsloth-zoo#1751)from .loader_utils import (DEFAULT_DEVICE_MAP,+ set_deepspeed_fused_lora_leaf_modules,+ set_deepspeed_moe_leaf_modules,OFFLOAD_EMBEDDING_AUTO,_exclude_rope_inv_freq_from_ddp,_get_fp8_mode_and_check_settings,@@ -3131,6 +3133,21 @@ def _decline_fused_lora_for_fsdp(model, reason = None) -> int:return n+def _fused_lora_layers(layers, mlp_function):+ """Decoder layers with a fused LoRA kernel installed (QKV, O or MLP)."""+ return [+ layer+ for layer in layers+ if getattr(getattr(layer, "self_attn", None), "apply_qkv", None) is apply_lora_qkv+ or getattr(getattr(layer, "self_attn", None), "apply_o", None) is apply_lora_o+ or any(+ getattr(getattr(getattr(layer, "mlp", None), name, None), "__func__", None)+ is mlp_function+ for name in ("forward", "_unsloth_forward")+ )+ ]++def _legacy_block_compile(model):"""Per-block compile for FastLanguageModel LoRA. Off before torch 2.11 unless =1: untracedfused LoRA (fast_lora.TRACE_LORA_FUNCTIONS) splits each block."""@@ -4043,6 +4060,7 @@ class FastLlamaModel:model, tokenizer = patch_tokenizer(model, tokenizer)model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)+ set_deepspeed_moe_leaf_modules(model)mark_auto_offload_for_ddp(model, _ddp_auto_swap)attach_offload_layers(model,@@ -5151,6 +5169,11 @@ class FastLlamaModel:f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "f"{n_qkv} QKV layers, {n_o} O layers and {n_mlp} MLP layers.{unfused_reason}",)+ if n_qkv + n_o + n_mlp > 0:+ # Only layers whose fused kernels were installed (the qwen2 fallback counts unfused QKV).+ set_deepspeed_fused_lora_leaf_modules(+ _fused_lora_layers(model.model.model.layers, _apply_lora_mlp)+ )patch_saving_functions(model)# Patch cross entropy loss labels; fixes #10.diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex c6b267da4d0..9f29a8109cb 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -244,6 +244,159 @@ def deepspeed_zero3_device_map(device_map):return None if is_automatic_device_map(device_map) else device_map+def deepspeed_zero_stage():+ """DeepSpeed ZeRO stage, else None. Unsloth loads before TrainingArguments, so `accelerate launch` env counts."""+ try:+ from transformers.integrations.deepspeed import deepspeed_config+ config = deepspeed_config()+ except Exception:+ config = None+ if config is None:+ try:+ from accelerate.state import AcceleratorState++ plugins = AcceleratorState._shared_state.get("deepspeed_plugins", None) or {}+ if not isinstance(plugins, dict):+ plugins = {"": plugins}+ for plugin in plugins.values():+ if (+ getattr(plugin, "selected", True)+ and getattr(plugin, "zero_stage", None) is not None+ ):+ return int(plugin.zero_stage)+ except Exception:+ pass+ if str(os.environ.get("ACCELERATE_USE_DEEPSPEED", "")).strip().lower() not in _TRUTHY:+ return None+ stage = str(os.environ.get("ACCELERATE_DEEPSPEED_ZERO_STAGE", "")).strip()+ if stage.isdigit():+ return int(stage)+ path = os.environ.get("ACCELERATE_DEEPSPEED_CONFIG_FILE", "")+ if not path:+ return None+ try:+ import json+ with open(path, "r", encoding = "utf-8") as f:+ config = json.load(f)+ except Exception:+ return None+ try:+ return int(config.get("zero_optimization", {}).get("stage"))+ except Exception:+ return None+++def deepspeed_zero3_requested():+ return deepspeed_zero_stage() == 3+++def sync_deepspeed_mixed_precision(args):+ """Resync an already-filled DeepSpeed config's fp16 / bf16 after Unsloth changed them, else+ transformers stops with "ds fp16.enabled=True vs hf fp16|fp16_full_eval=False"."""+ want = {+ "fp16": bool(getattr(args, "fp16", False) or getattr(args, "fp16_full_eval", False)),+ "bf16": bool(getattr(args, "bf16", False) or getattr(args, "bf16_full_eval", False)),+ }+ configs = []+ for holder, attr in (+ (getattr(args, "hf_deepspeed_config", None), "config"),+ (getattr(args, "deepspeed_plugin", None), "deepspeed_config"),+ ):+ config = getattr(holder, attr, None) if holder is not None else None+ if isinstance(config, dict) and all(config is not c for c in configs):+ configs.append(config)+ changed = []+ for config in configs:+ for key, enabled in want.items():+ section = config.get(key)+ if not isinstance(section, dict) or section.get("enabled", False) == "auto":+ continue+ if bool(section.get("enabled", False)) != enabled:+ section["enabled"] = enabled+ changed.append(key)+ hf_config = getattr(args, "hf_deepspeed_config", None)+ if hf_config is not None and isinstance(getattr(hf_config, "config", None), dict):+ # trainer_config_process already queued mismatches for the old flags; the config now agrees.+ mismatches = getattr(hf_config, "mismatches", None)+ if isinstance(mismatches, list):+ mismatches[:] = [+ m+ for m in mismatches+ if not m.startswith(("- ds fp16.enabled=", "- ds bf16.enabled="))+ ]+ if hasattr(hf_config, "_dtype"):+ # From the flags: accelerate adds absent fp16 / bf16 sections later without refreshing _dtype.+ hf_config._dtype = (+ torch.bfloat16 if want["bf16"] else torch.float16 if want["fp16"] else torch.float32+ )+ return changed+++def vlm_gradient_checkpointing_reentrant():+ """Only DDP needs non-reentrant (#3713); under ZeRO-3 non-reentrant fails torch's saved-tensor check."""+ return not is_distributed() or fsdp2_requested() or deepspeed_zero_stage() is not None+++def _is_moe_block(module):+ children = dict(module.named_children())+ return "experts" in children and ("gate" in children or "router" in children)+++def set_deepspeed_moe_leaf_modules(model):+ """Mark MoE blocks ZeRO-3 leaves (before `deepspeed.initialize`): per-expert hooks give mismatched all-gathers across ranks."""+ if model is None or not deepspeed_zero3_requested():+ return ()+ try:+ from deepspeed.utils import set_z3_leaf_modules+ except Exception:+ return ()+ classes = []+ for module in model.modules():+ if type(module) not in classes and _is_moe_block(module):+ classes.append(type(module))+ if not classes:+ return ()+ try:+ set_z3_leaf_modules(model, classes)+ except Exception as exc:+ print(f"Unsloth: could not mark MoE blocks as DeepSpeed ZeRO-3 leaf modules ({exc}).")+ return ()+ return tuple(cls.__name__ for cls in classes)+++def set_deepspeed_fused_lora_leaf_modules(layers):+ """Fused LoRA kernels read weights without calling their modules, so ZeRO-3's prefetch trace breaks unless layers are leaves."""+ layers = list(layers)+ for layer in layers:+ layer._unsloth_fused_lora_layer = True+ if not deepspeed_zero3_requested():+ return 0+ for layer in layers:+ layer._z3_leaf = True+ return len(layers)+++def apply_deepspeed_zero3_model_setup(model):+ """Redo load-time ZeRO-3 setup at trainer time, for models loaded before DeepSpeed was visible."""+ if not hasattr(model, "modules") or not deepspeed_zero3_requested():+ return+ try:+ set_deepspeed_moe_leaf_modules(model)+ set_deepspeed_fused_lora_leaf_modules(+ m for m in model.modules() if getattr(m, "_unsloth_fused_lora_layer", False)+ )+ except Exception as exc:+ print(f"Unsloth: could not apply the DeepSpeed ZeRO-3 model setup ({exc}).")+ if getattr(model, "_unsloth_vlm_gc_non_reentrant", False) and not getattr(+ model, "_unsloth_warned_zero3_gc", False+ ):+ model._unsloth_warned_zero3_gc = True+ print(+ "Unsloth: this model was loaded without DeepSpeed detected, so it uses non-reentrant gradient "+ "checkpointing, which can fail under ZeRO-3. Load the model under the DeepSpeed launch instead."+ )++UNSLOTH_DEVICE_MAP = "unsloth"# Same planner, different answer when it declines: every veto path ends in "sequential", which fills the first device to its whole free budget, wrong for a caller that picked several cards.diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex ad4175dfa15..00c73a9086a 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2578,6 +2578,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):extra_args = ""if "args" in call_args and "model" in call_args:mixed_precision = (+ "_unsloth_precision_requested = tuple(getattr(args, k, False) for k in ('fp16', 'bf16', 'fp16_full_eval', 'bf16_full_eval'))\n""use_bf16 = getattr(args, 'bf16', False)\n""if type(use_bf16) is not bool: use_bf16 = False\n""use_fp16 = getattr(args, 'fp16', False)\n"@@ -2695,6 +2696,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" args.fp16_full_eval = args.fp16\n")extra_args += eval_changes+ if "args" in call_args and "model" in call_args:+ # The switches above can contradict a DeepSpeed config already filled from the caller's flags.+ extra_args += (+ "if tuple(getattr(args, k, False) for k in ('fp16', 'bf16', 'fp16_full_eval', 'bf16_full_eval')) != _unsloth_precision_requested:\n"+ " from unsloth.models.loader_utils import sync_deepspeed_mixed_precision\n"+ " sync_deepspeed_mixed_precision(args)\n"+ "if model is not None:\n"+ " from unsloth.models.loader_utils import apply_deepspeed_zero3_model_setup\n"+ " apply_deepspeed_zero3_model_setup(model)\n"+ )if "model" in call_args:# No UNSLOTH_RETURN_LOGITS=1 for compute_metrics here: prediction_step forces it per eval call; here it leaked into training (#3071).diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 73e515ef359..4e68abc3772 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -142,6 +142,8 @@ from .remote_code_shims import apply_remote_code_shimsfrom .grouped_linear_lora import register_grouped_linear_lorafrom .loader_utils import (_bnb_bits_requested,+ vlm_gradient_checkpointing_reentrant,+ set_deepspeed_moe_leaf_modules,fsdp2_requested,DEFAULT_DEVICE_MAP,OFFLOAD_EMBEDDING_AUTO,@@ -5073,6 +5075,7 @@ class FastBaseModel:float32_mixed_precision = None,):reject_alora(model)+ set_deepspeed_moe_leaf_modules(model)from ..import_fixes import fix_transformers_voxtral_realtime_inplace_audio_addfix_transformers_voxtral_realtime_inplace_audio_add()@@ -5088,12 +5091,15 @@ class FastBaseModel:# VLMs can hit DDP "marked ready twice" with re-entrant checkpointing (#3713), so under DDP skip the offloaded/re-entrant checkpoint patch and default native checkpoint to non-reentrant, offloading via saved-tensor hooks instead.# FSDP2 has no DDP reducer to mark twice: each unit reduce-scatters in its own post-backward hook,# recompute included, so it keeps Unsloth's offloaded checkpointing (and unsloth_stream) as 1 GPU does.- use_reentrant = not is_distributed() or fsdp2_requested()+ use_reentrant = vlm_gradient_checkpointing_reentrant()# DeepSeek-V4.1 layers consume tensors from earlier layers; reentrant GC's no_grad pass drops those grads._gc_model_type = (getattr(getattr(model, "config", None), "model_type", "") or "").lower()_force_non_reentrant = _gc_model_type.startswith(_NON_REENTRANT_GC_MODEL_TYPES)if _force_non_reentrant:use_reentrant = False+ model._unsloth_vlm_gc_non_reentrant = bool(+ use_gradient_checkpointing and not use_reentrant and not _force_non_reentrant+ )if not use_reentrant:unpatch_unsloth_gradient_checkpointing()unpatch_unsloth_smart_gradient_checkpointing()