Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

DDP LoRA: default find_unused_parameters=False so gradient checkpointing does not crash (#13267)

Daniel Han0f5940b5d9d2
diff --git a/tests/test_ddp_find_unused_default.py b/tests/test_ddp_find_unused_default.py
new file mode 100644
index 00000000000..38d8b5cf7a4
--- /dev/null
+++ b/tests/test_ddp_find_unused_default.py
@@ -0,0 +1,269 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Unsloth trainers default DDP find_unused_parameters to False for a plain LoRA text model.
+
+transformers picks True for every non-PreTrainedModel (all PEFT models). The helper is loaded by
+path so the test stays CPU-only and does not import ``unsloth``.
+"""
+
+import importlib.util
+from pathlib import Path
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+peft = pytest.importorskip("peft")
+transformers = pytest.importorskip("transformers")
+
+ROOT = Path(__file__).resolve().parents[1]
+HELPER = ROOT / "unsloth" / "models" / "_ddp_defaults.py"
+
+
+def _helper():
+ assert HELPER.exists(), "unsloth/models/_ddp_defaults.py is missing"
+ spec = importlib.util.spec_from_file_location("_unsloth_ddp_defaults", HELPER)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module.unsloth_default_ddp_find_unused_parameters
+
+
+def _llama(**extra):
+ config = transformers.LlamaConfig(
+ vocab_size = 64,
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ **extra,
+ )
+ return transformers.LlamaForCausalLM(config)
+
+
+def _lora(model, **kwargs):
+ kwargs.setdefault("target_modules", ["q_proj", "v_proj", "gate_proj"])
+ return peft.get_peft_model(model, peft.LoraConfig(r = 4, lora_alpha = 8, **kwargs))
+
+
+def _args(**kwargs):
+ # TRL's configs (SFTConfig, GRPOConfig, ...) turn gradient checkpointing on by default.
+ kwargs.setdefault("gradient_checkpointing", True)
+ return transformers.TrainingArguments(output_dir = "unused", report_to = "none", **kwargs)
+
+
+def ddp_env(monkeypatch):
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.delenv("UNSLOTH_DDP_FIND_UNUSED_PARAMETERS_AUTO", raising = False)
+
+
+def test_plain_lora_defaults_to_false(ddp_env):
+ args = _args()
+ assert args.ddp_find_unused_parameters is None
+ assert _helper()(args, _lora(_llama())) is False
+ assert args.ddp_find_unused_parameters is False
+
+
+def test_no_gradient_checkpointing_keeps_default(ddp_env):
+ # Without checkpointing transformers' True works and tolerates an adapter a step never runs.
+ args = _args(gradient_checkpointing = False)
+ assert _helper()(args, _lora(_llama())) is None
+ assert args.ddp_find_unused_parameters is None
+
+
+def test_checkpointing_armed_on_model_only_defaults_to_false(ddp_env):
+ # Unsloth arms checkpointing on the model (for_training) even when the args leave it off.
+ model = _llama()
+ model.gradient_checkpointing_enable()
+ assert _helper()(_args(gradient_checkpointing = False), _lora(model)) is False
+
+
+def test_user_value_is_kept(ddp_env):
+ args = _args(ddp_find_unused_parameters = True)
+ assert _helper()(args, _lora(_llama())) is True
+
+
+def test_not_distributed_is_untouched(monkeypatch):
+ monkeypatch.setenv("WORLD_SIZE", "1")
+ args = _args()
+ assert _helper()(args, _lora(_llama())) is None
+
+
+def test_env_opt_out(ddp_env, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DDP_FIND_UNUSED_PARAMETERS_AUTO", "0")
+ args = _args()
+ assert _helper()(args, _lora(_llama())) is None
+
+
+ "make",
+ [
+ # MoE: unrouted experts get no gradient.
+ lambda: _lora(_llama(num_local_experts = 4)),
+ # modules_to_save: a full trainable copy Unsloth's fused paths may bypass.
+ lambda: _lora(_llama(), modules_to_save = ["lm_head"]),
+ lambda: _lora(_llama(), target_modules = ["q_proj", "lm_head"]),
+ lambda: _lora(_llama(), target_modules = ["q_proj", "embed_tokens"]),
+ # Full fine-tuning goes through transformers' own rule.
+ lambda: _llama(),
+ ],
+)
+def test_skippable_models_keep_transformers_default(ddp_env, make):
+ args = _args()
+ assert _helper()(args, make()) is None
+
+
+ "extra",
+ [
+ {"num_experts": 4},
+ {"n_routed_experts": 4},
+ {"moe_num_experts": 4},
+ {"num_experts_per_tok": 2},
+ {"layerdrop": 0.1},
+ {"encoder_layerdrop": 0.1},
+ {"decoder_layerdrop": 0.1},
+ {"add_cross_attention": True},
+ {"speech_encoder_layerdrop": 0.1},
+ {"is_encoder_decoder": True},
+ ],
+)
+def test_moe_and_layerdrop_config_keys_keep_default(ddp_env, extra):
+ model = _lora(_llama())
+ for key, value in extra.items():
+ setattr(model.get_base_model().config, key, value)
+ assert _helper()(_args(), model) is None
+
+
+def test_vision_sub_config_keeps_default(ddp_env):
+ model = _lora(_llama())
+ # A VLM loaded through a text class still carries its tower config.
+ model.get_base_model().config.vision_config = transformers.CLIPVisionConfig()
+ assert _helper()(_args(), model) is None
+
+
+def test_vision_config_under_text_config_keeps_default(ddp_env):
+ model = _lora(_llama())
+ outer = transformers.PretrainedConfig()
+ outer.text_config = model.get_base_model().config
+ outer.text_config.audio_config = transformers.PretrainedConfig()
+ model.get_base_model().config = outer
+ assert _helper()(_args(), model) is None
+
+
+def test_two_adapters_keep_default(ddp_env):
+ model = _lora(_llama())
+ model.add_adapter("second", peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["q_proj"]))
+ assert _helper()(_args(), model) is None
+
+
+def test_non_lora_peft_keeps_default(ddp_env):
+ model = peft.get_peft_model(
+ _llama(),
+ peft.IA3Config(target_modules = ["q_proj", "down_proj"], feedforward_modules = ["down_proj"]),
+ )
+ assert _helper()(_args(), model) is None
+
+
[email protected]("attr", ["target_parameters", "trainable_token_indices"])
+def test_extra_trainable_targets_keep_default(ddp_env, attr):
+ model = _lora(_llama())
+ cfg = next(iter(model.peft_config.values()))
+ object.__setattr__(
+ cfg, attr, ["mlp.experts.gate_up_proj"] if attr == "target_parameters" else [1, 2]
+ )
+ assert _helper()(_args(), model) is None
+
+
+def test_transformers_native_adapter_keeps_default(ddp_env):
+ # PreTrainedModel.add_adapter keeps the model a PreTrainedModel: transformers' own rule decides.
+ model = _llama()
+ model.add_adapter(peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["q_proj"]))
+ assert isinstance(model.peft_config, dict) and len(model.peft_config) == 1
+ assert _helper()(_args(), model) is None
+
+
+def test_non_lora_peft_type_keeps_default(ddp_env):
+ # Isolates the peft_type rule: LoRA-named tensors, but the adapter says it is something else.
+ model = _lora(_llama())
+ cfg = next(iter(model.peft_config.values()))
+ object.__setattr__(cfg, "peft_type", peft.PeftType.ADALORA)
+ assert _helper()(_args(), model) is None
+
+
+def test_modules_to_save_config_keeps_default(ddp_env):
+ model = _lora(_llama())
+ cfg = next(iter(model.peft_config.values()))
+ object.__setattr__(cfg, "modules_to_save", ["score"])
+ assert _helper()(_args(), model) is None
+
+
+def test_lora_on_linear_embedding_projection_keeps_default(ddp_env):
+ # An embedding-side Linear (e.g. GPT-NeoX's `embed_out` head on transformers 4.x) gets lora_A / lora_B
+ # tensors, so only the name rule keeps it out.
+ model = _llama()
+ model.model.embed_proj = torch.nn.Linear(16, 16, bias = False)
+ model = _lora(model, target_modules = ["q_proj", "embed_proj"])
+ assert any("embed_proj.lora_A" in n for n, _ in model.named_parameters())
+ assert _helper()(_args(), model) is None
+
+
+def test_layerdrop_in_a_composite_encoder_config_keeps_default(ddp_env):
+ # SpeechEncoderDecoder keeps Wav2Vec2's config (layerdrop 0.1) under config.encoder.
+ model = _lora(_llama())
+ model.get_base_model().config.encoder = types.SimpleNamespace(layerdrop = 0.1)
+ assert _helper()(_args(), model) is None
+
+
+def test_non_causal_lm_keeps_default(ddp_env):
+ # SeamlessM4T-style models carry generation-only submodules (t2u_model) a training forward never runs.
+ model = _llama()
+ model.__class__ = type("TinyForSpeechToSpeech", (type(model),), {})
+ assert _helper()(_args(), _lora(model)) is None
+
+
+def test_lora_on_an_output_head_alias_keeps_default(ddp_env):
+ # remote-code heads named output / output_layer: found through get_output_embeddings, not the name.
+ model = _llama()
+ model.output_layer = torch.nn.Linear(16, 64, bias = False)
+ model.get_output_embeddings = lambda: model.output_layer
+ model = _lora(model, target_modules = ["q_proj", "output_layer"])
+ assert _helper()(_args(), model) is None
+
+
+def test_dora_defaults_to_false(ddp_env):
+ assert _helper()(_args(), _lora(_llama(), use_dora = True)) is False
+
+
+def test_non_lora_trainable_param_keeps_default(ddp_env):
+ model = _lora(_llama())
+ model.base_model.model.model.norm.weight.requires_grad_(True)
+ assert _helper()(_args(), model) is None
+
+
+def test_reused_args_redecides(ddp_env):
+ args = _args()
+ helper = _helper()
+ assert helper(args, _lora(_llama())) is False
+ assert helper(args, _lora(_llama(num_local_experts = 4))) is None
+
+
+def test_reused_args_keep_a_value_set_after_the_auto_default(ddp_env):
+ args = _args()
+ helper = _helper()
+ assert helper(args, _lora(_llama())) is False
+ args.ddp_find_unused_parameters = True
+ assert helper(args, _lora(_llama())) is True
+
+
+def test_generated_trainer_calls_helper():
+ source = (ROOT / "unsloth" / "models" / "rl.py").read_text(encoding = "utf-8")
+ call = "unsloth_default_ddp_find_unused_parameters(args, model)"
+ assert call in source
+ # Before super().__init__: transformers builds the DDP kwargs inside Trainer.__init__ (5.x).
+ template_init = source.index("class Unsloth{RLTrainer_name}(_Unsloth{RLTrainer_name}):")
+ super_call = source.index("super().__init__({RLTrainer_call_args}", template_init)
+ assert template_init < source.index(call, template_init) < super_call
diff --git a/unsloth/models/_ddp_defaults.py b/unsloth/models/_ddp_defaults.py
new file mode 100644
index 00000000000..ef228f4eeac
--- /dev/null
+++ b/unsloth/models/_ddp_defaults.py
@@ -0,0 +1,187 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""DDP defaults for Unsloth trainers.
+
+transformers' Trainer passes ``find_unused_parameters=True`` to DDP whenever the model is not a
+``PreTrainedModel`` and ``ddp_find_unused_parameters`` is None, which is every PEFT model. Under reentrant
+gradient checkpointing that crashes on the first backward ("marked as ready twice"): the checkpointed
+forward runs without grad, DDP marks every LoRA weight unused, then the recompute marks them again.
+transformers already picks False for a ``PreTrainedModel`` under checkpointing; this does the same for a
+plain LoRA adapter on a dense text model, where every trainable parameter gets a gradient each step.
+Without checkpointing True works (and tolerates an adapter a step never runs), so it is left alone.
+"""
+
+import os
+
+__all__ = ["unsloth_default_ddp_find_unused_parameters"]
+
+_AUTO_FLAG = "_unsloth_auto_ddp_find_unused_parameters"
+# LoRA tensors that are used whenever their base layer runs.
+_LORA_PARAM_MARKERS = (".lora_A.", ".lora_B.", ".lora_magnitude_vector.")
+# Modules a step can skip (unrouted experts, a vision tower on a text batch, MTP heads), or that
+# Unsloth's fused loss / embedding paths may read by weight and bypass the LoRA forward.
+_SKIPPABLE_NAME_MARKERS = (
+ "expert",
+ "vision",
+ "visual",
+ "audio",
+ "image",
+ "projector",
+ "mtp",
+ "lm_head",
+ "embed",
+)
+_SUBMODEL_CONFIG_KEYS = ("vision_config", "audio_config", "vision_tower", "speech_config")
+_MOE_CONFIG_KEYS = (
+ "num_experts",
+ "num_local_experts",
+ "n_routed_experts",
+ "moe_num_experts",
+ "num_experts_per_tok",
+)
+
+
+def _is_distributed():
+ try:
+ import torch.distributed as dist
+ if dist.is_available() and dist.is_initialized():
+ return dist.get_world_size() > 1
+ except Exception:
+ pass
+ try:
+ return int(os.environ.get("WORLD_SIZE", "1")) > 1
+ except ValueError:
+ return False
+
+
+def _configs(config):
+ seen = []
+ stack = [config]
+ while stack:
+ c = stack.pop()
+ if c is None or any(c is s for s in seen):
+ continue
+ seen.append(c)
+ # encoder / decoder: composite configs (SpeechEncoderDecoder, VisionEncoderDecoder).
+ for key in ("text_config", "language_config", "llm_config", "encoder", "decoder"):
+ value = getattr(c, key, None)
+ if value is not None and not isinstance(
+ value, (str, bool, int, float, list, tuple, dict)
+ ):
+ stack.append(value)
+ return seen
+
+
+def _config_may_skip(config):
+ for c in _configs(config):
+ for key in _SUBMODEL_CONFIG_KEYS:
+ if getattr(c, key, None) is not None:
+ return True
+ for key in _MOE_CONFIG_KEYS:
+ value = getattr(c, key, None)
+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:
+ return True
+ # Optional cross-attention (GPT-2 style) only runs when encoder states are passed.
+ if getattr(c, "add_cross_attention", False) and not getattr(c, "is_encoder_decoder", False):
+ return True
+ # layerdrop, encoder_layerdrop, decoder_layerdrop, speech_encoder_layerdrop, ...
+ for key, value in getattr(c, "__dict__", {}).items():
+ if (
+ "layerdrop" in key
+ and isinstance(value, (int, float))
+ and not isinstance(value, bool)
+ and value > 0
+ ):
+ return True
+ return False
+
+
+def _gradient_checkpointing_on(args, model):
+ """True unless checkpointing is off both in the args (Trainer turns it on in train()) and on the model."""
+ if getattr(args, "gradient_checkpointing", False):
+ return True
+ try:
+ return any(getattr(m, "gradient_checkpointing", False) for m in model.modules())
+ except Exception:
+ return True
+
+
+def _lora_uses_every_trainable_param(model):
+ """True only when we can show every trainable tensor is a LoRA weight run on each forward."""
+ from transformers import PreTrainedModel
+
+ if isinstance(model, PreTrainedModel):
+ # transformers already decides for these (False under gradient checkpointing).
+ return False
+ peft_config = getattr(model, "peft_config", None)
+ if not isinstance(peft_config, dict) or len(peft_config) != 1:
+ return False
+ cfg = next(iter(peft_config.values()))
+ if (
+ str(getattr(getattr(cfg, "peft_type", None), "value", getattr(cfg, "peft_type", "")))
+ != "LORA"
+ ):
+ return False
+ for attr in ("modules_to_save", "target_parameters", "trainable_token_indices"):
+ if getattr(cfg, attr, None):
+ return False
+ base = getattr(model, "get_base_model", lambda: None)()
+ config = getattr(base, "config", None) or getattr(model, "config", None)
+ # Decoder-only causal LMs only: encoder-decoder and speech models can leave adapters idle (t2u_model).
+ if getattr(config, "is_encoder_decoder", False) or not type(base).__name__.endswith(
+ ("ForCausalLM", "LMHeadModel")
+ ):
+ return False
+ if _config_may_skip(config):
+ return False
+ # The output head under any name (output, output_layer, embed_out): fused losses read its weight.
+ try:
+ head = base.get_output_embeddings()
+ head_params = {id(p) for p in head.parameters()} if head is not None else set()
+ except Exception:
+ head_params = set()
+ n_trainable = 0
+ for name, param in model.named_parameters():
+ if not param.requires_grad:
+ continue
+ n_trainable += 1
+ if id(param) in head_params:
+ return False
+ if not any(m in name for m in _LORA_PARAM_MARKERS):
+ return False
+ lowered = name.lower()
+ if any(m in lowered for m in _SKIPPABLE_NAME_MARKERS):
+ return False
+ return n_trainable > 0
+
+
+def unsloth_default_ddp_find_unused_parameters(args, model):
+ """Set ``args.ddp_find_unused_parameters = False`` for a plain LoRA model under DDP with gradient
+ checkpointing when the user left it None.
+
+ Kept at None (transformers' True) for anything a step may skip: MoE / expert LoRA, vision or audio
+ towers, MTP heads, lm_head / embedding adapters, modules_to_save, several adapters, non-LoRA PEFT.
+ Returns the value now on ``args``.
+ """
+ if args is None or not hasattr(args, "ddp_find_unused_parameters"):
+ return None
+ if getattr(args, _AUTO_FLAG, False):
+ # Reused args object: our earlier False is not a user choice, re-decide; a value set since then wins.
+ if args.ddp_find_unused_parameters is False:
+ args.ddp_find_unused_parameters = None
+ object.__setattr__(args, _AUTO_FLAG, False)
+ if args.ddp_find_unused_parameters is not None:
+ return args.ddp_find_unused_parameters
+ if os.environ.get("UNSLOTH_DDP_FIND_UNUSED_PARAMETERS_AUTO", "1") == "0":
+ return None
+ if not _is_distributed():
+ return None
+ try:
+ safe = _gradient_checkpointing_on(args, model) and _lora_uses_every_trainable_param(model)
+ except Exception:
+ safe = False
+ if safe:
+ args.ddp_find_unused_parameters = False
+ object.__setattr__(args, _AUTO_FLAG, True)
+ return args.ddp_find_unused_parameters
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index bbecab934ff..17f7727d5f4 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -601,6 +601,12 @@ class Unsloth{RLTrainer_name}(_Unsloth{RLTrainer_name}):
if "model" in locals() and hasattr(model, "for_training"):
_use_gc = model._unsloth_gradient_checkpointing if hasattr(model, '_unsloth_gradient_checkpointing') else getattr(args, 'gradient_checkpointing', True)
model.for_training(use_gradient_checkpointing=_use_gc)
+ if "model" in locals():
+ try:
+ from unsloth.models._ddp_defaults import unsloth_default_ddp_find_unused_parameters
+ unsloth_default_ddp_find_unused_parameters(args, model)
+ except Exception:
+ pass
super().__init__({RLTrainer_call_args}{RLTrainer_kwargs})
if "model" in locals() and hasattr(model, "for_inference"):
model.for_inference()