Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
DDP LoRA: default find_unused_parameters=False so gradient checkpointing does not crash (#13267)
Daniel Han0f5940b5d9d2
diff --git a/tests/test_ddp_find_unused_default.py b/tests/test_ddp_find_unused_default.pynew file mode 100644index 00000000000..38d8b5cf7a4--- /dev/null+++ b/tests/test_ddp_find_unused_default.py@@ -0,0 +1,269 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Unsloth trainers default DDP find_unused_parameters to False for a plain LoRA text model.++transformers picks True for every non-PreTrainedModel (all PEFT models). The helper is loaded by+path so the test stays CPU-only and does not import ``unsloth``.+"""++import importlib.util+from pathlib import Path++import types++import pytest++torch = pytest.importorskip("torch")+peft = pytest.importorskip("peft")+transformers = pytest.importorskip("transformers")++ROOT = Path(__file__).resolve().parents[1]+HELPER = ROOT / "unsloth" / "models" / "_ddp_defaults.py"+++def _helper():+ assert HELPER.exists(), "unsloth/models/_ddp_defaults.py is missing"+ spec = importlib.util.spec_from_file_location("_unsloth_ddp_defaults", HELPER)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module.unsloth_default_ddp_find_unused_parameters+++def _llama(**extra):+ config = transformers.LlamaConfig(+ vocab_size = 64,+ hidden_size = 16,+ intermediate_size = 32,+ num_hidden_layers = 2,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ **extra,+ )+ return transformers.LlamaForCausalLM(config)+++def _lora(model, **kwargs):+ kwargs.setdefault("target_modules", ["q_proj", "v_proj", "gate_proj"])+ return peft.get_peft_model(model, peft.LoraConfig(r = 4, lora_alpha = 8, **kwargs))+++def _args(**kwargs):+ # TRL's configs (SFTConfig, GRPOConfig, ...) turn gradient checkpointing on by default.+ kwargs.setdefault("gradient_checkpointing", True)+ return transformers.TrainingArguments(output_dir = "unused", report_to = "none", **kwargs)+++def ddp_env(monkeypatch):+ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.delenv("UNSLOTH_DDP_FIND_UNUSED_PARAMETERS_AUTO", raising = False)+++def test_plain_lora_defaults_to_false(ddp_env):+ args = _args()+ assert args.ddp_find_unused_parameters is None+ assert _helper()(args, _lora(_llama())) is False+ assert args.ddp_find_unused_parameters is False+++def test_no_gradient_checkpointing_keeps_default(ddp_env):+ # Without checkpointing transformers' True works and tolerates an adapter a step never runs.+ args = _args(gradient_checkpointing = False)+ assert _helper()(args, _lora(_llama())) is None+ assert args.ddp_find_unused_parameters is None+++def test_checkpointing_armed_on_model_only_defaults_to_false(ddp_env):+ # Unsloth arms checkpointing on the model (for_training) even when the args leave it off.+ model = _llama()+ model.gradient_checkpointing_enable()+ assert _helper()(_args(gradient_checkpointing = False), _lora(model)) is False+++def test_user_value_is_kept(ddp_env):+ args = _args(ddp_find_unused_parameters = True)+ assert _helper()(args, _lora(_llama())) is True+++def test_not_distributed_is_untouched(monkeypatch):+ monkeypatch.setenv("WORLD_SIZE", "1")+ args = _args()+ assert _helper()(args, _lora(_llama())) is None+++def test_env_opt_out(ddp_env, monkeypatch):+ monkeypatch.setenv("UNSLOTH_DDP_FIND_UNUSED_PARAMETERS_AUTO", "0")+ args = _args()+ assert _helper()(args, _lora(_llama())) is None+++ "make",+ [+ # MoE: unrouted experts get no gradient.+ lambda: _lora(_llama(num_local_experts = 4)),+ # modules_to_save: a full trainable copy Unsloth's fused paths may bypass.+ lambda: _lora(_llama(), modules_to_save = ["lm_head"]),+ lambda: _lora(_llama(), target_modules = ["q_proj", "lm_head"]),+ lambda: _lora(_llama(), target_modules = ["q_proj", "embed_tokens"]),+ # Full fine-tuning goes through transformers' own rule.+ lambda: _llama(),+ ],+)+def test_skippable_models_keep_transformers_default(ddp_env, make):+ args = _args()+ assert _helper()(args, make()) is None+++ "extra",+ [+ {"num_experts": 4},+ {"n_routed_experts": 4},+ {"moe_num_experts": 4},+ {"num_experts_per_tok": 2},+ {"layerdrop": 0.1},+ {"encoder_layerdrop": 0.1},+ {"decoder_layerdrop": 0.1},+ {"add_cross_attention": True},+ {"speech_encoder_layerdrop": 0.1},+ {"is_encoder_decoder": True},+ ],+)+def test_moe_and_layerdrop_config_keys_keep_default(ddp_env, extra):+ model = _lora(_llama())+ for key, value in extra.items():+ setattr(model.get_base_model().config, key, value)+ assert _helper()(_args(), model) is None+++def test_vision_sub_config_keeps_default(ddp_env):+ model = _lora(_llama())+ # A VLM loaded through a text class still carries its tower config.+ model.get_base_model().config.vision_config = transformers.CLIPVisionConfig()+ assert _helper()(_args(), model) is None+++def test_vision_config_under_text_config_keeps_default(ddp_env):+ model = _lora(_llama())+ outer = transformers.PretrainedConfig()+ outer.text_config = model.get_base_model().config+ outer.text_config.audio_config = transformers.PretrainedConfig()+ model.get_base_model().config = outer+ assert _helper()(_args(), model) is None+++def test_two_adapters_keep_default(ddp_env):+ model = _lora(_llama())+ model.add_adapter("second", peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["q_proj"]))+ assert _helper()(_args(), model) is None+++def test_non_lora_peft_keeps_default(ddp_env):+ model = peft.get_peft_model(+ _llama(),+ peft.IA3Config(target_modules = ["q_proj", "down_proj"], feedforward_modules = ["down_proj"]),+ )+ assert _helper()(_args(), model) is None++[email protected]("attr", ["target_parameters", "trainable_token_indices"])+def test_extra_trainable_targets_keep_default(ddp_env, attr):+ model = _lora(_llama())+ cfg = next(iter(model.peft_config.values()))+ object.__setattr__(+ cfg, attr, ["mlp.experts.gate_up_proj"] if attr == "target_parameters" else [1, 2]+ )+ assert _helper()(_args(), model) is None+++def test_transformers_native_adapter_keeps_default(ddp_env):+ # PreTrainedModel.add_adapter keeps the model a PreTrainedModel: transformers' own rule decides.+ model = _llama()+ model.add_adapter(peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["q_proj"]))+ assert isinstance(model.peft_config, dict) and len(model.peft_config) == 1+ assert _helper()(_args(), model) is None+++def test_non_lora_peft_type_keeps_default(ddp_env):+ # Isolates the peft_type rule: LoRA-named tensors, but the adapter says it is something else.+ model = _lora(_llama())+ cfg = next(iter(model.peft_config.values()))+ object.__setattr__(cfg, "peft_type", peft.PeftType.ADALORA)+ assert _helper()(_args(), model) is None+++def test_modules_to_save_config_keeps_default(ddp_env):+ model = _lora(_llama())+ cfg = next(iter(model.peft_config.values()))+ object.__setattr__(cfg, "modules_to_save", ["score"])+ assert _helper()(_args(), model) is None+++def test_lora_on_linear_embedding_projection_keeps_default(ddp_env):+ # An embedding-side Linear (e.g. GPT-NeoX's `embed_out` head on transformers 4.x) gets lora_A / lora_B+ # tensors, so only the name rule keeps it out.+ model = _llama()+ model.model.embed_proj = torch.nn.Linear(16, 16, bias = False)+ model = _lora(model, target_modules = ["q_proj", "embed_proj"])+ assert any("embed_proj.lora_A" in n for n, _ in model.named_parameters())+ assert _helper()(_args(), model) is None+++def test_layerdrop_in_a_composite_encoder_config_keeps_default(ddp_env):+ # SpeechEncoderDecoder keeps Wav2Vec2's config (layerdrop 0.1) under config.encoder.+ model = _lora(_llama())+ model.get_base_model().config.encoder = types.SimpleNamespace(layerdrop = 0.1)+ assert _helper()(_args(), model) is None+++def test_non_causal_lm_keeps_default(ddp_env):+ # SeamlessM4T-style models carry generation-only submodules (t2u_model) a training forward never runs.+ model = _llama()+ model.__class__ = type("TinyForSpeechToSpeech", (type(model),), {})+ assert _helper()(_args(), _lora(model)) is None+++def test_lora_on_an_output_head_alias_keeps_default(ddp_env):+ # remote-code heads named output / output_layer: found through get_output_embeddings, not the name.+ model = _llama()+ model.output_layer = torch.nn.Linear(16, 64, bias = False)+ model.get_output_embeddings = lambda: model.output_layer+ model = _lora(model, target_modules = ["q_proj", "output_layer"])+ assert _helper()(_args(), model) is None+++def test_dora_defaults_to_false(ddp_env):+ assert _helper()(_args(), _lora(_llama(), use_dora = True)) is False+++def test_non_lora_trainable_param_keeps_default(ddp_env):+ model = _lora(_llama())+ model.base_model.model.model.norm.weight.requires_grad_(True)+ assert _helper()(_args(), model) is None+++def test_reused_args_redecides(ddp_env):+ args = _args()+ helper = _helper()+ assert helper(args, _lora(_llama())) is False+ assert helper(args, _lora(_llama(num_local_experts = 4))) is None+++def test_reused_args_keep_a_value_set_after_the_auto_default(ddp_env):+ args = _args()+ helper = _helper()+ assert helper(args, _lora(_llama())) is False+ args.ddp_find_unused_parameters = True+ assert helper(args, _lora(_llama())) is True+++def test_generated_trainer_calls_helper():+ source = (ROOT / "unsloth" / "models" / "rl.py").read_text(encoding = "utf-8")+ call = "unsloth_default_ddp_find_unused_parameters(args, model)"+ assert call in source+ # Before super().__init__: transformers builds the DDP kwargs inside Trainer.__init__ (5.x).+ template_init = source.index("class Unsloth{RLTrainer_name}(_Unsloth{RLTrainer_name}):")+ super_call = source.index("super().__init__({RLTrainer_call_args}", template_init)+ assert template_init < source.index(call, template_init) < super_calldiff --git a/unsloth/models/_ddp_defaults.py b/unsloth/models/_ddp_defaults.pynew file mode 100644index 00000000000..ef228f4eeac--- /dev/null+++ b/unsloth/models/_ddp_defaults.py@@ -0,0 +1,187 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""DDP defaults for Unsloth trainers.++transformers' Trainer passes ``find_unused_parameters=True`` to DDP whenever the model is not a+``PreTrainedModel`` and ``ddp_find_unused_parameters`` is None, which is every PEFT model. Under reentrant+gradient checkpointing that crashes on the first backward ("marked as ready twice"): the checkpointed+forward runs without grad, DDP marks every LoRA weight unused, then the recompute marks them again.+transformers already picks False for a ``PreTrainedModel`` under checkpointing; this does the same for a+plain LoRA adapter on a dense text model, where every trainable parameter gets a gradient each step.+Without checkpointing True works (and tolerates an adapter a step never runs), so it is left alone.+"""++import os++__all__ = ["unsloth_default_ddp_find_unused_parameters"]++_AUTO_FLAG = "_unsloth_auto_ddp_find_unused_parameters"+# LoRA tensors that are used whenever their base layer runs.+_LORA_PARAM_MARKERS = (".lora_A.", ".lora_B.", ".lora_magnitude_vector.")+# Modules a step can skip (unrouted experts, a vision tower on a text batch, MTP heads), or that+# Unsloth's fused loss / embedding paths may read by weight and bypass the LoRA forward.+_SKIPPABLE_NAME_MARKERS = (+ "expert",+ "vision",+ "visual",+ "audio",+ "image",+ "projector",+ "mtp",+ "lm_head",+ "embed",+)+_SUBMODEL_CONFIG_KEYS = ("vision_config", "audio_config", "vision_tower", "speech_config")+_MOE_CONFIG_KEYS = (+ "num_experts",+ "num_local_experts",+ "n_routed_experts",+ "moe_num_experts",+ "num_experts_per_tok",+)+++def _is_distributed():+ try:+ import torch.distributed as dist+ if dist.is_available() and dist.is_initialized():+ return dist.get_world_size() > 1+ except Exception:+ pass+ try:+ return int(os.environ.get("WORLD_SIZE", "1")) > 1+ except ValueError:+ return False+++def _configs(config):+ seen = []+ stack = [config]+ while stack:+ c = stack.pop()+ if c is None or any(c is s for s in seen):+ continue+ seen.append(c)+ # encoder / decoder: composite configs (SpeechEncoderDecoder, VisionEncoderDecoder).+ for key in ("text_config", "language_config", "llm_config", "encoder", "decoder"):+ value = getattr(c, key, None)+ if value is not None and not isinstance(+ value, (str, bool, int, float, list, tuple, dict)+ ):+ stack.append(value)+ return seen+++def _config_may_skip(config):+ for c in _configs(config):+ for key in _SUBMODEL_CONFIG_KEYS:+ if getattr(c, key, None) is not None:+ return True+ for key in _MOE_CONFIG_KEYS:+ value = getattr(c, key, None)+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:+ return True+ # Optional cross-attention (GPT-2 style) only runs when encoder states are passed.+ if getattr(c, "add_cross_attention", False) and not getattr(c, "is_encoder_decoder", False):+ return True+ # layerdrop, encoder_layerdrop, decoder_layerdrop, speech_encoder_layerdrop, ...+ for key, value in getattr(c, "__dict__", {}).items():+ if (+ "layerdrop" in key+ and isinstance(value, (int, float))+ and not isinstance(value, bool)+ and value > 0+ ):+ return True+ return False+++def _gradient_checkpointing_on(args, model):+ """True unless checkpointing is off both in the args (Trainer turns it on in train()) and on the model."""+ if getattr(args, "gradient_checkpointing", False):+ return True+ try:+ return any(getattr(m, "gradient_checkpointing", False) for m in model.modules())+ except Exception:+ return True+++def _lora_uses_every_trainable_param(model):+ """True only when we can show every trainable tensor is a LoRA weight run on each forward."""+ from transformers import PreTrainedModel++ if isinstance(model, PreTrainedModel):+ # transformers already decides for these (False under gradient checkpointing).+ return False+ peft_config = getattr(model, "peft_config", None)+ if not isinstance(peft_config, dict) or len(peft_config) != 1:+ return False+ cfg = next(iter(peft_config.values()))+ if (+ str(getattr(getattr(cfg, "peft_type", None), "value", getattr(cfg, "peft_type", "")))+ != "LORA"+ ):+ return False+ for attr in ("modules_to_save", "target_parameters", "trainable_token_indices"):+ if getattr(cfg, attr, None):+ return False+ base = getattr(model, "get_base_model", lambda: None)()+ config = getattr(base, "config", None) or getattr(model, "config", None)+ # Decoder-only causal LMs only: encoder-decoder and speech models can leave adapters idle (t2u_model).+ if getattr(config, "is_encoder_decoder", False) or not type(base).__name__.endswith(+ ("ForCausalLM", "LMHeadModel")+ ):+ return False+ if _config_may_skip(config):+ return False+ # The output head under any name (output, output_layer, embed_out): fused losses read its weight.+ try:+ head = base.get_output_embeddings()+ head_params = {id(p) for p in head.parameters()} if head is not None else set()+ except Exception:+ head_params = set()+ n_trainable = 0+ for name, param in model.named_parameters():+ if not param.requires_grad:+ continue+ n_trainable += 1+ if id(param) in head_params:+ return False+ if not any(m in name for m in _LORA_PARAM_MARKERS):+ return False+ lowered = name.lower()+ if any(m in lowered for m in _SKIPPABLE_NAME_MARKERS):+ return False+ return n_trainable > 0+++def unsloth_default_ddp_find_unused_parameters(args, model):+ """Set ``args.ddp_find_unused_parameters = False`` for a plain LoRA model under DDP with gradient+ checkpointing when the user left it None.++ Kept at None (transformers' True) for anything a step may skip: MoE / expert LoRA, vision or audio+ towers, MTP heads, lm_head / embedding adapters, modules_to_save, several adapters, non-LoRA PEFT.+ Returns the value now on ``args``.+ """+ if args is None or not hasattr(args, "ddp_find_unused_parameters"):+ return None+ if getattr(args, _AUTO_FLAG, False):+ # Reused args object: our earlier False is not a user choice, re-decide; a value set since then wins.+ if args.ddp_find_unused_parameters is False:+ args.ddp_find_unused_parameters = None+ object.__setattr__(args, _AUTO_FLAG, False)+ if args.ddp_find_unused_parameters is not None:+ return args.ddp_find_unused_parameters+ if os.environ.get("UNSLOTH_DDP_FIND_UNUSED_PARAMETERS_AUTO", "1") == "0":+ return None+ if not _is_distributed():+ return None+ try:+ safe = _gradient_checkpointing_on(args, model) and _lora_uses_every_trainable_param(model)+ except Exception:+ safe = False+ if safe:+ args.ddp_find_unused_parameters = False+ object.__setattr__(args, _AUTO_FLAG, True)+ return args.ddp_find_unused_parametersdiff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex bbecab934ff..17f7727d5f4 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -601,6 +601,12 @@ class Unsloth{RLTrainer_name}(_Unsloth{RLTrainer_name}):if "model" in locals() and hasattr(model, "for_training"):_use_gc = model._unsloth_gradient_checkpointing if hasattr(model, '_unsloth_gradient_checkpointing') else getattr(args, 'gradient_checkpointing', True)model.for_training(use_gradient_checkpointing=_use_gc)+ if "model" in locals():+ try:+ from unsloth.models._ddp_defaults import unsloth_default_ddp_find_unused_parameters+ unsloth_default_ddp_find_unused_parameters(args, model)+ except Exception:+ passsuper().__init__({RLTrainer_call_args}{RLTrainer_kwargs})if "model" in locals() and hasattr(model, "for_inference"):model.for_inference()