Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep generated RL configs on the reentrant gradient checkpoint path (#11519)
Daniel Han9f3153cc18a0
diff --git a/tests/test_gradient_checkpointing_reentrant.py b/tests/test_gradient_checkpointing_reentrant.pynew file mode 100644index 00000000000..118afcd4d3a--- /dev/null+++ b/tests/test_gradient_checkpointing_reentrant.py@@ -0,0 +1,132 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Generated RL configs must keep the reentrant gradient checkpoint path.++Unsloth gradient checkpointing needs `use_reentrant=True`. The non-reentrant+path recomputes every packed forward during backward and compares what each+pass saved, so a region packed compiled and recomputed eagerly aborts with++ CheckpointError: A different number of tensors was saved during the+ original forward and recomputation.++Two things push a config to non-reentrant, and only one of them was handled:++ TRL 0.27.0+ sets use_reentrant=False explicitly.++ transformers substitutes {"use_reentrant": False} whenever+ gradient_checkpointing_kwargs is None, which is what an older TRL leaves.++The second is not hypothetical. GKDConfig turns gradient_checkpointing on by+default, so knowledge distillation on TRL 0.25.1 reached the non-reentrant path+without ever asking for gradient checkpointing, and died on a Kaggle 2x T4+with 81 tensors saved in the forward against 79 in the recomputation.+"""++import re+import textwrap+++def _rl_source() -> str:+ import inspect+ from unsloth.models import rl+ return inspect.getsource(rl)+++def _config_post_snippet() -> str:+ src = _rl_source()+ match = re.search(r"RLConfig_post = \(\n(.*?)\n \)\n", src, re.S)+ assert match is not None, "RLConfig_post assignment not found"+ # The snippet is a concatenation of quoted lines; rebuild it the way the+ # module does rather than re-implementing the quoting.+ return eval("(\n" + match.group(1) + "\n)", {"__builtins__": {}})+++class _Config:+ def __init__(+ self,+ gradient_checkpointing = True,+ kwargs = None,+ ):+ self.gradient_checkpointing = gradient_checkpointing+ self.gradient_checkpointing_kwargs = kwargs+++def _run_post(config):+ snippet = textwrap.dedent(_config_post_snippet())+ body = "def _post(self):\n" + textwrap.indent(snippet, " ")+ ns = {}+ exec(compile(body, "<RLConfig_post>", "exec"), ns)+ ns["_post"](config)+ return config+++def test_a_config_that_never_set_the_kwargs_gets_reentrant_pinned():+ # This is the case transformers would otherwise fill in with False.+ config = _run_post(_Config(kwargs = None))+ assert config.gradient_checkpointing_kwargs == {"use_reentrant": True}+++def test_an_explicit_false_is_overridden():+ config = _run_post(_Config(kwargs = {"use_reentrant": False}))+ assert config.gradient_checkpointing_kwargs == {"use_reentrant": True}+++def test_other_checkpoint_kwargs_are_preserved():+ config = _run_post(_Config(kwargs = {"determinism_check": "none"}))+ assert config.gradient_checkpointing_kwargs == {+ "determinism_check": "none",+ "use_reentrant": True,+ }+++def test_a_config_asking_for_context_fn_is_left_alone():+ # torch/utils/checkpoint.py raises "Passing `context_fn` or `debug` is only+ # supported when use_reentrant=False" as soon as a checkpointed forward+ # runs, so pinning here would turn a working setup into a crash.+ sentinel = object()+ config = _run_post(_Config(kwargs = {"use_reentrant": False, "context_fn": sentinel}))+ assert config.gradient_checkpointing_kwargs == {+ "use_reentrant": False,+ "context_fn": sentinel,+ }+++def test_a_config_asking_for_debug_is_left_alone():+ config = _run_post(_Config(kwargs = {"use_reentrant": False, "debug": True}))+ assert config.gradient_checkpointing_kwargs == {"use_reentrant": False, "debug": True}+++def test_a_falsy_debug_does_not_block_the_pin():+ # debug=False is the torch default, so it is not a non-reentrant request.+ config = _run_post(_Config(kwargs = {"debug": False}))+ assert config.gradient_checkpointing_kwargs == {"debug": False, "use_reentrant": True}+++def test_checkpointing_off_is_left_completely_alone():+ # transformers never reads these kwargs in that case, so touching them+ # would only widen the blast radius.+ config = _run_post(_Config(gradient_checkpointing = False, kwargs = None))+ assert config.gradient_checkpointing_kwargs is None+++def test_the_pin_is_not_gated_on_a_trl_version():+ # The previous guard only ran for TRL 0.27.0+, which is exactly why older+ # TRL leaked through to the non-reentrant path.+ src = _rl_source()+ match = re.search(r"RLConfig_post = \(\n(.*?)\n \)\n", src, re.S)+ assert match is not None+ preceding = src[: match.start()]+ tail = preceding[-600:]+ assert 'Version("0.27.0")' not in tail, (+ "RLConfig_post is gated on a TRL version again; older TRL leaves "+ "gradient_checkpointing_kwargs as None and transformers then picks "+ "use_reentrant=False for it"+ )+++def test_the_snippet_pins_true_rather_than_deleting_the_key():+ snippet = _config_post_snippet()+ assert "use_reentrant'] = True" in snippet or '"use_reentrant"] = True' in snippet, snippet+ assert "del " not in snippet, (+ "deleting the key is not enough: a config that never set it leaves "+ "None, and transformers substitutes False"+ )diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex d3839b34dfa..b227bc83958 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -2800,15 +2800,42 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):RLConfig_extra_args = extra_argsRLConfig_call_args = call_args- # TRL 0.27.0+ forces use_reentrant=False in gradient_checkpointing_kwargs, but Unsloth gradient checkpointing requires True, so remove the setting after super().__init__() applies it.- RLConfig_post = ""- if trl_version >= Version("0.27.0"):- RLConfig_post = (- " # Unsloth: Remove use_reentrant=False forced by TRL 0.27.0+\n"- " if getattr(self, 'gradient_checkpointing_kwargs', None) is not None:\n"- " if 'use_reentrant' in self.gradient_checkpointing_kwargs:\n"- " del self.gradient_checkpointing_kwargs['use_reentrant']\n"- )+ # Unsloth gradient checkpointing requires the reentrant path. The+ # non-reentrant one recomputes every packed forward during backward and+ # compares what each pass saved, so a region packed compiled and recomputed+ # eagerly aborts the backward with "A different number of tensors was saved+ # during the original forward and recomputation".+ #+ # Two different things push it to non-reentrant, and deleting the key only+ # answers one of them:+ #+ # TRL 0.27.0+ sets use_reentrant=False explicitly, which the delete below+ # used to handle on its own.+ #+ # transformers substitutes {"use_reentrant": False} whenever+ # gradient_checkpointing_kwargs is None (see gradient_checkpointing_enable+ # in modeling_utils). A config on older TRL never sets the key at all, so+ # there is nothing to delete and transformers picks False for us.+ #+ # So the value is pinned rather than removed, for every TRL version. This is+ # not hypothetical on older TRL: GKDConfig turns gradient_checkpointing on by+ # default, so a distillation run reaches the non-reentrant path without ever+ # having asked for gradient checkpointing at all.+ #+ # Only touched when gradient checkpointing is actually on, since otherwise+ # transformers never reads these kwargs.+ # A config asking for context_fn or debug is left alone. torch accepts+ # neither under use_reentrant=True and raises as soon as a checkpointed+ # forward runs, so pinning those would turn a working non-reentrant setup+ # into a crash. determinism_check carries over fine and is not excluded.+ RLConfig_post = (+ " # Unsloth: keep the reentrant checkpoint path\n"+ " if getattr(self, 'gradient_checkpointing', False):\n"+ " _gc_kwargs = getattr(self, 'gradient_checkpointing_kwargs', None) or {}\n"+ " if _gc_kwargs.get('context_fn') is None and not _gc_kwargs.get('debug', False):\n"+ " _gc_kwargs['use_reentrant'] = True\n"+ " self.gradient_checkpointing_kwargs = _gc_kwargs\n"+ )RLTrainer_extras = patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, imports