Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep generated RL configs on the reentrant gradient checkpoint path (#11519)

Daniel Han9f3153cc18a0
diff --git a/tests/test_gradient_checkpointing_reentrant.py b/tests/test_gradient_checkpointing_reentrant.py
new file mode 100644
index 00000000000..118afcd4d3a
--- /dev/null
+++ b/tests/test_gradient_checkpointing_reentrant.py
@@ -0,0 +1,132 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Generated RL configs must keep the reentrant gradient checkpoint path.
+
+Unsloth gradient checkpointing needs `use_reentrant=True`. The non-reentrant
+path recomputes every packed forward during backward and compares what each
+pass saved, so a region packed compiled and recomputed eagerly aborts with
+
+ CheckpointError: A different number of tensors was saved during the
+ original forward and recomputation.
+
+Two things push a config to non-reentrant, and only one of them was handled:
+
+ TRL 0.27.0+ sets use_reentrant=False explicitly.
+
+ transformers substitutes {"use_reentrant": False} whenever
+ gradient_checkpointing_kwargs is None, which is what an older TRL leaves.
+
+The second is not hypothetical. GKDConfig turns gradient_checkpointing on by
+default, so knowledge distillation on TRL 0.25.1 reached the non-reentrant path
+without ever asking for gradient checkpointing, and died on a Kaggle 2x T4
+with 81 tensors saved in the forward against 79 in the recomputation.
+"""
+
+import re
+import textwrap
+
+
+def _rl_source() -> str:
+ import inspect
+ from unsloth.models import rl
+ return inspect.getsource(rl)
+
+
+def _config_post_snippet() -> str:
+ src = _rl_source()
+ match = re.search(r"RLConfig_post = \(\n(.*?)\n \)\n", src, re.S)
+ assert match is not None, "RLConfig_post assignment not found"
+ # The snippet is a concatenation of quoted lines; rebuild it the way the
+ # module does rather than re-implementing the quoting.
+ return eval("(\n" + match.group(1) + "\n)", {"__builtins__": {}})
+
+
+class _Config:
+ def __init__(
+ self,
+ gradient_checkpointing = True,
+ kwargs = None,
+ ):
+ self.gradient_checkpointing = gradient_checkpointing
+ self.gradient_checkpointing_kwargs = kwargs
+
+
+def _run_post(config):
+ snippet = textwrap.dedent(_config_post_snippet())
+ body = "def _post(self):\n" + textwrap.indent(snippet, " ")
+ ns = {}
+ exec(compile(body, "<RLConfig_post>", "exec"), ns)
+ ns["_post"](config)
+ return config
+
+
+def test_a_config_that_never_set_the_kwargs_gets_reentrant_pinned():
+ # This is the case transformers would otherwise fill in with False.
+ config = _run_post(_Config(kwargs = None))
+ assert config.gradient_checkpointing_kwargs == {"use_reentrant": True}
+
+
+def test_an_explicit_false_is_overridden():
+ config = _run_post(_Config(kwargs = {"use_reentrant": False}))
+ assert config.gradient_checkpointing_kwargs == {"use_reentrant": True}
+
+
+def test_other_checkpoint_kwargs_are_preserved():
+ config = _run_post(_Config(kwargs = {"determinism_check": "none"}))
+ assert config.gradient_checkpointing_kwargs == {
+ "determinism_check": "none",
+ "use_reentrant": True,
+ }
+
+
+def test_a_config_asking_for_context_fn_is_left_alone():
+ # torch/utils/checkpoint.py raises "Passing `context_fn` or `debug` is only
+ # supported when use_reentrant=False" as soon as a checkpointed forward
+ # runs, so pinning here would turn a working setup into a crash.
+ sentinel = object()
+ config = _run_post(_Config(kwargs = {"use_reentrant": False, "context_fn": sentinel}))
+ assert config.gradient_checkpointing_kwargs == {
+ "use_reentrant": False,
+ "context_fn": sentinel,
+ }
+
+
+def test_a_config_asking_for_debug_is_left_alone():
+ config = _run_post(_Config(kwargs = {"use_reentrant": False, "debug": True}))
+ assert config.gradient_checkpointing_kwargs == {"use_reentrant": False, "debug": True}
+
+
+def test_a_falsy_debug_does_not_block_the_pin():
+ # debug=False is the torch default, so it is not a non-reentrant request.
+ config = _run_post(_Config(kwargs = {"debug": False}))
+ assert config.gradient_checkpointing_kwargs == {"debug": False, "use_reentrant": True}
+
+
+def test_checkpointing_off_is_left_completely_alone():
+ # transformers never reads these kwargs in that case, so touching them
+ # would only widen the blast radius.
+ config = _run_post(_Config(gradient_checkpointing = False, kwargs = None))
+ assert config.gradient_checkpointing_kwargs is None
+
+
+def test_the_pin_is_not_gated_on_a_trl_version():
+ # The previous guard only ran for TRL 0.27.0+, which is exactly why older
+ # TRL leaked through to the non-reentrant path.
+ src = _rl_source()
+ match = re.search(r"RLConfig_post = \(\n(.*?)\n \)\n", src, re.S)
+ assert match is not None
+ preceding = src[: match.start()]
+ tail = preceding[-600:]
+ assert 'Version("0.27.0")' not in tail, (
+ "RLConfig_post is gated on a TRL version again; older TRL leaves "
+ "gradient_checkpointing_kwargs as None and transformers then picks "
+ "use_reentrant=False for it"
+ )
+
+
+def test_the_snippet_pins_true_rather_than_deleting_the_key():
+ snippet = _config_post_snippet()
+ assert "use_reentrant'] = True" in snippet or '"use_reentrant"] = True' in snippet, snippet
+ assert "del " not in snippet, (
+ "deleting the key is not enough: a config that never set it leaves "
+ "None, and transformers substitutes False"
+ )
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index d3839b34dfa..b227bc83958 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -2800,15 +2800,42 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
RLConfig_extra_args = extra_args
RLConfig_call_args = call_args
- # TRL 0.27.0+ forces use_reentrant=False in gradient_checkpointing_kwargs, but Unsloth gradient checkpointing requires True, so remove the setting after super().__init__() applies it.
- RLConfig_post = ""
- if trl_version >= Version("0.27.0"):
- RLConfig_post = (
- " # Unsloth: Remove use_reentrant=False forced by TRL 0.27.0+\n"
- " if getattr(self, 'gradient_checkpointing_kwargs', None) is not None:\n"
- " if 'use_reentrant' in self.gradient_checkpointing_kwargs:\n"
- " del self.gradient_checkpointing_kwargs['use_reentrant']\n"
- )
+ # Unsloth gradient checkpointing requires the reentrant path. The
+ # non-reentrant one recomputes every packed forward during backward and
+ # compares what each pass saved, so a region packed compiled and recomputed
+ # eagerly aborts the backward with "A different number of tensors was saved
+ # during the original forward and recomputation".
+ #
+ # Two different things push it to non-reentrant, and deleting the key only
+ # answers one of them:
+ #
+ # TRL 0.27.0+ sets use_reentrant=False explicitly, which the delete below
+ # used to handle on its own.
+ #
+ # transformers substitutes {"use_reentrant": False} whenever
+ # gradient_checkpointing_kwargs is None (see gradient_checkpointing_enable
+ # in modeling_utils). A config on older TRL never sets the key at all, so
+ # there is nothing to delete and transformers picks False for us.
+ #
+ # So the value is pinned rather than removed, for every TRL version. This is
+ # not hypothetical on older TRL: GKDConfig turns gradient_checkpointing on by
+ # default, so a distillation run reaches the non-reentrant path without ever
+ # having asked for gradient checkpointing at all.
+ #
+ # Only touched when gradient checkpointing is actually on, since otherwise
+ # transformers never reads these kwargs.
+ # A config asking for context_fn or debug is left alone. torch accepts
+ # neither under use_reentrant=True and raises as soon as a checkpointed
+ # forward runs, so pinning those would turn a working non-reentrant setup
+ # into a crash. determinism_check carries over fine and is not excluded.
+ RLConfig_post = (
+ " # Unsloth: keep the reentrant checkpoint path\n"
+ " if getattr(self, 'gradient_checkpointing', False):\n"
+ " _gc_kwargs = getattr(self, 'gradient_checkpointing_kwargs', None) or {}\n"
+ " if _gc_kwargs.get('context_fn') is None and not _gc_kwargs.get('debug', False):\n"
+ " _gc_kwargs['use_reentrant'] = True\n"
+ " self.gradient_checkpointing_kwargs = _gc_kwargs\n"
+ )
RLTrainer_extras = patch_functions(
RLTrainer, trainer_file, RLTrainer_name, all_imports, imports