Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Freeze the setup-time heap during Trainer.train(); add resume parity and memory-flat GPU tests (#13332)
Daniel Hanabfe3142cb3b
diff --git a/tests/_shared/tiny_sft_runner.py b/tests/_shared/tiny_sft_runner.pynew file mode 100644index 00000000000..fb53fade78e--- /dev/null+++ b/tests/_shared/tiny_sft_runner.py@@ -0,0 +1,137 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Tiny LoRA SFT run in its own process, for the GPU resume-parity and memory-flat tests.++ python tiny_sft_runner.py --out DIR [--resume CKPT] [--save-steps N] [--max-steps M]+ [--memory-probe WARMUP] [--leak-bytes B]++Writes DIR/result.json (per-step loss / grad_norm / learning_rate, memory trace) and+DIR/adapter.pt (trainable weights after train()).+"""++import argparse+import gc+import json+import os++MODEL = os.environ.get("UNSLOTH_TINY_TEST_MODEL", "trl-internal-testing/tiny-Qwen3ForCausalLM")+++def main():+ p = argparse.ArgumentParser()+ p.add_argument("--out", required = True)+ p.add_argument("--max-steps", type = int, default = 12)+ p.add_argument("--save-steps", type = int, default = 0)+ p.add_argument("--resume", default = None)+ p.add_argument("--memory-probe", type = int, default = -1, help = "warmup steps; -1 = off")+ p.add_argument("--leak-bytes", type = int, default = 0, help = "inject a per-step CUDA leak")+ a = p.parse_args()+ os.makedirs(a.out, exist_ok = True)++ import unsloth # noqa: F401 (must precede transformers / trl)+ from unsloth import FastLanguageModel+ import torch+ from datasets import Dataset+ from transformers import TrainerCallback+ from trl import SFTConfig, SFTTrainer++ # V100 / T4 have no bf16: train in fp16 there, as the notebooks do.+ use_bf16 = torch.cuda.is_bf16_supported()+ model, tokenizer = FastLanguageModel.from_pretrained(+ MODEL,+ max_seq_length = 64,+ load_in_4bit = False,+ dtype = torch.bfloat16 if use_bf16 else torch.float16,+ )+ model = FastLanguageModel.get_peft_model(+ model,+ r = 8,+ lora_alpha = 16,+ lora_dropout = 0,+ target_modules = [+ "q_proj",+ "k_proj",+ "v_proj",+ "o_proj",+ "gate_proj",+ "up_proj",+ "down_proj",+ ],+ random_state = 3407,+ use_gradient_checkpointing = "unsloth",+ )+ if tokenizer.pad_token is None:+ tokenizer.pad_token = tokenizer.eos_token+ texts = [+ f"Question {i}: what is {i} plus {3 * i % 17}? Answer: {i + 3 * i % 17}."+ for i in range(256)+ ]+ dataset = Dataset.from_dict({"text": texts})++ trace = {"allocated": [], "step": []}+ leaked = []++ class Probe(TrainerCallback):+ def on_step_end(self, args, state, control, **kwargs):+ if a.leak_bytes:+ leaked.append(torch.empty(a.leak_bytes, dtype = torch.uint8, device = "cuda"))+ if a.memory_probe >= 0 and state.global_step > a.memory_probe:+ gc.collect()+ torch.cuda.synchronize()+ trace["step"].append(state.global_step)+ trace["allocated"].append(torch.cuda.memory_allocated())++ if a.memory_probe >= 0:+ torch.cuda.memory._record_memory_history(max_entries = 100000)++ config = SFTConfig(+ output_dir = os.path.join(a.out, "trainer"),+ dataset_text_field = "text",+ max_length = 64,+ per_device_train_batch_size = 4,+ gradient_accumulation_steps = 2,+ max_steps = a.max_steps,+ learning_rate = 1e-3,+ warmup_steps = 2,+ lr_scheduler_type = "linear",+ weight_decay = 0.01,+ optim = "adamw_torch",+ logging_steps = 1,+ save_strategy = "steps" if a.save_steps else "no",+ save_steps = a.save_steps or 500,+ report_to = "none",+ seed = 3407,+ bf16 = use_bf16,+ fp16 = not use_bf16,+ dataset_num_proc = 1,+ dataloader_num_workers = 0,+ )+ trainer = SFTTrainer(+ model = model,+ processing_class = tokenizer,+ train_dataset = dataset,+ args = config,+ callbacks = [Probe()],+ )+ trainer.train(resume_from_checkpoint = a.resume)++ steps = [+ {k: h[k] for k in ("step", "loss", "grad_norm", "learning_rate") if k in h}+ for h in trainer.state.log_history+ if "loss" in h+ ]+ adapter = {n: t.detach().float().cpu() for n, t in model.named_parameters() if t.requires_grad}+ torch.save(adapter, os.path.join(a.out, "adapter.pt"))+ result = {"steps": steps, "trace": trace}+ if a.memory_probe >= 0:+ try:+ torch.cuda.memory._dump_snapshot(os.path.join(a.out, "memory_snapshot.pickle"))+ finally:+ torch.cuda.memory._record_memory_history(enabled = None)+ with open(os.path.join(a.out, "result.json"), "w") as f:+ json.dump(result, f)+++if __name__ == "__main__":+ main()diff --git a/tests/test_gc_train_scope.py b/tests/test_gc_train_scope.pynew file mode 100644index 00000000000..611dc55bbed--- /dev/null+++ b/tests/test_gc_train_scope.py@@ -0,0 +1,221 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""gc.freeze() scope around Trainer.train(): CPU only, the module is loaded by path (no unsloth import)."""++import ast+import gc+import importlib.util+import os+import weakref++import pytest++_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+_SCOPE_PATH = os.path.join(_ROOT, "unsloth", "models", "_gc_train_scope.py")+_UTILS_PATH = os.path.join(_ROOT, "unsloth", "models", "_utils.py")+++def _load_scope():+ spec = importlib.util.spec_from_file_location("unsloth_gc_train_scope_under_test", _SCOPE_PATH)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def scope(monkeypatch):+ monkeypatch.delenv("UNSLOTH_GC_FREEZE", raising = False)+ if gc.get_freeze_count() != 0:+ pytest.skip("something in this process already froze the heap")+ module = _load_scope()+ yield module+ gc.unfreeze()+ assert module._depth == 0+++class _Cycle:+ def __init__(self):+ self.me = self+++def _make_trainer(scope, body = None):+ class FakeTrainer:+ def train(self, *args, **kwargs):+ self.seen = gc.get_freeze_count()+ if body is not None:+ return body(self, *args, **kwargs)+ return "done"++ assert scope.patch_trainer_train_gc_scope(FakeTrainer) is True+ return FakeTrainer+++def test_frozen_during_train_unfrozen_after(scope):+ Trainer = _make_trainer(scope)+ t = Trainer()+ assert t.train() == "done"+ assert t.seen > 0+ assert gc.get_freeze_count() == 0++[email protected]("exc_type", [RuntimeError, KeyboardInterrupt])+def test_unfrozen_after_exception(scope, exc_type):+ def body(self):+ assert gc.get_freeze_count() > 0+ raise exc_type("boom")++ Trainer = _make_trainer(scope, body)+ with pytest.raises(exc_type):+ Trainer().train()+ assert gc.get_freeze_count() == 0+ assert scope._depth == 0+++def test_nested_train_freezes_once(scope, monkeypatch):+ calls = {"freeze": 0, "unfreeze": 0}+ real_freeze, real_unfreeze = gc.freeze, gc.unfreeze++ def freeze():+ calls["freeze"] += 1+ real_freeze()++ def unfreeze():+ calls["unfreeze"] += 1+ real_unfreeze()++ monkeypatch.setattr(scope.gc, "freeze", freeze)+ monkeypatch.setattr(scope.gc, "unfreeze", unfreeze)++ inner_after = {}++ def body(self, depth = 0):+ if depth < 2:+ # Same shape as the RL resume guard / per-instance wrappers: call train() again.+ type(self).train(self, depth + 1)+ inner_after[depth] = gc.get_freeze_count()+ return depth++ Trainer = _make_trainer(scope, body)+ Trainer().train()+ assert calls == {"freeze": 1, "unfreeze": 1}+ # The inner returns must not have unfrozen the outer scope.+ assert inner_after[0] > 0 and inner_after[1] > 0+ assert gc.get_freeze_count() == 0+++def test_wrapper_captured_after_install_freezes_once(scope, monkeypatch):+ calls = {"freeze": 0}+ real_freeze = gc.freeze++ def freeze():+ calls["freeze"] += 1+ real_freeze()++ monkeypatch.setattr(scope.gc, "freeze", freeze)+ Base = _make_trainer(scope)++ class Sub(Base):+ pass++ original = Sub.train # what rl.py's resume guard captures++ def guarded(self, *args, **kwargs):+ with scope.unsloth_gc_train_scope():+ return original(self, *args, **kwargs)++ Sub.train = guarded+ t = Sub()+ t.train()+ assert t.seen > 0+ assert calls["freeze"] == 1+ assert gc.get_freeze_count() == 0+++def test_external_freeze_is_left_alone(scope):+ gc.collect()+ gc.freeze()+ before = gc.get_freeze_count()+ assert before > 0+ Trainer = _make_trainer(scope)+ t = Trainer()+ t.train()+ # Not ours: still frozen after train(), and train() did not freeze anything on top.+ assert t.seen == before+ assert gc.get_freeze_count() == before+++def test_gc_enabled_state_preserved(scope):+ was = gc.isenabled()+ try:+ gc.disable()+ Trainer = _make_trainer(scope)+ t = Trainer()+ t.train()+ assert t.seen > 0+ assert not gc.isenabled()+ finally:+ if was:+ gc.enable()+ Trainer().train()+ assert gc.isenabled() == was++[email protected]("value", ["0", "false", "OFF"])+def test_opt_out(scope, monkeypatch, value):+ monkeypatch.setenv("UNSLOTH_GC_FREEZE", value)+ Trainer = _make_trainer(scope)+ t = Trainer()+ t.train()+ assert t.seen == 0+++def test_install_is_idempotent(scope):+ Trainer = _make_trainer(scope)+ wrapped = Trainer.train+ assert scope.patch_trainer_train_gc_scope(Trainer) is False+ assert Trainer.train is wrapped++ class Sub(Trainer):+ pass++ # A subclass without its own train() is covered through the base; nothing to wrap.+ assert scope.patch_trainer_train_gc_scope(Sub) is False+ assert "train" not in Sub.__dict__+++def test_cycle_dropped_after_train_is_collected(scope):+ obj = _Cycle()+ ref = weakref.ref(obj)++ def body(self):+ # While frozen, a dropped pre-existing cycle is not collectable: this is why we must unfreeze.+ holder = self.holder+ self.holder = None+ del holder+ gc.collect()+ self.alive_while_frozen = ref() is not None++ Trainer = _make_trainer(scope, body)+ t = Trainer()+ t.holder = obj+ del obj+ t.train()+ assert t.alive_while_frozen+ # Notebook pattern after training: `del model; gc.collect()` must free it.+ gc.collect()+ assert ref() is None+++def test_utils_installs_on_transformers_trainer_at_import():+ with open(_UTILS_PATH, encoding = "utf-8") as f:+ tree = ast.parse(f.read())+ # A module-level call, so it runs before llama.py's PatchFastRL captures `train` for the+ # RL resume guard (a later install would be bypassed by that captured reference).+ found = False+ for node in tree.body:+ if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):+ func = node.value.func+ if isinstance(func, ast.Name) and func.id == "_patch_trainer_train_gc_scope":+ found = ast.unparse(node.value.args[0]) == "transformers.trainer.Trainer"+ assert founddiff --git a/tests/test_memory_flat_gpu.py b/tests/test_memory_flat_gpu.pynew file mode 100644index 00000000000..9f20365c0b8--- /dev/null+++ b/tests/test_memory_flat_gpu.py@@ -0,0 +1,88 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Allocated CUDA memory must stay flat over a LoRA SFT run once it has warmed up.++After WARMUP steps, every step ends with gc.collect() + synchronize and records+torch.cuda.memory_allocated(). The least-squares slope over the remaining steps, times the+number of steps, must stay under BOUND_BYTES. On failure the CUDA memory snapshot from the run+(torch.cuda.memory._dump_snapshot, open it at https://pytorch.org/memory_viz) is named in the+message.++A second run injects a 1 MiB per-step leak to show the check can fail.+Each run is a subprocess (tests/_shared/tiny_sft_runner.py).+"""++import json+import os+import subprocess+import sys++import pytest++from real_accelerator import has_real_cuda++# `gpu`: deselected by the default addopts, CI runs it in the `-m 'gpu or slow'` leg.+pytestmark = [+ pytest.mark.gpu,+ pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA GPU"),+]++_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+_RUNNER = os.path.join(_ROOT, "tests", "_shared", "tiny_sft_runner.py")+STEPS = 60+WARMUP = 10+BOUND_BYTES = 256 * 1024 # total drift allowed over the measured window+LEAK_BYTES = 1024 * 1024+++def _run(out, *args):+ env = dict(os.environ)+ env["PYTHONPATH"] = _ROOT + os.pathsep + env.get("PYTHONPATH", "")+ proc = subprocess.run(+ [+ sys.executable,+ _RUNNER,+ "--out",+ str(out),+ "--max-steps",+ str(STEPS),+ "--memory-probe",+ str(WARMUP),+ *args,+ ],+ env = env,+ capture_output = True,+ text = True,+ timeout = 1200,+ )+ assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:]+ with open(os.path.join(out, "result.json")) as f:+ return json.load(f)+++def _drift(result):+ xs, ys = result["trace"]["step"], result["trace"]["allocated"]+ assert len(xs) >= STEPS - WARMUP - 1, f"only {len(xs)} memory samples"+ n = len(xs)+ mx, my = sum(xs) / n, sum(ys) / n+ slope = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sum((x - mx) ** 2 for x in xs)+ return slope * (xs[-1] - xs[0]), slope, min(ys), max(ys)+++def test_allocated_memory_is_flat_after_warmup(tmp_path):+ result = _run(tmp_path / "clean")+ drift, slope, lo, hi = _drift(result)+ snapshot = tmp_path / "clean" / "memory_snapshot.pickle"+ assert drift < BOUND_BYTES, (+ f"allocated memory grew {drift / 2**20:.2f} MiB over {STEPS - WARMUP} steps "+ f"({slope / 1024:.1f} KiB/step, range {lo} .. {hi} bytes); snapshot: {snapshot}"+ )+++def test_check_catches_an_injected_leak(tmp_path):+ result = _run(tmp_path / "leak", "--leak-bytes", str(LEAK_BYTES))+ drift, slope, _, _ = _drift(result)+ assert (+ drift >= BOUND_BYTES+ ), f"a {LEAK_BYTES} B/step leak was not flagged (slope {slope:.0f} B/step)"diff --git a/tests/test_resume_parity_gpu.py b/tests/test_resume_parity_gpu.pynew file mode 100644index 00000000000..8b1f6a241e3--- /dev/null+++ b/tests/test_resume_parity_gpu.py@@ -0,0 +1,119 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Resume from a checkpoint must continue the run it was saved from.++2N steps straight vs N steps + checkpoint + resume in a fresh process + N steps: per-step loss,+grad norm and learning rate over the resumed half, and the final adapter weights, must match the+straight run. The tolerance is set from an A/A pair of straight runs (run-to-run noise), with a+small floor so a bit-exact A/A does not demand bit-exact resume.++Each run is a subprocess (tests/_shared/tiny_sft_runner.py), so the resume really starts cold.+"""++import json+import os+import shutil+import subprocess+import sys++import pytest++from real_accelerator import has_real_cuda++# `gpu`: deselected by the default addopts, CI runs it in the `-m 'gpu or slow'` leg.+pytestmark = [+ pytest.mark.gpu,+ pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA GPU"),+]++_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+_RUNNER = os.path.join(_ROOT, "tests", "_shared", "tiny_sft_runner.py")+N = 6+# The gate is max(floor, 4 x A/A noise). On an H100 / B200 the A/A pair and the resume are both+# bit-exact, so the floors only absorb nondeterminism the A/A pair happened not to show.+FLOOR = {"loss": 1e-4, "grad_norm": 1e-5, "learning_rate": 1e-12, "adapter": 1e-5}+++def _run(out, *args):+ env = dict(os.environ)+ env["PYTHONPATH"] = _ROOT + os.pathsep + env.get("PYTHONPATH", "")+ proc = subprocess.run(+ [sys.executable, _RUNNER, "--out", str(out), "--max-steps", str(2 * N), *args],+ env = env,+ capture_output = True,+ text = True,+ timeout = 1200,+ )+ assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:]+ with open(os.path.join(out, "result.json")) as f:+ result = json.load(f)+ import torch++ result["adapter"] = torch.load(os.path.join(out, "adapter.pt"))+ return result++[email protected](scope = "module")+def runs(tmp_path_factory):+ base = tmp_path_factory.mktemp("resume_parity")+ straight = _run(base / "A", "--save-steps", str(N))+ straight_again = _run(base / "B", "--save-steps", str(N))+ checkpoint = base / "A" / "trainer" / f"checkpoint-{N}"+ assert checkpoint.is_dir(), sorted(os.listdir(base / "A" / "trainer"))+ resumed = _run(base / "R", "--resume", str(checkpoint))+ return {"base": base, "A": straight, "B": straight_again, "R": resumed, "ckpt": checkpoint}+++def _series(result, key):+ return {s["step"]: s[key] for s in result["steps"] if key in s}+++def _max_diff(x, y, key):+ a, b = _series(x, key), _series(y, key)+ steps = [s for s in range(N + 1, 2 * N + 1)]+ missing = [s for s in steps if s not in a or s not in b]+ assert not missing, f"{key}: steps {missing} not logged"+ return max(abs(a[s] - b[s]) for s in steps)+++def _adapter_diff(x, y):+ assert x.keys() == y.keys() and len(x) > 0+ return max((x[k] - y[k]).abs().max().item() for k in x)+++def _check(runs, resumed):+ report, bad = {}, []+ for key in ("loss", "grad_norm", "learning_rate"):+ noise = _max_diff(runs["A"], runs["B"], key)+ got = _max_diff(runs["A"], resumed, key)+ tol = max(FLOOR[key], 4 * noise)+ report[key] = (got, noise, tol)+ if got > tol:+ bad.append(key)+ noise = _adapter_diff(runs["A"]["adapter"], runs["B"]["adapter"])+ got = _adapter_diff(runs["A"]["adapter"], resumed["adapter"])+ tol = max(FLOOR["adapter"], 4 * noise)+ report["adapter"] = (got, noise, tol)+ if got > tol:+ bad.append("adapter")+ return bad, report+++def test_resume_matches_straight_run(runs):+ bad, report = _check(runs, runs["R"])+ assert not bad, f"resume diverged on {bad}; (max diff, A/A noise, tolerance): {report}"+++def test_comparison_catches_a_resume_without_optimizer_state(runs):+ # Same resume with the optimizer state removed from the checkpoint: Adam restarts from zero+ # moments, which a working comparison must flag.+ broken = runs["base"] / f"checkpoint-{N}-no-optimizer"+ shutil.copytree(runs["ckpt"], broken)+ removed = [p for p in os.listdir(broken) if p.startswith("optimizer")]+ assert removed, os.listdir(broken)+ for name in removed:+ os.remove(broken / name)+ resumed = _run(runs["base"] / "R_broken", "--resume", str(broken))+ bad, report = _check(runs, resumed)+ assert bad, f"a resume without optimizer state passed the comparison: {report}"diff --git a/unsloth/models/_gc_train_scope.py b/unsloth/models/_gc_train_scope.pynew file mode 100644index 00000000000..7bd2554f120--- /dev/null+++ b/unsloth/models/_gc_train_scope.py@@ -0,0 +1,102 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Freeze the setup-time heap for the duration of `Trainer.train()`.++Libraries call `gc.collect()` several times inside `train()`. Each full collection walks every+tracked object, and after loading a model, tokenizer, dataset and compiled code that heap is+large: about 0.5 s per pass, 2 to 3 s per `train()` on a 0.6B / 1B LoRA run. `gc.freeze()` moves+everything alive at train start into the permanent generation, so those passes only see objects+created during training.++The freeze must be undone when `train()` exits (normally, by exception or by KeyboardInterrupt):+frozen objects are never collected, so a cyclic object dropped after training (the usual+`del model; gc.collect()` in a notebook) would otherwise keep its memory forever.++Rules:+- Only the outermost `train()` freezes; nested calls (resume guards, per-instance wrappers) are+ counted and do nothing.+- If something is already frozen at entry, the caller owns the freeze: Python cannot unfreeze+ only our objects, so we leave the heap alone and do not unfreeze on exit.+- `gc.isenabled()` is never touched.+- `UNSLOTH_GC_FREEZE=0` turns it off (read on each outermost entry).+"""++import functools+import gc+import os+import threading++__all__ = [+ "unsloth_gc_train_scope",+ "patch_trainer_train_gc_scope",+]++_lock = threading.Lock()+_depth = 0+_owns_freeze = False+++def _gc_freeze_enabled():+ value = os.environ.get("UNSLOTH_GC_FREEZE", "1").strip().lower()+ return value not in ("0", "false", "no", "off")+++def _enter():+ global _depth, _owns_freeze+ with _lock:+ _depth += 1+ if _depth != 1:+ return+ _owns_freeze = False+ try:+ if not _gc_freeze_enabled() or gc.get_freeze_count() > 0:+ return+ gc.collect()+ gc.freeze()+ _owns_freeze = True+ except BaseException:+ # KeyboardInterrupt inside gc.collect(): leave the counter as we found it.+ _depth -= 1+ raise+++def _exit():+ global _depth, _owns_freeze+ with _lock:+ _depth -= 1+ if _depth != 0:+ return+ if _owns_freeze:+ _owns_freeze = False+ gc.unfreeze()+++class unsloth_gc_train_scope:+ """Context manager: `gc.collect(); gc.freeze()` on the outermost entry, `gc.unfreeze()` on exit."""++ __slots__ = ()++ def __enter__(self):+ _enter()+ return self++ def __exit__(self, exc_type, exc, tb):+ _exit()+ return False+++def patch_trainer_train_gc_scope(trainer_class):+ """Wrap `trainer_class.train` once so every call runs inside `unsloth_gc_train_scope`."""+ original_train = trainer_class.__dict__.get("train")+ if original_train is None or getattr(original_train, "_unsloth_gc_train_scope", False):+ return False++ @functools.wraps(original_train)+ def train(self, *args, **kwargs):+ with unsloth_gc_train_scope():+ return original_train(self, *args, **kwargs)++ train._unsloth_gc_train_scope = True+ trainer_class.train = train+ return Truediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex a7018de4c7e..ec0ab6524cf 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -3571,6 +3571,12 @@ import transformers.trainertransformers.trainer.get_model_param_count = get_model_param_count+# gc.freeze() the setup-time heap for each outermost Trainer.train(). Installed at import, before+# PatchFastRL (llama.py) captures `train` for the RL resume guard, so every trainer goes through it.+from ._gc_train_scope import patch_trainer_train_gc_scope as _patch_trainer_train_gc_scope++_patch_trainer_train_gc_scope(transformers.trainer.Trainer)+def patch_mistral_nemo_config(config):if "head_dim (" not in config:@@ -5360,6 +5366,9 @@ def patch_gradient_accumulation_fix(Trainer):Trainer.__init__ = _unsloth_trainer_initTrainer._unsloth_init_wrapped_for_accelerate_gas = True+ # Idempotent: already installed at import unless Trainer was reloaded since.+ _patch_trainer_train_gc_scope(Trainer)+# Real children only: PEFT's LoraModel forwards `base_model` to the backbone, skipping the loss head._LOSS_KWARGS_CHILDREN = ("base_model", "model") + _UNSLOTH_WRAPPED_MODULE_ATTRS