Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Freeze the setup-time heap during Trainer.train(); add resume parity and memory-flat GPU tests (#13332)

Daniel Hanabfe3142cb3b
diff --git a/tests/_shared/tiny_sft_runner.py b/tests/_shared/tiny_sft_runner.py
new file mode 100644
index 00000000000..fb53fade78e
--- /dev/null
+++ b/tests/_shared/tiny_sft_runner.py
@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Tiny LoRA SFT run in its own process, for the GPU resume-parity and memory-flat tests.
+
+ python tiny_sft_runner.py --out DIR [--resume CKPT] [--save-steps N] [--max-steps M]
+ [--memory-probe WARMUP] [--leak-bytes B]
+
+Writes DIR/result.json (per-step loss / grad_norm / learning_rate, memory trace) and
+DIR/adapter.pt (trainable weights after train()).
+"""
+
+import argparse
+import gc
+import json
+import os
+
+MODEL = os.environ.get("UNSLOTH_TINY_TEST_MODEL", "trl-internal-testing/tiny-Qwen3ForCausalLM")
+
+
+def main():
+ p = argparse.ArgumentParser()
+ p.add_argument("--out", required = True)
+ p.add_argument("--max-steps", type = int, default = 12)
+ p.add_argument("--save-steps", type = int, default = 0)
+ p.add_argument("--resume", default = None)
+ p.add_argument("--memory-probe", type = int, default = -1, help = "warmup steps; -1 = off")
+ p.add_argument("--leak-bytes", type = int, default = 0, help = "inject a per-step CUDA leak")
+ a = p.parse_args()
+ os.makedirs(a.out, exist_ok = True)
+
+ import unsloth # noqa: F401 (must precede transformers / trl)
+ from unsloth import FastLanguageModel
+ import torch
+ from datasets import Dataset
+ from transformers import TrainerCallback
+ from trl import SFTConfig, SFTTrainer
+
+ # V100 / T4 have no bf16: train in fp16 there, as the notebooks do.
+ use_bf16 = torch.cuda.is_bf16_supported()
+ model, tokenizer = FastLanguageModel.from_pretrained(
+ MODEL,
+ max_seq_length = 64,
+ load_in_4bit = False,
+ dtype = torch.bfloat16 if use_bf16 else torch.float16,
+ )
+ model = FastLanguageModel.get_peft_model(
+ model,
+ r = 8,
+ lora_alpha = 16,
+ lora_dropout = 0,
+ target_modules = [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "gate_proj",
+ "up_proj",
+ "down_proj",
+ ],
+ random_state = 3407,
+ use_gradient_checkpointing = "unsloth",
+ )
+ if tokenizer.pad_token is None:
+ tokenizer.pad_token = tokenizer.eos_token
+ texts = [
+ f"Question {i}: what is {i} plus {3 * i % 17}? Answer: {i + 3 * i % 17}."
+ for i in range(256)
+ ]
+ dataset = Dataset.from_dict({"text": texts})
+
+ trace = {"allocated": [], "step": []}
+ leaked = []
+
+ class Probe(TrainerCallback):
+ def on_step_end(self, args, state, control, **kwargs):
+ if a.leak_bytes:
+ leaked.append(torch.empty(a.leak_bytes, dtype = torch.uint8, device = "cuda"))
+ if a.memory_probe >= 0 and state.global_step > a.memory_probe:
+ gc.collect()
+ torch.cuda.synchronize()
+ trace["step"].append(state.global_step)
+ trace["allocated"].append(torch.cuda.memory_allocated())
+
+ if a.memory_probe >= 0:
+ torch.cuda.memory._record_memory_history(max_entries = 100000)
+
+ config = SFTConfig(
+ output_dir = os.path.join(a.out, "trainer"),
+ dataset_text_field = "text",
+ max_length = 64,
+ per_device_train_batch_size = 4,
+ gradient_accumulation_steps = 2,
+ max_steps = a.max_steps,
+ learning_rate = 1e-3,
+ warmup_steps = 2,
+ lr_scheduler_type = "linear",
+ weight_decay = 0.01,
+ optim = "adamw_torch",
+ logging_steps = 1,
+ save_strategy = "steps" if a.save_steps else "no",
+ save_steps = a.save_steps or 500,
+ report_to = "none",
+ seed = 3407,
+ bf16 = use_bf16,
+ fp16 = not use_bf16,
+ dataset_num_proc = 1,
+ dataloader_num_workers = 0,
+ )
+ trainer = SFTTrainer(
+ model = model,
+ processing_class = tokenizer,
+ train_dataset = dataset,
+ args = config,
+ callbacks = [Probe()],
+ )
+ trainer.train(resume_from_checkpoint = a.resume)
+
+ steps = [
+ {k: h[k] for k in ("step", "loss", "grad_norm", "learning_rate") if k in h}
+ for h in trainer.state.log_history
+ if "loss" in h
+ ]
+ adapter = {n: t.detach().float().cpu() for n, t in model.named_parameters() if t.requires_grad}
+ torch.save(adapter, os.path.join(a.out, "adapter.pt"))
+ result = {"steps": steps, "trace": trace}
+ if a.memory_probe >= 0:
+ try:
+ torch.cuda.memory._dump_snapshot(os.path.join(a.out, "memory_snapshot.pickle"))
+ finally:
+ torch.cuda.memory._record_memory_history(enabled = None)
+ with open(os.path.join(a.out, "result.json"), "w") as f:
+ json.dump(result, f)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_gc_train_scope.py b/tests/test_gc_train_scope.py
new file mode 100644
index 00000000000..611dc55bbed
--- /dev/null
+++ b/tests/test_gc_train_scope.py
@@ -0,0 +1,221 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""gc.freeze() scope around Trainer.train(): CPU only, the module is loaded by path (no unsloth import)."""
+
+import ast
+import gc
+import importlib.util
+import os
+import weakref
+
+import pytest
+
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_SCOPE_PATH = os.path.join(_ROOT, "unsloth", "models", "_gc_train_scope.py")
+_UTILS_PATH = os.path.join(_ROOT, "unsloth", "models", "_utils.py")
+
+
+def _load_scope():
+ spec = importlib.util.spec_from_file_location("unsloth_gc_train_scope_under_test", _SCOPE_PATH)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def scope(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_GC_FREEZE", raising = False)
+ if gc.get_freeze_count() != 0:
+ pytest.skip("something in this process already froze the heap")
+ module = _load_scope()
+ yield module
+ gc.unfreeze()
+ assert module._depth == 0
+
+
+class _Cycle:
+ def __init__(self):
+ self.me = self
+
+
+def _make_trainer(scope, body = None):
+ class FakeTrainer:
+ def train(self, *args, **kwargs):
+ self.seen = gc.get_freeze_count()
+ if body is not None:
+ return body(self, *args, **kwargs)
+ return "done"
+
+ assert scope.patch_trainer_train_gc_scope(FakeTrainer) is True
+ return FakeTrainer
+
+
+def test_frozen_during_train_unfrozen_after(scope):
+ Trainer = _make_trainer(scope)
+ t = Trainer()
+ assert t.train() == "done"
+ assert t.seen > 0
+ assert gc.get_freeze_count() == 0
+
+
[email protected]("exc_type", [RuntimeError, KeyboardInterrupt])
+def test_unfrozen_after_exception(scope, exc_type):
+ def body(self):
+ assert gc.get_freeze_count() > 0
+ raise exc_type("boom")
+
+ Trainer = _make_trainer(scope, body)
+ with pytest.raises(exc_type):
+ Trainer().train()
+ assert gc.get_freeze_count() == 0
+ assert scope._depth == 0
+
+
+def test_nested_train_freezes_once(scope, monkeypatch):
+ calls = {"freeze": 0, "unfreeze": 0}
+ real_freeze, real_unfreeze = gc.freeze, gc.unfreeze
+
+ def freeze():
+ calls["freeze"] += 1
+ real_freeze()
+
+ def unfreeze():
+ calls["unfreeze"] += 1
+ real_unfreeze()
+
+ monkeypatch.setattr(scope.gc, "freeze", freeze)
+ monkeypatch.setattr(scope.gc, "unfreeze", unfreeze)
+
+ inner_after = {}
+
+ def body(self, depth = 0):
+ if depth < 2:
+ # Same shape as the RL resume guard / per-instance wrappers: call train() again.
+ type(self).train(self, depth + 1)
+ inner_after[depth] = gc.get_freeze_count()
+ return depth
+
+ Trainer = _make_trainer(scope, body)
+ Trainer().train()
+ assert calls == {"freeze": 1, "unfreeze": 1}
+ # The inner returns must not have unfrozen the outer scope.
+ assert inner_after[0] > 0 and inner_after[1] > 0
+ assert gc.get_freeze_count() == 0
+
+
+def test_wrapper_captured_after_install_freezes_once(scope, monkeypatch):
+ calls = {"freeze": 0}
+ real_freeze = gc.freeze
+
+ def freeze():
+ calls["freeze"] += 1
+ real_freeze()
+
+ monkeypatch.setattr(scope.gc, "freeze", freeze)
+ Base = _make_trainer(scope)
+
+ class Sub(Base):
+ pass
+
+ original = Sub.train # what rl.py's resume guard captures
+
+ def guarded(self, *args, **kwargs):
+ with scope.unsloth_gc_train_scope():
+ return original(self, *args, **kwargs)
+
+ Sub.train = guarded
+ t = Sub()
+ t.train()
+ assert t.seen > 0
+ assert calls["freeze"] == 1
+ assert gc.get_freeze_count() == 0
+
+
+def test_external_freeze_is_left_alone(scope):
+ gc.collect()
+ gc.freeze()
+ before = gc.get_freeze_count()
+ assert before > 0
+ Trainer = _make_trainer(scope)
+ t = Trainer()
+ t.train()
+ # Not ours: still frozen after train(), and train() did not freeze anything on top.
+ assert t.seen == before
+ assert gc.get_freeze_count() == before
+
+
+def test_gc_enabled_state_preserved(scope):
+ was = gc.isenabled()
+ try:
+ gc.disable()
+ Trainer = _make_trainer(scope)
+ t = Trainer()
+ t.train()
+ assert t.seen > 0
+ assert not gc.isenabled()
+ finally:
+ if was:
+ gc.enable()
+ Trainer().train()
+ assert gc.isenabled() == was
+
+
[email protected]("value", ["0", "false", "OFF"])
+def test_opt_out(scope, monkeypatch, value):
+ monkeypatch.setenv("UNSLOTH_GC_FREEZE", value)
+ Trainer = _make_trainer(scope)
+ t = Trainer()
+ t.train()
+ assert t.seen == 0
+
+
+def test_install_is_idempotent(scope):
+ Trainer = _make_trainer(scope)
+ wrapped = Trainer.train
+ assert scope.patch_trainer_train_gc_scope(Trainer) is False
+ assert Trainer.train is wrapped
+
+ class Sub(Trainer):
+ pass
+
+ # A subclass without its own train() is covered through the base; nothing to wrap.
+ assert scope.patch_trainer_train_gc_scope(Sub) is False
+ assert "train" not in Sub.__dict__
+
+
+def test_cycle_dropped_after_train_is_collected(scope):
+ obj = _Cycle()
+ ref = weakref.ref(obj)
+
+ def body(self):
+ # While frozen, a dropped pre-existing cycle is not collectable: this is why we must unfreeze.
+ holder = self.holder
+ self.holder = None
+ del holder
+ gc.collect()
+ self.alive_while_frozen = ref() is not None
+
+ Trainer = _make_trainer(scope, body)
+ t = Trainer()
+ t.holder = obj
+ del obj
+ t.train()
+ assert t.alive_while_frozen
+ # Notebook pattern after training: `del model; gc.collect()` must free it.
+ gc.collect()
+ assert ref() is None
+
+
+def test_utils_installs_on_transformers_trainer_at_import():
+ with open(_UTILS_PATH, encoding = "utf-8") as f:
+ tree = ast.parse(f.read())
+ # A module-level call, so it runs before llama.py's PatchFastRL captures `train` for the
+ # RL resume guard (a later install would be bypassed by that captured reference).
+ found = False
+ for node in tree.body:
+ if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):
+ func = node.value.func
+ if isinstance(func, ast.Name) and func.id == "_patch_trainer_train_gc_scope":
+ found = ast.unparse(node.value.args[0]) == "transformers.trainer.Trainer"
+ assert found
diff --git a/tests/test_memory_flat_gpu.py b/tests/test_memory_flat_gpu.py
new file mode 100644
index 00000000000..9f20365c0b8
--- /dev/null
+++ b/tests/test_memory_flat_gpu.py
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Allocated CUDA memory must stay flat over a LoRA SFT run once it has warmed up.
+
+After WARMUP steps, every step ends with gc.collect() + synchronize and records
+torch.cuda.memory_allocated(). The least-squares slope over the remaining steps, times the
+number of steps, must stay under BOUND_BYTES. On failure the CUDA memory snapshot from the run
+(torch.cuda.memory._dump_snapshot, open it at https://pytorch.org/memory_viz) is named in the
+message.
+
+A second run injects a 1 MiB per-step leak to show the check can fail.
+Each run is a subprocess (tests/_shared/tiny_sft_runner.py).
+"""
+
+import json
+import os
+import subprocess
+import sys
+
+import pytest
+
+from real_accelerator import has_real_cuda
+
+# `gpu`: deselected by the default addopts, CI runs it in the `-m 'gpu or slow'` leg.
+pytestmark = [
+ pytest.mark.gpu,
+ pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA GPU"),
+]
+
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_RUNNER = os.path.join(_ROOT, "tests", "_shared", "tiny_sft_runner.py")
+STEPS = 60
+WARMUP = 10
+BOUND_BYTES = 256 * 1024 # total drift allowed over the measured window
+LEAK_BYTES = 1024 * 1024
+
+
+def _run(out, *args):
+ env = dict(os.environ)
+ env["PYTHONPATH"] = _ROOT + os.pathsep + env.get("PYTHONPATH", "")
+ proc = subprocess.run(
+ [
+ sys.executable,
+ _RUNNER,
+ "--out",
+ str(out),
+ "--max-steps",
+ str(STEPS),
+ "--memory-probe",
+ str(WARMUP),
+ *args,
+ ],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 1200,
+ )
+ assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:]
+ with open(os.path.join(out, "result.json")) as f:
+ return json.load(f)
+
+
+def _drift(result):
+ xs, ys = result["trace"]["step"], result["trace"]["allocated"]
+ assert len(xs) >= STEPS - WARMUP - 1, f"only {len(xs)} memory samples"
+ n = len(xs)
+ mx, my = sum(xs) / n, sum(ys) / n
+ slope = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sum((x - mx) ** 2 for x in xs)
+ return slope * (xs[-1] - xs[0]), slope, min(ys), max(ys)
+
+
+def test_allocated_memory_is_flat_after_warmup(tmp_path):
+ result = _run(tmp_path / "clean")
+ drift, slope, lo, hi = _drift(result)
+ snapshot = tmp_path / "clean" / "memory_snapshot.pickle"
+ assert drift < BOUND_BYTES, (
+ f"allocated memory grew {drift / 2**20:.2f} MiB over {STEPS - WARMUP} steps "
+ f"({slope / 1024:.1f} KiB/step, range {lo} .. {hi} bytes); snapshot: {snapshot}"
+ )
+
+
+def test_check_catches_an_injected_leak(tmp_path):
+ result = _run(tmp_path / "leak", "--leak-bytes", str(LEAK_BYTES))
+ drift, slope, _, _ = _drift(result)
+ assert (
+ drift >= BOUND_BYTES
+ ), f"a {LEAK_BYTES} B/step leak was not flagged (slope {slope:.0f} B/step)"
diff --git a/tests/test_resume_parity_gpu.py b/tests/test_resume_parity_gpu.py
new file mode 100644
index 00000000000..8b1f6a241e3
--- /dev/null
+++ b/tests/test_resume_parity_gpu.py
@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Resume from a checkpoint must continue the run it was saved from.
+
+2N steps straight vs N steps + checkpoint + resume in a fresh process + N steps: per-step loss,
+grad norm and learning rate over the resumed half, and the final adapter weights, must match the
+straight run. The tolerance is set from an A/A pair of straight runs (run-to-run noise), with a
+small floor so a bit-exact A/A does not demand bit-exact resume.
+
+Each run is a subprocess (tests/_shared/tiny_sft_runner.py), so the resume really starts cold.
+"""
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+
+import pytest
+
+from real_accelerator import has_real_cuda
+
+# `gpu`: deselected by the default addopts, CI runs it in the `-m 'gpu or slow'` leg.
+pytestmark = [
+ pytest.mark.gpu,
+ pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA GPU"),
+]
+
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_RUNNER = os.path.join(_ROOT, "tests", "_shared", "tiny_sft_runner.py")
+N = 6
+# The gate is max(floor, 4 x A/A noise). On an H100 / B200 the A/A pair and the resume are both
+# bit-exact, so the floors only absorb nondeterminism the A/A pair happened not to show.
+FLOOR = {"loss": 1e-4, "grad_norm": 1e-5, "learning_rate": 1e-12, "adapter": 1e-5}
+
+
+def _run(out, *args):
+ env = dict(os.environ)
+ env["PYTHONPATH"] = _ROOT + os.pathsep + env.get("PYTHONPATH", "")
+ proc = subprocess.run(
+ [sys.executable, _RUNNER, "--out", str(out), "--max-steps", str(2 * N), *args],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 1200,
+ )
+ assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:]
+ with open(os.path.join(out, "result.json")) as f:
+ result = json.load(f)
+ import torch
+
+ result["adapter"] = torch.load(os.path.join(out, "adapter.pt"))
+ return result
+
+
[email protected](scope = "module")
+def runs(tmp_path_factory):
+ base = tmp_path_factory.mktemp("resume_parity")
+ straight = _run(base / "A", "--save-steps", str(N))
+ straight_again = _run(base / "B", "--save-steps", str(N))
+ checkpoint = base / "A" / "trainer" / f"checkpoint-{N}"
+ assert checkpoint.is_dir(), sorted(os.listdir(base / "A" / "trainer"))
+ resumed = _run(base / "R", "--resume", str(checkpoint))
+ return {"base": base, "A": straight, "B": straight_again, "R": resumed, "ckpt": checkpoint}
+
+
+def _series(result, key):
+ return {s["step"]: s[key] for s in result["steps"] if key in s}
+
+
+def _max_diff(x, y, key):
+ a, b = _series(x, key), _series(y, key)
+ steps = [s for s in range(N + 1, 2 * N + 1)]
+ missing = [s for s in steps if s not in a or s not in b]
+ assert not missing, f"{key}: steps {missing} not logged"
+ return max(abs(a[s] - b[s]) for s in steps)
+
+
+def _adapter_diff(x, y):
+ assert x.keys() == y.keys() and len(x) > 0
+ return max((x[k] - y[k]).abs().max().item() for k in x)
+
+
+def _check(runs, resumed):
+ report, bad = {}, []
+ for key in ("loss", "grad_norm", "learning_rate"):
+ noise = _max_diff(runs["A"], runs["B"], key)
+ got = _max_diff(runs["A"], resumed, key)
+ tol = max(FLOOR[key], 4 * noise)
+ report[key] = (got, noise, tol)
+ if got > tol:
+ bad.append(key)
+ noise = _adapter_diff(runs["A"]["adapter"], runs["B"]["adapter"])
+ got = _adapter_diff(runs["A"]["adapter"], resumed["adapter"])
+ tol = max(FLOOR["adapter"], 4 * noise)
+ report["adapter"] = (got, noise, tol)
+ if got > tol:
+ bad.append("adapter")
+ return bad, report
+
+
+def test_resume_matches_straight_run(runs):
+ bad, report = _check(runs, runs["R"])
+ assert not bad, f"resume diverged on {bad}; (max diff, A/A noise, tolerance): {report}"
+
+
+def test_comparison_catches_a_resume_without_optimizer_state(runs):
+ # Same resume with the optimizer state removed from the checkpoint: Adam restarts from zero
+ # moments, which a working comparison must flag.
+ broken = runs["base"] / f"checkpoint-{N}-no-optimizer"
+ shutil.copytree(runs["ckpt"], broken)
+ removed = [p for p in os.listdir(broken) if p.startswith("optimizer")]
+ assert removed, os.listdir(broken)
+ for name in removed:
+ os.remove(broken / name)
+ resumed = _run(runs["base"] / "R_broken", "--resume", str(broken))
+ bad, report = _check(runs, resumed)
+ assert bad, f"a resume without optimizer state passed the comparison: {report}"
diff --git a/unsloth/models/_gc_train_scope.py b/unsloth/models/_gc_train_scope.py
new file mode 100644
index 00000000000..7bd2554f120
--- /dev/null
+++ b/unsloth/models/_gc_train_scope.py
@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Freeze the setup-time heap for the duration of `Trainer.train()`.
+
+Libraries call `gc.collect()` several times inside `train()`. Each full collection walks every
+tracked object, and after loading a model, tokenizer, dataset and compiled code that heap is
+large: about 0.5 s per pass, 2 to 3 s per `train()` on a 0.6B / 1B LoRA run. `gc.freeze()` moves
+everything alive at train start into the permanent generation, so those passes only see objects
+created during training.
+
+The freeze must be undone when `train()` exits (normally, by exception or by KeyboardInterrupt):
+frozen objects are never collected, so a cyclic object dropped after training (the usual
+`del model; gc.collect()` in a notebook) would otherwise keep its memory forever.
+
+Rules:
+- Only the outermost `train()` freezes; nested calls (resume guards, per-instance wrappers) are
+ counted and do nothing.
+- If something is already frozen at entry, the caller owns the freeze: Python cannot unfreeze
+ only our objects, so we leave the heap alone and do not unfreeze on exit.
+- `gc.isenabled()` is never touched.
+- `UNSLOTH_GC_FREEZE=0` turns it off (read on each outermost entry).
+"""
+
+import functools
+import gc
+import os
+import threading
+
+__all__ = [
+ "unsloth_gc_train_scope",
+ "patch_trainer_train_gc_scope",
+]
+
+_lock = threading.Lock()
+_depth = 0
+_owns_freeze = False
+
+
+def _gc_freeze_enabled():
+ value = os.environ.get("UNSLOTH_GC_FREEZE", "1").strip().lower()
+ return value not in ("0", "false", "no", "off")
+
+
+def _enter():
+ global _depth, _owns_freeze
+ with _lock:
+ _depth += 1
+ if _depth != 1:
+ return
+ _owns_freeze = False
+ try:
+ if not _gc_freeze_enabled() or gc.get_freeze_count() > 0:
+ return
+ gc.collect()
+ gc.freeze()
+ _owns_freeze = True
+ except BaseException:
+ # KeyboardInterrupt inside gc.collect(): leave the counter as we found it.
+ _depth -= 1
+ raise
+
+
+def _exit():
+ global _depth, _owns_freeze
+ with _lock:
+ _depth -= 1
+ if _depth != 0:
+ return
+ if _owns_freeze:
+ _owns_freeze = False
+ gc.unfreeze()
+
+
+class unsloth_gc_train_scope:
+ """Context manager: `gc.collect(); gc.freeze()` on the outermost entry, `gc.unfreeze()` on exit."""
+
+ __slots__ = ()
+
+ def __enter__(self):
+ _enter()
+ return self
+
+ def __exit__(self, exc_type, exc, tb):
+ _exit()
+ return False
+
+
+def patch_trainer_train_gc_scope(trainer_class):
+ """Wrap `trainer_class.train` once so every call runs inside `unsloth_gc_train_scope`."""
+ original_train = trainer_class.__dict__.get("train")
+ if original_train is None or getattr(original_train, "_unsloth_gc_train_scope", False):
+ return False
+
+ @functools.wraps(original_train)
+ def train(self, *args, **kwargs):
+ with unsloth_gc_train_scope():
+ return original_train(self, *args, **kwargs)
+
+ train._unsloth_gc_train_scope = True
+ trainer_class.train = train
+ return True
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index a7018de4c7e..ec0ab6524cf 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -3571,6 +3571,12 @@ import transformers.trainer
transformers.trainer.get_model_param_count = get_model_param_count
+# gc.freeze() the setup-time heap for each outermost Trainer.train(). Installed at import, before
+# PatchFastRL (llama.py) captures `train` for the RL resume guard, so every trainer goes through it.
+from ._gc_train_scope import patch_trainer_train_gc_scope as _patch_trainer_train_gc_scope
+
+_patch_trainer_train_gc_scope(transformers.trainer.Trainer)
+
def patch_mistral_nemo_config(config):
if "head_dim (" not in config:
@@ -5360,6 +5366,9 @@ def patch_gradient_accumulation_fix(Trainer):
Trainer.__init__ = _unsloth_trainer_init
Trainer._unsloth_init_wrapped_for_accelerate_gas = True
+ # Idempotent: already installed at import unless Trainer was reloaded since.
+ _patch_trainer_train_gc_scope(Trainer)
+
# Real children only: PEFT's LoraModel forwards `base_model` to the backbone, skipping the loss head.
_LOSS_KWARGS_CHILDREN = ("base_model", "model") + _UNSLOTH_WRAPPED_MODULE_ATTRS