Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop ignoring the Memory tab's gradient checkpointing setting (#10666)

Nilaye726fb1f5639
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index deb6d86e3d5..4444cf3ca79 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -51,7 +51,7 @@ import math
from loggers import get_logger
import time
from pathlib import Path
-from typing import Any, Dict, List, Optional, Callable
+from typing import Any, Dict, List, Optional, Callable, Union
from datasets import Dataset
from core.training.eval_dataset import evaluation_enabled
from utils.datasets.audio_decode import ensure_audio_decoding
@@ -255,6 +255,25 @@ def _dataset_has_audio_column(dataset) -> Optional[bool]:
return False if saw_a_usable_value else None
+# Marks an omitted mode, which keeps the loaded one; a literal default would overwrite it.
+_UNSET = object()
+
+
+def normalize_gradient_checkpointing(value) -> Union[str, bool]:
+ if isinstance(value, str):
+ text = value.strip().lower()
+ if text in ("", "unsloth"):
+ return "unsloth"
+ if text in ("true", "1", "yes"):
+ return True
+ if text in ("false", "0", "no", "none", "off"):
+ return False
+ elif value in (True, False, "unsloth"):
+ return value
+ logger.warning(f"Invalid gradient_checkpointing value: {value}, defaulting to 'unsloth'")
+ return "unsloth"
+
+
class UnslothTrainer:
def __new__(cls, *args, **kwargs):
if cls is UnslothTrainer and should_use_mlx_training_backend():
@@ -278,6 +297,7 @@ class UnslothTrainer:
self.is_audio = False
self.is_audio_vlm = False
self._audio_type = None
+ self._use_gradient_checkpointing = "unsloth"
# True until a probe says otherwise, so a path that never probes cannot trip the inconclusive-detection guard.
self._audio_type_known = True
self._is_dataset_audio = False
@@ -793,10 +813,14 @@ class UnslothTrainer:
local_files_only: bool = False,
actual_model_repo_id: Optional[str] = None,
model_revision: Optional[str] = None,
+ use_gradient_checkpointing: Union[str, bool] = "unsloth",
) -> bool:
"""Load model for training (supports both text and vision models)"""
self.load_in_4bit = load_in_4bit
self.trust_remote_code = trust_remote_code
+ # The loader installs the checkpointing implementation; a full finetune never reinstalls it.
+ use_gradient_checkpointing = normalize_gradient_checkpointing(use_gradient_checkpointing)
+ self._use_gradient_checkpointing = use_gradient_checkpointing
lookup_name = model_load_name or model_name
self.model_load_error = None
try:
@@ -953,6 +977,7 @@ class UnslothTrainer:
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
logger.info("Loaded CSM audio model")
@@ -973,6 +998,7 @@ class UnslothTrainer:
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
self.model.generation_config.language = "<|en|>"
self.model.generation_config.task = "transcribe"
@@ -993,6 +1019,7 @@ class UnslothTrainer:
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
logger.info(f"Loaded {self._audio_type} audio model (FastLanguageModel)")
@@ -1030,6 +1057,7 @@ class UnslothTrainer:
full_finetuning = full_finetuning,
token = hf_token,
trust_remote_code = trust_remote_code,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
logger.info("Loaded Spark-TTS (bicodec) model")
@@ -1045,6 +1073,7 @@ class UnslothTrainer:
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
logger.info("Loaded OuteTTS (dac) model (FastModel)")
@@ -1061,6 +1090,7 @@ class UnslothTrainer:
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
logger.info("Loaded audio VLM model (FastModel)")
@@ -1076,6 +1106,7 @@ class UnslothTrainer:
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
logger.info("Loaded vision model")
@@ -1103,6 +1134,7 @@ class UnslothTrainer:
trust_remote_code = trust_remote_code,
revision = model_revision,
use_exact_model_name = model_revision is not None,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
logger.info("Loaded text model")
@@ -1122,7 +1154,9 @@ class UnslothTrainer:
return False
if full_finetuning:
- self.model.for_training()
+ self.model.for_training(
+ use_gradient_checkpointing = use_gradient_checkpointing,
+ )
self._update_progress(status_message = "Model loaded successfully")
logger.info("Model loaded successfully")
@@ -1151,6 +1185,7 @@ class UnslothTrainer:
local_files_only = local_files_only,
actual_model_repo_id = actual_model_repo_id,
model_revision = model_revision,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
error_msg = str(e)
error_lower = error_msg.lower()
@@ -1208,7 +1243,7 @@ class UnslothTrainer:
lora_r: int = 16,
lora_alpha: int = 16,
lora_dropout: float = 0.0,
- use_gradient_checkpointing: str = "unsloth",
+ use_gradient_checkpointing: Union[str, bool] = _UNSET,
use_rslora: bool = False,
use_loftq: bool = False,
use_dora: bool = False,
@@ -1218,11 +1253,26 @@ class UnslothTrainer:
if self.model is None:
raise ValueError("Model not loaded. Call load_model() first.")
+ if use_gradient_checkpointing is _UNSET:
+ use_gradient_checkpointing = self._use_gradient_checkpointing
+ else:
+ use_gradient_checkpointing = normalize_gradient_checkpointing(
+ use_gradient_checkpointing
+ )
+
if not use_lora:
+ if use_gradient_checkpointing != self._use_gradient_checkpointing:
+ logger.warning(
+ f"Gradient checkpointing {use_gradient_checkpointing} was requested after the "
+ f"model was loaded with {self._use_gradient_checkpointing}. A full finetune "
+ f"keeps the loaded mode, so pass it to load_model() instead."
+ )
self._update_progress(status_message = "Full finetuning mode - no LoRA adapters")
logger.info("Full finetuning mode - training all parameters\n")
return True
+ self._use_gradient_checkpointing = use_gradient_checkpointing
+
# LoRA/QLoRA. "all-linear" is a PEFT keyword targeting every linear layer.
if isinstance(target_modules, list) and "all-linear" in target_modules:
if len(target_modules) == 1:
@@ -1242,25 +1292,6 @@ class UnslothTrainer:
"down_proj",
]
- if isinstance(use_gradient_checkpointing, str):
- use_gradient_checkpointing = use_gradient_checkpointing.strip().lower()
- if use_gradient_checkpointing == "" or use_gradient_checkpointing == "unsloth":
- use_gradient_checkpointing = "unsloth"
- elif use_gradient_checkpointing in ("true", "1", "yes"):
- use_gradient_checkpointing = True
- elif use_gradient_checkpointing in ("false", "0", "no", "none", "off"):
- use_gradient_checkpointing = False
- else:
- logger.warning(
- f"Invalid gradient_checkpointing value: {use_gradient_checkpointing}, defaulting to 'unsloth'"
- )
- use_gradient_checkpointing = "unsloth"
- elif use_gradient_checkpointing not in (True, False, "unsloth"):
- logger.warning(
- f"Invalid gradient_checkpointing type/value: {use_gradient_checkpointing}, defaulting to 'unsloth'"
- )
- use_gradient_checkpointing = "unsloth"
-
if self.model is None:
error_msg = "Model is None - model was not loaded properly"
logger.error(error_msg)
@@ -3896,7 +3927,10 @@ class UnslothTrainer:
from backend.data_utils import DeepSeekOCRDataCollator
logger.info("Configuring DeepSeek OCR data collator...\n")
- FastVisionModel.for_training(self.model)
+ FastVisionModel.for_training(
+ self.model,
+ use_gradient_checkpointing = self._use_gradient_checkpointing,
+ )
# (image_size, base_size, crop_mode) is a coupled preset: changing image_size alone desyncs the
# per-crop grid from num_queries.
if training_args.get("vision_image_size") is not None:
@@ -3923,6 +3957,13 @@ class UnslothTrainer:
elif self.is_audio_vlm and not raw_text_mode:
# Audio VLM collator (e.g. Gemma 3N), mirrors the Gemma3N_(4B)-Audio notebook.
logger.info("Configuring audio VLM data collator...\n")
+ from unsloth import FastModel
+
+ # The module flags decide whether a layer checkpoints, as in the image VLM branch.
+ FastModel.for_training(
+ self.model,
+ use_gradient_checkpointing = self._use_gradient_checkpointing,
+ )
processor = self.tokenizer
audio_col_name = getattr(self, "_audio_vlm_audio_col", "audio")
@@ -3962,7 +4003,10 @@ class UnslothTrainer:
logger.info("Using UnslothVisionDataCollator for vision model\n")
from unsloth.trainer import UnslothVisionDataCollator
- FastVisionModel.for_training(self.model)
+ FastVisionModel.for_training(
+ self.model,
+ use_gradient_checkpointing = self._use_gradient_checkpointing,
+ )
vision_image_size = training_args.get("vision_image_size")
if vision_image_size is None:
data_collator = UnslothVisionDataCollator(self.model, self.tokenizer)
@@ -4006,6 +4050,8 @@ class UnslothTrainer:
serial_as_none = False,
),
"max_seq_length": training_args.get("max_seq_length", 2048),
+ # TRL defaults this on, and train() re-enables checkpointing from it.
+ "gradient_checkpointing": bool(self._use_gradient_checkpointing),
}
if training_args.get("enable_tensorboard", False):
config_args["logging_dir"] = str(
@@ -4089,8 +4135,6 @@ class UnslothTrainer:
{
"optim": optim_value,
"lr_scheduler_type": lr_scheduler_type_value,
- "gradient_checkpointing": True,
- "gradient_checkpointing_kwargs": {"use_reentrant": False},
"max_grad_norm": 0.3,
"remove_unused_columns": False,
"dataset_text_field": "",
@@ -4098,6 +4142,8 @@ class UnslothTrainer:
"max_length": training_args.get("max_seq_length", 2048),
}
)
+ if config_args["gradient_checkpointing"]:
+ config_args["gradient_checkpointing_kwargs"] = {"use_reentrant": False}
else:
is_cpt = training_args.get("is_cpt", False)
self.is_cpt = is_cpt
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index 60587badd08..1f751026363 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -631,6 +631,10 @@ class TrainingProgress:
is_run_summary: bool = False
+# Marks an omitted mode, which keeps the loaded one; a literal default would overwrite it.
+_UNSET = object()
+
+
class _MLXTrainerAdapter:
"""Adapts the legacy UnslothTrainer API to the shared Unsloth MLX worker path."""
@@ -695,6 +699,7 @@ class _MLXTrainerAdapter:
trust_remote_code: bool = False,
full_finetuning: bool = False,
gpu_ids: Optional[list[int]] = None,
+ use_gradient_checkpointing: Union[str, bool] = "unsloth",
) -> bool:
self.model_name = model_name
self.max_seq_length = max_seq_length
@@ -730,6 +735,7 @@ class _MLXTrainerAdapter:
"is_dataset_audio": bool(is_dataset_audio),
"trust_remote_code": bool(trust_remote_code),
"gpu_ids": gpu_ids,
+ "gradient_checkpointing": use_gradient_checkpointing,
}
self._update_progress(
is_training = False,
@@ -753,11 +759,14 @@ class _MLXTrainerAdapter:
lora_r: int = 16,
lora_alpha: int = 16,
lora_dropout: float = 0.0,
- use_gradient_checkpointing: Union[str, bool] = "unsloth",
+ use_gradient_checkpointing: Union[str, bool] = _UNSET,
use_rslora: bool = False,
use_loftq: bool = False,
use_dora: bool = False,
) -> bool:
+ if use_gradient_checkpointing is _UNSET:
+ # This entry overrides load_model's, so default to the mode recorded there.
+ use_gradient_checkpointing = self._model_config.get("gradient_checkpointing", "unsloth")
self._peft_config = {
"use_lora": bool(use_lora),
"lora_r": lora_r,
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index ff5912b4aae..acf4613d085 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -4035,6 +4035,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
local_files_only = model_local_only,
actual_model_repo_id = config.get("actual_model_repo_id"),
model_revision = model_revision,
+ use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
)
fallback_error = (
_model_cache_fallback_error(config, trainer.model_load_error)
@@ -4099,6 +4100,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
local_files_only = model_local_only,
actual_model_repo_id = config.get("actual_model_repo_id"),
model_revision = model_revision,
+ use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
)
finally:
_load_watchdog_stop.set()
@@ -4189,7 +4191,10 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
)
else:
_send_status(event_queue, "Preparing model for full finetuning...")
- success = trainer.prepare_model_for_training(use_lora = False)
+ success = trainer.prepare_model_for_training(
+ use_lora = False,
+ use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
+ )
if not success or trainer.should_stop:
if trainer.should_stop:
diff --git a/studio/backend/tests/test_audio_lora_logging.py b/studio/backend/tests/test_audio_lora_logging.py
index 84b1f7a63c0..e57c39bb844 100644
--- a/studio/backend/tests/test_audio_lora_logging.py
+++ b/studio/backend/tests/test_audio_lora_logging.py
@@ -7,12 +7,27 @@ import ast
import sys
from pathlib import Path
from types import ModuleType, SimpleNamespace
+from typing import Union
from unittest.mock import Mock
import pytest
import structlog
+def _module_level_dependencies(tree):
+ """Defaults and annotations are evaluated at def time, so exec needs the module-level
+ names the signature reads."""
+ return [
+ node
+ for node in tree.body
+ if (isinstance(node, ast.FunctionDef) and node.name == "normalize_gradient_checkpointing")
+ or (
+ isinstance(node, ast.Assign)
+ and any(isinstance(t, ast.Name) and t.id == "_UNSET" for t in node.targets)
+ )
+ ]
+
+
@pytest.mark.parametrize("audio_type", ["csm", "bicodec", "dac", "snac", "audio_vlm"])
def test_audio_lora_setup_accepts_real_structlog(monkeypatch, audio_type):
source = Path(__file__).resolve().parents[1] / "core/training/trainer.py"
@@ -25,14 +40,15 @@ def test_audio_lora_setup_accepts_real_structlog(monkeypatch, audio_type):
for n in trainer.body
if isinstance(n, ast.FunctionDef) and n.name == "prepare_model_for_training"
)
- namespace = {"logger": structlog.get_logger()}
+ namespace = {"logger": structlog.get_logger(), "Union": Union}
adapter = Mock(return_value = SimpleNamespace(config = object()))
fast_model = SimpleNamespace(get_peft_model = adapter)
module = ModuleType("unsloth")
module.FastModel = fast_model
monkeypatch.setitem(sys.modules, "unsloth", module)
namespace["FastLanguageModel"] = fast_model
- exec(compile(ast.Module(body = [method], type_ignores = []), str(source), "exec"), namespace)
+ body = _module_level_dependencies(tree) + [method]
+ exec(compile(ast.Module(body = body, type_ignores = []), str(source), "exec"), namespace)
model = SimpleNamespace(config = object())
progress = Mock()
instance = SimpleNamespace(
@@ -42,6 +58,7 @@ def test_audio_lora_setup_accepts_real_structlog(monkeypatch, audio_type):
is_vlm = False,
should_stop = False,
_update_progress = progress,
+ _use_gradient_checkpointing = "unsloth",
)
assert namespace["prepare_model_for_training"](instance) is True
diff --git a/studio/backend/tests/test_vision_gradient_checkpointing.py b/studio/backend/tests/test_vision_gradient_checkpointing.py
new file mode 100644
index 00000000000..0da187dbfbf
--- /dev/null
+++ b/studio/backend/tests/test_vision_gradient_checkpointing.py
@@ -0,0 +1,462 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The Memory tab's Gradient Checkpointing choice must reach the loader and SFTConfig."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+import ast # noqa: E402
+import importlib # noqa: E402
+import inspect # noqa: E402
+import types # noqa: E402
+from unittest.mock import MagicMock # noqa: E402
+
+import pytest # noqa: E402
+
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(name, attrs):
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001 - stub unusable imports
+ pass
+ _STUBBED.append(name)
+ mod = types.ModuleType(name)
+ mod.__spec__ = None
+ for attr in attrs:
+ setattr(mod, attr, MagicMock())
+ sys.modules[name] = mod
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, mod)
+
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+
+from core.training import trainer as tmod # noqa: E402
+
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+
+class _Stop(Exception):
+ """Unwinds ``_train_worker`` once the SFT config has been captured."""
+
+
+class _FakeModel:
+ config = types.SimpleNamespace()
+
+ def parameters(self):
+ return iter(())
+
+ def modules(self):
+ return iter(())
+
+ def train(self):
+ return self
+
+
+def run_sft_training(monkeypatch):
+ """Drive the real SFT branches of ``_train_worker`` on CPU and report what they built."""
+ seen = {"config_args": None, "for_training": []}
+
+ class _FakeVisionModel:
+ @staticmethod
+ def for_training(model, use_gradient_checkpointing = True):
+ seen["for_training"].append(use_gradient_checkpointing)
+ return model
+
+ @staticmethod
+ def get_peft_model(model, **kwargs):
+ return model
+
+ def _capture_sft_config(**config_args):
+ seen["config_args"] = config_args
+ raise _Stop
+
+ fake_unsloth = types.ModuleType("unsloth")
+ fake_unsloth.__spec__ = None
+ fake_unsloth.FastModel = _FakeVisionModel
+ fake_unsloth.FastVisionModel = _FakeVisionModel
+ fake_unsloth.FastLanguageModel = _FakeVisionModel
+ fake_unsloth_trainer = types.ModuleType("unsloth.trainer")
+ fake_unsloth_trainer.__spec__ = None
+ fake_unsloth_trainer.UnslothVisionDataCollator = MagicMock()
+ fake_unsloth.trainer = fake_unsloth_trainer
+
+ monkeypatch.setitem(sys.modules, "unsloth", fake_unsloth)
+ monkeypatch.setitem(sys.modules, "unsloth.trainer", fake_unsloth_trainer)
+ monkeypatch.setattr(tmod, "should_use_mlx_training_backend", lambda *a, **k: False)
+ monkeypatch.setattr(tmod, "FastVisionModel", _FakeVisionModel)
+ monkeypatch.setattr(tmod, "FastLanguageModel", _FakeVisionModel)
+ monkeypatch.setattr(tmod, "SFTConfig", _capture_sft_config)
+ monkeypatch.setattr(tmod, "is_bfloat16_supported", lambda: False)
+
+ def _run(
+ gradient_checkpointing,
+ *,
+ is_audio_vlm = False,
+ use_lora = True,
+ text = False,
+ ):
+ t = tmod.UnslothTrainer()
+ t.model = _FakeModel()
+ t.tokenizer = types.SimpleNamespace()
+ t.model_name = "unsloth/Qwen2-VL-7B-Instruct"
+ t.is_vlm = not (is_audio_vlm or text)
+ t.is_audio_vlm = is_audio_vlm
+ # Stands in for the skipped load_model(), which the worker gives the same mode.
+ t._use_gradient_checkpointing = tmod.normalize_gradient_checkpointing(
+ gradient_checkpointing
+ )
+ monkeypatch.setattr(
+ t,
+ "_configure_online_tokenization",
+ lambda **kwargs: types.SimpleNamespace(enabled = False),
+ raising = True,
+ )
+
+ assert t.prepare_model_for_training(
+ use_lora = use_lora,
+ use_gradient_checkpointing = gradient_checkpointing,
+ )
+ dataset = [{"text": "hi"}] if text else [{"messages": []}]
+ t._train_worker(
+ {"dataset": dataset, "final_format": "text" if text else "vision"},
+ max_steps = 1,
+ )
+
+ assert seen["config_args"] is not None, "the SFT config was never built"
+ return seen
+
+ return _run
+
+
+def test_none_reaches_both_the_sft_config_and_for_training(run_sft_training):
+ seen = run_sft_training("none")
+
+ assert (
+ seen["config_args"]["gradient_checkpointing"] is False
+ ), "the vision branch re-enabled gradient checkpointing after the user turned it off"
+ assert (
+ "gradient_checkpointing_kwargs" not in seen["config_args"]
+ ), "checkpointing kwargs were sent for a run that does not checkpoint"
+ assert seen["for_training"] == [False], (
+ "for_training re-flagged every layer, leaving them without a "
+ "_gradient_checkpointing_func"
+ )
+
+
[email protected]("use_lora", [True, False])
+def test_audio_vlm_honours_none_as_well(run_sft_training, use_lora):
+ seen = run_sft_training("none", is_audio_vlm = True, use_lora = use_lora)
+
+ assert seen["config_args"]["gradient_checkpointing"] is False
+ assert "gradient_checkpointing_kwargs" not in seen["config_args"]
+ # SFTConfig never turns checkpointing off, so the flags load_model set must be reapplied.
+ assert seen["for_training"] == [False], (
+ "the audio VLM branch left the module gradient_checkpointing flags alone, "
+ "so the run checkpoints despite the Memory tab saying None"
+ )
+
+
+ ("choice", "mode"), [("unsloth", "unsloth"), ("true", True), ("", "unsloth")]
+)
[email protected]("is_audio_vlm", [False, True])
+def test_the_default_path_still_checkpoints(run_sft_training, choice, mode, is_audio_vlm):
+ seen = run_sft_training(choice, is_audio_vlm = is_audio_vlm)
+
+ assert seen["config_args"]["gradient_checkpointing"] is True
+ assert seen["config_args"]["gradient_checkpointing_kwargs"] == {"use_reentrant": False}
+ assert seen["for_training"] == [mode]
+
+
+def test_full_finetuning_honours_none_too(run_sft_training):
+ seen = run_sft_training("none", use_lora = False)
+
+ assert seen["config_args"]["gradient_checkpointing"] is False
+ assert "gradient_checkpointing_kwargs" not in seen["config_args"]
+ assert seen["for_training"] == [False]
+
+
[email protected](("choice", "enabled"), [("none", False), ("unsloth", True)])
[email protected]("use_lora", [True, False])
+def test_a_text_run_carries_the_choice_into_sft_config(run_sft_training, choice, enabled, use_lora):
+ """TRL defaults checkpointing on, and train() re-enables it from SFTConfig."""
+ seen = run_sft_training(choice, text = True, use_lora = use_lora)
+
+ assert seen["config_args"]["gradient_checkpointing"] is enabled
+
+
+def load_model_run(monkeypatch):
+ """Drive the real ``load_model`` against a stubbed loader and report what it passed."""
+ seen = {"loads": [], "for_training": [], "stamp": None}
+
+ def _record_for_training(use_gradient_checkpointing = True):
+ seen["for_training"].append(use_gradient_checkpointing)
+
+ class _FakeLoader:
+ fail_once_with = None
+
+ @classmethod
+ def from_pretrained(cls, **kwargs):
+ seen["loads"].append(kwargs)
+ if cls.fail_once_with is not None:
+ error, cls.fail_once_with = cls.fail_once_with, None
+ raise error
+ model = _FakeModel()
+ # As post_patch_model does; real text LoRA loads record it in get_peft_model instead.
+ seen["stamp"] = kwargs.get("use_gradient_checkpointing")
+ model._unsloth_gradient_checkpointing = seen["stamp"]
+ model.for_training = _record_for_training
+ return model, types.SimpleNamespace(image_processor = object())
+
+ @staticmethod
+ def get_peft_model(
+ model,
+ use_gradient_checkpointing = "unsloth",
+ **kwargs,
+ ):
+ # Adding adapters re-records the mode.
+ seen["stamp"] = use_gradient_checkpointing
+ model._unsloth_gradient_checkpointing = use_gradient_checkpointing
+ return model
+
+ fake_unsloth = types.ModuleType("unsloth")
+ fake_unsloth.__spec__ = None
+ fake_unsloth.FastModel = _FakeLoader
+ fake_unsloth.FastVisionModel = _FakeLoader
+ fake_unsloth.FastLanguageModel = _FakeLoader
+
+ monkeypatch.setitem(sys.modules, "unsloth", fake_unsloth)
+ monkeypatch.setattr(tmod, "should_use_mlx_training_backend", lambda *a, **k: False)
+ monkeypatch.setattr(tmod, "FastVisionModel", _FakeLoader)
+ monkeypatch.setattr(tmod, "FastLanguageModel", _FakeLoader)
+ monkeypatch.setattr(tmod, "clear_gpu_cache", lambda *a, **k: None)
+ monkeypatch.setattr(tmod, "get_device_map", lambda *a, **k: "cuda:0")
+ monkeypatch.setattr(tmod, "get_visible_gpu_count", lambda *a, **k: 1)
+ monkeypatch.setattr(tmod, "raise_if_offloaded", lambda *a, **k: None)
+ monkeypatch.setattr(tmod, "restore_hf_cache_repo_identity", lambda *a, **k: None)
+ monkeypatch.setattr(tmod, "is_bfloat16_supported", lambda: False)
+
+ def _run(
+ gradient_checkpointing,
+ *,
+ full_finetuning = False,
+ kind = "vision",
+ fail_once = None,
+ ):
+ audio_type = "audio_vlm" if kind == "audio_vlm" else None
+ monkeypatch.setattr(tmod, "detect_audio_type_checked", lambda *a, **k: (audio_type, True))
+ monkeypatch.setattr(tmod, "is_vision_model", lambda *a, **k: kind == "vision")
+ _FakeLoader.fail_once_with = fail_once
+
+ t = tmod.UnslothTrainer()
+ monkeypatch.setattr(t, "_cleanup_audio_artifacts", lambda: None, raising = True)
+ assert t.load_model(
+ model_name = "unsloth/Qwen2-VL-7B-Instruct",
+ max_seq_length = 2048,
+ is_dataset_image = kind == "vision",
+ is_dataset_audio = kind == "audio_vlm",
+ local_files_only = True,
+ full_finetuning = full_finetuning,
+ use_gradient_checkpointing = gradient_checkpointing,
+ )
+ return t, seen
+
+ return _run
+
+
+ ("choice", "mode"), [("none", False), ("true", True), ("unsloth", "unsloth")]
+)
[email protected]("kind", ["vision", "audio_vlm", "text"])
[email protected]("full_finetuning", [True, False])
+def test_the_loader_is_told_the_choice(load_model_run, kind, full_finetuning, choice, mode):
+ """A full finetune never revisits the loaded mode, which must be True, False or "unsloth"."""
+ _, seen = load_model_run(choice, full_finetuning = full_finetuning, kind = kind)
+
+ assert seen["loads"][-1]["use_gradient_checkpointing"] is mode, (
+ "the model was not loaded with the chosen mode, so the trainer restores another one "
+ "at train() and the Memory tab is ignored"
+ )
+ if full_finetuning:
+ assert seen["for_training"] == [mode]
+
+
+def test_the_retry_after_a_source_code_failure_keeps_the_choice(load_model_run):
+ """The retry reloads the model, so it re-records the mode and must not lose it."""
+ _, seen = load_model_run(
+ "none",
+ full_finetuning = True,
+ fail_once = OSError("could not get source code"),
+ )
+
+ assert len(seen["loads"]) == 2, "the retry never ran, so this asserts nothing"
+ assert seen["loads"][-1]["use_gradient_checkpointing"] is False
+
+
[email protected]("kind", ["vision", "audio_vlm", "text"])
[email protected]("use_lora", [True, False])
+def test_a_later_mode_applies_only_where_adapters_are_added(
+ load_model_run, monkeypatch, use_lora, kind
+):
+ """LoRA re-applies a prepare-time mode; a full finetune keeps the loaded one."""
+ t, seen = load_model_run("unsloth", full_finetuning = not use_lora, kind = kind)
+ warnings = []
+ monkeypatch.setattr(tmod.logger, "warning", lambda message, *a, **k: warnings.append(message))
+
+ assert t.prepare_model_for_training(use_lora = use_lora, use_gradient_checkpointing = "none")
+
+ expected = False if use_lora else "unsloth"
+ assert t._use_gradient_checkpointing == expected
+ assert seen["stamp"] == expected
+ assert any("load_model" in w for w in warnings) is not use_lora
+
+
+def _cli_load_model_calls():
+ cli = _BACKEND.parent.parent / "unsloth_cli" / "commands" / "train.py"
+ tree = ast.parse(cli.read_text(encoding = "utf-8"))
+ calls = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "load_model"
+ and getattr(node.func.value, "id", None) == "trainer"
+ ]
+ assert calls, "the CLI no longer loads a model here"
+ return calls
+
+
+def test_the_cli_chooses_the_mode_when_it_loads():
+ """The CLI must load with the configured mode, and load a full finetune as one."""
+ calls = _cli_load_model_calls()
+
+ for node in calls:
+ forwarded = [k for k in node.keywords if k.arg == "use_gradient_checkpointing"]
+ assert forwarded, (
+ f"train.py line {node.lineno} loads without the configured gradient checkpointing, "
+ "so the run is built with a different mode"
+ )
+ value = forwarded[0].value
+ assert (
+ ast.unparse(value) == "cfg.training.gradient_checkpointing"
+ ), f"train.py line {node.lineno} does not pass the configured mode through unchanged"
+ full = [k.value for k in node.keywords if k.arg == "full_finetuning"]
+ assert (
+ full and ast.unparse(full[0]) == "not use_lora"
+ ), f"train.py line {node.lineno} loads a full finetune as if adapters will follow"
+
+
+def test_both_trainers_accept_what_the_cli_loads_with():
+ """The CLI builds either trainer from one call site, so its keywords must bind to both."""
+ from core.training.training import _MLXTrainerAdapter
+
+ keyword_sets = [{k.arg for k in node.keywords if k.arg} for node in _cli_load_model_calls()]
+
+ for trainer_class in (tmod.UnslothTrainer, _MLXTrainerAdapter):
+ signature = inspect.signature(trainer_class.load_model)
+ for keywords in keyword_sets:
+ signature.bind(object(), **{name: None for name in keywords})
+
+
+def test_the_mlx_adapter_keeps_the_mode_it_was_loaded_with():
+ """The PEFT entry wins in the worker config, so an omitted mode keeps the loaded one."""
+ from core.training.training import _MLXTrainerAdapter
+
+ adapter = _MLXTrainerAdapter()
+ assert adapter.load_model(model_name = "unsloth/Qwen3-4B", use_gradient_checkpointing = False)
+ assert adapter.prepare_model_for_training(use_lora = True)
+
+ assert adapter._peft_config["gradient_checkpointing"] is False
+ assert adapter._model_config["gradient_checkpointing"] is False
+
+
+def test_the_mlx_adapter_rejects_load_arguments_it_would_drop():
+ """An unknown load keyword must raise rather than silently never reach the worker."""
+ from core.training.training import _MLXTrainerAdapter
+
+ signature = inspect.signature(_MLXTrainerAdapter.load_model)
+
+ with pytest.raises(TypeError):
+ signature.bind(object(), model_name = "unsloth/Qwen3-4B", model_revision = "abc123")
+
+
[email protected]("full_finetuning", [True, False])
+def test_omitting_the_argument_keeps_the_choice(load_model_run, full_finetuning):
+ t, _ = load_model_run("none", full_finetuning = full_finetuning)
+
+ assert t.prepare_model_for_training(use_lora = not full_finetuning)
+
+ assert t.model._unsloth_gradient_checkpointing is False
+ assert t._use_gradient_checkpointing is False
+
+
+def _trainer_calls_to(method):
+ worker = _BACKEND / "core" / "training" / "worker.py"
+ tree = ast.parse(worker.read_text(encoding = "utf-8"))
+ return [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == method
+ and getattr(node.func.value, "id", None) == "trainer"
+ ]
+
+
+def _reads_the_memory_tab(keyword):
+ """Old configs lack the key, so the ``"unsloth"`` fallback is part of the contract."""
+ node = keyword.value
+ if not (isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "get"):
+ return False
+ if getattr(node.func.value, "id", None) != "config":
+ return False
+ return [getattr(a, "value", None) for a in node.args] == [
+ "gradient_checkpointing",
+ "unsloth",
+ ]
+
+
+ ("method", "expected"),
+ [("prepare_model_for_training", 3), ("load_model", 2)],
+)
+def test_the_worker_forwards_the_setting_on_every_call(method, expected):
+ """An omitted or literal argument ignores the Memory tab."""
+ calls = _trainer_calls_to(method)
+
+ assert (
+ len(calls) >= expected
+ ), f"expected at least {expected} trainer.{method} calls, found {len(calls)}"
+ unforwarded = [
+ node.lineno
+ for node in calls
+ if not any(
+ k.arg == "use_gradient_checkpointing" and _reads_the_memory_tab(k)
+ for k in node.keywords
+ )
+ ]
+ assert not unforwarded, (
+ f"worker.py lines {unforwarded} call {method} without forwarding the config's "
+ "gradient_checkpointing; that run ignores the Memory tab and checkpoints anyway. "
+ "Rewriting that expression means teaching this check the new shape, not dropping it."
+ )
diff --git a/unsloth_cli/commands/train.py b/unsloth_cli/commands/train.py
index 2175cf0eccd..4993988c539 100644
--- a/unsloth_cli/commands/train.py
+++ b/unsloth_cli/commands/train.py
@@ -124,7 +124,9 @@ def train(
model_name = cfg.model,
max_seq_length = cfg.training.max_seq_length,
load_in_4bit = cfg.training.load_in_4bit if use_lora else False,
+ full_finetuning = not use_lora,
hf_token = hf_token,
+ use_gradient_checkpointing = cfg.training.gradient_checkpointing,
):
typer.echo("Model load failed", err = True)
raise typer.Exit(code = 1)