Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop ignoring the Memory tab's gradient checkpointing setting (#10666)
Nilaye726fb1f5639
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex deb6d86e3d5..4444cf3ca79 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -51,7 +51,7 @@ import mathfrom loggers import get_loggerimport timefrom pathlib import Path-from typing import Any, Dict, List, Optional, Callable+from typing import Any, Dict, List, Optional, Callable, Unionfrom datasets import Datasetfrom core.training.eval_dataset import evaluation_enabledfrom utils.datasets.audio_decode import ensure_audio_decoding@@ -255,6 +255,25 @@ def _dataset_has_audio_column(dataset) -> Optional[bool]:return False if saw_a_usable_value else None+# Marks an omitted mode, which keeps the loaded one; a literal default would overwrite it.+_UNSET = object()+++def normalize_gradient_checkpointing(value) -> Union[str, bool]:+ if isinstance(value, str):+ text = value.strip().lower()+ if text in ("", "unsloth"):+ return "unsloth"+ if text in ("true", "1", "yes"):+ return True+ if text in ("false", "0", "no", "none", "off"):+ return False+ elif value in (True, False, "unsloth"):+ return value+ logger.warning(f"Invalid gradient_checkpointing value: {value}, defaulting to 'unsloth'")+ return "unsloth"++class UnslothTrainer:def __new__(cls, *args, **kwargs):if cls is UnslothTrainer and should_use_mlx_training_backend():@@ -278,6 +297,7 @@ class UnslothTrainer:self.is_audio = Falseself.is_audio_vlm = Falseself._audio_type = None+ self._use_gradient_checkpointing = "unsloth"# True until a probe says otherwise, so a path that never probes cannot trip the inconclusive-detection guard.self._audio_type_known = Trueself._is_dataset_audio = False@@ -793,10 +813,14 @@ class UnslothTrainer:local_files_only: bool = False,actual_model_repo_id: Optional[str] = None,model_revision: Optional[str] = None,+ use_gradient_checkpointing: Union[str, bool] = "unsloth",) -> bool:"""Load model for training (supports both text and vision models)"""self.load_in_4bit = load_in_4bitself.trust_remote_code = trust_remote_code+ # The loader installs the checkpointing implementation; a full finetune never reinstalls it.+ use_gradient_checkpointing = normalize_gradient_checkpointing(use_gradient_checkpointing)+ self._use_gradient_checkpointing = use_gradient_checkpointinglookup_name = model_load_name or model_nameself.model_load_error = Nonetry:@@ -953,6 +977,7 @@ class UnslothTrainer:trust_remote_code = trust_remote_code,revision = model_revision,use_exact_model_name = model_revision is not None,+ use_gradient_checkpointing = use_gradient_checkpointing,)logger.info("Loaded CSM audio model")@@ -973,6 +998,7 @@ class UnslothTrainer:trust_remote_code = trust_remote_code,revision = model_revision,use_exact_model_name = model_revision is not None,+ use_gradient_checkpointing = use_gradient_checkpointing,)self.model.generation_config.language = "<|en|>"self.model.generation_config.task = "transcribe"@@ -993,6 +1019,7 @@ class UnslothTrainer:trust_remote_code = trust_remote_code,revision = model_revision,use_exact_model_name = model_revision is not None,+ use_gradient_checkpointing = use_gradient_checkpointing,)logger.info(f"Loaded {self._audio_type} audio model (FastLanguageModel)")@@ -1030,6 +1057,7 @@ class UnslothTrainer:full_finetuning = full_finetuning,token = hf_token,trust_remote_code = trust_remote_code,+ use_gradient_checkpointing = use_gradient_checkpointing,)logger.info("Loaded Spark-TTS (bicodec) model")@@ -1045,6 +1073,7 @@ class UnslothTrainer:trust_remote_code = trust_remote_code,revision = model_revision,use_exact_model_name = model_revision is not None,+ use_gradient_checkpointing = use_gradient_checkpointing,)logger.info("Loaded OuteTTS (dac) model (FastModel)")@@ -1061,6 +1090,7 @@ class UnslothTrainer:trust_remote_code = trust_remote_code,revision = model_revision,use_exact_model_name = model_revision is not None,+ use_gradient_checkpointing = use_gradient_checkpointing,)logger.info("Loaded audio VLM model (FastModel)")@@ -1076,6 +1106,7 @@ class UnslothTrainer:trust_remote_code = trust_remote_code,revision = model_revision,use_exact_model_name = model_revision is not None,+ use_gradient_checkpointing = use_gradient_checkpointing,)logger.info("Loaded vision model")@@ -1103,6 +1134,7 @@ class UnslothTrainer:trust_remote_code = trust_remote_code,revision = model_revision,use_exact_model_name = model_revision is not None,+ use_gradient_checkpointing = use_gradient_checkpointing,)logger.info("Loaded text model")@@ -1122,7 +1154,9 @@ class UnslothTrainer:return Falseif full_finetuning:- self.model.for_training()+ self.model.for_training(+ use_gradient_checkpointing = use_gradient_checkpointing,+ )self._update_progress(status_message = "Model loaded successfully")logger.info("Model loaded successfully")@@ -1151,6 +1185,7 @@ class UnslothTrainer:local_files_only = local_files_only,actual_model_repo_id = actual_model_repo_id,model_revision = model_revision,+ use_gradient_checkpointing = use_gradient_checkpointing,)error_msg = str(e)error_lower = error_msg.lower()@@ -1208,7 +1243,7 @@ class UnslothTrainer:lora_r: int = 16,lora_alpha: int = 16,lora_dropout: float = 0.0,- use_gradient_checkpointing: str = "unsloth",+ use_gradient_checkpointing: Union[str, bool] = _UNSET,use_rslora: bool = False,use_loftq: bool = False,use_dora: bool = False,@@ -1218,11 +1253,26 @@ class UnslothTrainer:if self.model is None:raise ValueError("Model not loaded. Call load_model() first.")+ if use_gradient_checkpointing is _UNSET:+ use_gradient_checkpointing = self._use_gradient_checkpointing+ else:+ use_gradient_checkpointing = normalize_gradient_checkpointing(+ use_gradient_checkpointing+ )+if not use_lora:+ if use_gradient_checkpointing != self._use_gradient_checkpointing:+ logger.warning(+ f"Gradient checkpointing {use_gradient_checkpointing} was requested after the "+ f"model was loaded with {self._use_gradient_checkpointing}. A full finetune "+ f"keeps the loaded mode, so pass it to load_model() instead."+ )self._update_progress(status_message = "Full finetuning mode - no LoRA adapters")logger.info("Full finetuning mode - training all parameters\n")return True+ self._use_gradient_checkpointing = use_gradient_checkpointing+# LoRA/QLoRA. "all-linear" is a PEFT keyword targeting every linear layer.if isinstance(target_modules, list) and "all-linear" in target_modules:if len(target_modules) == 1:@@ -1242,25 +1292,6 @@ class UnslothTrainer:"down_proj",]- if isinstance(use_gradient_checkpointing, str):- use_gradient_checkpointing = use_gradient_checkpointing.strip().lower()- if use_gradient_checkpointing == "" or use_gradient_checkpointing == "unsloth":- use_gradient_checkpointing = "unsloth"- elif use_gradient_checkpointing in ("true", "1", "yes"):- use_gradient_checkpointing = True- elif use_gradient_checkpointing in ("false", "0", "no", "none", "off"):- use_gradient_checkpointing = False- else:- logger.warning(- f"Invalid gradient_checkpointing value: {use_gradient_checkpointing}, defaulting to 'unsloth'"- )- use_gradient_checkpointing = "unsloth"- elif use_gradient_checkpointing not in (True, False, "unsloth"):- logger.warning(- f"Invalid gradient_checkpointing type/value: {use_gradient_checkpointing}, defaulting to 'unsloth'"- )- use_gradient_checkpointing = "unsloth"-if self.model is None:error_msg = "Model is None - model was not loaded properly"logger.error(error_msg)@@ -3896,7 +3927,10 @@ class UnslothTrainer:from backend.data_utils import DeepSeekOCRDataCollatorlogger.info("Configuring DeepSeek OCR data collator...\n")- FastVisionModel.for_training(self.model)+ FastVisionModel.for_training(+ self.model,+ use_gradient_checkpointing = self._use_gradient_checkpointing,+ )# (image_size, base_size, crop_mode) is a coupled preset: changing image_size alone desyncs the# per-crop grid from num_queries.if training_args.get("vision_image_size") is not None:@@ -3923,6 +3957,13 @@ class UnslothTrainer:elif self.is_audio_vlm and not raw_text_mode:# Audio VLM collator (e.g. Gemma 3N), mirrors the Gemma3N_(4B)-Audio notebook.logger.info("Configuring audio VLM data collator...\n")+ from unsloth import FastModel++ # The module flags decide whether a layer checkpoints, as in the image VLM branch.+ FastModel.for_training(+ self.model,+ use_gradient_checkpointing = self._use_gradient_checkpointing,+ )processor = self.tokenizeraudio_col_name = getattr(self, "_audio_vlm_audio_col", "audio")@@ -3962,7 +4003,10 @@ class UnslothTrainer:logger.info("Using UnslothVisionDataCollator for vision model\n")from unsloth.trainer import UnslothVisionDataCollator- FastVisionModel.for_training(self.model)+ FastVisionModel.for_training(+ self.model,+ use_gradient_checkpointing = self._use_gradient_checkpointing,+ )vision_image_size = training_args.get("vision_image_size")if vision_image_size is None:data_collator = UnslothVisionDataCollator(self.model, self.tokenizer)@@ -4006,6 +4050,8 @@ class UnslothTrainer:serial_as_none = False,),"max_seq_length": training_args.get("max_seq_length", 2048),+ # TRL defaults this on, and train() re-enables checkpointing from it.+ "gradient_checkpointing": bool(self._use_gradient_checkpointing),}if training_args.get("enable_tensorboard", False):config_args["logging_dir"] = str(@@ -4089,8 +4135,6 @@ class UnslothTrainer:{"optim": optim_value,"lr_scheduler_type": lr_scheduler_type_value,- "gradient_checkpointing": True,- "gradient_checkpointing_kwargs": {"use_reentrant": False},"max_grad_norm": 0.3,"remove_unused_columns": False,"dataset_text_field": "",@@ -4098,6 +4142,8 @@ class UnslothTrainer:"max_length": training_args.get("max_seq_length", 2048),})+ if config_args["gradient_checkpointing"]:+ config_args["gradient_checkpointing_kwargs"] = {"use_reentrant": False}else:is_cpt = training_args.get("is_cpt", False)self.is_cpt = is_cptdiff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.pyindex 60587badd08..1f751026363 100644--- a/studio/backend/core/training/training.py+++ b/studio/backend/core/training/training.py@@ -631,6 +631,10 @@ class TrainingProgress:is_run_summary: bool = False+# Marks an omitted mode, which keeps the loaded one; a literal default would overwrite it.+_UNSET = object()++class _MLXTrainerAdapter:"""Adapts the legacy UnslothTrainer API to the shared Unsloth MLX worker path."""@@ -695,6 +699,7 @@ class _MLXTrainerAdapter:trust_remote_code: bool = False,full_finetuning: bool = False,gpu_ids: Optional[list[int]] = None,+ use_gradient_checkpointing: Union[str, bool] = "unsloth",) -> bool:self.model_name = model_nameself.max_seq_length = max_seq_length@@ -730,6 +735,7 @@ class _MLXTrainerAdapter:"is_dataset_audio": bool(is_dataset_audio),"trust_remote_code": bool(trust_remote_code),"gpu_ids": gpu_ids,+ "gradient_checkpointing": use_gradient_checkpointing,}self._update_progress(is_training = False,@@ -753,11 +759,14 @@ class _MLXTrainerAdapter:lora_r: int = 16,lora_alpha: int = 16,lora_dropout: float = 0.0,- use_gradient_checkpointing: Union[str, bool] = "unsloth",+ use_gradient_checkpointing: Union[str, bool] = _UNSET,use_rslora: bool = False,use_loftq: bool = False,use_dora: bool = False,) -> bool:+ if use_gradient_checkpointing is _UNSET:+ # This entry overrides load_model's, so default to the mode recorded there.+ use_gradient_checkpointing = self._model_config.get("gradient_checkpointing", "unsloth")self._peft_config = {"use_lora": bool(use_lora),"lora_r": lora_r,diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex ff5912b4aae..acf4613d085 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -4035,6 +4035,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->local_files_only = model_local_only,actual_model_repo_id = config.get("actual_model_repo_id"),model_revision = model_revision,+ use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),)fallback_error = (_model_cache_fallback_error(config, trainer.model_load_error)@@ -4099,6 +4100,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->local_files_only = model_local_only,actual_model_repo_id = config.get("actual_model_repo_id"),model_revision = model_revision,+ use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),)finally:_load_watchdog_stop.set()@@ -4189,7 +4191,10 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->)else:_send_status(event_queue, "Preparing model for full finetuning...")- success = trainer.prepare_model_for_training(use_lora = False)+ success = trainer.prepare_model_for_training(+ use_lora = False,+ use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),+ )if not success or trainer.should_stop:if trainer.should_stop:diff --git a/studio/backend/tests/test_audio_lora_logging.py b/studio/backend/tests/test_audio_lora_logging.pyindex 84b1f7a63c0..e57c39bb844 100644--- a/studio/backend/tests/test_audio_lora_logging.py+++ b/studio/backend/tests/test_audio_lora_logging.py@@ -7,12 +7,27 @@ import astimport sysfrom pathlib import Pathfrom types import ModuleType, SimpleNamespace+from typing import Unionfrom unittest.mock import Mockimport pytestimport structlog+def _module_level_dependencies(tree):+ """Defaults and annotations are evaluated at def time, so exec needs the module-level+ names the signature reads."""+ return [+ node+ for node in tree.body+ if (isinstance(node, ast.FunctionDef) and node.name == "normalize_gradient_checkpointing")+ or (+ isinstance(node, ast.Assign)+ and any(isinstance(t, ast.Name) and t.id == "_UNSET" for t in node.targets)+ )+ ]++@pytest.mark.parametrize("audio_type", ["csm", "bicodec", "dac", "snac", "audio_vlm"])def test_audio_lora_setup_accepts_real_structlog(monkeypatch, audio_type):source = Path(__file__).resolve().parents[1] / "core/training/trainer.py"@@ -25,14 +40,15 @@ def test_audio_lora_setup_accepts_real_structlog(monkeypatch, audio_type):for n in trainer.bodyif isinstance(n, ast.FunctionDef) and n.name == "prepare_model_for_training")- namespace = {"logger": structlog.get_logger()}+ namespace = {"logger": structlog.get_logger(), "Union": Union}adapter = Mock(return_value = SimpleNamespace(config = object()))fast_model = SimpleNamespace(get_peft_model = adapter)module = ModuleType("unsloth")module.FastModel = fast_modelmonkeypatch.setitem(sys.modules, "unsloth", module)namespace["FastLanguageModel"] = fast_model- exec(compile(ast.Module(body = [method], type_ignores = []), str(source), "exec"), namespace)+ body = _module_level_dependencies(tree) + [method]+ exec(compile(ast.Module(body = body, type_ignores = []), str(source), "exec"), namespace)model = SimpleNamespace(config = object())progress = Mock()instance = SimpleNamespace(@@ -42,6 +58,7 @@ def test_audio_lora_setup_accepts_real_structlog(monkeypatch, audio_type):is_vlm = False,should_stop = False,_update_progress = progress,+ _use_gradient_checkpointing = "unsloth",)assert namespace["prepare_model_for_training"](instance) is Truediff --git a/studio/backend/tests/test_vision_gradient_checkpointing.py b/studio/backend/tests/test_vision_gradient_checkpointing.pynew file mode 100644index 00000000000..0da187dbfbf--- /dev/null+++ b/studio/backend/tests/test_vision_gradient_checkpointing.py@@ -0,0 +1,462 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The Memory tab's Gradient Checkpointing choice must reach the loader and SFTConfig."""++from __future__ import annotations++import sys+from pathlib import Path++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++import ast # noqa: E402+import importlib # noqa: E402+import inspect # noqa: E402+import types # noqa: E402+from unittest.mock import MagicMock # noqa: E402++import pytest # noqa: E402+++_STUBBED: list[str] = []+++def _stub_if_missing(name, attrs):+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001 - stub unusable imports+ pass+ _STUBBED.append(name)+ mod = types.ModuleType(name)+ mod.__spec__ = None+ for attr in attrs:+ setattr(mod, attr, MagicMock())+ sys.modules[name] = mod+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, mod)+++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))++from core.training import trainer as tmod # noqa: E402++for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)+++class _Stop(Exception):+ """Unwinds ``_train_worker`` once the SFT config has been captured."""+++class _FakeModel:+ config = types.SimpleNamespace()++ def parameters(self):+ return iter(())++ def modules(self):+ return iter(())++ def train(self):+ return self+++def run_sft_training(monkeypatch):+ """Drive the real SFT branches of ``_train_worker`` on CPU and report what they built."""+ seen = {"config_args": None, "for_training": []}++ class _FakeVisionModel:+ @staticmethod+ def for_training(model, use_gradient_checkpointing = True):+ seen["for_training"].append(use_gradient_checkpointing)+ return model++ @staticmethod+ def get_peft_model(model, **kwargs):+ return model++ def _capture_sft_config(**config_args):+ seen["config_args"] = config_args+ raise _Stop++ fake_unsloth = types.ModuleType("unsloth")+ fake_unsloth.__spec__ = None+ fake_unsloth.FastModel = _FakeVisionModel+ fake_unsloth.FastVisionModel = _FakeVisionModel+ fake_unsloth.FastLanguageModel = _FakeVisionModel+ fake_unsloth_trainer = types.ModuleType("unsloth.trainer")+ fake_unsloth_trainer.__spec__ = None+ fake_unsloth_trainer.UnslothVisionDataCollator = MagicMock()+ fake_unsloth.trainer = fake_unsloth_trainer++ monkeypatch.setitem(sys.modules, "unsloth", fake_unsloth)+ monkeypatch.setitem(sys.modules, "unsloth.trainer", fake_unsloth_trainer)+ monkeypatch.setattr(tmod, "should_use_mlx_training_backend", lambda *a, **k: False)+ monkeypatch.setattr(tmod, "FastVisionModel", _FakeVisionModel)+ monkeypatch.setattr(tmod, "FastLanguageModel", _FakeVisionModel)+ monkeypatch.setattr(tmod, "SFTConfig", _capture_sft_config)+ monkeypatch.setattr(tmod, "is_bfloat16_supported", lambda: False)++ def _run(+ gradient_checkpointing,+ *,+ is_audio_vlm = False,+ use_lora = True,+ text = False,+ ):+ t = tmod.UnslothTrainer()+ t.model = _FakeModel()+ t.tokenizer = types.SimpleNamespace()+ t.model_name = "unsloth/Qwen2-VL-7B-Instruct"+ t.is_vlm = not (is_audio_vlm or text)+ t.is_audio_vlm = is_audio_vlm+ # Stands in for the skipped load_model(), which the worker gives the same mode.+ t._use_gradient_checkpointing = tmod.normalize_gradient_checkpointing(+ gradient_checkpointing+ )+ monkeypatch.setattr(+ t,+ "_configure_online_tokenization",+ lambda **kwargs: types.SimpleNamespace(enabled = False),+ raising = True,+ )++ assert t.prepare_model_for_training(+ use_lora = use_lora,+ use_gradient_checkpointing = gradient_checkpointing,+ )+ dataset = [{"text": "hi"}] if text else [{"messages": []}]+ t._train_worker(+ {"dataset": dataset, "final_format": "text" if text else "vision"},+ max_steps = 1,+ )++ assert seen["config_args"] is not None, "the SFT config was never built"+ return seen++ return _run+++def test_none_reaches_both_the_sft_config_and_for_training(run_sft_training):+ seen = run_sft_training("none")++ assert (+ seen["config_args"]["gradient_checkpointing"] is False+ ), "the vision branch re-enabled gradient checkpointing after the user turned it off"+ assert (+ "gradient_checkpointing_kwargs" not in seen["config_args"]+ ), "checkpointing kwargs were sent for a run that does not checkpoint"+ assert seen["for_training"] == [False], (+ "for_training re-flagged every layer, leaving them without a "+ "_gradient_checkpointing_func"+ )++[email protected]("use_lora", [True, False])+def test_audio_vlm_honours_none_as_well(run_sft_training, use_lora):+ seen = run_sft_training("none", is_audio_vlm = True, use_lora = use_lora)++ assert seen["config_args"]["gradient_checkpointing"] is False+ assert "gradient_checkpointing_kwargs" not in seen["config_args"]+ # SFTConfig never turns checkpointing off, so the flags load_model set must be reapplied.+ assert seen["for_training"] == [False], (+ "the audio VLM branch left the module gradient_checkpointing flags alone, "+ "so the run checkpoints despite the Memory tab saying None"+ )+++ ("choice", "mode"), [("unsloth", "unsloth"), ("true", True), ("", "unsloth")]+)[email protected]("is_audio_vlm", [False, True])+def test_the_default_path_still_checkpoints(run_sft_training, choice, mode, is_audio_vlm):+ seen = run_sft_training(choice, is_audio_vlm = is_audio_vlm)++ assert seen["config_args"]["gradient_checkpointing"] is True+ assert seen["config_args"]["gradient_checkpointing_kwargs"] == {"use_reentrant": False}+ assert seen["for_training"] == [mode]+++def test_full_finetuning_honours_none_too(run_sft_training):+ seen = run_sft_training("none", use_lora = False)++ assert seen["config_args"]["gradient_checkpointing"] is False+ assert "gradient_checkpointing_kwargs" not in seen["config_args"]+ assert seen["for_training"] == [False]++[email protected](("choice", "enabled"), [("none", False), ("unsloth", True)])[email protected]("use_lora", [True, False])+def test_a_text_run_carries_the_choice_into_sft_config(run_sft_training, choice, enabled, use_lora):+ """TRL defaults checkpointing on, and train() re-enables it from SFTConfig."""+ seen = run_sft_training(choice, text = True, use_lora = use_lora)++ assert seen["config_args"]["gradient_checkpointing"] is enabled+++def load_model_run(monkeypatch):+ """Drive the real ``load_model`` against a stubbed loader and report what it passed."""+ seen = {"loads": [], "for_training": [], "stamp": None}++ def _record_for_training(use_gradient_checkpointing = True):+ seen["for_training"].append(use_gradient_checkpointing)++ class _FakeLoader:+ fail_once_with = None++ @classmethod+ def from_pretrained(cls, **kwargs):+ seen["loads"].append(kwargs)+ if cls.fail_once_with is not None:+ error, cls.fail_once_with = cls.fail_once_with, None+ raise error+ model = _FakeModel()+ # As post_patch_model does; real text LoRA loads record it in get_peft_model instead.+ seen["stamp"] = kwargs.get("use_gradient_checkpointing")+ model._unsloth_gradient_checkpointing = seen["stamp"]+ model.for_training = _record_for_training+ return model, types.SimpleNamespace(image_processor = object())++ @staticmethod+ def get_peft_model(+ model,+ use_gradient_checkpointing = "unsloth",+ **kwargs,+ ):+ # Adding adapters re-records the mode.+ seen["stamp"] = use_gradient_checkpointing+ model._unsloth_gradient_checkpointing = use_gradient_checkpointing+ return model++ fake_unsloth = types.ModuleType("unsloth")+ fake_unsloth.__spec__ = None+ fake_unsloth.FastModel = _FakeLoader+ fake_unsloth.FastVisionModel = _FakeLoader+ fake_unsloth.FastLanguageModel = _FakeLoader++ monkeypatch.setitem(sys.modules, "unsloth", fake_unsloth)+ monkeypatch.setattr(tmod, "should_use_mlx_training_backend", lambda *a, **k: False)+ monkeypatch.setattr(tmod, "FastVisionModel", _FakeLoader)+ monkeypatch.setattr(tmod, "FastLanguageModel", _FakeLoader)+ monkeypatch.setattr(tmod, "clear_gpu_cache", lambda *a, **k: None)+ monkeypatch.setattr(tmod, "get_device_map", lambda *a, **k: "cuda:0")+ monkeypatch.setattr(tmod, "get_visible_gpu_count", lambda *a, **k: 1)+ monkeypatch.setattr(tmod, "raise_if_offloaded", lambda *a, **k: None)+ monkeypatch.setattr(tmod, "restore_hf_cache_repo_identity", lambda *a, **k: None)+ monkeypatch.setattr(tmod, "is_bfloat16_supported", lambda: False)++ def _run(+ gradient_checkpointing,+ *,+ full_finetuning = False,+ kind = "vision",+ fail_once = None,+ ):+ audio_type = "audio_vlm" if kind == "audio_vlm" else None+ monkeypatch.setattr(tmod, "detect_audio_type_checked", lambda *a, **k: (audio_type, True))+ monkeypatch.setattr(tmod, "is_vision_model", lambda *a, **k: kind == "vision")+ _FakeLoader.fail_once_with = fail_once++ t = tmod.UnslothTrainer()+ monkeypatch.setattr(t, "_cleanup_audio_artifacts", lambda: None, raising = True)+ assert t.load_model(+ model_name = "unsloth/Qwen2-VL-7B-Instruct",+ max_seq_length = 2048,+ is_dataset_image = kind == "vision",+ is_dataset_audio = kind == "audio_vlm",+ local_files_only = True,+ full_finetuning = full_finetuning,+ use_gradient_checkpointing = gradient_checkpointing,+ )+ return t, seen++ return _run+++ ("choice", "mode"), [("none", False), ("true", True), ("unsloth", "unsloth")]+)[email protected]("kind", ["vision", "audio_vlm", "text"])[email protected]("full_finetuning", [True, False])+def test_the_loader_is_told_the_choice(load_model_run, kind, full_finetuning, choice, mode):+ """A full finetune never revisits the loaded mode, which must be True, False or "unsloth"."""+ _, seen = load_model_run(choice, full_finetuning = full_finetuning, kind = kind)++ assert seen["loads"][-1]["use_gradient_checkpointing"] is mode, (+ "the model was not loaded with the chosen mode, so the trainer restores another one "+ "at train() and the Memory tab is ignored"+ )+ if full_finetuning:+ assert seen["for_training"] == [mode]+++def test_the_retry_after_a_source_code_failure_keeps_the_choice(load_model_run):+ """The retry reloads the model, so it re-records the mode and must not lose it."""+ _, seen = load_model_run(+ "none",+ full_finetuning = True,+ fail_once = OSError("could not get source code"),+ )++ assert len(seen["loads"]) == 2, "the retry never ran, so this asserts nothing"+ assert seen["loads"][-1]["use_gradient_checkpointing"] is False++[email protected]("kind", ["vision", "audio_vlm", "text"])[email protected]("use_lora", [True, False])+def test_a_later_mode_applies_only_where_adapters_are_added(+ load_model_run, monkeypatch, use_lora, kind+):+ """LoRA re-applies a prepare-time mode; a full finetune keeps the loaded one."""+ t, seen = load_model_run("unsloth", full_finetuning = not use_lora, kind = kind)+ warnings = []+ monkeypatch.setattr(tmod.logger, "warning", lambda message, *a, **k: warnings.append(message))++ assert t.prepare_model_for_training(use_lora = use_lora, use_gradient_checkpointing = "none")++ expected = False if use_lora else "unsloth"+ assert t._use_gradient_checkpointing == expected+ assert seen["stamp"] == expected+ assert any("load_model" in w for w in warnings) is not use_lora+++def _cli_load_model_calls():+ cli = _BACKEND.parent.parent / "unsloth_cli" / "commands" / "train.py"+ tree = ast.parse(cli.read_text(encoding = "utf-8"))+ calls = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "load_model"+ and getattr(node.func.value, "id", None) == "trainer"+ ]+ assert calls, "the CLI no longer loads a model here"+ return calls+++def test_the_cli_chooses_the_mode_when_it_loads():+ """The CLI must load with the configured mode, and load a full finetune as one."""+ calls = _cli_load_model_calls()++ for node in calls:+ forwarded = [k for k in node.keywords if k.arg == "use_gradient_checkpointing"]+ assert forwarded, (+ f"train.py line {node.lineno} loads without the configured gradient checkpointing, "+ "so the run is built with a different mode"+ )+ value = forwarded[0].value+ assert (+ ast.unparse(value) == "cfg.training.gradient_checkpointing"+ ), f"train.py line {node.lineno} does not pass the configured mode through unchanged"+ full = [k.value for k in node.keywords if k.arg == "full_finetuning"]+ assert (+ full and ast.unparse(full[0]) == "not use_lora"+ ), f"train.py line {node.lineno} loads a full finetune as if adapters will follow"+++def test_both_trainers_accept_what_the_cli_loads_with():+ """The CLI builds either trainer from one call site, so its keywords must bind to both."""+ from core.training.training import _MLXTrainerAdapter++ keyword_sets = [{k.arg for k in node.keywords if k.arg} for node in _cli_load_model_calls()]++ for trainer_class in (tmod.UnslothTrainer, _MLXTrainerAdapter):+ signature = inspect.signature(trainer_class.load_model)+ for keywords in keyword_sets:+ signature.bind(object(), **{name: None for name in keywords})+++def test_the_mlx_adapter_keeps_the_mode_it_was_loaded_with():+ """The PEFT entry wins in the worker config, so an omitted mode keeps the loaded one."""+ from core.training.training import _MLXTrainerAdapter++ adapter = _MLXTrainerAdapter()+ assert adapter.load_model(model_name = "unsloth/Qwen3-4B", use_gradient_checkpointing = False)+ assert adapter.prepare_model_for_training(use_lora = True)++ assert adapter._peft_config["gradient_checkpointing"] is False+ assert adapter._model_config["gradient_checkpointing"] is False+++def test_the_mlx_adapter_rejects_load_arguments_it_would_drop():+ """An unknown load keyword must raise rather than silently never reach the worker."""+ from core.training.training import _MLXTrainerAdapter++ signature = inspect.signature(_MLXTrainerAdapter.load_model)++ with pytest.raises(TypeError):+ signature.bind(object(), model_name = "unsloth/Qwen3-4B", model_revision = "abc123")++[email protected]("full_finetuning", [True, False])+def test_omitting_the_argument_keeps_the_choice(load_model_run, full_finetuning):+ t, _ = load_model_run("none", full_finetuning = full_finetuning)++ assert t.prepare_model_for_training(use_lora = not full_finetuning)++ assert t.model._unsloth_gradient_checkpointing is False+ assert t._use_gradient_checkpointing is False+++def _trainer_calls_to(method):+ worker = _BACKEND / "core" / "training" / "worker.py"+ tree = ast.parse(worker.read_text(encoding = "utf-8"))+ return [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == method+ and getattr(node.func.value, "id", None) == "trainer"+ ]+++def _reads_the_memory_tab(keyword):+ """Old configs lack the key, so the ``"unsloth"`` fallback is part of the contract."""+ node = keyword.value+ if not (isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "get"):+ return False+ if getattr(node.func.value, "id", None) != "config":+ return False+ return [getattr(a, "value", None) for a in node.args] == [+ "gradient_checkpointing",+ "unsloth",+ ]+++ ("method", "expected"),+ [("prepare_model_for_training", 3), ("load_model", 2)],+)+def test_the_worker_forwards_the_setting_on_every_call(method, expected):+ """An omitted or literal argument ignores the Memory tab."""+ calls = _trainer_calls_to(method)++ assert (+ len(calls) >= expected+ ), f"expected at least {expected} trainer.{method} calls, found {len(calls)}"+ unforwarded = [+ node.lineno+ for node in calls+ if not any(+ k.arg == "use_gradient_checkpointing" and _reads_the_memory_tab(k)+ for k in node.keywords+ )+ ]+ assert not unforwarded, (+ f"worker.py lines {unforwarded} call {method} without forwarding the config's "+ "gradient_checkpointing; that run ignores the Memory tab and checkpoints anyway. "+ "Rewriting that expression means teaching this check the new shape, not dropping it."+ )diff --git a/unsloth_cli/commands/train.py b/unsloth_cli/commands/train.pyindex 2175cf0eccd..4993988c539 100644--- a/unsloth_cli/commands/train.py+++ b/unsloth_cli/commands/train.py@@ -124,7 +124,9 @@ def train(model_name = cfg.model,max_seq_length = cfg.training.max_seq_length,load_in_4bit = cfg.training.load_in_4bit if use_lora else False,+ full_finetuning = not use_lora,hf_token = hf_token,+ use_gradient_checkpointing = cfg.training.gradient_checkpointing,):typer.echo("Model load failed", err = True)raise typer.Exit(code = 1)