Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Export the trained weights when a full fine-tune loaded from a local folder is saved to GGUF (#11601)

Nilayae3077a09f2d
diff --git a/tests/saving/test_export_dispatch.py b/tests/saving/test_export_dispatch.py
index b31258cc488..e0fc1022ee7 100644
--- a/tests/saving/test_export_dispatch.py
+++ b/tests/saving/test_export_dispatch.py
@@ -210,6 +210,78 @@ def test_non_peft_gguf_uses_checkpoint_as_input_not_output(
assert tokenizer.saved_to == [str(checkpoint)]
+ "full_finetuning, state_dict",
+ [
+ (True, None),
+ (True, {"lm_head.weight": "consolidated"}),
+ (False, {"lm_head.weight": "consolidated"}),
+ ],
+)
+def test_gguf_converts_trained_or_supplied_weights_not_source_checkpoint(
+ monkeypatch, tmp_path, full_finetuning, state_dict
+):
+ checkpoint = tmp_path / "checkpoint"
+ checkpoint.mkdir()
+ requested = tmp_path / "export" / "model"
+ requested.parent.mkdir()
+ model = _FakeModel()
+ model.config = type(
+ "cfg",
+ (),
+ {
+ "_name_or_path": str(checkpoint),
+ "architectures": ["LlamaForCausalLM"],
+ "model_type": "llama",
+ },
+ )()
+ model._unsloth_full_finetuning = full_finetuning
+ model.saved_to = []
+ model.save_pretrained = lambda path, **kwargs: model.saved_to.append(
+ (path, kwargs.get("state_dict"))
+ )
+ tokenizer = _FakeTokenizer()
+ seen = {}
+
+ monkeypatch.setattr(save_mod, "_is_vlm", lambda _model: False)
+ monkeypatch.setattr(save_mod, "_is_gpt_oss", lambda _model: False)
+ monkeypatch.setattr(save_mod, "fix_tokenizer_bos_token", lambda _tokenizer: (False, None))
+ monkeypatch.setattr(save_mod, "_resolve_imatrix_file", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(save_mod, "dtype_from_config", lambda _config: save_mod.torch.float16)
+ monkeypatch.setattr(save_mod, "create_ollama_modelfile", lambda *_args, **_kwargs: None)
+ preflight = []
+ monkeypatch.setattr(
+ save_mod,
+ "_preflight_gguf_disk",
+ lambda **kwargs: preflight.append((kwargs["needs_merge"], kwargs["state_dict"]))
+ or (kwargs["save_directory"], True),
+ )
+
+ def _save_to_gguf(**kwargs):
+ seen.update(kwargs)
+ output = tmp_path / "export" / "model_gguf" / "model.F16.gguf"
+ output.parent.mkdir()
+ output.write_bytes(b"GGUF")
+ return [str(output)], True, False
+
+ monkeypatch.setattr(save_mod, "save_to_gguf", _save_to_gguf)
+
+ save_mod.unsloth_save_pretrained_gguf(
+ model,
+ str(requested),
+ tokenizer = tokenizer,
+ quantization_method = "f16",
+ state_dict = state_dict,
+ )
+
+ assert seen["model_directory"] == str(requested)
+ assert model.saved_to == [(str(requested), state_dict)]
+ assert tokenizer.saved_to == [str(requested)]
+ assert os.listdir(checkpoint) == []
+ assert preflight == [(True, state_dict)]
+ assert save_mod._gguf_model_input_directory(model, str(requested), state_dict) == str(requested)
+
+
# The above rejection points users at push_to_hub_gguf(save_method='lora'), so that path has to work; it is only ever
# exercised here.
diff --git a/tests/saving/test_gguf_disk_preflight.py b/tests/saving/test_gguf_disk_preflight.py
index 4e615900edb..5e3346b9a80 100644
--- a/tests/saving/test_gguf_disk_preflight.py
+++ b/tests/saving/test_gguf_disk_preflight.py
@@ -1326,6 +1326,14 @@ class TestFallbackCheckpointDtype:
model.config = type("cfg", (), {"_name_or_path": str(tmp_path)})()
assert S._fallback_checkpoint_extra_bytes(model) == 0
+ def test_a_supplied_state_dict_is_what_gets_sized(self, sized_from_parameters, tmp_path):
+ torch = sized_from_parameters
+ model = torch.nn.Linear(8, 8, dtype = torch.bfloat16)
+ model.config = type("cfg", (), {"_name_or_path": str(tmp_path)})()
+ state_dict = {k: v.float() for k, v in model.state_dict().items()}
+ n_parameters = sum(p.numel() for p in model.parameters())
+ assert S._fallback_checkpoint_extra_bytes(model, state_dict) == n_parameters * 2
+
def test_an_unmeasurable_model_adds_nothing(self):
assert S._fallback_checkpoint_extra_bytes(_FakeModel()) == 0
@@ -1937,7 +1945,7 @@ class TestEachFilesystemIsChargedForWhatItHolds:
source = inspect.getsource(S.unsloth_save_pretrained_gguf)
fallback = source.split("Saving directly without LoRA merge")[1]
- assert "self.save_pretrained(save_directory)" in fallback
+ assert "self.save_pretrained(save_directory" in fallback
assert "merge_and_overwrite_lora" not in fallback
def test_an_export_writing_no_merge_is_not_charged_the_reserve(self, split):
@@ -3397,7 +3405,7 @@ class TestTheGgufPreflightIsToldTheModelDtype:
monkeypatch.setattr(
S,
"_fallback_checkpoint_extra_bytes",
- lambda model: asked.append(None) or 0,
+ lambda model, *_: asked.append(None) or 0,
)
wrong = estimate(quantization_methods = ["f16", "q4_k_m"], first_conversion = "f16")
right = estimate(quantization_methods = ["f16", "q4_k_m"], first_conversion = "bf16")
@@ -3472,7 +3480,7 @@ class TestThePrewarmedCacheIsChargedToItsOwnFilesystem:
monkeypatch.setattr(
S, "_filesystem_id", lambda path: devices.get(str(path), state["cache_device"])
)
- monkeypatch.setattr(S, "_fallback_checkpoint_extra_bytes", lambda model: 0)
+ monkeypatch.setattr(S, "_fallback_checkpoint_extra_bytes", lambda model, *_: 0)
monkeypatch.setattr(S, "IS_KAGGLE_ENVIRONMENT", False)
monkeypatch.setattr(S, "IS_COLAB_ENVIRONMENT", False)
monkeypatch.delenv("UNSLOTH_DISK_PREFLIGHT", raising = False)
@@ -3565,7 +3573,7 @@ class TestAnUnsupportedBF16IsNormalizedBeforeEstimating:
monkeypatch.setattr(S, "estimate_gguf_export_bytes", estimate)
monkeypatch.setattr(S, "free_bytes", lambda path: 1000 * GB)
monkeypatch.setattr(S, "kaggle_tmp_redirect", lambda *a, **k: ("model", None))
- monkeypatch.setattr(S, "_fallback_checkpoint_extra_bytes", lambda model: 0)
+ monkeypatch.setattr(S, "_fallback_checkpoint_extra_bytes", lambda model, *_: 0)
monkeypatch.delenv("UNSLOTH_DISK_PREFLIGHT", raising = False)
return asked
@@ -3666,7 +3674,7 @@ class TestTheCacheIsChargedOnTheConversionFilesystem:
lambda path: state["conversion_free"] if str(path) == self.WORK else 1000 * GB,
)
monkeypatch.setattr(S, "kaggle_tmp_redirect", lambda *a, **k: ("model", None))
- monkeypatch.setattr(S, "_fallback_checkpoint_extra_bytes", lambda model: 0)
+ monkeypatch.setattr(S, "_fallback_checkpoint_extra_bytes", lambda model, *_: 0)
monkeypatch.setattr(S, "_gguf_conversion_directory", lambda directory: self.WORK)
monkeypatch.setattr(S, "_hub_cache_directory", lambda: self.CACHE)
monkeypatch.setattr(S, "IS_KAGGLE_ENVIRONMENT", False)
@@ -3696,6 +3704,23 @@ class TestTheCacheIsChargedOnTheConversionFilesystem:
state.update(conversion_free = self.BASE + self.CONVERSION)
assert self._preflight() == ("model", True)
+ def test_a_supplied_state_dict_reaches_both_sizing_helpers(self, state, monkeypatch):
+ state.update(conversion_free = self.BASE + self.CONVERSION)
+ state_dict = {"lm_head.weight": "trained"}
+ seen = []
+ monkeypatch.setattr(
+ S, "_fallback_checkpoint_extra_bytes", lambda model, sd: seen.append(sd) or 0
+ )
+ monkeypatch.setattr(
+ S,
+ "_gguf_model_input_directory",
+ lambda model, directory, sd: seen.append(sd) or directory,
+ )
+ S._preflight_gguf_disk(
+ _FakeModel(), "model", "f32", first_conversion = "f32", state_dict = state_dict
+ )
+ assert seen == [state_dict, state_dict]
+
def test_a_cache_elsewhere_is_not_charged_here(self, state):
"""The premise reversed: only a cache on THIS disk costs the pre-warm."""
state.update(cache_device = 3)
@@ -3768,7 +3793,7 @@ class TestACacheOnAnotherFilesystemIsNotChargedToTheOutputDisk:
S, "_filesystem_id", lambda path: devices.get(str(path), state["cache_device"])
)
monkeypatch.setattr(S, "_hub_cache_directory", lambda: self.ELSEWHERE)
- monkeypatch.setattr(S, "_fallback_checkpoint_extra_bytes", lambda model: 0)
+ monkeypatch.setattr(S, "_fallback_checkpoint_extra_bytes", lambda model, *_: 0)
monkeypatch.setattr(S, "IS_KAGGLE_ENVIRONMENT", False)
monkeypatch.setattr(S, "IS_COLAB_ENVIRONMENT", False)
monkeypatch.delenv("UNSLOTH_DISK_PREFLIGHT", raising = False)
diff --git a/tests/test_grpo_autocast_per_trainer.py b/tests/test_grpo_autocast_per_trainer.py
index f280c935e4d..01209b5ec97 100644
--- a/tests/test_grpo_autocast_per_trainer.py
+++ b/tests/test_grpo_autocast_per_trainer.py
@@ -331,7 +331,11 @@ def test_an_unstamped_model_still_takes_the_environments_finetuning_mode():
def test_the_loaders_stamp_the_full_finetuning_answer_on_the_model():
- for rel in ("unsloth/models/loader.py", "unsloth/models/vision.py"):
+ for rel in (
+ "unsloth/models/loader.py",
+ "unsloth/models/vision.py",
+ "unsloth/models/sentence_transformer.py",
+ ):
src = (REPO_ROOT / rel).read_text(encoding = "utf-8")
assert "_mark_full_finetuning(" in src, rel
diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py
index 931eb10343d..f63b855c02f 100644
--- a/unsloth/models/sentence_transformer.py
+++ b/unsloth/models/sentence_transformer.py
@@ -23,6 +23,7 @@ from ._utils import (
resolve_model_class,
resolve_encoder_attention_implementation,
maybe_prefetch_hf_snapshot,
+ _mark_full_finetuning,
)
import inspect
import json
@@ -1622,6 +1623,7 @@ class FastSentenceTransformer(FastModel):
)
st_model._unsloth_fast_encoder = True
+ _mark_full_finetuning(st_model[0].auto_model, full_finetuning)
st_model._compile_mode = compile_mode
st_model._dtype = dtype
st_model._load_in_4bit = load_in_4bit
diff --git a/unsloth/save.py b/unsloth/save.py
index 48e2b2213bd..14958d28126 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -3248,27 +3248,32 @@ def _imatrix_is_enabled(imatrix_file):
return imatrix_file is not None and imatrix_file is not False
-def _gguf_writes_16bit_checkpoint(model):
- """Whether a GGUF export writes a full 16-bit checkpoint before converting. A PEFT model is merged into one. A non-PEFT model reuses an existing checkpoint when `_name_or_path` names a directory, and otherwise falls back to `save_pretrained`, which writes the same two bytes per parameter; sizing that fallback at zero is what lets an export pass the preflight and then fill the disk. A module-level helper rather than a local, because the caller snapshots `locals()` into the kwargs of `unsloth_generic_save`."""
+def _gguf_reuses_loaded_checkpoint(model, state_dict = None):
+ """Whether a non-PEFT GGUF export converts the folder the model was loaded from. Only when the weights to export are that folder's: full finetuning trains them in place, and a `state_dict` replaces them."""
if isinstance(model, (PeftModel, PeftModelForCausalLM)):
- return True
+ return False
+ if state_dict is not None or getattr(model, "_unsloth_full_finetuning", False):
+ return False
name_or_path = getattr(getattr(model, "config", None), "_name_or_path", None)
try:
- return not (name_or_path and os.path.isdir(str(name_or_path)))
+ return bool(name_or_path and os.path.isdir(str(name_or_path)))
except Exception:
- return True
+ return False
-def _fallback_checkpoint_extra_bytes(model):
- """Bytes the non-PEFT fallback checkpoint costs ON TOP of the 16-bit estimate. `estimate_gguf_export_bytes` budgets two bytes per logical parameter, which is what a LoRA merge writes, but the fallback calls `self.save_pretrained` with no cast, so a model loaded with `dtype = torch.float32` writes four. Measured from the parameters' real storage so a mixed-dtype model is not priced off its largest tensor, and clamped at zero: this can only ask for more room, never less."""
+def _gguf_writes_16bit_checkpoint(model, state_dict = None):
+ """Whether a GGUF export writes a full 16-bit checkpoint before converting: a PEFT merge, or the `save_pretrained` fallback of a non-PEFT model that cannot reuse its loaded checkpoint. Sizing that fallback at zero is what lets an export pass the preflight and then fill the disk. A module-level helper rather than a local, because the caller snapshots `locals()` into the kwargs of `unsloth_generic_save`."""
+ return not _gguf_reuses_loaded_checkpoint(model, state_dict)
+
+
+def _fallback_checkpoint_extra_bytes(model, state_dict = None):
+ """Bytes the non-PEFT fallback checkpoint costs ON TOP of the 16-bit estimate. `estimate_gguf_export_bytes` budgets two bytes per logical parameter, which is what a LoRA merge writes, but the fallback calls `self.save_pretrained` with no cast, so a model loaded with `dtype = torch.float32` writes four. Measured from the supplied `state_dict`, else the parameters' real storage, so a mixed-dtype model is not priced off its largest tensor, and clamped at zero: this can only ask for more room, never less."""
if isinstance(model, (PeftModel, PeftModelForCausalLM)):
return 0
- if not _gguf_writes_16bit_checkpoint(model):
+ if not _gguf_writes_16bit_checkpoint(model, state_dict):
return 0
try:
- actual = 0
- for parameter in model.parameters():
- actual += parameter.numel() * parameter.element_size()
+ actual = _full_model_checkpoint_bytes(model, state_dict)
return max(0, actual - model_16bit_bytes(model))
except Exception:
return 0
@@ -3332,16 +3337,14 @@ def _gguf_conversion_directory(model_directory):
return cwd if _directory_is_writable(cwd) else model_directory
-def _gguf_model_input_directory(model, save_directory):
- """The folder the converter reads, which is not always `save_directory`. A non-PEFT model whose `_name_or_path` names a directory is converted from that checkpoint, which `unsloth_save_pretrained_gguf` assigns to `save_directory` before calling `save_to_gguf`; the same condition `_gguf_writes_16bit_checkpoint` uses. It matters only in the unwritable-CWD fallback, where the intermediate GGUF lands beside the reused checkpoint rather than the requested output, and the two can be on different filesystems."""
- if isinstance(model, (PeftModel, PeftModelForCausalLM)):
- return save_directory
- name_or_path = getattr(getattr(model, "config", None), "_name_or_path", None)
- try:
- if name_or_path and os.path.isdir(str(name_or_path)):
- return str(name_or_path)
- except Exception:
- pass
+def _gguf_model_input_directory(
+ model,
+ save_directory,
+ state_dict = None,
+):
+ """The folder the converter reads, which is not always `save_directory`: a reused loaded checkpoint, which `unsloth_save_pretrained_gguf` assigns to `save_directory` before calling `save_to_gguf`. It matters only in the unwritable-CWD fallback, where the intermediate GGUF lands beside the reused checkpoint rather than the requested output, and the two can be on different filesystems."""
+ if _gguf_reuses_loaded_checkpoint(model, state_dict):
+ return str(model.config._name_or_path)
return save_directory
@@ -3406,6 +3409,7 @@ def _preflight_gguf_disk(
has_imatrix = False,
needs_merge = True,
merge_is_disposable = False,
+ state_dict = None,
):
"""Refuse a GGUF export that cannot fit, before it writes a single byte. Returns `(directory, prewarm_ok)`. `directory` differs from the input only when a Kaggle kernel's tiny working directory was swapped for the large /tmp overlay, and then it says so once. `prewarm_ok` is False when the export fits only without pre-warming the Hugging Face cache with the base model. A GGUF export peaks at more than "the model, twice": it caches the full-precision base, writes the 16-bit HF merge, then an intermediate GGUF at the source dtype, then each requested quant, with every earlier artefact still on disk. Gemma4 (26B A4B) Vision, Gemma4 (31B) Vision and Qwen3 32B each trained, ran inference and completed `merged_16bit` before dying partway through a GGUF shard, because the check in front of them had sized the job at two copies. Dropping the pre-warm is tried before refusing, because the merge downloads what it needs either way. `merge_is_disposable` says the merge is this export's own throwaway, so `_free_merge_if_disk_is_tight` may delete it once the intermediate GGUF exists and the peak becomes the larger of two phases rather than their sum; defaults off, which is what every caller got before. Never blocks on a guess: an unmeasurable model or disk returns the directory untouched. UNSLOTH_DISK_PREFLIGHT=0 disables."""
if os.environ.get("UNSLOTH_DISK_PREFLIGHT", "1").strip().lower() in (
@@ -3457,7 +3461,7 @@ def _preflight_gguf_disk(
)
# The estimate prices the checkpoint at 2 bytes per parameter, but the non-PEFT fallback writes the model's own dtype, so an fp32 model needs the difference. Zero for a 16-bit model.
if need > 0 and needs_merge:
- extra = _fallback_checkpoint_extra_bytes(model)
+ extra = _fallback_checkpoint_extra_bytes(model, state_dict)
need += extra
need_with_cache += extra
# The same estimate without the checkpoint: the `_gguf` sibling's intermediate plus every quant. Used only when that sibling sits on a smaller filesystem. Its own try, so an estimator that cannot answer leaves the main guard standing.
@@ -3547,7 +3551,7 @@ def _preflight_gguf_disk(
)
# Resolved before the split is priced, because where the conversion lands decides which filesystem it is charged to.
conversion_directory = _gguf_conversion_directory(
- _gguf_model_input_directory(model, save_directory)
+ _gguf_model_input_directory(model, save_directory, state_dict)
)
# Cleared when the cache shares a filesystem with room for the export but not a cached base too: dropping the optional half beats failing. The message travels with the flag, since more than one filesystem can set it and each has to name the one it measured.
@@ -3893,9 +3897,10 @@ def unsloth_save_pretrained_gguf(
# Resolved rather than left at the default, which says "f16" while the export asks the config.
model_dtype = _gguf_source_dtype(self),
has_imatrix = _imatrix_is_enabled(imatrix_file),
- needs_merge = _gguf_writes_16bit_checkpoint(self),
+ needs_merge = _gguf_writes_16bit_checkpoint(self, state_dict),
# The same flag save_to_gguf reclaims on. Where a non-PEFT model reuses its own checkpoint the flag is cleared below on the same condition, so the two cannot disagree.
merge_is_disposable = merge_is_disposable,
+ state_dict = state_dict,
)
arguments = dict(locals())
@@ -3971,9 +3976,9 @@ def unsloth_save_pretrained_gguf(
f"{_offloaded_parameter_hint(self)}"
) from e
else:
- # Non-PEFT model: the checkpoint already exists, so point save_to_gguf at the original path instead of re-saving into a temp subdir.
+ # Non-PEFT model: convert the loaded checkpoint in place when it still holds the weights to export.
original_path = getattr(self.config, "_name_or_path", None)
- if original_path and os.path.isdir(original_path):
+ if _gguf_reuses_loaded_checkpoint(self, state_dict):
print(
f"Unsloth: Model is not a PEFT model. Using existing checkpoint at {original_path}"
)
@@ -3988,7 +3993,7 @@ def unsloth_save_pretrained_gguf(
os.makedirs(save_directory, exist_ok = True)
# `gguf_directory` can point anywhere, and freeing bytes on one filesystem does nothing for a quantize pass writing to another: without this the merge could be deleted for a destination it cannot help, data gone and the export still out of space.
try:
- self.save_pretrained(save_directory)
+ self.save_pretrained(save_directory, state_dict = state_dict)
if tokenizer is not None:
tokenizer.save_pretrained(save_directory)
except Exception as e: