Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

save_pretrained_gguf: stop rewriting a reused local checkpoint's tokenizer with a BOS-stripped template (#14095)

Leo Borcherding5aa5f7bb83e9
diff --git a/tests/saving/test_export_dispatch.py b/tests/saving/test_export_dispatch.py
index 1652a74546d..4abf75bb54b 100644
--- a/tests/saving/test_export_dispatch.py
+++ b/tests/saving/test_export_dispatch.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import inspect
import os
+import shutil
import pytest
@@ -178,7 +179,7 @@ def test_non_peft_gguf_uses_checkpoint_as_input_not_output(
def _save_to_gguf(**kwargs):
seen.update(kwargs)
output = tmp_path / "export" / "model_gguf" / "model.Q8_0.gguf"
- output.parent.mkdir()
+ output.parent.mkdir(exist_ok = True)
output.write_bytes(b"GGUF")
return [str(output)], True, False
@@ -192,10 +193,12 @@ def test_non_peft_gguf_uses_checkpoint_as_input_not_output(
quantization_method = "q8_0",
)
- assert seen["model_directory"] == str(checkpoint)
+ staged = seen["model_directory"]
+ assert os.path.dirname(staged) == f"{requested}_gguf"
assert seen["gguf_directory"] == f"{requested}_gguf"
assert result["gguf_directory"] == f"{requested}_gguf"
- assert tokenizer.saved_to == [str(checkpoint)]
+ assert tokenizer.saved_to == [staged]
+ assert not os.path.exists(staged)
@pytest.mark.parametrize(
@@ -574,3 +577,220 @@ def test_every_converter_child_env_goes_through_the_token_boundary():
assert not offenders, (
"these build a child env without applying the token boundary: " + ", ".join(offenders)
)
+
+
+def test_non_peft_gguf_leaves_the_users_checkpoint_as_it_was(monkeypatch, tmp_path):
+ """The BOS-stripped template and the sentencepiece fix are for the converter only: written into
+ the loaded checkpoint, they changed how that folder prompts in transformers afterwards."""
+ checkpoint = tmp_path / "checkpoint"
+ checkpoint.mkdir()
+ (checkpoint / "tokenizer_config.json").write_text('{"chat_template": "{{ bos_token }}x"}')
+ (checkpoint / "model.safetensors").write_bytes(b"WEIGHTS")
+ before = {p.name: p.read_bytes() for p in checkpoint.iterdir()}
+ model = _FakeModel()
+ model.config = type(
+ "cfg",
+ (),
+ {
+ "_name_or_path": str(checkpoint),
+ "architectures": ["LlamaForCausalLM"],
+ "model_type": "llama",
+ },
+ )()
+
+ class StrippingTokenizer(_FakeTokenizer):
+ def save_pretrained(self, path):
+ super().save_pretrained(path)
+ with open(os.path.join(path, "tokenizer_config.json"), "w") as f:
+ f.write('{"chat_template": "x"}')
+
+ converter_saw = {}
+
+ def _save_to_gguf(**kwargs):
+ folder = kwargs["model_directory"]
+ converter_saw.update(
+ {name: open(os.path.join(folder, name), "rb").read() for name in os.listdir(folder)}
+ )
+ output = tmp_path / "export" / "model_gguf" / "model.Q8_0.gguf"
+ output.parent.mkdir(exist_ok = True)
+ output.write_bytes(b"GGUF")
+ return [str(output)], True, False
+
+ (tmp_path / "export").mkdir()
+ monkeypatch.setattr(save_mod, "_is_vlm", lambda _model: False)
+ monkeypatch.setattr(save_mod, "_is_gpt_oss", lambda _model: False)
+ monkeypatch.setattr(save_mod, "fix_tokenizer_bos_token", lambda _tokenizer: (True, None))
+ monkeypatch.setattr(save_mod, "_resolve_imatrix_file", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(save_mod, "dtype_from_config", lambda _config: save_mod.torch.float16)
+ monkeypatch.setattr(save_mod, "create_ollama_modelfile", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(save_mod, "save_to_gguf", _save_to_gguf)
+
+ save_mod.unsloth_save_pretrained_gguf(
+ model,
+ str(tmp_path / "export" / "model"),
+ tokenizer = StrippingTokenizer(),
+ quantization_method = "q8_0",
+ )
+
+ assert converter_saw["tokenizer_config.json"] == b'{"chat_template": "x"}'
+ assert converter_saw["model.safetensors"] == b"WEIGHTS"
+ assert {p.name: p.read_bytes() for p in checkpoint.iterdir()} == before
+
+
[email protected]("fails_at", ["tokenizer_save", "quantization_method"])
+def test_non_peft_gguf_removes_the_staged_copy_when_export_fails_before_converting(
+ monkeypatch, tmp_path, fails_at
+):
+ checkpoint = tmp_path / "checkpoint"
+ checkpoint.mkdir()
+ (checkpoint / "model.safetensors").write_bytes(b"WEIGHTS")
+ model = _FakeModel()
+ model.config = type(
+ "cfg",
+ (),
+ {
+ "_name_or_path": str(checkpoint),
+ "architectures": ["LlamaForCausalLM"],
+ "model_type": "llama",
+ },
+ )()
+
+ class BrokenTokenizer(_FakeTokenizer):
+ def save_pretrained(self, path):
+ raise OSError("tokenizer save failed")
+
+ monkeypatch.setattr(save_mod, "_is_vlm", lambda _model: False)
+ monkeypatch.setattr(save_mod, "_is_gpt_oss", lambda _model: False)
+ monkeypatch.setattr(save_mod, "fix_tokenizer_bos_token", lambda _tokenizer: (False, None))
+ monkeypatch.setattr(save_mod, "_resolve_imatrix_file", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(save_mod, "dtype_from_config", lambda _config: save_mod.torch.float16)
+ monkeypatch.setattr(save_mod, "save_to_gguf", lambda **kw: pytest.fail("converter reached"))
+
+ requested = tmp_path / "export" / "model"
+ (tmp_path / "export").mkdir()
+ with pytest.raises((OSError, TypeError)):
+ save_mod.unsloth_save_pretrained_gguf(
+ model,
+ str(requested),
+ tokenizer = BrokenTokenizer() if fails_at == "tokenizer_save" else _FakeTokenizer(),
+ quantization_method = "q8_0" if fails_at == "tokenizer_save" else 8,
+ )
+
+ gguf_directory = f"{requested}_gguf"
+ leftovers = os.listdir(gguf_directory) if os.path.isdir(gguf_directory) else []
+ assert not [name for name in leftovers if name.startswith(".unsloth_gguf_source_")]
+ assert os.listdir(checkpoint) == ["model.safetensors"]
+
+
+def test_staged_checkpoint_links_weights_and_big_files_but_copies_the_tokenizer(
+ monkeypatch, tmp_path
+):
+ checkpoint = tmp_path / "checkpoint"
+ checkpoint.mkdir()
+ (checkpoint / "model.safetensors").write_bytes(b"WEIGHTS")
+ (checkpoint / "tf_model.h5").write_bytes(b"x" * 64)
+ (checkpoint / "tokenizer_config.json").write_text("{}")
+ (checkpoint / "tokenizer.bin").write_bytes(b"VOCAB")
+ monkeypatch.setattr(save_mod, "_GGUF_LINK_MIN_BYTES", 32)
+
+ staged = save_mod._stage_reused_checkpoint(str(checkpoint), str(tmp_path / "out_gguf"))
+ try:
+ for name in ("model.safetensors", "tf_model.h5"):
+ assert os.path.samefile(os.path.join(staged, name), checkpoint / name)
+ for name in ("tokenizer_config.json", "tokenizer.bin"):
+ assert not os.path.samefile(os.path.join(staged, name), checkpoint / name)
+ shutil.copyfile(checkpoint / "tokenizer.bin", os.path.join(staged, "tokenizer.bin"))
+ with open(os.path.join(staged, "tokenizer_config.json"), "w") as f:
+ f.write("rewritten")
+ assert (checkpoint / "tokenizer_config.json").read_text() == "{}"
+ finally:
+ save_mod._drop_staged_checkpoint(staged)
+ assert not os.path.exists(staged)
+
+
+def test_staged_checkpoint_copies_tokenizer_subfolders_and_resolves_a_relative_source(
+ monkeypatch, tmp_path
+):
+ checkpoint = tmp_path / "checkpoint"
+ (checkpoint / "additional_chat_templates").mkdir(parents = True)
+ (checkpoint / "additional_chat_templates" / "tool_use.jinja").write_text("{{ bos_token }}")
+ (checkpoint / "model.safetensors").write_bytes(b"WEIGHTS")
+ monkeypatch.chdir(tmp_path)
+
+ def _cross_device(*_args):
+ raise OSError("EXDEV")
+
+ # The symlink fallback, as for a GGUF folder on another filesystem.
+ monkeypatch.setattr(save_mod.os, "link", _cross_device)
+ staged = save_mod._stage_reused_checkpoint("checkpoint", str(tmp_path / "out_gguf"))
+ try:
+ assert open(os.path.join(staged, "model.safetensors"), "rb").read() == b"WEIGHTS"
+ with open(os.path.join(staged, "additional_chat_templates", "tool_use.jinja"), "w") as f:
+ f.write("stripped")
+ template = checkpoint / "additional_chat_templates" / "tool_use.jinja"
+ assert template.read_text() == "{{ bos_token }}"
+ finally:
+ save_mod._drop_staged_checkpoint(staged)
+
+
[email protected]("links", ["hard", "symbolic", "none"])
+def test_preflight_prices_a_staged_copy_only_when_nothing_can_link(monkeypatch, tmp_path, links):
+ checkpoint = tmp_path / "checkpoint"
+ checkpoint.mkdir()
+ (checkpoint / "model.safetensors").write_bytes(b"W" * 100)
+ (checkpoint / "tokenizer.json").write_text("{}")
+ model = _FakeModel()
+ model.config = type("cfg", (), {"_name_or_path": str(checkpoint)})()
+ monkeypatch.setattr(save_mod, "_gguf_reuses_loaded_checkpoint", lambda *_args: True)
+
+ def _refused(*_args, **_kwargs):
+ raise OSError("not supported here")
+
+ if links != "hard":
+ monkeypatch.setattr(save_mod.os, "link", _refused)
+ if links == "none":
+ monkeypatch.setattr(save_mod.os, "symlink", _refused)
+ copied = save_mod._staged_checkpoint_copy_bytes(model, str(tmp_path / "out" / "model"))
+ assert copied == (100 if links == "none" else 0)
+ assert not list(tmp_path.rglob(".unsloth_gguf_link_probe_*"))
+
+
+def test_a_failed_staging_restores_the_callers_chat_template(monkeypatch, tmp_path):
+ checkpoint = tmp_path / "checkpoint"
+ checkpoint.mkdir()
+ (checkpoint / "model.safetensors").write_bytes(b"WEIGHTS")
+ model = _FakeModel()
+ model.config = type(
+ "cfg",
+ (),
+ {
+ "_name_or_path": str(checkpoint),
+ "architectures": ["LlamaForCausalLM"],
+ "model_type": "llama",
+ },
+ )()
+ tokenizer = _FakeTokenizer()
+ tokenizer.chat_template = "{{ bos_token }}x"
+
+ def _strip(tok):
+ tok.chat_template = "x"
+ return True, None
+
+ def _no_room(*_args):
+ raise OSError("cannot create the GGUF folder")
+
+ monkeypatch.setattr(save_mod, "_is_vlm", lambda _model: False)
+ monkeypatch.setattr(save_mod, "_is_gpt_oss", lambda _model: False)
+ monkeypatch.setattr(save_mod, "fix_tokenizer_bos_token", _strip)
+ monkeypatch.setattr(save_mod, "_resolve_imatrix_file", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(save_mod, "_stage_reused_checkpoint", _no_room)
+ monkeypatch.setattr(save_mod, "save_to_gguf", lambda **kw: pytest.fail("converter reached"))
+ (tmp_path / "export").mkdir()
+ with pytest.raises(OSError):
+ save_mod.unsloth_save_pretrained_gguf(
+ model,
+ str(tmp_path / "export" / "model"),
+ tokenizer = tokenizer,
+ quantization_method = "q8_0",
+ )
+ assert tokenizer.chat_template == "{{ bos_token }}x"
diff --git a/tests/saving/test_gguf_disk_preflight.py b/tests/saving/test_gguf_disk_preflight.py
index 4816839e5b8..cb8a3870e87 100644
--- a/tests/saving/test_gguf_disk_preflight.py
+++ b/tests/saving/test_gguf_disk_preflight.py
@@ -2573,9 +2573,9 @@ class TestTheFallbackFollowsTheReusedCheckpoint:
def __init__(self, directory):
self.config = type("cfg", (), {"_name_or_path": directory})()
- def test_the_reused_checkpoint_is_the_input_folder(self, tmp_path):
+ def test_the_reused_checkpoint_is_staged_inside_the_gguf_folder(self, tmp_path):
model = self._NonPeftFromDisk(str(tmp_path))
- assert S._gguf_model_input_directory(model, "model") == str(tmp_path)
+ assert S._gguf_model_input_directory(model, "model") == "model_gguf"
def test_a_merged_model_still_reads_the_output(self, tmp_path, monkeypatch):
"""A PEFT merge writes the checkpoint into `save_directory` and converts it."""
@@ -2589,8 +2589,8 @@ class TestTheFallbackFollowsTheReusedCheckpoint:
def test_a_model_with_no_config_reads_the_output(self):
assert S._gguf_model_input_directory(_FakeModel(), "model") == "model"
- def test_the_probe_measures_the_checkpoints_filesystem(self, tmp_path, monkeypatch):
- """End to end: the refusal names the checkpoint, not the requested output."""
+ def test_the_probe_measures_the_staged_inputs_filesystem(self, tmp_path, monkeypatch):
+ """End to end: the refusal names the GGUF folder holding the staged copy, never the checkpoint."""
checkpoint = tmp_path / "base"
checkpoint.mkdir()
model = self._NonPeftFromDisk(str(checkpoint))
@@ -2603,7 +2603,7 @@ class TestTheFallbackFollowsTheReusedCheckpoint:
monkeypatch.setattr(
S,
"free_bytes",
- lambda path: 1 * GB if str(path) == str(checkpoint) else 1000 * GB,
+ lambda path: 1 * GB if str(path) == "output_gguf" else 1000 * GB,
)
monkeypatch.setattr(S, "kaggle_tmp_redirect", lambda *a, **k: (a[0], None))
monkeypatch.setattr(
@@ -2617,8 +2617,8 @@ class TestTheFallbackFollowsTheReusedCheckpoint:
model, "output", "q4_k_m", first_conversion = "bf16", needs_merge = False
)
message = str(error.value)
- assert str(checkpoint) in message
- assert "1.0GB free" in message and "20.0GB" in message
+ assert "output_gguf" in message and str(checkpoint) not in message
+ assert "1.0GB free" in message
def test_the_converter_really_falls_back_to_its_input_folder(self):
"""The redirect is only worth following while llama_cpp does this."""
diff --git a/tests/saving/test_push_to_hub_gguf_single_commit.py b/tests/saving/test_push_to_hub_gguf_single_commit.py
index e16f3aeb07c..e0c648d1f4e 100644
--- a/tests/saving/test_push_to_hub_gguf_single_commit.py
+++ b/tests/saving/test_push_to_hub_gguf_single_commit.py
@@ -20,7 +20,7 @@ class _RecordingApi:
def record(*args, **kwargs):
if name == "create_commit":
readme = next(op for op in kwargs["operations"] if op.path_in_repo == "README.md")
- kwargs["readme"] = open(readme.path_or_fileobj, encoding = "utf-8").read()
+ kwargs["readme"] = readme.path_or_fileobj.decode("utf-8")
_RecordingApi.calls.append((name, kwargs))
if name == "create_commit" and kwargs.get("create_pr"):
return SimpleNamespace(pr_url = "https://huggingface.co/u/my-model/discussions/1")
@@ -121,3 +121,21 @@ def test_datasets_go_in_the_same_commit(push, monkeypatch, datasets):
expected = [datasets] if isinstance(datasets, str) else datasets
assert card.data.datasets == expected
assert "gguf" in card.data.tags
+
+
+def test_readme_is_not_written_into_the_save_directory(push, tmp_path, monkeypatch):
+ """For a non-PEFT model the returned save directory is the user's own checkpoint."""
+ checkpoint = tmp_path / "checkpoint"
+ checkpoint.mkdir()
+ (checkpoint / "README.md").write_text("my model card")
+
+ def _convert_reusing_checkpoint(**kwargs):
+ result = _fake_convert(**kwargs)
+ result["save_directory"] = str(checkpoint)
+ return result
+
+ monkeypatch.setattr(save_mod, "unsloth_save_pretrained_gguf", _convert_reusing_checkpoint)
+ calls = push()
+ commit = next(kw for name, kw in calls if name == "create_commit")
+ assert "Unsloth" in commit["readme"]
+ assert (checkpoint / "README.md").read_text() == "my model card"
diff --git a/tests/test_gguf_disk_headroom.py b/tests/test_gguf_disk_headroom.py
index 5ccf85a88e9..048fa74b7b4 100644
--- a/tests/test_gguf_disk_headroom.py
+++ b/tests/test_gguf_disk_headroom.py
@@ -361,7 +361,8 @@ def test_the_non_peft_branch_marks_the_checkpoint_as_not_disposable(
checkpoint.mkdir()
model = _FakeModel(str(checkpoint))
seen = _run_export(monkeypatch, save_mod, tmp_path, model)
- assert seen["model_directory"] == str(checkpoint)
+ # Converted from a staged copy inside the GGUF folder, still never reclaimable.
+ assert os.path.dirname(seen["model_directory"]) == str(tmp_path / "export" / "model_gguf")
assert seen["merge_is_disposable"] is False
diff --git a/unsloth/save.py b/unsloth/save.py
index 2a5ec462436..c1ba0e0aa37 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -3644,6 +3644,109 @@ def _fallback_checkpoint_extra_bytes(model, state_dict = None):
return 0
+_GGUF_LINKED_WEIGHT_SUFFIXES = (".safetensors", ".bin", ".pt", ".pth", ".npz")
+# Larger than any tokenizer file: other big artifacts (tf_model.h5, an old .gguf) are linked, not copied.
+_GGUF_LINK_MIN_BYTES = 256 * 1024**2
+# Always copied: save_pretrained rewrites them, and a slow tokenizer's shutil.copyfile raises SameFileError on a hard link.
+_GGUF_COPIED_PREFIXES = ("tokenizer", "vocab", "spiece", "sentencepiece", "merges")
+
+
+def _gguf_stage_links(name, size):
+ if name.lower().startswith(_GGUF_COPIED_PREFIXES):
+ return False
+ return name.endswith(_GGUF_LINKED_WEIGHT_SUFFIXES) or size >= _GGUF_LINK_MIN_BYTES
+
+
+def _staged_checkpoint_copy_bytes(
+ model,
+ save_directory,
+ state_dict = None,
+):
+ """Bytes `_stage_reused_checkpoint` will COPY rather than link: every linked file's size when neither a hard link (another filesystem, FAT/exFAT) nor a symlink (Windows without Developer Mode) can be made from the reused checkpoint into the GGUF folder, else zero. Never raises; unmeasurable is zero."""
+ import tempfile
+ try:
+ if not _gguf_reuses_loaded_checkpoint(model, state_dict):
+ return 0
+ source = os.path.abspath(
+ str(getattr(model, "_unsloth_modelscope_snapshot", None) or model.config._name_or_path)
+ )
+ linked = []
+ for root, _dirs, files in os.walk(source):
+ for name in files:
+ path = os.path.join(root, name)
+ size = os.path.getsize(path)
+ if _gguf_stage_links(name, size):
+ linked.append((path, size))
+ if not linked:
+ return 0
+ probe = os.path.abspath(_gguf_output_directory(save_directory))
+ while not os.path.isdir(probe):
+ parent = os.path.dirname(probe)
+ if parent == probe:
+ return 0
+ probe = parent
+ scratch = tempfile.mkdtemp(prefix = ".unsloth_gguf_link_probe_", dir = probe)
+ try:
+ for link in (os.link, os.symlink):
+ try:
+ link(linked[0][0], os.path.join(scratch, link.__name__))
+ return 0
+ except OSError:
+ continue
+ finally:
+ shutil.rmtree(scratch, ignore_errors = True)
+ return sum(size for _path, size in linked)
+ except Exception:
+ return 0
+
+
+def _stage_reused_checkpoint(source, gguf_directory):
+ """A throwaway copy of the user's checkpoint folder for the converter to read. Weights are
+ linked; everything else is copied, subfolders included, because the tokenizer save (e.g.
+ additional_chat_templates/) and fix_sentencepiece_gguf rewrite files there and a link would
+ carry the write back into the user's checkpoint."""
+ import tempfile
+
+ # A relative source would make every symlink below resolve against the staging folder.
+ source = os.path.abspath(source)
+ os.makedirs(gguf_directory, exist_ok = True)
+ staging = tempfile.mkdtemp(prefix = ".unsloth_gguf_source_", dir = gguf_directory)
+ staging_real = os.path.realpath(staging)
+
+ def _stage(src_dir, dst_dir):
+ for name in os.listdir(src_dir):
+ src = os.path.join(src_dir, name)
+ dst = os.path.join(dst_dir, name)
+ if os.path.isdir(src):
+ if (staging_real + os.sep).startswith(os.path.realpath(src) + os.sep):
+ continue # the GGUF output folder sits inside the checkpoint
+ os.mkdir(dst)
+ _stage(src, dst)
+ elif _gguf_stage_links(name, os.path.getsize(src)):
+ for link in (os.link, os.symlink):
+ try:
+ link(src, dst)
+ break
+ except OSError:
+ continue
+ else:
+ shutil.copy2(src, dst)
+ else:
+ shutil.copy2(src, dst)
+
+ try:
+ _stage(source, staging)
+ except BaseException:
+ shutil.rmtree(staging, ignore_errors = True)
+ raise
+ return staging
+
+
+def _drop_staged_checkpoint(staged_directory):
+ if staged_directory is not None:
+ shutil.rmtree(staged_directory, ignore_errors = True)
+
+
def _gguf_output_directory(save_directory):
"""Where the GGUF files land: a SIBLING of `save_directory`, not a child. One definition for the preflight and for the export itself, so the disk measured cannot drift from the disk written."""
return f"{save_directory}_gguf"
@@ -3707,11 +3810,9 @@ def _gguf_model_input_directory(
save_directory,
state_dict = None,
):
- """The folder the converter reads, which is not always `save_directory`: a reused loaded checkpoint, which `unsloth_save_pretrained_gguf` assigns to `save_directory` before calling `save_to_gguf`. It matters only in the unwritable-CWD fallback, where the intermediate GGUF lands beside the reused checkpoint rather than the requested output, and the two can be on different filesystems."""
+ """The folder the converter reads, which is not always `save_directory`: a reused loaded checkpoint is converted from a staged copy inside the GGUF output folder. It matters only in the unwritable-CWD fallback, where the intermediate GGUF lands beside the converter's input."""
if _gguf_reuses_loaded_checkpoint(model, state_dict):
- return str(
- getattr(model, "_unsloth_modelscope_snapshot", None) or model.config._name_or_path
- )
+ return _gguf_output_directory(save_directory)
return save_directory
@@ -3841,6 +3942,12 @@ def _preflight_gguf_disk(
)
except Exception:
need_sibling = 0
+ # Lands in the `_gguf` sibling with the conversion, so it is charged to both figures.
+ staged_copy = _staged_checkpoint_copy_bytes(model, save_directory, state_dict)
+ need += staged_copy
+ need_with_cache += staged_copy
+ if need_sibling > 0:
+ need_sibling += staged_copy
# The first phase a disposable merge peaks at: merge plus intermediate GGUF, before any quant. quantization_methods = () still prices the intermediate, which must exist to be read.
try:
need_merge_phase = estimate_gguf_export_bytes(
@@ -4378,6 +4485,7 @@ def unsloth_save_pretrained_gguf(
preexisting_weights = None
is_peft_model = isinstance(self, PeftModelForCausalLM) or isinstance(self, PeftModel)
+ staged_directory = None
# The flag holds for both branches that write the weights; the middle branch reuses the user's own checkpoint and clears it, whatever the caller asked for.
if is_peft_model:
@@ -4400,12 +4508,21 @@ def unsloth_save_pretrained_gguf(
print(
f"Unsloth: Model is not a PEFT model. Using existing checkpoint at {original_path}"
)
- save_directory = original_path
+ # A staged copy, so the BOS-stripped tokenizer never lands in the user's checkpoint.
+ try:
+ staged_directory = _stage_reused_checkpoint(original_path, gguf_directory)
+ save_directory = staged_directory
+ # Persist tokenizer fixes (BOS stripping) so the GGUF converter reads the corrected chat template.
+ if tokenizer is not None:
+ tokenizer.save_pretrained(save_directory)
+ except BaseException:
+ _drop_staged_checkpoint(staged_directory)
+ # The caller keeps its tokenizer: the restore below is never reached.
+ for _template_owner, _old_template in old_chat_templates:
+ _template_owner.chat_template = _old_template
+ raise
# The user's own checkpoint, not an intermediate: without this a save_pretrained_gguf on a tight disk would delete the model it was handed.
merge_is_disposable = False
- # Persist tokenizer fixes (BOS stripping) so the GGUF converter reads the corrected chat template.
- if tokenizer is not None:
- tokenizer.save_pretrained(save_directory)
else:
print("Unsloth: Model is not a PEFT model. Saving directly without LoRA merge...")
os.makedirs(save_directory, exist_ok = True)
@@ -4422,67 +4539,75 @@ def unsloth_save_pretrained_gguf(
f"{_offloaded_parameter_hint(self)}"
) from e
- if fix_bos_token:
- for _template_owner, _old_template in old_chat_templates:
- _template_owner.chat_template = _old_template
+ # Until the conversion's own finally, an error here would leave the staged copy in the GGUF folder.
+ try:
+ if fix_bos_token:
+ for _template_owner, _old_template in old_chat_templates:
+ _template_owner.chat_template = _old_template
- if is_processor:
- tokenizer = tokenizer.tokenizer
+ if is_processor:
+ tokenizer = tokenizer.tokenizer
- for _ in range(3):
- import gc
- gc.collect()
- clean_gpu_cache()
+ for _ in range(3):
+ import gc
+ gc.collect()
+ clean_gpu_cache()
- try:
- model_dtype = dtype_from_config(self.config)
- model_type = self.config.model_type
- if type(model_dtype) is str:
- assert model_dtype == "float16" or model_dtype == "bfloat16"
- elif model_dtype == torch.float16:
+ try:
+ model_dtype = dtype_from_config(self.config)
+ model_type = self.config.model_type
+ if type(model_dtype) is str:
+ assert model_dtype == "float16" or model_dtype == "bfloat16"
+ elif model_dtype == torch.float16:
+ model_dtype = "float16"
+ elif model_dtype == torch.bfloat16:
+ model_dtype = "bfloat16"
+ else:
+ raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16")
+ except Exception as e:
+ print(f"Unsloth: Could not determine dtype ({e}), defaulting to float16")
model_dtype = "float16"
- elif model_dtype == torch.bfloat16:
- model_dtype = "bfloat16"
- else:
- raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16")
- except Exception as e:
- print(f"Unsloth: Could not determine dtype ({e}), defaulting to float16")
- model_dtype = "float16"
- print("Unsloth: Converting to GGUF format...")
+ print("Unsloth: Converting to GGUF format...")
- quantization_methods = []
- if quantization_method is not None:
- if isinstance(quantization_method, list):
- pass
- elif isinstance(quantization_method, str):
- quantization_method = [
- quantization_method,
- ]
- elif isinstance(quantization_method, tuple):
- quantization_method = list(quantization_method)
- else:
- raise TypeError(
- "Unsloth: quantization_method can only be a string or a list of strings"
- )
- for i, quant_method in enumerate(quantization_method):
- if quant_method is None:
- quant_method = "q8_0"
+ quantization_methods = []
+ if quantization_method is not None:
+ if isinstance(quantization_method, list):
+ pass
+ elif isinstance(quantization_method, str):
+ quantization_method = [
+ quantization_method,
+ ]
+ elif isinstance(quantization_method, tuple):
+ quantization_method = list(quantization_method)
else:
- quant_method = quant_method.lower()
- if quant_method == "not_quantized":
- quant_method = "f16"
- elif quant_method == "fast_quantized":
- quant_method = "q8_0"
- elif quant_method == "quantized":
- quant_method = "q4_k_m"
- quantization_methods.append(quant_method.lower())
+ raise TypeError(
+ "Unsloth: quantization_method can only be a string or a list of strings"
+ )
+ for i, quant_method in enumerate(quantization_method):
+ if quant_method is None:
+ quant_method = "q8_0"
+ else:
+ quant_method = quant_method.lower()
+ if quant_method == "not_quantized":
+ quant_method = "f16"
+ elif quant_method == "fast_quantized":
+ quant_method = "q8_0"
+ elif quant_method == "quantized":
+ quant_method = "q4_k_m"
+ quantization_methods.append(quant_method.lower())
+ except BaseException:
+ _drop_staged_checkpoint(staged_directory)
+ raise
try:
from .tokenizer_utils import fix_sentencepiece_gguf
fix_sentencepiece_gguf(save_directory)
except Exception as e:
logger.warning(f"Unsloth: fix_sentencepiece_gguf skipped ({type(e).__name__}): {e}")
+ except BaseException:
+ _drop_staged_checkpoint(staged_directory)
+ raise
try:
all_file_locations, want_full_precision, is_vlm_update = save_to_gguf(
@@ -4520,6 +4645,10 @@ def unsloth_save_pretrained_gguf(
) from e
else:
raise RuntimeError(f"Unsloth: GGUF conversion failed: {_describe_exception(e)}") from e
+ finally:
+ if staged_directory is not None:
+ _drop_staged_checkpoint(staged_directory)
+ save_directory = original_path
modelfile_location = None
ollama_success = False
@@ -5051,11 +5180,12 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi
'[<img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20made%20with%20love.png" width="200"/>](https://github.com/unslothai/unsloth)\n'
)
- readme_path = os.path.join(actual_save_directory, "README.md")
- with open(readme_path, "w", encoding = "utf-8") as f:
- f.write(readme_content)
-
- operations.append(CommitOperationAdd(path_in_repo = "README.md", path_or_fileobj = readme_path))
+ # Uploaded from memory: for a non-PEFT model the save directory is the user's own checkpoint, whose README this must not overwrite.
+ operations.append(
+ CommitOperationAdd(
+ path_in_repo = "README.md", path_or_fileobj = readme_content.encode("utf-8")
+ )
+ )
commit = api.create_commit(
repo_id = full_repo_id,