Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
save_pretrained_gguf: stop rewriting a reused local checkpoint's tokenizer with a BOS-stripped template (#14095)
Leo Borcherding5aa5f7bb83e9
diff --git a/tests/saving/test_export_dispatch.py b/tests/saving/test_export_dispatch.pyindex 1652a74546d..4abf75bb54b 100644--- a/tests/saving/test_export_dispatch.py+++ b/tests/saving/test_export_dispatch.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport inspectimport os+import shutilimport pytest@@ -178,7 +179,7 @@ def test_non_peft_gguf_uses_checkpoint_as_input_not_output(def _save_to_gguf(**kwargs):seen.update(kwargs)output = tmp_path / "export" / "model_gguf" / "model.Q8_0.gguf"- output.parent.mkdir()+ output.parent.mkdir(exist_ok = True)output.write_bytes(b"GGUF")return [str(output)], True, False@@ -192,10 +193,12 @@ def test_non_peft_gguf_uses_checkpoint_as_input_not_output(quantization_method = "q8_0",)- assert seen["model_directory"] == str(checkpoint)+ staged = seen["model_directory"]+ assert os.path.dirname(staged) == f"{requested}_gguf"assert seen["gguf_directory"] == f"{requested}_gguf"assert result["gguf_directory"] == f"{requested}_gguf"- assert tokenizer.saved_to == [str(checkpoint)]+ assert tokenizer.saved_to == [staged]+ assert not os.path.exists(staged)@pytest.mark.parametrize(@@ -574,3 +577,220 @@ def test_every_converter_child_env_goes_through_the_token_boundary():assert not offenders, ("these build a child env without applying the token boundary: " + ", ".join(offenders))+++def test_non_peft_gguf_leaves_the_users_checkpoint_as_it_was(monkeypatch, tmp_path):+ """The BOS-stripped template and the sentencepiece fix are for the converter only: written into+ the loaded checkpoint, they changed how that folder prompts in transformers afterwards."""+ checkpoint = tmp_path / "checkpoint"+ checkpoint.mkdir()+ (checkpoint / "tokenizer_config.json").write_text('{"chat_template": "{{ bos_token }}x"}')+ (checkpoint / "model.safetensors").write_bytes(b"WEIGHTS")+ before = {p.name: p.read_bytes() for p in checkpoint.iterdir()}+ model = _FakeModel()+ model.config = type(+ "cfg",+ (),+ {+ "_name_or_path": str(checkpoint),+ "architectures": ["LlamaForCausalLM"],+ "model_type": "llama",+ },+ )()++ class StrippingTokenizer(_FakeTokenizer):+ def save_pretrained(self, path):+ super().save_pretrained(path)+ with open(os.path.join(path, "tokenizer_config.json"), "w") as f:+ f.write('{"chat_template": "x"}')++ converter_saw = {}++ def _save_to_gguf(**kwargs):+ folder = kwargs["model_directory"]+ converter_saw.update(+ {name: open(os.path.join(folder, name), "rb").read() for name in os.listdir(folder)}+ )+ output = tmp_path / "export" / "model_gguf" / "model.Q8_0.gguf"+ output.parent.mkdir(exist_ok = True)+ output.write_bytes(b"GGUF")+ return [str(output)], True, False++ (tmp_path / "export").mkdir()+ monkeypatch.setattr(save_mod, "_is_vlm", lambda _model: False)+ monkeypatch.setattr(save_mod, "_is_gpt_oss", lambda _model: False)+ monkeypatch.setattr(save_mod, "fix_tokenizer_bos_token", lambda _tokenizer: (True, None))+ monkeypatch.setattr(save_mod, "_resolve_imatrix_file", lambda *_args, **_kwargs: None)+ monkeypatch.setattr(save_mod, "dtype_from_config", lambda _config: save_mod.torch.float16)+ monkeypatch.setattr(save_mod, "create_ollama_modelfile", lambda *_args, **_kwargs: None)+ monkeypatch.setattr(save_mod, "save_to_gguf", _save_to_gguf)++ save_mod.unsloth_save_pretrained_gguf(+ model,+ str(tmp_path / "export" / "model"),+ tokenizer = StrippingTokenizer(),+ quantization_method = "q8_0",+ )++ assert converter_saw["tokenizer_config.json"] == b'{"chat_template": "x"}'+ assert converter_saw["model.safetensors"] == b"WEIGHTS"+ assert {p.name: p.read_bytes() for p in checkpoint.iterdir()} == before++[email protected]("fails_at", ["tokenizer_save", "quantization_method"])+def test_non_peft_gguf_removes_the_staged_copy_when_export_fails_before_converting(+ monkeypatch, tmp_path, fails_at+):+ checkpoint = tmp_path / "checkpoint"+ checkpoint.mkdir()+ (checkpoint / "model.safetensors").write_bytes(b"WEIGHTS")+ model = _FakeModel()+ model.config = type(+ "cfg",+ (),+ {+ "_name_or_path": str(checkpoint),+ "architectures": ["LlamaForCausalLM"],+ "model_type": "llama",+ },+ )()++ class BrokenTokenizer(_FakeTokenizer):+ def save_pretrained(self, path):+ raise OSError("tokenizer save failed")++ monkeypatch.setattr(save_mod, "_is_vlm", lambda _model: False)+ monkeypatch.setattr(save_mod, "_is_gpt_oss", lambda _model: False)+ monkeypatch.setattr(save_mod, "fix_tokenizer_bos_token", lambda _tokenizer: (False, None))+ monkeypatch.setattr(save_mod, "_resolve_imatrix_file", lambda *_args, **_kwargs: None)+ monkeypatch.setattr(save_mod, "dtype_from_config", lambda _config: save_mod.torch.float16)+ monkeypatch.setattr(save_mod, "save_to_gguf", lambda **kw: pytest.fail("converter reached"))++ requested = tmp_path / "export" / "model"+ (tmp_path / "export").mkdir()+ with pytest.raises((OSError, TypeError)):+ save_mod.unsloth_save_pretrained_gguf(+ model,+ str(requested),+ tokenizer = BrokenTokenizer() if fails_at == "tokenizer_save" else _FakeTokenizer(),+ quantization_method = "q8_0" if fails_at == "tokenizer_save" else 8,+ )++ gguf_directory = f"{requested}_gguf"+ leftovers = os.listdir(gguf_directory) if os.path.isdir(gguf_directory) else []+ assert not [name for name in leftovers if name.startswith(".unsloth_gguf_source_")]+ assert os.listdir(checkpoint) == ["model.safetensors"]+++def test_staged_checkpoint_links_weights_and_big_files_but_copies_the_tokenizer(+ monkeypatch, tmp_path+):+ checkpoint = tmp_path / "checkpoint"+ checkpoint.mkdir()+ (checkpoint / "model.safetensors").write_bytes(b"WEIGHTS")+ (checkpoint / "tf_model.h5").write_bytes(b"x" * 64)+ (checkpoint / "tokenizer_config.json").write_text("{}")+ (checkpoint / "tokenizer.bin").write_bytes(b"VOCAB")+ monkeypatch.setattr(save_mod, "_GGUF_LINK_MIN_BYTES", 32)++ staged = save_mod._stage_reused_checkpoint(str(checkpoint), str(tmp_path / "out_gguf"))+ try:+ for name in ("model.safetensors", "tf_model.h5"):+ assert os.path.samefile(os.path.join(staged, name), checkpoint / name)+ for name in ("tokenizer_config.json", "tokenizer.bin"):+ assert not os.path.samefile(os.path.join(staged, name), checkpoint / name)+ shutil.copyfile(checkpoint / "tokenizer.bin", os.path.join(staged, "tokenizer.bin"))+ with open(os.path.join(staged, "tokenizer_config.json"), "w") as f:+ f.write("rewritten")+ assert (checkpoint / "tokenizer_config.json").read_text() == "{}"+ finally:+ save_mod._drop_staged_checkpoint(staged)+ assert not os.path.exists(staged)+++def test_staged_checkpoint_copies_tokenizer_subfolders_and_resolves_a_relative_source(+ monkeypatch, tmp_path+):+ checkpoint = tmp_path / "checkpoint"+ (checkpoint / "additional_chat_templates").mkdir(parents = True)+ (checkpoint / "additional_chat_templates" / "tool_use.jinja").write_text("{{ bos_token }}")+ (checkpoint / "model.safetensors").write_bytes(b"WEIGHTS")+ monkeypatch.chdir(tmp_path)++ def _cross_device(*_args):+ raise OSError("EXDEV")++ # The symlink fallback, as for a GGUF folder on another filesystem.+ monkeypatch.setattr(save_mod.os, "link", _cross_device)+ staged = save_mod._stage_reused_checkpoint("checkpoint", str(tmp_path / "out_gguf"))+ try:+ assert open(os.path.join(staged, "model.safetensors"), "rb").read() == b"WEIGHTS"+ with open(os.path.join(staged, "additional_chat_templates", "tool_use.jinja"), "w") as f:+ f.write("stripped")+ template = checkpoint / "additional_chat_templates" / "tool_use.jinja"+ assert template.read_text() == "{{ bos_token }}"+ finally:+ save_mod._drop_staged_checkpoint(staged)++[email protected]("links", ["hard", "symbolic", "none"])+def test_preflight_prices_a_staged_copy_only_when_nothing_can_link(monkeypatch, tmp_path, links):+ checkpoint = tmp_path / "checkpoint"+ checkpoint.mkdir()+ (checkpoint / "model.safetensors").write_bytes(b"W" * 100)+ (checkpoint / "tokenizer.json").write_text("{}")+ model = _FakeModel()+ model.config = type("cfg", (), {"_name_or_path": str(checkpoint)})()+ monkeypatch.setattr(save_mod, "_gguf_reuses_loaded_checkpoint", lambda *_args: True)++ def _refused(*_args, **_kwargs):+ raise OSError("not supported here")++ if links != "hard":+ monkeypatch.setattr(save_mod.os, "link", _refused)+ if links == "none":+ monkeypatch.setattr(save_mod.os, "symlink", _refused)+ copied = save_mod._staged_checkpoint_copy_bytes(model, str(tmp_path / "out" / "model"))+ assert copied == (100 if links == "none" else 0)+ assert not list(tmp_path.rglob(".unsloth_gguf_link_probe_*"))+++def test_a_failed_staging_restores_the_callers_chat_template(monkeypatch, tmp_path):+ checkpoint = tmp_path / "checkpoint"+ checkpoint.mkdir()+ (checkpoint / "model.safetensors").write_bytes(b"WEIGHTS")+ model = _FakeModel()+ model.config = type(+ "cfg",+ (),+ {+ "_name_or_path": str(checkpoint),+ "architectures": ["LlamaForCausalLM"],+ "model_type": "llama",+ },+ )()+ tokenizer = _FakeTokenizer()+ tokenizer.chat_template = "{{ bos_token }}x"++ def _strip(tok):+ tok.chat_template = "x"+ return True, None++ def _no_room(*_args):+ raise OSError("cannot create the GGUF folder")++ monkeypatch.setattr(save_mod, "_is_vlm", lambda _model: False)+ monkeypatch.setattr(save_mod, "_is_gpt_oss", lambda _model: False)+ monkeypatch.setattr(save_mod, "fix_tokenizer_bos_token", _strip)+ monkeypatch.setattr(save_mod, "_resolve_imatrix_file", lambda *_args, **_kwargs: None)+ monkeypatch.setattr(save_mod, "_stage_reused_checkpoint", _no_room)+ monkeypatch.setattr(save_mod, "save_to_gguf", lambda **kw: pytest.fail("converter reached"))+ (tmp_path / "export").mkdir()+ with pytest.raises(OSError):+ save_mod.unsloth_save_pretrained_gguf(+ model,+ str(tmp_path / "export" / "model"),+ tokenizer = tokenizer,+ quantization_method = "q8_0",+ )+ assert tokenizer.chat_template == "{{ bos_token }}x"diff --git a/tests/saving/test_gguf_disk_preflight.py b/tests/saving/test_gguf_disk_preflight.pyindex 4816839e5b8..cb8a3870e87 100644--- a/tests/saving/test_gguf_disk_preflight.py+++ b/tests/saving/test_gguf_disk_preflight.py@@ -2573,9 +2573,9 @@ class TestTheFallbackFollowsTheReusedCheckpoint:def __init__(self, directory):self.config = type("cfg", (), {"_name_or_path": directory})()- def test_the_reused_checkpoint_is_the_input_folder(self, tmp_path):+ def test_the_reused_checkpoint_is_staged_inside_the_gguf_folder(self, tmp_path):model = self._NonPeftFromDisk(str(tmp_path))- assert S._gguf_model_input_directory(model, "model") == str(tmp_path)+ assert S._gguf_model_input_directory(model, "model") == "model_gguf"def test_a_merged_model_still_reads_the_output(self, tmp_path, monkeypatch):"""A PEFT merge writes the checkpoint into `save_directory` and converts it."""@@ -2589,8 +2589,8 @@ class TestTheFallbackFollowsTheReusedCheckpoint:def test_a_model_with_no_config_reads_the_output(self):assert S._gguf_model_input_directory(_FakeModel(), "model") == "model"- def test_the_probe_measures_the_checkpoints_filesystem(self, tmp_path, monkeypatch):- """End to end: the refusal names the checkpoint, not the requested output."""+ def test_the_probe_measures_the_staged_inputs_filesystem(self, tmp_path, monkeypatch):+ """End to end: the refusal names the GGUF folder holding the staged copy, never the checkpoint."""checkpoint = tmp_path / "base"checkpoint.mkdir()model = self._NonPeftFromDisk(str(checkpoint))@@ -2603,7 +2603,7 @@ class TestTheFallbackFollowsTheReusedCheckpoint:monkeypatch.setattr(S,"free_bytes",- lambda path: 1 * GB if str(path) == str(checkpoint) else 1000 * GB,+ lambda path: 1 * GB if str(path) == "output_gguf" else 1000 * GB,)monkeypatch.setattr(S, "kaggle_tmp_redirect", lambda *a, **k: (a[0], None))monkeypatch.setattr(@@ -2617,8 +2617,8 @@ class TestTheFallbackFollowsTheReusedCheckpoint:model, "output", "q4_k_m", first_conversion = "bf16", needs_merge = False)message = str(error.value)- assert str(checkpoint) in message- assert "1.0GB free" in message and "20.0GB" in message+ assert "output_gguf" in message and str(checkpoint) not in message+ assert "1.0GB free" in messagedef test_the_converter_really_falls_back_to_its_input_folder(self):"""The redirect is only worth following while llama_cpp does this."""diff --git a/tests/saving/test_push_to_hub_gguf_single_commit.py b/tests/saving/test_push_to_hub_gguf_single_commit.pyindex e16f3aeb07c..e0c648d1f4e 100644--- a/tests/saving/test_push_to_hub_gguf_single_commit.py+++ b/tests/saving/test_push_to_hub_gguf_single_commit.py@@ -20,7 +20,7 @@ class _RecordingApi:def record(*args, **kwargs):if name == "create_commit":readme = next(op for op in kwargs["operations"] if op.path_in_repo == "README.md")- kwargs["readme"] = open(readme.path_or_fileobj, encoding = "utf-8").read()+ kwargs["readme"] = readme.path_or_fileobj.decode("utf-8")_RecordingApi.calls.append((name, kwargs))if name == "create_commit" and kwargs.get("create_pr"):return SimpleNamespace(pr_url = "https://huggingface.co/u/my-model/discussions/1")@@ -121,3 +121,21 @@ def test_datasets_go_in_the_same_commit(push, monkeypatch, datasets):expected = [datasets] if isinstance(datasets, str) else datasetsassert card.data.datasets == expectedassert "gguf" in card.data.tags+++def test_readme_is_not_written_into_the_save_directory(push, tmp_path, monkeypatch):+ """For a non-PEFT model the returned save directory is the user's own checkpoint."""+ checkpoint = tmp_path / "checkpoint"+ checkpoint.mkdir()+ (checkpoint / "README.md").write_text("my model card")++ def _convert_reusing_checkpoint(**kwargs):+ result = _fake_convert(**kwargs)+ result["save_directory"] = str(checkpoint)+ return result++ monkeypatch.setattr(save_mod, "unsloth_save_pretrained_gguf", _convert_reusing_checkpoint)+ calls = push()+ commit = next(kw for name, kw in calls if name == "create_commit")+ assert "Unsloth" in commit["readme"]+ assert (checkpoint / "README.md").read_text() == "my model card"diff --git a/tests/test_gguf_disk_headroom.py b/tests/test_gguf_disk_headroom.pyindex 5ccf85a88e9..048fa74b7b4 100644--- a/tests/test_gguf_disk_headroom.py+++ b/tests/test_gguf_disk_headroom.py@@ -361,7 +361,8 @@ def test_the_non_peft_branch_marks_the_checkpoint_as_not_disposable(checkpoint.mkdir()model = _FakeModel(str(checkpoint))seen = _run_export(monkeypatch, save_mod, tmp_path, model)- assert seen["model_directory"] == str(checkpoint)+ # Converted from a staged copy inside the GGUF folder, still never reclaimable.+ assert os.path.dirname(seen["model_directory"]) == str(tmp_path / "export" / "model_gguf")assert seen["merge_is_disposable"] is Falsediff --git a/unsloth/save.py b/unsloth/save.pyindex 2a5ec462436..c1ba0e0aa37 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -3644,6 +3644,109 @@ def _fallback_checkpoint_extra_bytes(model, state_dict = None):return 0+_GGUF_LINKED_WEIGHT_SUFFIXES = (".safetensors", ".bin", ".pt", ".pth", ".npz")+# Larger than any tokenizer file: other big artifacts (tf_model.h5, an old .gguf) are linked, not copied.+_GGUF_LINK_MIN_BYTES = 256 * 1024**2+# Always copied: save_pretrained rewrites them, and a slow tokenizer's shutil.copyfile raises SameFileError on a hard link.+_GGUF_COPIED_PREFIXES = ("tokenizer", "vocab", "spiece", "sentencepiece", "merges")+++def _gguf_stage_links(name, size):+ if name.lower().startswith(_GGUF_COPIED_PREFIXES):+ return False+ return name.endswith(_GGUF_LINKED_WEIGHT_SUFFIXES) or size >= _GGUF_LINK_MIN_BYTES+++def _staged_checkpoint_copy_bytes(+ model,+ save_directory,+ state_dict = None,+):+ """Bytes `_stage_reused_checkpoint` will COPY rather than link: every linked file's size when neither a hard link (another filesystem, FAT/exFAT) nor a symlink (Windows without Developer Mode) can be made from the reused checkpoint into the GGUF folder, else zero. Never raises; unmeasurable is zero."""+ import tempfile+ try:+ if not _gguf_reuses_loaded_checkpoint(model, state_dict):+ return 0+ source = os.path.abspath(+ str(getattr(model, "_unsloth_modelscope_snapshot", None) or model.config._name_or_path)+ )+ linked = []+ for root, _dirs, files in os.walk(source):+ for name in files:+ path = os.path.join(root, name)+ size = os.path.getsize(path)+ if _gguf_stage_links(name, size):+ linked.append((path, size))+ if not linked:+ return 0+ probe = os.path.abspath(_gguf_output_directory(save_directory))+ while not os.path.isdir(probe):+ parent = os.path.dirname(probe)+ if parent == probe:+ return 0+ probe = parent+ scratch = tempfile.mkdtemp(prefix = ".unsloth_gguf_link_probe_", dir = probe)+ try:+ for link in (os.link, os.symlink):+ try:+ link(linked[0][0], os.path.join(scratch, link.__name__))+ return 0+ except OSError:+ continue+ finally:+ shutil.rmtree(scratch, ignore_errors = True)+ return sum(size for _path, size in linked)+ except Exception:+ return 0+++def _stage_reused_checkpoint(source, gguf_directory):+ """A throwaway copy of the user's checkpoint folder for the converter to read. Weights are+ linked; everything else is copied, subfolders included, because the tokenizer save (e.g.+ additional_chat_templates/) and fix_sentencepiece_gguf rewrite files there and a link would+ carry the write back into the user's checkpoint."""+ import tempfile++ # A relative source would make every symlink below resolve against the staging folder.+ source = os.path.abspath(source)+ os.makedirs(gguf_directory, exist_ok = True)+ staging = tempfile.mkdtemp(prefix = ".unsloth_gguf_source_", dir = gguf_directory)+ staging_real = os.path.realpath(staging)++ def _stage(src_dir, dst_dir):+ for name in os.listdir(src_dir):+ src = os.path.join(src_dir, name)+ dst = os.path.join(dst_dir, name)+ if os.path.isdir(src):+ if (staging_real + os.sep).startswith(os.path.realpath(src) + os.sep):+ continue # the GGUF output folder sits inside the checkpoint+ os.mkdir(dst)+ _stage(src, dst)+ elif _gguf_stage_links(name, os.path.getsize(src)):+ for link in (os.link, os.symlink):+ try:+ link(src, dst)+ break+ except OSError:+ continue+ else:+ shutil.copy2(src, dst)+ else:+ shutil.copy2(src, dst)++ try:+ _stage(source, staging)+ except BaseException:+ shutil.rmtree(staging, ignore_errors = True)+ raise+ return staging+++def _drop_staged_checkpoint(staged_directory):+ if staged_directory is not None:+ shutil.rmtree(staged_directory, ignore_errors = True)++def _gguf_output_directory(save_directory):"""Where the GGUF files land: a SIBLING of `save_directory`, not a child. One definition for the preflight and for the export itself, so the disk measured cannot drift from the disk written."""return f"{save_directory}_gguf"@@ -3707,11 +3810,9 @@ def _gguf_model_input_directory(save_directory,state_dict = None,):- """The folder the converter reads, which is not always `save_directory`: a reused loaded checkpoint, which `unsloth_save_pretrained_gguf` assigns to `save_directory` before calling `save_to_gguf`. It matters only in the unwritable-CWD fallback, where the intermediate GGUF lands beside the reused checkpoint rather than the requested output, and the two can be on different filesystems."""+ """The folder the converter reads, which is not always `save_directory`: a reused loaded checkpoint is converted from a staged copy inside the GGUF output folder. It matters only in the unwritable-CWD fallback, where the intermediate GGUF lands beside the converter's input."""if _gguf_reuses_loaded_checkpoint(model, state_dict):- return str(- getattr(model, "_unsloth_modelscope_snapshot", None) or model.config._name_or_path- )+ return _gguf_output_directory(save_directory)return save_directory@@ -3841,6 +3942,12 @@ def _preflight_gguf_disk()except Exception:need_sibling = 0+ # Lands in the `_gguf` sibling with the conversion, so it is charged to both figures.+ staged_copy = _staged_checkpoint_copy_bytes(model, save_directory, state_dict)+ need += staged_copy+ need_with_cache += staged_copy+ if need_sibling > 0:+ need_sibling += staged_copy# The first phase a disposable merge peaks at: merge plus intermediate GGUF, before any quant. quantization_methods = () still prices the intermediate, which must exist to be read.try:need_merge_phase = estimate_gguf_export_bytes(@@ -4378,6 +4485,7 @@ def unsloth_save_pretrained_gguf(preexisting_weights = Noneis_peft_model = isinstance(self, PeftModelForCausalLM) or isinstance(self, PeftModel)+ staged_directory = None# The flag holds for both branches that write the weights; the middle branch reuses the user's own checkpoint and clears it, whatever the caller asked for.if is_peft_model:@@ -4400,12 +4508,21 @@ def unsloth_save_pretrained_gguf(print(f"Unsloth: Model is not a PEFT model. Using existing checkpoint at {original_path}")- save_directory = original_path+ # A staged copy, so the BOS-stripped tokenizer never lands in the user's checkpoint.+ try:+ staged_directory = _stage_reused_checkpoint(original_path, gguf_directory)+ save_directory = staged_directory+ # Persist tokenizer fixes (BOS stripping) so the GGUF converter reads the corrected chat template.+ if tokenizer is not None:+ tokenizer.save_pretrained(save_directory)+ except BaseException:+ _drop_staged_checkpoint(staged_directory)+ # The caller keeps its tokenizer: the restore below is never reached.+ for _template_owner, _old_template in old_chat_templates:+ _template_owner.chat_template = _old_template+ raise# The user's own checkpoint, not an intermediate: without this a save_pretrained_gguf on a tight disk would delete the model it was handed.merge_is_disposable = False- # Persist tokenizer fixes (BOS stripping) so the GGUF converter reads the corrected chat template.- if tokenizer is not None:- tokenizer.save_pretrained(save_directory)else:print("Unsloth: Model is not a PEFT model. Saving directly without LoRA merge...")os.makedirs(save_directory, exist_ok = True)@@ -4422,67 +4539,75 @@ def unsloth_save_pretrained_gguf(f"{_offloaded_parameter_hint(self)}") from e- if fix_bos_token:- for _template_owner, _old_template in old_chat_templates:- _template_owner.chat_template = _old_template+ # Until the conversion's own finally, an error here would leave the staged copy in the GGUF folder.+ try:+ if fix_bos_token:+ for _template_owner, _old_template in old_chat_templates:+ _template_owner.chat_template = _old_template- if is_processor:- tokenizer = tokenizer.tokenizer+ if is_processor:+ tokenizer = tokenizer.tokenizer- for _ in range(3):- import gc- gc.collect()- clean_gpu_cache()+ for _ in range(3):+ import gc+ gc.collect()+ clean_gpu_cache()- try:- model_dtype = dtype_from_config(self.config)- model_type = self.config.model_type- if type(model_dtype) is str:- assert model_dtype == "float16" or model_dtype == "bfloat16"- elif model_dtype == torch.float16:+ try:+ model_dtype = dtype_from_config(self.config)+ model_type = self.config.model_type+ if type(model_dtype) is str:+ assert model_dtype == "float16" or model_dtype == "bfloat16"+ elif model_dtype == torch.float16:+ model_dtype = "float16"+ elif model_dtype == torch.bfloat16:+ model_dtype = "bfloat16"+ else:+ raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16")+ except Exception as e:+ print(f"Unsloth: Could not determine dtype ({e}), defaulting to float16")model_dtype = "float16"- elif model_dtype == torch.bfloat16:- model_dtype = "bfloat16"- else:- raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16")- except Exception as e:- print(f"Unsloth: Could not determine dtype ({e}), defaulting to float16")- model_dtype = "float16"- print("Unsloth: Converting to GGUF format...")+ print("Unsloth: Converting to GGUF format...")- quantization_methods = []- if quantization_method is not None:- if isinstance(quantization_method, list):- pass- elif isinstance(quantization_method, str):- quantization_method = [- quantization_method,- ]- elif isinstance(quantization_method, tuple):- quantization_method = list(quantization_method)- else:- raise TypeError(- "Unsloth: quantization_method can only be a string or a list of strings"- )- for i, quant_method in enumerate(quantization_method):- if quant_method is None:- quant_method = "q8_0"+ quantization_methods = []+ if quantization_method is not None:+ if isinstance(quantization_method, list):+ pass+ elif isinstance(quantization_method, str):+ quantization_method = [+ quantization_method,+ ]+ elif isinstance(quantization_method, tuple):+ quantization_method = list(quantization_method)else:- quant_method = quant_method.lower()- if quant_method == "not_quantized":- quant_method = "f16"- elif quant_method == "fast_quantized":- quant_method = "q8_0"- elif quant_method == "quantized":- quant_method = "q4_k_m"- quantization_methods.append(quant_method.lower())+ raise TypeError(+ "Unsloth: quantization_method can only be a string or a list of strings"+ )+ for i, quant_method in enumerate(quantization_method):+ if quant_method is None:+ quant_method = "q8_0"+ else:+ quant_method = quant_method.lower()+ if quant_method == "not_quantized":+ quant_method = "f16"+ elif quant_method == "fast_quantized":+ quant_method = "q8_0"+ elif quant_method == "quantized":+ quant_method = "q4_k_m"+ quantization_methods.append(quant_method.lower())+ except BaseException:+ _drop_staged_checkpoint(staged_directory)+ raisetry:from .tokenizer_utils import fix_sentencepiece_gguffix_sentencepiece_gguf(save_directory)except Exception as e:logger.warning(f"Unsloth: fix_sentencepiece_gguf skipped ({type(e).__name__}): {e}")+ except BaseException:+ _drop_staged_checkpoint(staged_directory)+ raisetry:all_file_locations, want_full_precision, is_vlm_update = save_to_gguf(@@ -4520,6 +4645,10 @@ def unsloth_save_pretrained_gguf() from eelse:raise RuntimeError(f"Unsloth: GGUF conversion failed: {_describe_exception(e)}") from e+ finally:+ if staged_directory is not None:+ _drop_staged_checkpoint(staged_directory)+ save_directory = original_pathmodelfile_location = Noneollama_success = False@@ -5051,11 +5180,12 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi'[<img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20made%20with%20love.png" width="200"/>](https://github.com/unslothai/unsloth)\n')- readme_path = os.path.join(actual_save_directory, "README.md")- with open(readme_path, "w", encoding = "utf-8") as f:- f.write(readme_content)-- operations.append(CommitOperationAdd(path_in_repo = "README.md", path_or_fileobj = readme_path))+ # Uploaded from memory: for a non-PEFT model the save directory is the user's own checkpoint, whose README this must not overwrite.+ operations.append(+ CommitOperationAdd(+ path_in_repo = "README.md", path_or_fileobj = readme_content.encode("utf-8")+ )+ )commit = api.create_commit(repo_id = full_repo_id,