Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Restore facts the comment trim cut from the unsloth package (#10328)
Daniel Hanca42c015f8e1
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex a8897a0cf16..b4d09653b5f 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -81,9 +81,11 @@ propagate_torchao_fix_to_subprocesses()check_transformers_dependency_versions()check_fbgemm_gpu_version()torchvision_compatibility_check()-# Must precede `import unsloth_zoo`: its temporary_patches reach transformers.processing_utils ->-# audio_utils -> torchaudio, so a torchaudio raising at extension init takes the whole unsloth-# import down before the late guard would have neutralised it.+# Ahead of `import unsloth_zoo` below, deliberately not down with the other import fixes: unsloth_zoo's+# temporary_patches reach transformers.processing_utils, which imports transformers.audio_utils, which+# imports torchaudio, so a torchaudio that raises at extension init takes the whole unsloth import down at+# that `import unsloth_zoo` line, long before the late block would have neutralised it. Measured:+# Kaggle-Muse_Glimmer_(30B)-GRPO died at cell 4 with the guard present but not yet run.disable_torchaudio_if_cuda_mismatched()fix_diffusers_warnings()fix_huggingface_hub()diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex c161babd18a..fb6926a884c 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -3199,8 +3199,10 @@ _PEFT_CONVERSION_SYMBOLS = {),}-# Of those, the ones peft calls rather than merely imports. An inert body on a REAL-# transformers would be called and answer wrongly, so those get a placeholder that says so.+# Of those, the ones peft calls rather than merely imports. The stubs are deliberately inert: on+# transformers <5 the whole module is ours and peft's converter never runs. An inert body on a REAL+# transformers is a different matter, since peft would call it and get a wrong answer, so those get a+# placeholder that says what is wrong instead._PEFT_CONVERSION_RUNTIME_SYMBOLS = frozenset(("transformers.core_model_loading.dot_natural_key",diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 312baa66b53..242270099b3 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -142,7 +142,8 @@ try:# If an earlier `import bitsandbytes` died inside __init__, CPython evicts only the parent from# sys.modules and keeps its submodules, so this retry re-executes __init__ without rebinding- # bnb.functional. `import x.y as z` reads sys.modules directly and survives that.+ # bnb.functional. `import x.y as z` reads sys.modules directly and survives that, plain attribute+ # access does not.import bitsandbytes.functional as bnb_functionalexcept Exception:# device_type.py already degrades to 16bit/full finetuning when bnb is missing (gfx906, whosediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 9b61396fbe6..d1d38afda1d 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -272,8 +272,9 @@ def _unsloth_install_pretrain_detector(model):return modelmarker = getattr(model, "_unsloth_pretrain_marker", None)if isinstance(marker, dict):- # A live hook is already recording: keep it and do NOT clear seen, since a grad-enabled probe- # may already have flagged the poisoned cache and a re-entrant call must not erase that.+ # A live hook is already recording: keep it (no duplicates) and do NOT clear seen, since a+ # grad-enabled probe may already have flagged the poisoned cache and a re-entrant+ # get_peft_model / patch_peft_model call must not erase that before train() resets it.if "hook" in marker:return model# Marker exists but its hook was torn down, so reinstall fresh and reset seen.diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex ac169165e71..7df2605565d 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -240,7 +240,8 @@ def planner_quantization_kwargs(from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULESexcept Exception:# Built on every quantized load, planning or not, so an older unsloth_zoo must not turn a 4bit load- # into an ImportError. One without the shared list predates the planner that consumes it.+ # into an ImportError. One without the shared list predates the planner that consumes it, so this+ # plan was going to decline anyway.return kwargskwargs["llm_int8_skip_modules"] = SKIP_QUANTIZATION_MODULES + list(extra_skip_modules or [])return kwargsdiff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex f339146a129..2de3fa219cc 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -1519,7 +1519,8 @@ def _wrap_sft_evaluate_cap(trainer_cls):finally:container[key] = previous# evaluate() with no split falls back to the one stored on the trainer, which a caller can install- # after construction, where the constructor's cap can no longer see it.+ # after construction, where the constructor's cap can no longer see it. Every eval during+ # training comes through here too.stored = getattr(self, keyword, None) if keyword == "eval_dataset" else Noneif stored is None:return original(self, *args, **kwargs)@@ -1842,7 +1843,9 @@ def _install_grpo_hidden_states_forward_wrapper(model):raisereturn forward_without_hidden_states()# The signature advertised the parameter but the forward refuses the value: retry without the- # minimisation, through the same fallback, rather than lose hidden states over it.+ # minimisation, through the same fallback, rather than lose hidden states over it: a forward that+ # splats **kwargs into a sub-module can refuse the logits limiter and the hidden states one after+ # the other.forward_kwargs.pop(logits_kwarg, None)logits_kwarg = Nonetry:@@ -2432,14 +2435,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" _cols = getattr(_ds, 'column_names', None)\n"" if isinstance(_cols, dict):\n"" _cols = [_c for _v in _cols.values() for _c in (_v or [])]\n"- # A packed split is out: TRL skips truncation when packing, and cutting input_ids under a+ # A packed split is out: TRL skips truncation when packing+ # (`if args.max_length is not None and not packing`), and cutting input_ids under a# seq_lengths that still describes the old row is worse than not cutting." if 'seq_lengths' in (_cols or ()): return False\n"" return bool(_cols) and 'input_ids' in _cols\n"" except Exception:\n"" return False\n"# Read a row BACK: every predicate above is a guess about what the dataset will do, and this is the- # one check that observes it.+ # one check that observes it. A split with no input_ids is raw, so prep tokenizes it with+ # the cap and it is fine." _unsloth_cap = args.max_length\n"# TRL slices [-max_length:] for keep_end, which callers use when the completion sits at the tail of# a long prompt; always keeping the prefix trained on the wrong half of every row.@@ -2450,8 +2455,13 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" _unsloth_known_mode = _unsloth_truncation_mode in ('keep_start', 'keep_end')\n"" _unsloth_slice = slice(-_unsloth_cap, None) if _unsloth_keep_end else slice(None, _unsloth_cap)\n"# Resolved outside the truncation block, since the fallback reads it even under- # skip_prepare_dataset. eval_packing is separate from packing, so packing=False with- # eval_packing=True lands here and TRL packs the eval split instead of truncating it.+ # skip_prepare_dataset. eval_packing is separate from packing:+ # packing = args.packing if args.eval_packing is None else args.eval_packing+ # (sft_trainer.py), so packing=False with eval_packing=True reaches this branch, which is+ # gated on `not args.packing`. TRL then packs the eval split instead of truncating it, and+ # every strategy owns the overflow: `wrapped` concatenates the stream before chunking,+ # `bfd_split` splits an overlength example into more chunks. Cutting rows at the cap first+ # throws that away." _unsloth_eval_packing = getattr(args, 'packing', False) if getattr(args, 'eval_packing', None) is None else getattr(args, 'eval_packing')\n"" _unsloth_completion_only = getattr(args, 'completion_only_loss', None)\n"# Column names first, a row only if free: on a one-shot stream this probe ate the first@@ -2481,9 +2491,10 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):# Parked on args so the late evaluate()/predict() cap reads the SAME value: it resolves from# the train schema, and disagreeing with the collator leaves an all -100 row in." args._unsloth_completion_only_loss = _unsloth_completion_only\n"- # EVERY row, not the first: a short row 0 before a long row 5000 read as within the cap. A- # map-style split is read in full; a stream cannot be rewound, so a bounded prefix is all- # there is and the check says so.+ # EVERY row, not the first: a short row 0 before a long row 5000 read as within the cap, and+ # in the fallback branch nothing downstream truncates it. A map-style split is read in+ # full; a stream cannot be rewound, so a bounded prefix is all there is and the check says+ # so." _UNSLOTH_SCAN_ROWS = 1024\n"" def _unsloth_within_cap(_ds):\n"" if _ds is None: return True\n"@@ -2523,7 +2534,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" if not _unsloth_skip_prepare:\n"# Honour TRL's preparation map_kwargs so a large pre-tokenized dataset is not rewritten# single-process, through the same helper as every map site: the config layer writes serial as- # dataset_num_proc = 1, and datasets >= 4.1 builds a Pool(1) for it.+ # dataset_num_proc = 1, and datasets >= 4.1 builds a Pool(1) for it, forking a tokenizer+ # worker on the host that asked for none." _unsloth_map_kw = {}\n"" try:\n"" try:\n"@@ -2889,8 +2901,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"logging_steps": 1,"max_seq_length": None,"num_generations": 8,- # steps_per_generation would otherwise default to ga_steps, which is wrong, and- # generation_batch_size clashes with it.+ # "steps_per_generation": 1, # Otherwise defaults to ga_steps, which is wrong+ # "generation_batch_size": None, # Useless. If steps_per_generation is set, generation_batch_size clashes"top_k": None,"vllm_mode": "colocate","generation_kwargs": {},diff --git a/unsloth/save.py b/unsloth/save.pyindex d3da157a764..aba5449830e 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -4180,16 +4180,18 @@ def _preflight_gguf_disk(# Cleared when the cache shares a filesystem with room for the export but not a cached base too: dropping# the optional half beats failing. The message travels with the flag, since more than one filesystem can- # set it.+ # set it and each has to name the one it measured.sibling_prewarm_ok = Trueprewarm_drop_message = None# Resolved once, above the split, since the cache need not be on either filesystem. Zero whenever the pre-# warm cannot run, which leaves every branch below inert.cache_extra = max(0, need_with_cache - need)cache_directory = _hub_cache_directory() if cache_extra > 0 else None- # The cache lands wherever HF_HOME points, not necessarily save_directory's disk. Charging it here drops- # the pre-warm on a disk that had room, and the next export re-downloads the base. It only LOWERS the- # figure and only the pre-warm reads it.+ # The cache copy lands wherever the cache is, which need not be the filesystem holding `save_directory`:+ # `HF_HOME` on a data volume is the ordinary layout on a machine with more than one disk. Charging it here+ # drops the pre-warm on a disk that had room, and the next export re-downloads the base. It only LOWERS+ # the figure and only the pre-warm reads it, so nothing that fit before is refused now. Unresolvable+ # leaves it charged here, where it was charged before.cache_here = cache_extraif (cache_extra > 0@@ -5642,8 +5644,8 @@ def _resolve_imatrix_file(model, imatrix_file, token, dest_dir):f"(got {type(imatrix_file).__name__}).")- # imatrix_file=True auto-resolves from the upstream Unsloth GGUF repo; hf_hub_download is imported here- # since nothing else needs it.+ # imatrix_file=True auto-resolves from the upstream Unsloth GGUF repo. HfApi is the module-level import+ # (save.py top); hf_hub_download is imported here since nothing else needs it.from huggingface_hub import hf_hub_downloadif token is None:diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.pyindex 31749818f31..bf8b4e5753f 100644--- a/unsloth/tokenizer_utils.py+++ b/unsloth/tokenizer_utils.py@@ -557,9 +557,10 @@ def _load_correct_tokenizer(# Kaggle's /tmp seems to have an 80GB limit, so use it.cache_dir = os.path.join(KAGGLE_TMP, cache_dir)elif cache_dir == "huggingface_tokenizers_cache":- # This default name is Colab/Kaggle-only; elsewhere use the HF default cache, and keep a- # caller-supplied cache_dir so the tokenizer loads from the prefetch-warmed dir.+ # This default name is Colab/Kaggle-only; elsewhere use the HF default cache.cache_dir = None+ # else: keep a caller-supplied cache_dir so the tokenizer loads from the prefetch-warmed dir instead+ # of risking an in-process Hub/Xet transfer.# Try the slow tokenizer first and fall back to fast only, mainly to solve Deepseek models with no# tokenizer.model file.