Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Restore facts the comment trim cut from the unsloth package (#10328)

Daniel Hanca42c015f8e1
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index a8897a0cf16..b4d09653b5f 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -81,9 +81,11 @@ propagate_torchao_fix_to_subprocesses()
check_transformers_dependency_versions()
check_fbgemm_gpu_version()
torchvision_compatibility_check()
-# Must precede `import unsloth_zoo`: its temporary_patches reach transformers.processing_utils ->
-# audio_utils -> torchaudio, so a torchaudio raising at extension init takes the whole unsloth
-# import down before the late guard would have neutralised it.
+# Ahead of `import unsloth_zoo` below, deliberately not down with the other import fixes: unsloth_zoo's
+# temporary_patches reach transformers.processing_utils, which imports transformers.audio_utils, which
+# imports torchaudio, so a torchaudio that raises at extension init takes the whole unsloth import down at
+# that `import unsloth_zoo` line, long before the late block would have neutralised it. Measured:
+# Kaggle-Muse_Glimmer_(30B)-GRPO died at cell 4 with the guard present but not yet run.
disable_torchaudio_if_cuda_mismatched()
fix_diffusers_warnings()
fix_huggingface_hub()
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index c161babd18a..fb6926a884c 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -3199,8 +3199,10 @@ _PEFT_CONVERSION_SYMBOLS = {
),
}
-# Of those, the ones peft calls rather than merely imports. An inert body on a REAL
-# transformers would be called and answer wrongly, so those get a placeholder that says so.
+# Of those, the ones peft calls rather than merely imports. The stubs are deliberately inert: on
+# transformers <5 the whole module is ours and peft's converter never runs. An inert body on a REAL
+# transformers is a different matter, since peft would call it and get a wrong answer, so those get a
+# placeholder that says what is wrong instead.
_PEFT_CONVERSION_RUNTIME_SYMBOLS = frozenset(
(
"transformers.core_model_loading.dot_natural_key",
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 312baa66b53..242270099b3 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -142,7 +142,8 @@ try:
# If an earlier `import bitsandbytes` died inside __init__, CPython evicts only the parent from
# sys.modules and keeps its submodules, so this retry re-executes __init__ without rebinding
- # bnb.functional. `import x.y as z` reads sys.modules directly and survives that.
+ # bnb.functional. `import x.y as z` reads sys.modules directly and survives that, plain attribute
+ # access does not.
import bitsandbytes.functional as bnb_functional
except Exception:
# device_type.py already degrades to 16bit/full finetuning when bnb is missing (gfx906, whose
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 9b61396fbe6..d1d38afda1d 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -272,8 +272,9 @@ def _unsloth_install_pretrain_detector(model):
return model
marker = getattr(model, "_unsloth_pretrain_marker", None)
if isinstance(marker, dict):
- # A live hook is already recording: keep it and do NOT clear seen, since a grad-enabled probe
- # may already have flagged the poisoned cache and a re-entrant call must not erase that.
+ # A live hook is already recording: keep it (no duplicates) and do NOT clear seen, since a
+ # grad-enabled probe may already have flagged the poisoned cache and a re-entrant
+ # get_peft_model / patch_peft_model call must not erase that before train() resets it.
if "hook" in marker:
return model
# Marker exists but its hook was torn down, so reinstall fresh and reset seen.
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index ac169165e71..7df2605565d 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -240,7 +240,8 @@ def planner_quantization_kwargs(
from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES
except Exception:
# Built on every quantized load, planning or not, so an older unsloth_zoo must not turn a 4bit load
- # into an ImportError. One without the shared list predates the planner that consumes it.
+ # into an ImportError. One without the shared list predates the planner that consumes it, so this
+ # plan was going to decline anyway.
return kwargs
kwargs["llm_int8_skip_modules"] = SKIP_QUANTIZATION_MODULES + list(extra_skip_modules or [])
return kwargs
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index f339146a129..2de3fa219cc 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -1519,7 +1519,8 @@ def _wrap_sft_evaluate_cap(trainer_cls):
finally:
container[key] = previous
# evaluate() with no split falls back to the one stored on the trainer, which a caller can install
- # after construction, where the constructor's cap can no longer see it.
+ # after construction, where the constructor's cap can no longer see it. Every eval during
+ # training comes through here too.
stored = getattr(self, keyword, None) if keyword == "eval_dataset" else None
if stored is None:
return original(self, *args, **kwargs)
@@ -1842,7 +1843,9 @@ def _install_grpo_hidden_states_forward_wrapper(model):
raise
return forward_without_hidden_states()
# The signature advertised the parameter but the forward refuses the value: retry without the
- # minimisation, through the same fallback, rather than lose hidden states over it.
+ # minimisation, through the same fallback, rather than lose hidden states over it: a forward that
+ # splats **kwargs into a sub-module can refuse the logits limiter and the hidden states one after
+ # the other.
forward_kwargs.pop(logits_kwarg, None)
logits_kwarg = None
try:
@@ -2432,14 +2435,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" _cols = getattr(_ds, 'column_names', None)\n"
" if isinstance(_cols, dict):\n"
" _cols = [_c for _v in _cols.values() for _c in (_v or [])]\n"
- # A packed split is out: TRL skips truncation when packing, and cutting input_ids under a
+ # A packed split is out: TRL skips truncation when packing
+ # (`if args.max_length is not None and not packing`), and cutting input_ids under a
# seq_lengths that still describes the old row is worse than not cutting.
" if 'seq_lengths' in (_cols or ()): return False\n"
" return bool(_cols) and 'input_ids' in _cols\n"
" except Exception:\n"
" return False\n"
# Read a row BACK: every predicate above is a guess about what the dataset will do, and this is the
- # one check that observes it.
+ # one check that observes it. A split with no input_ids is raw, so prep tokenizes it with
+ # the cap and it is fine.
" _unsloth_cap = args.max_length\n"
# TRL slices [-max_length:] for keep_end, which callers use when the completion sits at the tail of
# a long prompt; always keeping the prefix trained on the wrong half of every row.
@@ -2450,8 +2455,13 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" _unsloth_known_mode = _unsloth_truncation_mode in ('keep_start', 'keep_end')\n"
" _unsloth_slice = slice(-_unsloth_cap, None) if _unsloth_keep_end else slice(None, _unsloth_cap)\n"
# Resolved outside the truncation block, since the fallback reads it even under
- # skip_prepare_dataset. eval_packing is separate from packing, so packing=False with
- # eval_packing=True lands here and TRL packs the eval split instead of truncating it.
+ # skip_prepare_dataset. eval_packing is separate from packing:
+ # packing = args.packing if args.eval_packing is None else args.eval_packing
+ # (sft_trainer.py), so packing=False with eval_packing=True reaches this branch, which is
+ # gated on `not args.packing`. TRL then packs the eval split instead of truncating it, and
+ # every strategy owns the overflow: `wrapped` concatenates the stream before chunking,
+ # `bfd_split` splits an overlength example into more chunks. Cutting rows at the cap first
+ # throws that away.
" _unsloth_eval_packing = getattr(args, 'packing', False) if getattr(args, 'eval_packing', None) is None else getattr(args, 'eval_packing')\n"
" _unsloth_completion_only = getattr(args, 'completion_only_loss', None)\n"
# Column names first, a row only if free: on a one-shot stream this probe ate the first
@@ -2481,9 +2491,10 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
# Parked on args so the late evaluate()/predict() cap reads the SAME value: it resolves from
# the train schema, and disagreeing with the collator leaves an all -100 row in.
" args._unsloth_completion_only_loss = _unsloth_completion_only\n"
- # EVERY row, not the first: a short row 0 before a long row 5000 read as within the cap. A
- # map-style split is read in full; a stream cannot be rewound, so a bounded prefix is all
- # there is and the check says so.
+ # EVERY row, not the first: a short row 0 before a long row 5000 read as within the cap, and
+ # in the fallback branch nothing downstream truncates it. A map-style split is read in
+ # full; a stream cannot be rewound, so a bounded prefix is all there is and the check says
+ # so.
" _UNSLOTH_SCAN_ROWS = 1024\n"
" def _unsloth_within_cap(_ds):\n"
" if _ds is None: return True\n"
@@ -2523,7 +2534,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" if not _unsloth_skip_prepare:\n"
# Honour TRL's preparation map_kwargs so a large pre-tokenized dataset is not rewritten
# single-process, through the same helper as every map site: the config layer writes serial as
- # dataset_num_proc = 1, and datasets >= 4.1 builds a Pool(1) for it.
+ # dataset_num_proc = 1, and datasets >= 4.1 builds a Pool(1) for it, forking a tokenizer
+ # worker on the host that asked for none.
" _unsloth_map_kw = {}\n"
" try:\n"
" try:\n"
@@ -2889,8 +2901,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"logging_steps": 1,
"max_seq_length": None,
"num_generations": 8,
- # steps_per_generation would otherwise default to ga_steps, which is wrong, and
- # generation_batch_size clashes with it.
+ # "steps_per_generation": 1, # Otherwise defaults to ga_steps, which is wrong
+ # "generation_batch_size": None, # Useless. If steps_per_generation is set, generation_batch_size clashes
"top_k": None,
"vllm_mode": "colocate",
"generation_kwargs": {},
diff --git a/unsloth/save.py b/unsloth/save.py
index d3da157a764..aba5449830e 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -4180,16 +4180,18 @@ def _preflight_gguf_disk(
# Cleared when the cache shares a filesystem with room for the export but not a cached base too: dropping
# the optional half beats failing. The message travels with the flag, since more than one filesystem can
- # set it.
+ # set it and each has to name the one it measured.
sibling_prewarm_ok = True
prewarm_drop_message = None
# Resolved once, above the split, since the cache need not be on either filesystem. Zero whenever the pre-
# warm cannot run, which leaves every branch below inert.
cache_extra = max(0, need_with_cache - need)
cache_directory = _hub_cache_directory() if cache_extra > 0 else None
- # The cache lands wherever HF_HOME points, not necessarily save_directory's disk. Charging it here drops
- # the pre-warm on a disk that had room, and the next export re-downloads the base. It only LOWERS the
- # figure and only the pre-warm reads it.
+ # The cache copy lands wherever the cache is, which need not be the filesystem holding `save_directory`:
+ # `HF_HOME` on a data volume is the ordinary layout on a machine with more than one disk. Charging it here
+ # drops the pre-warm on a disk that had room, and the next export re-downloads the base. It only LOWERS
+ # the figure and only the pre-warm reads it, so nothing that fit before is refused now. Unresolvable
+ # leaves it charged here, where it was charged before.
cache_here = cache_extra
if (
cache_extra > 0
@@ -5642,8 +5644,8 @@ def _resolve_imatrix_file(model, imatrix_file, token, dest_dir):
f"(got {type(imatrix_file).__name__})."
)
- # imatrix_file=True auto-resolves from the upstream Unsloth GGUF repo; hf_hub_download is imported here
- # since nothing else needs it.
+ # imatrix_file=True auto-resolves from the upstream Unsloth GGUF repo. HfApi is the module-level import
+ # (save.py top); hf_hub_download is imported here since nothing else needs it.
from huggingface_hub import hf_hub_download
if token is None:
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index 31749818f31..bf8b4e5753f 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -557,9 +557,10 @@ def _load_correct_tokenizer(
# Kaggle's /tmp seems to have an 80GB limit, so use it.
cache_dir = os.path.join(KAGGLE_TMP, cache_dir)
elif cache_dir == "huggingface_tokenizers_cache":
- # This default name is Colab/Kaggle-only; elsewhere use the HF default cache, and keep a
- # caller-supplied cache_dir so the tokenizer loads from the prefetch-warmed dir.
+ # This default name is Colab/Kaggle-only; elsewhere use the HF default cache.
cache_dir = None
+ # else: keep a caller-supplied cache_dir so the tokenizer loads from the prefetch-warmed dir instead
+ # of risking an in-process Hub/Xet transfer.
# Try the slow tokenizer first and fall back to fast only, mainly to solve Deepseek models with no
# tokenizer.model file.