Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
SFT: pick padded batches + block compile over padding-free when micro batches are host bound (#15185)
Daniel Han6988ffc8dfdf
diff --git a/tests/test_padding_choice.py b/tests/test_padding_choice.pynew file mode 100644index 00000000000..4d43b21eb70--- /dev/null+++ b/tests/test_padding_choice.py@@ -0,0 +1,287 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""unsloth/utils/padding_choice.py: the padded vs padding-free rule, on CPU.++The measured table is real data (Colab A100 / L4, Kaggle T4, Qwen3 0.6B / 4B LoRA r16 16 bit, 3 reps,+torch 2.14.1; 4 bit agreed within the A/A band): padded + block compile throughput relative to padding-free. The rule must+never pick padded where padded lost.+"""++import importlib.util+import pathlib+import sys+from types import SimpleNamespace++import pytest++ROOT = pathlib.Path(__file__).resolve().parents[1]+++def _load():+ spec = importlib.util.spec_from_file_location(+ "_unsloth_padding_choice", ROOT / "unsloth" / "utils" / "padding_choice.py"+ )+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return module+++pc = _load()++# Qwen3 LoRA linear FLOPs per token (gradient checkpointing) and attention FLOPs per pair, from mfu.py.+Q06 = (3.26e9, 9.18e5)+Q4 = (2.34e10, 2.36e6)+PEAK = {"A100": 312.0, "L4": 121.0, "T4": 65.0}+# (gpu, model, padded tokens, padded pairs per micro batch, measured padded/padding-free - 1 in %)+# Alpaca (short) at batch 2 / 8 / 16 and 1-4k token documents at batch 2 / 8.+ALP2, ALP8, ALP16 = (472, 6.4e4), (2883, 6.6e5), (6665, 1.6e6)+LONG2, LONG8 = (6176, 9.9e6), (30299, 5.8e7)+MEASURED = [+ ("A100", Q06, ALP2, +15.3), ("A100", Q06, ALP8, +11.0), ("A100", Q06, ALP16, +9.7),+ ("A100", Q06, LONG2, -1.3), ("A100", Q06, LONG8, -28.2),+ ("A100", Q4, ALP2, +16.2), ("A100", Q4, ALP8, +0.7), ("A100", Q4, ALP16, -42.5),+ ("A100", Q4, LONG2, -12.8), ("A100", Q4, LONG8, -33.0),+ ("L4", Q06, ALP2, +17.0), ("L4", Q06, ALP8, -1.3), ("L4", Q06, ALP16, -55.7),+ ("L4", Q06, LONG2, -15.8), ("L4", Q06, LONG8, -34.2),+ ("L4", Q4, ALP2, +3.0), ("L4", Q4, ALP8, -44.4), ("L4", Q4, ALP16, -57.3),+ ("L4", Q4, LONG2, -14.4), ("L4", Q4, LONG8, -32.5),+ ("T4", Q06, ALP2, +31.2), ("T4", Q06, ALP8, -15.8), ("T4", Q06, ALP16, -44.3),+ ("T4", Q06, LONG2, -12.6), ("T4", Q06, LONG8, -31.0),+ ("T4", Q4, ALP2, -20.8), ("T4", Q4, ALP8, -38.4), ("T4", Q4, ALP16, -53.8),+ ("T4", Q4, LONG2, -13.9), ("T4", Q4, LONG8, -32.1),+] # fmt: skip++[email protected]("gpu, flops, tokens, gain", MEASURED)+def test_rule_never_picks_padded_where_it_lost(gpu, flops, tokens, gain):+ use_pf, reason, gpu_s = pc.choose_padding(+ flops[0], flops[1], tokens[0], tokens[1], PEAK[gpu], total_microbatches = 10**9+ )+ if gain < 5.0:+ assert use_pf, (gpu, tokens, gain, reason, gpu_s)+++def test_rule_picks_padded_in_clear_host_bound_cells():+ wins = [+ (gpu, f, t)+ for gpu, f, t, gain in MEASURED+ if gain >= 14.0 and not (gpu == "L4" and f is Q4)+ ]+ assert wins+ for gpu, f, t in wins:+ use_pf, reason, _ = pc.choose_padding(f[0], f[1], t[0], t[1], PEAK[gpu], 10**9)+ assert not use_pf, (gpu, t, reason)+++def test_short_runs_keep_padding_free():+ # 60 steps x grad accum 4 cannot repay the extra block compile.+ use_pf, reason, _ = pc.choose_padding(*Q06, *ALP2, PEAK["A100"], total_microbatches = 240)+ assert use_pf and "too short" in reason+ use_pf, _, _ = pc.choose_padding(*Q06, *ALP2, PEAK["A100"], total_microbatches = 2000)+ assert not use_pf+ # 4 bit compiles far longer, so needs a longer run.+ use_pf, _, _ = pc.choose_padding(+ *Q06, *ALP2, PEAK["A100"], total_microbatches = 2000, load_in_4bit = True+ )+ assert use_pf+ use_pf, _, _ = pc.choose_padding(+ *Q06, *ALP2, PEAK["A100"], total_microbatches = 6000, load_in_4bit = True+ )+ assert not use_pf+ use_pf, _, _ = pc.choose_padding(*Q06, *ALP2, PEAK["A100"], total_microbatches = None)+ assert use_pf++[email protected]("peak", [None, 0, -1.0])+def test_unknown_gpu_keeps_padding_free(peak):+ assert pc.choose_padding(*Q06, *ALP2, peak, 10**9)[0] is True+++def test_effective_max_length_follows_trainer_precedence():+ import dataclasses++ @dataclasses.dataclass+ class Cfg:+ max_length: int = 1024++ model = SimpleNamespace(max_seq_length = 4096)+ # SFTConfig's default is replaced by the model length, as the generated SFTTrainer init does.+ assert pc._effective_max_length(model, Cfg()) == 4096+ # A caller-set cap below the model length wins; above it, the model length wins.+ assert pc._effective_max_length(model, Cfg(max_length = 512)) == 512+ assert pc._effective_max_length(model, Cfg(max_length = 8192)) == 4096+ # No model length: the config value is used as is.+ assert pc._effective_max_length(SimpleNamespace(), Cfg()) == 1024+ assert pc._effective_max_length(SimpleNamespace(), Cfg(max_length = None)) is None+++def test_default_max_length_does_not_clip_long_rows():+ # Codex P2 case: default max_length 1024, model loaded at 4096, rows of 2000 / 4000 tokens on A100.+ import dataclasses++ @dataclasses.dataclass+ class Cfg:+ max_length: int = 1024++ cap = pc._effective_max_length(SimpleNamespace(max_seq_length = 4096), Cfg())+ _, padded, pairs = pc.estimate_microbatch_tokens([2000, 4000] * 50, 2, cap)+ assert pc.choose_padding(*Q06, padded, pairs, PEAK["A100"], 10**9)[0] is True+ _, padded, pairs = pc.estimate_microbatch_tokens([2000, 4000] * 50, 2, 1024)+ assert (+ pc.choose_padding(*Q06, padded, pairs, PEAK["A100"], 10**9)[0] is False+ ) # the old clipped estimate+++def test_estimate_microbatch_tokens():+ real, padded, pairs = pc.estimate_microbatch_tokens([100] * 50, 4)+ assert real == padded == 400 and pairs == 4 * 100 * 101 / 2+ real, padded, _ = pc.estimate_microbatch_tokens([10, 1000], 8, max_length = 500)+ assert padded == pytest.approx(+ 8 * 500, rel = 0.02+ ) # 8 draws of 2 lengths: max is almost always 500+ assert real < padded+ assert pc.estimate_microbatch_tokens([], 2) is None+ assert pc.estimate_microbatch_tokens([5, 6], 0) is None+ # deterministic+ assert pc.estimate_microbatch_tokens(list(range(1, 300)), 3) == pc.estimate_microbatch_tokens(+ list(range(1, 300)), 3+ )+++ "value, mode",+ [+ (None, "auto"), ("auto", "auto"), ("", "auto"), ("padding_free", "padding_free"),+ ("0", "padding_free"), ("padded", "padded"), ("PADDED", "padded"), ("junk", "auto"),+ ],+) # fmt: skip+def test_mode(monkeypatch, value, mode):+ if value is None:+ monkeypatch.delenv("UNSLOTH_PADDING_CHOICE", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_PADDING_CHOICE", value)+ assert pc.padding_choice_mode() == mode+++def test_env_overrides_and_failures(monkeypatch):+ monkeypatch.setenv("UNSLOTH_PADDING_CHOICE", "padded")+ assert pc.decide_auto_padding_free(None, None, None, None)[0] is False+ monkeypatch.setenv("UNSLOTH_PADDING_CHOICE", "padding_free")+ assert pc.decide_auto_padding_free(None, None, None, None)[0] is True+ monkeypatch.setenv("UNSLOTH_PADDING_CHOICE", "auto")+ assert pc.decide_auto_padding_free(None, None, None, None)[0] is True+ assert pc.decide_auto_padding_free("org/model", None, None, None)[0] is True++ def boom(*a, **k):+ raise RuntimeError("x")++ monkeypatch.setattr(pc, "_decide", boom)+ use_pf, reason = pc.decide_auto_padding_free(object(), None, None, None)+ assert use_pf is True and "RuntimeError" in reason+++class _Tok:+ chat_template = "x"++ def __call__(+ self,+ texts,+ add_special_tokens = False,+ ):+ return {"input_ids": [t.split() for t in texts]}++ def apply_chat_template(+ self,+ messages,+ tokenize = False,+ ):+ return " ".join(m["content"] for m in messages)+++def test_row_texts_formats():+ tok = _Tok()+ cfg = SimpleNamespace(dataset_text_field = "text")+ rows = [{"text": "a b c"}, {"text": "d"}]+ assert pc._row_texts(rows, ["text"], cfg, None, tok) == ["a b c", "d"]+ rows = [+ {+ "messages": [+ {"role": "user", "content": "hi there"},+ {"role": "assistant", "content": "yo"},+ ]+ }+ ]+ assert pc._row_texts(rows, ["messages"], cfg, None, tok) == ["hi there yo"]+ rows = [{"prompt": "p q", "completion": " r"}]+ assert pc._row_texts(rows, ["prompt", "completion"], cfg, None, tok) == ["p q r"]+ rows = [{"instruction": "x", "output": "y z"}]+ assert pc._row_texts(rows, ["instruction", "output"], cfg, None, tok) is None+ fmt = lambda row: row["instruction"] + " " + row["output"]+ assert pc._row_texts(rows, ["instruction", "output"], cfg, fmt, tok) == ["x y z"]+ assert pc._row_texts(rows, ["instruction", "output"], cfg, lambda r: [fmt(r)], tok) == ["x y z"]+ assert pc._row_texts(rows, ["instruction", "output"], cfg, lambda r: 3, tok) is None+++def test_sample_lengths():+ datasets = pytest.importorskip("datasets")+ tok = _Tok()+ cfg = SimpleNamespace(dataset_text_field = "text")+ ds = datasets.Dataset.from_dict({"text": ["a b c", "d", "e f"]})+ assert sorted(pc._sample_lengths(ds, cfg, tok, None)) == [1, 2, 3]+ ds = datasets.Dataset.from_dict({"input_ids": [[1, 2], [3], [4, 5, 6, 7]]})+ assert sorted(pc._sample_lengths(ds, cfg, None, None)) == [1, 2, 4]+ big = datasets.Dataset.from_dict({"text": ["w " * (i % 7 + 1) for i in range(5000)]})+ lengths = pc._sample_lengths(big, cfg, tok, None)+ assert len(lengths) == pc._SAMPLE_ROWS+ assert pc._sample_lengths(None, cfg, tok, None) is None+ assert pc._sample_lengths(datasets.Dataset.from_dict({"text": []}), cfg, tok, None) is None+ iterable = datasets.Dataset.from_dict({"text": ["a"]}).to_iterable_dataset()+ assert pc._sample_lengths(iterable, cfg, tok, None) is None+++def test_total_microbatches():+ cfg = SimpleNamespace(+ per_device_train_batch_size = 2, gradient_accumulation_steps = 4, max_steps = 60+ )+ assert pc._total_microbatches(cfg, 52000) == 240+ cfg = SimpleNamespace(+ per_device_train_batch_size = 2,+ gradient_accumulation_steps = 4,+ max_steps = -1,+ num_train_epochs = 1,+ )+ assert pc._total_microbatches(cfg, 52001) == 26001+ assert pc._total_microbatches(cfg, None) is None+++def test_moe_detection():+ assert pc._is_moe(SimpleNamespace(num_experts = 8))+ assert pc._is_moe(SimpleNamespace(n_routed_experts = 64))+ assert not pc._is_moe(SimpleNamespace(num_experts = 0, hidden_size = 8))+ assert not pc._is_moe(SimpleNamespace(hidden_size = 8))+++def test_trainer_wires_the_choice():+ src = (ROOT / "unsloth" / "trainer.py").read_text()+ assert "_auto_padding_choice(" in src+ assert "from unsloth.utils.padding_choice import decide_auto_padding_free" in src++[email protected]("platform, hip, cuda, available, reason", [+ ("linux", "6.2.41133", None, True, "ROCm"),+ ("win32", None, "13.0", True, "Windows"),+ ("win32", "6.2", None, True, "ROCm"),+ ("linux", None, None, False, "not an NVIDIA CUDA device"),+ ("darwin", None, None, False, "not an NVIDIA CUDA device"),+]) # fmt: skip+def test_non_cuda_linux_keeps_padding_free(monkeypatch, platform, hip, cuda, available, reason):+ torch = pytest.importorskip("torch")+ monkeypatch.setenv("UNSLOTH_PADDING_CHOICE", "auto")+ monkeypatch.setattr(pc.sys, "platform", platform)+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)+ monkeypatch.setattr(torch.version, "cuda", cuda, raising = False)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: available)+ use_pf, why = pc.decide_auto_padding_free(torch.nn.Linear(2, 2), SimpleNamespace(), None, None)+ assert use_pf is True and reason in why, whydiff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 6b88ba0682f..8058052c7d8 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -215,6 +215,23 @@ def _should_auto_padding_free(config) -> bool:return getattr(config, "padding_free", None) is None+def _auto_padding_choice(+ original_init, model, config, train_dataset, processing_class, args, kwargs+):+ """(use_padding_free, reason) for an unset `padding_free` (unsloth/utils/padding_choice.py)."""+ try:+ from unsloth.utils.padding_choice import decide_auto_padding_free+ try:+ bound = inspect.signature(original_init).bind_partial(None, *args, **kwargs).arguments+ except Exception:+ bound = kwargs+ return decide_auto_padding_free(+ model, config, train_dataset, processing_class, bound.get("formatting_func")+ )+ except Exception as exc:+ return True, f"padding choice skipped ({type(exc).__name__})"++def _disable_sample_packing(config):if config is None:return@@ -1987,9 +2004,21 @@ def _patch_sft_trainer_auto_packing(trl_module):if padding_free_requested:configure_padding_free(config_arg)elif _should_auto_padding_free(config_arg):- configure_padding_free(config_arg)- auto_padding_free_active = True- logger.info("Unsloth: Padding-free batching auto-enabled for SFTTrainer instance.")+ use_padding_free, choice_reason = _auto_padding_choice(+ original_init, model, config_arg, train_dataset, processing_class, args, kwargs+ )+ if use_padding_free:+ configure_padding_free(config_arg)+ auto_padding_free_active = True+ logger.info(+ "Unsloth: Padding-free batching auto-enabled for SFTTrainer instance."+ )+ else:+ # Left unset (None), exactly as UNSLOTH_DISABLE_AUTO_PADDING_FREE=1 leaves it.+ print(+ f"Unsloth: Padded batches + per-block compile chosen over padding-free "+ f"({choice_reason}). Set UNSLOTH_PADDING_CHOICE=padding_free to override."+ )try:original_init(self, *args, **kwargs)diff --git a/unsloth/utils/padding_choice.py b/unsloth/utils/padding_choice.pynew file mode 100644index 00000000000..cd1ab56b7f9--- /dev/null+++ b/unsloth/utils/padding_choice.py@@ -0,0 +1,329 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Choose between padding-free and padded batches for SFT when the user left `padding_free` unset.++Padding-free removes pad tokens, but its varlen batches skip the per-block compile, so each step+launches more kernels from Python. Padded batches pay for their pad tokens on the GPU but run the+compiled blocks. Which one is faster depends on whether a micro batch is bound by the host or by the+GPU: measured on A100, L4 and T4 with Qwen3 0.6B / 4B LoRA (16 bit and 4 bit), padded + block+compile was 10-17% faster only when a padded micro batch costs the GPU well under the ~0.25-0.4 s+the host spends launching it, and up to 57% slower once the GPU is the bottleneck.++Rule (all estimates, every unknown keeps padding-free, the old default):+ padded GPU time = (linear FLOPs per token * padded tokens + attention FLOPs per pair * pairs)+ / (datasheet 16 bit peak * _EFFICIENCY)+ pick padded iff padded GPU time < _MAX_PADDED_GPU_S+ and the run has enough micro batches to repay the extra block compile+ (about 25 s in 16 bit, 100 s in 4 bit) at ~_SAVED_S_PER_MICROBATCH each.++Only Linux + NVIDIA CUDA is ever switched: ROCm / HIP, Windows, XPU, MPS and CPU keep padding-free+(no measurements there yet), unless `UNSLOTH_PADDING_CHOICE=padded` is set explicitly.++`UNSLOTH_PADDING_CHOICE=padding_free` restores the previous behaviour (always padding-free),+`=padded` forces padded batches, `=auto` (default) applies the rule. An explicit+`padding_free=True/False` on the config always wins.+"""++import logging+import math+import os+import random+import sys++__all__ = [+ "padding_choice_mode",+ "estimate_microbatch_tokens",+ "choose_padding",+ "decide_auto_padding_free",+]++logger = logging.getLogger(__name__)++_EFFICIENCY = (+ 0.5 # fraction of the datasheet peak a LoRA step reaches when GPU bound (A100 measured 0.5)+)+_MAX_PADDED_GPU_S = 0.10 # padded micro batch GPU estimate below which the host dominates+_SAVED_S_PER_MICROBATCH = 0.04 # host time block compile saved per micro batch where it won+_EXTRA_COMPILE_S_16BIT = 25.0+_EXTRA_COMPILE_S_4BIT = 100.0+_REPAY_FACTOR = 2.0+_SAMPLE_ROWS = 512+_SIM_BATCHES = 256+++def padding_choice_mode():+ mode = os.environ.get("UNSLOTH_PADDING_CHOICE", "auto").strip().lower()+ if mode in ("padding_free", "padding-free", "packed", "0", "off", "false", "no"):+ return "padding_free"+ if mode in ("padded", "pad", "block_compile"):+ return "padded"+ return "auto"+++def estimate_microbatch_tokens(+ lengths,+ batch_size,+ max_length = None,+ seed = 0,+):+ """Expected (real tokens, padded tokens, attention pairs when padded) per micro batch.++ Batches are drawn uniformly from `lengths` (the trainer shuffles); padded = batch * longest.+ """+ lengths = [int(x) for x in lengths if x is not None and int(x) > 0]+ if not lengths or batch_size < 1:+ return None+ if max_length:+ lengths = [min(x, int(max_length)) for x in lengths]+ rng = random.Random(seed)+ real = padded = pairs = 0+ for _ in range(_SIM_BATCHES):+ batch = [lengths[rng.randrange(len(lengths))] for _ in range(batch_size)]+ longest = max(batch)+ real += sum(batch)+ padded += batch_size * longest+ pairs += batch_size * longest * (longest + 1) / 2+ n = _SIM_BATCHES+ return real / n, padded / n, pairs / n+++def choose_padding(+ linear_flops_per_token,+ attention_flops_per_pair,+ padded_tokens,+ padded_pairs,+ peak_tflops,+ total_microbatches = None,+ load_in_4bit = False,+):+ """Pure rule. Returns (use_padding_free: bool, reason: str, padded_gpu_s: float | None)."""+ if not peak_tflops or peak_tflops <= 0 or not linear_flops_per_token or not padded_tokens:+ return True, "no estimate", None+ flops = linear_flops_per_token * padded_tokens + (attention_flops_per_pair or 0) * (+ padded_pairs or 0+ )+ gpu_s = flops / (peak_tflops * 1e12 * _EFFICIENCY)+ if gpu_s >= _MAX_PADDED_GPU_S:+ return True, f"GPU bound (padded micro batch ~{gpu_s * 1e3:.0f} ms of GPU work)", gpu_s+ extra = _EXTRA_COMPILE_S_4BIT if load_in_4bit else _EXTRA_COMPILE_S_16BIT+ needed = math.ceil(_REPAY_FACTOR * extra / _SAVED_S_PER_MICROBATCH)+ if total_microbatches is None or total_microbatches < needed:+ return (+ True,+ f"run too short to repay the block compile ({total_microbatches} < {needed} micro batches)",+ gpu_s,+ )+ return False, f"host bound (padded micro batch ~{gpu_s * 1e3:.0f} ms of GPU work)", gpu_s+++# ---------------------------------------------------------------------------------------------+# Feature gathering. Every failure returns None, which keeps padding-free.+++def _sample_indices(+ n,+ k = _SAMPLE_ROWS,+ seed = 0,+):+ if n <= k:+ return list(range(n))+ return sorted(random.Random(seed).sample(range(n), k))+++def _tokenizer_of(processing_class):+ tok = getattr(processing_class, "tokenizer", None) or processing_class+ return tok if callable(tok) else None+++def _row_texts(rows, columns, config, formatting_func, tok):+ """One string per row as SFTTrainer would build it, or None when the format is not known."""+ if formatting_func is not None:+ out = []+ for row in rows:+ text = formatting_func(row)+ if isinstance(text, (list, tuple)):+ text = text[0] if text else ""+ if not isinstance(text, str):+ return None+ out.append(text)+ return out+ field = getattr(config, "dataset_text_field", None) or "text"+ if field in columns:+ texts = [row[field] for row in rows]+ return texts if all(isinstance(t, str) for t in texts) else None+ chat = getattr(tok, "apply_chat_template", None)+ if "messages" in columns or "conversations" in columns:+ key = "messages" if "messages" in columns else "conversations"+ if chat is None or getattr(tok, "chat_template", None) is None:+ return None+ return [chat(row[key], tokenize = False) for row in rows]+ if "prompt" in columns and "completion" in columns:+ out = []+ for row in rows:+ p, c = row["prompt"], row["completion"]+ if isinstance(p, str) and isinstance(c, str):+ out.append(p + c)+ elif isinstance(p, list) and isinstance(c, list) and chat is not None:+ out.append(chat(p + c, tokenize = False))+ else:+ return None+ return out+ return None+++def _sample_lengths(dataset, config, processing_class, formatting_func):+ if dataset is None or isinstance(dataset, dict):+ return None+ try:+ n = len(dataset)+ except Exception:+ return None # IterableDataset: no cheap sample+ if n == 0:+ return None+ columns = list(getattr(dataset, "column_names", None) or [])+ idx = _sample_indices(n)+ if "input_ids" in columns and formatting_func is None:+ return [len(x) for x in dataset.select(idx)["input_ids"]]+ tok = _tokenizer_of(processing_class)+ if tok is None:+ return None+ rows = dataset.select(idx)+ rows = [rows[i] for i in range(len(rows))]+ texts = _row_texts(rows, columns, config, formatting_func, tok)+ if not texts:+ return None+ return [len(x) for x in tok(texts, add_special_tokens = False)["input_ids"]]+++def _block_compile_installed(model):+ try:+ from ..models._block_compile import _MARK, _OFF, _enabled+ except Exception:+ return False+ if not _enabled():+ return False+ for module in model.modules():+ if (+ getattr(type(module).forward, _MARK, False)+ and module.__dict__.get("_unsloth_block_mode", None) != _OFF+ ):+ return True+ return False+++def _is_moe(config):+ config = getattr(config, "get_text_config", lambda: config)()+ for key in ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts"):+ if (getattr(config, key, None) or 0) > 1:+ return True+ return False+++def _is_4bit(model):+ if getattr(model, "is_loaded_in_4bit", False):+ return True+ return any(type(m).__name__ == "Linear4bit" for m in model.modules())+++def _total_microbatches(config, n_rows):+ bs = int(getattr(config, "per_device_train_batch_size", 1) or 1)+ ga = int(getattr(config, "gradient_accumulation_steps", 1) or 1)+ max_steps = getattr(config, "max_steps", -1)+ if max_steps is not None and max_steps > 0:+ return int(max_steps) * ga+ epochs = float(getattr(config, "num_train_epochs", 1) or 1)+ if not n_rows:+ return None+ return int(epochs * math.ceil(n_rows / bs))+++def _effective_max_length(model, config):+ """The row cap training will use. This runs before the generated SFTTrainer init, which+ replaces SFTConfig's default `max_length` with the model's `max_seq_length` and keeps only a+ caller-set value below it (unsloth/models/rl.py), so mirror that precedence here."""+ given = getattr(config, "max_length", None)+ default = None+ try:+ import dataclasses+ for field in dataclasses.fields(type(config)):+ if field.name == "max_length":+ default = field.default+ except Exception:+ pass+ explicit = given if (given or 0) > 0 and given != default else None+ model_length = getattr(model, "max_seq_length", None)+ if model_length:+ return min(int(model_length), int(explicit)) if explicit else int(model_length)+ return given+++def decide_auto_padding_free(+ model,+ config,+ train_dataset,+ processing_class,+ formatting_func = None,+):+ """True = padding-free (old default), False = padded batches. Never raises."""+ mode = padding_choice_mode()+ if mode == "padding_free":+ return True, "UNSLOTH_PADDING_CHOICE=padding_free"+ if mode == "padded":+ return False, "UNSLOTH_PADDING_CHOICE=padded"+ try:+ return _decide(model, config, train_dataset, processing_class, formatting_func)+ except Exception as exc: # the old default on any surprise+ logger.info(f"Unsloth: padding choice skipped ({exc}).")+ return True, f"padding choice skipped ({type(exc).__name__})"+++def _decide(model, config, train_dataset, processing_class, formatting_func):+ import torch++ if model is None or isinstance(model, str) or not hasattr(model, "modules"):+ return True, "model not loaded"+ # Measured only on Linux + NVIDIA CUDA. ROCm / HIP (torch.cuda.is_available() is True there),+ # Windows (slower host, triton-windows compile), XPU, MPS and CPU keep padding-free unchanged.+ if getattr(torch.version, "hip", None):+ return True, "ROCm / HIP device (not measured)"+ if sys.platform == "win32":+ return True, "Windows (not measured)"+ if not torch.cuda.is_available() or not getattr(torch.version, "cuda", None):+ return True, "not an NVIDIA CUDA device"+ if int(os.environ.get("WORLD_SIZE", "1") or 1) > 1:+ return True, "distributed run"+ model_config = getattr(model, "config", None)+ if model_config is None or _is_moe(model_config):+ return True, "MoE or unknown config"+ if not _block_compile_installed(model):+ return True, "per-block compile not installed"+ bs = int(getattr(config, "per_device_train_batch_size", 1) or 1)+ lengths = _sample_lengths(train_dataset, config, processing_class, formatting_func)+ if not lengths:+ return True, "dataset lengths unknown"+ est = estimate_microbatch_tokens(lengths, bs, _effective_max_length(model, config))+ if est is None:+ return True, "dataset lengths unknown"+ _, padded_tokens, padded_pairs = est+ from .mfu import count_attention_flops_per_pair, count_linear_flops_per_token, get_peak_tflops++ linear, _ = count_linear_flops_per_token(model, gradient_checkpointing = True)+ attention = count_attention_flops_per_pair(model_config, gradient_checkpointing = True)+ dtype = torch.bfloat16 if getattr(config, "bf16", False) else torch.float16+ device = next((p.device for p in model.parameters() if p.device.type == "cuda"), None)+ peak = get_peak_tflops(device, dtype)+ try:+ n_rows = len(train_dataset)+ except Exception:+ n_rows = None+ use_pf, reason, _ = choose_padding(+ linear,+ attention,+ padded_tokens,+ padded_pairs,+ peak,+ total_microbatches = _total_microbatches(config, n_rows),+ load_in_4bit = _is_4bit(model),+ )+ return use_pf, reason