Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

SFT: pick padded batches + block compile over padding-free when micro batches are host bound (#15185)

Daniel Han6988ffc8dfdf
diff --git a/tests/test_padding_choice.py b/tests/test_padding_choice.py
new file mode 100644
index 00000000000..4d43b21eb70
--- /dev/null
+++ b/tests/test_padding_choice.py
@@ -0,0 +1,287 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""unsloth/utils/padding_choice.py: the padded vs padding-free rule, on CPU.
+
+The measured table is real data (Colab A100 / L4, Kaggle T4, Qwen3 0.6B / 4B LoRA r16 16 bit, 3 reps,
+torch 2.14.1; 4 bit agreed within the A/A band): padded + block compile throughput relative to padding-free. The rule must
+never pick padded where padded lost.
+"""
+
+import importlib.util
+import pathlib
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+
+
+def _load():
+ spec = importlib.util.spec_from_file_location(
+ "_unsloth_padding_choice", ROOT / "unsloth" / "utils" / "padding_choice.py"
+ )
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+pc = _load()
+
+# Qwen3 LoRA linear FLOPs per token (gradient checkpointing) and attention FLOPs per pair, from mfu.py.
+Q06 = (3.26e9, 9.18e5)
+Q4 = (2.34e10, 2.36e6)
+PEAK = {"A100": 312.0, "L4": 121.0, "T4": 65.0}
+# (gpu, model, padded tokens, padded pairs per micro batch, measured padded/padding-free - 1 in %)
+# Alpaca (short) at batch 2 / 8 / 16 and 1-4k token documents at batch 2 / 8.
+ALP2, ALP8, ALP16 = (472, 6.4e4), (2883, 6.6e5), (6665, 1.6e6)
+LONG2, LONG8 = (6176, 9.9e6), (30299, 5.8e7)
+MEASURED = [
+ ("A100", Q06, ALP2, +15.3), ("A100", Q06, ALP8, +11.0), ("A100", Q06, ALP16, +9.7),
+ ("A100", Q06, LONG2, -1.3), ("A100", Q06, LONG8, -28.2),
+ ("A100", Q4, ALP2, +16.2), ("A100", Q4, ALP8, +0.7), ("A100", Q4, ALP16, -42.5),
+ ("A100", Q4, LONG2, -12.8), ("A100", Q4, LONG8, -33.0),
+ ("L4", Q06, ALP2, +17.0), ("L4", Q06, ALP8, -1.3), ("L4", Q06, ALP16, -55.7),
+ ("L4", Q06, LONG2, -15.8), ("L4", Q06, LONG8, -34.2),
+ ("L4", Q4, ALP2, +3.0), ("L4", Q4, ALP8, -44.4), ("L4", Q4, ALP16, -57.3),
+ ("L4", Q4, LONG2, -14.4), ("L4", Q4, LONG8, -32.5),
+ ("T4", Q06, ALP2, +31.2), ("T4", Q06, ALP8, -15.8), ("T4", Q06, ALP16, -44.3),
+ ("T4", Q06, LONG2, -12.6), ("T4", Q06, LONG8, -31.0),
+ ("T4", Q4, ALP2, -20.8), ("T4", Q4, ALP8, -38.4), ("T4", Q4, ALP16, -53.8),
+ ("T4", Q4, LONG2, -13.9), ("T4", Q4, LONG8, -32.1),
+] # fmt: skip
+
+
[email protected]("gpu, flops, tokens, gain", MEASURED)
+def test_rule_never_picks_padded_where_it_lost(gpu, flops, tokens, gain):
+ use_pf, reason, gpu_s = pc.choose_padding(
+ flops[0], flops[1], tokens[0], tokens[1], PEAK[gpu], total_microbatches = 10**9
+ )
+ if gain < 5.0:
+ assert use_pf, (gpu, tokens, gain, reason, gpu_s)
+
+
+def test_rule_picks_padded_in_clear_host_bound_cells():
+ wins = [
+ (gpu, f, t)
+ for gpu, f, t, gain in MEASURED
+ if gain >= 14.0 and not (gpu == "L4" and f is Q4)
+ ]
+ assert wins
+ for gpu, f, t in wins:
+ use_pf, reason, _ = pc.choose_padding(f[0], f[1], t[0], t[1], PEAK[gpu], 10**9)
+ assert not use_pf, (gpu, t, reason)
+
+
+def test_short_runs_keep_padding_free():
+ # 60 steps x grad accum 4 cannot repay the extra block compile.
+ use_pf, reason, _ = pc.choose_padding(*Q06, *ALP2, PEAK["A100"], total_microbatches = 240)
+ assert use_pf and "too short" in reason
+ use_pf, _, _ = pc.choose_padding(*Q06, *ALP2, PEAK["A100"], total_microbatches = 2000)
+ assert not use_pf
+ # 4 bit compiles far longer, so needs a longer run.
+ use_pf, _, _ = pc.choose_padding(
+ *Q06, *ALP2, PEAK["A100"], total_microbatches = 2000, load_in_4bit = True
+ )
+ assert use_pf
+ use_pf, _, _ = pc.choose_padding(
+ *Q06, *ALP2, PEAK["A100"], total_microbatches = 6000, load_in_4bit = True
+ )
+ assert not use_pf
+ use_pf, _, _ = pc.choose_padding(*Q06, *ALP2, PEAK["A100"], total_microbatches = None)
+ assert use_pf
+
+
[email protected]("peak", [None, 0, -1.0])
+def test_unknown_gpu_keeps_padding_free(peak):
+ assert pc.choose_padding(*Q06, *ALP2, peak, 10**9)[0] is True
+
+
+def test_effective_max_length_follows_trainer_precedence():
+ import dataclasses
+
+ @dataclasses.dataclass
+ class Cfg:
+ max_length: int = 1024
+
+ model = SimpleNamespace(max_seq_length = 4096)
+ # SFTConfig's default is replaced by the model length, as the generated SFTTrainer init does.
+ assert pc._effective_max_length(model, Cfg()) == 4096
+ # A caller-set cap below the model length wins; above it, the model length wins.
+ assert pc._effective_max_length(model, Cfg(max_length = 512)) == 512
+ assert pc._effective_max_length(model, Cfg(max_length = 8192)) == 4096
+ # No model length: the config value is used as is.
+ assert pc._effective_max_length(SimpleNamespace(), Cfg()) == 1024
+ assert pc._effective_max_length(SimpleNamespace(), Cfg(max_length = None)) is None
+
+
+def test_default_max_length_does_not_clip_long_rows():
+ # Codex P2 case: default max_length 1024, model loaded at 4096, rows of 2000 / 4000 tokens on A100.
+ import dataclasses
+
+ @dataclasses.dataclass
+ class Cfg:
+ max_length: int = 1024
+
+ cap = pc._effective_max_length(SimpleNamespace(max_seq_length = 4096), Cfg())
+ _, padded, pairs = pc.estimate_microbatch_tokens([2000, 4000] * 50, 2, cap)
+ assert pc.choose_padding(*Q06, padded, pairs, PEAK["A100"], 10**9)[0] is True
+ _, padded, pairs = pc.estimate_microbatch_tokens([2000, 4000] * 50, 2, 1024)
+ assert (
+ pc.choose_padding(*Q06, padded, pairs, PEAK["A100"], 10**9)[0] is False
+ ) # the old clipped estimate
+
+
+def test_estimate_microbatch_tokens():
+ real, padded, pairs = pc.estimate_microbatch_tokens([100] * 50, 4)
+ assert real == padded == 400 and pairs == 4 * 100 * 101 / 2
+ real, padded, _ = pc.estimate_microbatch_tokens([10, 1000], 8, max_length = 500)
+ assert padded == pytest.approx(
+ 8 * 500, rel = 0.02
+ ) # 8 draws of 2 lengths: max is almost always 500
+ assert real < padded
+ assert pc.estimate_microbatch_tokens([], 2) is None
+ assert pc.estimate_microbatch_tokens([5, 6], 0) is None
+ # deterministic
+ assert pc.estimate_microbatch_tokens(list(range(1, 300)), 3) == pc.estimate_microbatch_tokens(
+ list(range(1, 300)), 3
+ )
+
+
+ "value, mode",
+ [
+ (None, "auto"), ("auto", "auto"), ("", "auto"), ("padding_free", "padding_free"),
+ ("0", "padding_free"), ("padded", "padded"), ("PADDED", "padded"), ("junk", "auto"),
+ ],
+) # fmt: skip
+def test_mode(monkeypatch, value, mode):
+ if value is None:
+ monkeypatch.delenv("UNSLOTH_PADDING_CHOICE", raising = False)
+ else:
+ monkeypatch.setenv("UNSLOTH_PADDING_CHOICE", value)
+ assert pc.padding_choice_mode() == mode
+
+
+def test_env_overrides_and_failures(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_PADDING_CHOICE", "padded")
+ assert pc.decide_auto_padding_free(None, None, None, None)[0] is False
+ monkeypatch.setenv("UNSLOTH_PADDING_CHOICE", "padding_free")
+ assert pc.decide_auto_padding_free(None, None, None, None)[0] is True
+ monkeypatch.setenv("UNSLOTH_PADDING_CHOICE", "auto")
+ assert pc.decide_auto_padding_free(None, None, None, None)[0] is True
+ assert pc.decide_auto_padding_free("org/model", None, None, None)[0] is True
+
+ def boom(*a, **k):
+ raise RuntimeError("x")
+
+ monkeypatch.setattr(pc, "_decide", boom)
+ use_pf, reason = pc.decide_auto_padding_free(object(), None, None, None)
+ assert use_pf is True and "RuntimeError" in reason
+
+
+class _Tok:
+ chat_template = "x"
+
+ def __call__(
+ self,
+ texts,
+ add_special_tokens = False,
+ ):
+ return {"input_ids": [t.split() for t in texts]}
+
+ def apply_chat_template(
+ self,
+ messages,
+ tokenize = False,
+ ):
+ return " ".join(m["content"] for m in messages)
+
+
+def test_row_texts_formats():
+ tok = _Tok()
+ cfg = SimpleNamespace(dataset_text_field = "text")
+ rows = [{"text": "a b c"}, {"text": "d"}]
+ assert pc._row_texts(rows, ["text"], cfg, None, tok) == ["a b c", "d"]
+ rows = [
+ {
+ "messages": [
+ {"role": "user", "content": "hi there"},
+ {"role": "assistant", "content": "yo"},
+ ]
+ }
+ ]
+ assert pc._row_texts(rows, ["messages"], cfg, None, tok) == ["hi there yo"]
+ rows = [{"prompt": "p q", "completion": " r"}]
+ assert pc._row_texts(rows, ["prompt", "completion"], cfg, None, tok) == ["p q r"]
+ rows = [{"instruction": "x", "output": "y z"}]
+ assert pc._row_texts(rows, ["instruction", "output"], cfg, None, tok) is None
+ fmt = lambda row: row["instruction"] + " " + row["output"]
+ assert pc._row_texts(rows, ["instruction", "output"], cfg, fmt, tok) == ["x y z"]
+ assert pc._row_texts(rows, ["instruction", "output"], cfg, lambda r: [fmt(r)], tok) == ["x y z"]
+ assert pc._row_texts(rows, ["instruction", "output"], cfg, lambda r: 3, tok) is None
+
+
+def test_sample_lengths():
+ datasets = pytest.importorskip("datasets")
+ tok = _Tok()
+ cfg = SimpleNamespace(dataset_text_field = "text")
+ ds = datasets.Dataset.from_dict({"text": ["a b c", "d", "e f"]})
+ assert sorted(pc._sample_lengths(ds, cfg, tok, None)) == [1, 2, 3]
+ ds = datasets.Dataset.from_dict({"input_ids": [[1, 2], [3], [4, 5, 6, 7]]})
+ assert sorted(pc._sample_lengths(ds, cfg, None, None)) == [1, 2, 4]
+ big = datasets.Dataset.from_dict({"text": ["w " * (i % 7 + 1) for i in range(5000)]})
+ lengths = pc._sample_lengths(big, cfg, tok, None)
+ assert len(lengths) == pc._SAMPLE_ROWS
+ assert pc._sample_lengths(None, cfg, tok, None) is None
+ assert pc._sample_lengths(datasets.Dataset.from_dict({"text": []}), cfg, tok, None) is None
+ iterable = datasets.Dataset.from_dict({"text": ["a"]}).to_iterable_dataset()
+ assert pc._sample_lengths(iterable, cfg, tok, None) is None
+
+
+def test_total_microbatches():
+ cfg = SimpleNamespace(
+ per_device_train_batch_size = 2, gradient_accumulation_steps = 4, max_steps = 60
+ )
+ assert pc._total_microbatches(cfg, 52000) == 240
+ cfg = SimpleNamespace(
+ per_device_train_batch_size = 2,
+ gradient_accumulation_steps = 4,
+ max_steps = -1,
+ num_train_epochs = 1,
+ )
+ assert pc._total_microbatches(cfg, 52001) == 26001
+ assert pc._total_microbatches(cfg, None) is None
+
+
+def test_moe_detection():
+ assert pc._is_moe(SimpleNamespace(num_experts = 8))
+ assert pc._is_moe(SimpleNamespace(n_routed_experts = 64))
+ assert not pc._is_moe(SimpleNamespace(num_experts = 0, hidden_size = 8))
+ assert not pc._is_moe(SimpleNamespace(hidden_size = 8))
+
+
+def test_trainer_wires_the_choice():
+ src = (ROOT / "unsloth" / "trainer.py").read_text()
+ assert "_auto_padding_choice(" in src
+ assert "from unsloth.utils.padding_choice import decide_auto_padding_free" in src
+
+
[email protected]("platform, hip, cuda, available, reason", [
+ ("linux", "6.2.41133", None, True, "ROCm"),
+ ("win32", None, "13.0", True, "Windows"),
+ ("win32", "6.2", None, True, "ROCm"),
+ ("linux", None, None, False, "not an NVIDIA CUDA device"),
+ ("darwin", None, None, False, "not an NVIDIA CUDA device"),
+]) # fmt: skip
+def test_non_cuda_linux_keeps_padding_free(monkeypatch, platform, hip, cuda, available, reason):
+ torch = pytest.importorskip("torch")
+ monkeypatch.setenv("UNSLOTH_PADDING_CHOICE", "auto")
+ monkeypatch.setattr(pc.sys, "platform", platform)
+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)
+ monkeypatch.setattr(torch.version, "cuda", cuda, raising = False)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: available)
+ use_pf, why = pc.decide_auto_padding_free(torch.nn.Linear(2, 2), SimpleNamespace(), None, None)
+ assert use_pf is True and reason in why, why
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index 6b88ba0682f..8058052c7d8 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -215,6 +215,23 @@ def _should_auto_padding_free(config) -> bool:
return getattr(config, "padding_free", None) is None
+def _auto_padding_choice(
+ original_init, model, config, train_dataset, processing_class, args, kwargs
+):
+ """(use_padding_free, reason) for an unset `padding_free` (unsloth/utils/padding_choice.py)."""
+ try:
+ from unsloth.utils.padding_choice import decide_auto_padding_free
+ try:
+ bound = inspect.signature(original_init).bind_partial(None, *args, **kwargs).arguments
+ except Exception:
+ bound = kwargs
+ return decide_auto_padding_free(
+ model, config, train_dataset, processing_class, bound.get("formatting_func")
+ )
+ except Exception as exc:
+ return True, f"padding choice skipped ({type(exc).__name__})"
+
+
def _disable_sample_packing(config):
if config is None:
return
@@ -1987,9 +2004,21 @@ def _patch_sft_trainer_auto_packing(trl_module):
if padding_free_requested:
configure_padding_free(config_arg)
elif _should_auto_padding_free(config_arg):
- configure_padding_free(config_arg)
- auto_padding_free_active = True
- logger.info("Unsloth: Padding-free batching auto-enabled for SFTTrainer instance.")
+ use_padding_free, choice_reason = _auto_padding_choice(
+ original_init, model, config_arg, train_dataset, processing_class, args, kwargs
+ )
+ if use_padding_free:
+ configure_padding_free(config_arg)
+ auto_padding_free_active = True
+ logger.info(
+ "Unsloth: Padding-free batching auto-enabled for SFTTrainer instance."
+ )
+ else:
+ # Left unset (None), exactly as UNSLOTH_DISABLE_AUTO_PADDING_FREE=1 leaves it.
+ print(
+ f"Unsloth: Padded batches + per-block compile chosen over padding-free "
+ f"({choice_reason}). Set UNSLOTH_PADDING_CHOICE=padding_free to override."
+ )
try:
original_init(self, *args, **kwargs)
diff --git a/unsloth/utils/padding_choice.py b/unsloth/utils/padding_choice.py
new file mode 100644
index 00000000000..cd1ab56b7f9
--- /dev/null
+++ b/unsloth/utils/padding_choice.py
@@ -0,0 +1,329 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Choose between padding-free and padded batches for SFT when the user left `padding_free` unset.
+
+Padding-free removes pad tokens, but its varlen batches skip the per-block compile, so each step
+launches more kernels from Python. Padded batches pay for their pad tokens on the GPU but run the
+compiled blocks. Which one is faster depends on whether a micro batch is bound by the host or by the
+GPU: measured on A100, L4 and T4 with Qwen3 0.6B / 4B LoRA (16 bit and 4 bit), padded + block
+compile was 10-17% faster only when a padded micro batch costs the GPU well under the ~0.25-0.4 s
+the host spends launching it, and up to 57% slower once the GPU is the bottleneck.
+
+Rule (all estimates, every unknown keeps padding-free, the old default):
+ padded GPU time = (linear FLOPs per token * padded tokens + attention FLOPs per pair * pairs)
+ / (datasheet 16 bit peak * _EFFICIENCY)
+ pick padded iff padded GPU time < _MAX_PADDED_GPU_S
+ and the run has enough micro batches to repay the extra block compile
+ (about 25 s in 16 bit, 100 s in 4 bit) at ~_SAVED_S_PER_MICROBATCH each.
+
+Only Linux + NVIDIA CUDA is ever switched: ROCm / HIP, Windows, XPU, MPS and CPU keep padding-free
+(no measurements there yet), unless `UNSLOTH_PADDING_CHOICE=padded` is set explicitly.
+
+`UNSLOTH_PADDING_CHOICE=padding_free` restores the previous behaviour (always padding-free),
+`=padded` forces padded batches, `=auto` (default) applies the rule. An explicit
+`padding_free=True/False` on the config always wins.
+"""
+
+import logging
+import math
+import os
+import random
+import sys
+
+__all__ = [
+ "padding_choice_mode",
+ "estimate_microbatch_tokens",
+ "choose_padding",
+ "decide_auto_padding_free",
+]
+
+logger = logging.getLogger(__name__)
+
+_EFFICIENCY = (
+ 0.5 # fraction of the datasheet peak a LoRA step reaches when GPU bound (A100 measured 0.5)
+)
+_MAX_PADDED_GPU_S = 0.10 # padded micro batch GPU estimate below which the host dominates
+_SAVED_S_PER_MICROBATCH = 0.04 # host time block compile saved per micro batch where it won
+_EXTRA_COMPILE_S_16BIT = 25.0
+_EXTRA_COMPILE_S_4BIT = 100.0
+_REPAY_FACTOR = 2.0
+_SAMPLE_ROWS = 512
+_SIM_BATCHES = 256
+
+
+def padding_choice_mode():
+ mode = os.environ.get("UNSLOTH_PADDING_CHOICE", "auto").strip().lower()
+ if mode in ("padding_free", "padding-free", "packed", "0", "off", "false", "no"):
+ return "padding_free"
+ if mode in ("padded", "pad", "block_compile"):
+ return "padded"
+ return "auto"
+
+
+def estimate_microbatch_tokens(
+ lengths,
+ batch_size,
+ max_length = None,
+ seed = 0,
+):
+ """Expected (real tokens, padded tokens, attention pairs when padded) per micro batch.
+
+ Batches are drawn uniformly from `lengths` (the trainer shuffles); padded = batch * longest.
+ """
+ lengths = [int(x) for x in lengths if x is not None and int(x) > 0]
+ if not lengths or batch_size < 1:
+ return None
+ if max_length:
+ lengths = [min(x, int(max_length)) for x in lengths]
+ rng = random.Random(seed)
+ real = padded = pairs = 0
+ for _ in range(_SIM_BATCHES):
+ batch = [lengths[rng.randrange(len(lengths))] for _ in range(batch_size)]
+ longest = max(batch)
+ real += sum(batch)
+ padded += batch_size * longest
+ pairs += batch_size * longest * (longest + 1) / 2
+ n = _SIM_BATCHES
+ return real / n, padded / n, pairs / n
+
+
+def choose_padding(
+ linear_flops_per_token,
+ attention_flops_per_pair,
+ padded_tokens,
+ padded_pairs,
+ peak_tflops,
+ total_microbatches = None,
+ load_in_4bit = False,
+):
+ """Pure rule. Returns (use_padding_free: bool, reason: str, padded_gpu_s: float | None)."""
+ if not peak_tflops or peak_tflops <= 0 or not linear_flops_per_token or not padded_tokens:
+ return True, "no estimate", None
+ flops = linear_flops_per_token * padded_tokens + (attention_flops_per_pair or 0) * (
+ padded_pairs or 0
+ )
+ gpu_s = flops / (peak_tflops * 1e12 * _EFFICIENCY)
+ if gpu_s >= _MAX_PADDED_GPU_S:
+ return True, f"GPU bound (padded micro batch ~{gpu_s * 1e3:.0f} ms of GPU work)", gpu_s
+ extra = _EXTRA_COMPILE_S_4BIT if load_in_4bit else _EXTRA_COMPILE_S_16BIT
+ needed = math.ceil(_REPAY_FACTOR * extra / _SAVED_S_PER_MICROBATCH)
+ if total_microbatches is None or total_microbatches < needed:
+ return (
+ True,
+ f"run too short to repay the block compile ({total_microbatches} < {needed} micro batches)",
+ gpu_s,
+ )
+ return False, f"host bound (padded micro batch ~{gpu_s * 1e3:.0f} ms of GPU work)", gpu_s
+
+
+# ---------------------------------------------------------------------------------------------
+# Feature gathering. Every failure returns None, which keeps padding-free.
+
+
+def _sample_indices(
+ n,
+ k = _SAMPLE_ROWS,
+ seed = 0,
+):
+ if n <= k:
+ return list(range(n))
+ return sorted(random.Random(seed).sample(range(n), k))
+
+
+def _tokenizer_of(processing_class):
+ tok = getattr(processing_class, "tokenizer", None) or processing_class
+ return tok if callable(tok) else None
+
+
+def _row_texts(rows, columns, config, formatting_func, tok):
+ """One string per row as SFTTrainer would build it, or None when the format is not known."""
+ if formatting_func is not None:
+ out = []
+ for row in rows:
+ text = formatting_func(row)
+ if isinstance(text, (list, tuple)):
+ text = text[0] if text else ""
+ if not isinstance(text, str):
+ return None
+ out.append(text)
+ return out
+ field = getattr(config, "dataset_text_field", None) or "text"
+ if field in columns:
+ texts = [row[field] for row in rows]
+ return texts if all(isinstance(t, str) for t in texts) else None
+ chat = getattr(tok, "apply_chat_template", None)
+ if "messages" in columns or "conversations" in columns:
+ key = "messages" if "messages" in columns else "conversations"
+ if chat is None or getattr(tok, "chat_template", None) is None:
+ return None
+ return [chat(row[key], tokenize = False) for row in rows]
+ if "prompt" in columns and "completion" in columns:
+ out = []
+ for row in rows:
+ p, c = row["prompt"], row["completion"]
+ if isinstance(p, str) and isinstance(c, str):
+ out.append(p + c)
+ elif isinstance(p, list) and isinstance(c, list) and chat is not None:
+ out.append(chat(p + c, tokenize = False))
+ else:
+ return None
+ return out
+ return None
+
+
+def _sample_lengths(dataset, config, processing_class, formatting_func):
+ if dataset is None or isinstance(dataset, dict):
+ return None
+ try:
+ n = len(dataset)
+ except Exception:
+ return None # IterableDataset: no cheap sample
+ if n == 0:
+ return None
+ columns = list(getattr(dataset, "column_names", None) or [])
+ idx = _sample_indices(n)
+ if "input_ids" in columns and formatting_func is None:
+ return [len(x) for x in dataset.select(idx)["input_ids"]]
+ tok = _tokenizer_of(processing_class)
+ if tok is None:
+ return None
+ rows = dataset.select(idx)
+ rows = [rows[i] for i in range(len(rows))]
+ texts = _row_texts(rows, columns, config, formatting_func, tok)
+ if not texts:
+ return None
+ return [len(x) for x in tok(texts, add_special_tokens = False)["input_ids"]]
+
+
+def _block_compile_installed(model):
+ try:
+ from ..models._block_compile import _MARK, _OFF, _enabled
+ except Exception:
+ return False
+ if not _enabled():
+ return False
+ for module in model.modules():
+ if (
+ getattr(type(module).forward, _MARK, False)
+ and module.__dict__.get("_unsloth_block_mode", None) != _OFF
+ ):
+ return True
+ return False
+
+
+def _is_moe(config):
+ config = getattr(config, "get_text_config", lambda: config)()
+ for key in ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts"):
+ if (getattr(config, key, None) or 0) > 1:
+ return True
+ return False
+
+
+def _is_4bit(model):
+ if getattr(model, "is_loaded_in_4bit", False):
+ return True
+ return any(type(m).__name__ == "Linear4bit" for m in model.modules())
+
+
+def _total_microbatches(config, n_rows):
+ bs = int(getattr(config, "per_device_train_batch_size", 1) or 1)
+ ga = int(getattr(config, "gradient_accumulation_steps", 1) or 1)
+ max_steps = getattr(config, "max_steps", -1)
+ if max_steps is not None and max_steps > 0:
+ return int(max_steps) * ga
+ epochs = float(getattr(config, "num_train_epochs", 1) or 1)
+ if not n_rows:
+ return None
+ return int(epochs * math.ceil(n_rows / bs))
+
+
+def _effective_max_length(model, config):
+ """The row cap training will use. This runs before the generated SFTTrainer init, which
+ replaces SFTConfig's default `max_length` with the model's `max_seq_length` and keeps only a
+ caller-set value below it (unsloth/models/rl.py), so mirror that precedence here."""
+ given = getattr(config, "max_length", None)
+ default = None
+ try:
+ import dataclasses
+ for field in dataclasses.fields(type(config)):
+ if field.name == "max_length":
+ default = field.default
+ except Exception:
+ pass
+ explicit = given if (given or 0) > 0 and given != default else None
+ model_length = getattr(model, "max_seq_length", None)
+ if model_length:
+ return min(int(model_length), int(explicit)) if explicit else int(model_length)
+ return given
+
+
+def decide_auto_padding_free(
+ model,
+ config,
+ train_dataset,
+ processing_class,
+ formatting_func = None,
+):
+ """True = padding-free (old default), False = padded batches. Never raises."""
+ mode = padding_choice_mode()
+ if mode == "padding_free":
+ return True, "UNSLOTH_PADDING_CHOICE=padding_free"
+ if mode == "padded":
+ return False, "UNSLOTH_PADDING_CHOICE=padded"
+ try:
+ return _decide(model, config, train_dataset, processing_class, formatting_func)
+ except Exception as exc: # the old default on any surprise
+ logger.info(f"Unsloth: padding choice skipped ({exc}).")
+ return True, f"padding choice skipped ({type(exc).__name__})"
+
+
+def _decide(model, config, train_dataset, processing_class, formatting_func):
+ import torch
+
+ if model is None or isinstance(model, str) or not hasattr(model, "modules"):
+ return True, "model not loaded"
+ # Measured only on Linux + NVIDIA CUDA. ROCm / HIP (torch.cuda.is_available() is True there),
+ # Windows (slower host, triton-windows compile), XPU, MPS and CPU keep padding-free unchanged.
+ if getattr(torch.version, "hip", None):
+ return True, "ROCm / HIP device (not measured)"
+ if sys.platform == "win32":
+ return True, "Windows (not measured)"
+ if not torch.cuda.is_available() or not getattr(torch.version, "cuda", None):
+ return True, "not an NVIDIA CUDA device"
+ if int(os.environ.get("WORLD_SIZE", "1") or 1) > 1:
+ return True, "distributed run"
+ model_config = getattr(model, "config", None)
+ if model_config is None or _is_moe(model_config):
+ return True, "MoE or unknown config"
+ if not _block_compile_installed(model):
+ return True, "per-block compile not installed"
+ bs = int(getattr(config, "per_device_train_batch_size", 1) or 1)
+ lengths = _sample_lengths(train_dataset, config, processing_class, formatting_func)
+ if not lengths:
+ return True, "dataset lengths unknown"
+ est = estimate_microbatch_tokens(lengths, bs, _effective_max_length(model, config))
+ if est is None:
+ return True, "dataset lengths unknown"
+ _, padded_tokens, padded_pairs = est
+ from .mfu import count_attention_flops_per_pair, count_linear_flops_per_token, get_peak_tflops
+
+ linear, _ = count_linear_flops_per_token(model, gradient_checkpointing = True)
+ attention = count_attention_flops_per_pair(model_config, gradient_checkpointing = True)
+ dtype = torch.bfloat16 if getattr(config, "bf16", False) else torch.float16
+ device = next((p.device for p in model.parameters() if p.device.type == "cuda"), None)
+ peak = get_peak_tflops(device, dtype)
+ try:
+ n_rows = len(train_dataset)
+ except Exception:
+ n_rows = None
+ use_pf, reason, _ = choose_padding(
+ linear,
+ attention,
+ padded_tokens,
+ padded_pairs,
+ peak,
+ total_microbatches = _total_microbatches(config, n_rows),
+ load_in_4bit = _is_4bit(model),
+ )
+ return use_pf, reason