Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

[perf] Qwen3.5-9B LoRA SFT with unsloth-cli.py on a B200: 0.84 s to 0.77 s per step at 1 GPU, the model-independent half of #10744 (#11238)

Hexu Zhaoa801caee39d6
diff --git a/tests/test_fla_autotuner_fast_path.py b/tests/test_fla_autotuner_fast_path.py
new file mode 100644
index 00000000000..733293af56b
--- /dev/null
+++ b/tests/test_fla_autotuner_fast_path.py
@@ -0,0 +1,262 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""PR #11238, commit 7b45cc8f9: `patch_fla_autotuner_fast_path` rebinds
+`fla.ops.utils.cache.CachedAutotuner.run` to launch with a latched config.
+
+Each test compares the patched path against the ORIGINAL `CachedAutotuner.run` on the same
+inputs, so "same kernel, same config, same numerics" is measured, not asserted.
+
+ PYTHONPATH=<tree> CUDA_VISIBLE_DEVICES=<n> pytest -q test_fla_autotuner_fast_path.py
+"""
+
+import importlib
+
+import pytest
+
+# These runners do not all ship torch/triton/fla; skip rather than erroring at collection.
+torch = pytest.importorskip("torch")
+triton = pytest.importorskip("triton")
+tl = pytest.importorskip("triton.language")
+fla_cache = pytest.importorskip("fla.ops.utils.cache")
+U = pytest.importorskip("unsloth.models._utils")
+
+PATCHER = getattr(U, "patch_fla_autotuner_fast_path", None)
+needs_patch = pytest.mark.skipif(
+ PATCHER is None, reason = "base tree: no patch_fla_autotuner_fast_path"
+)
+CUDA = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a GPU")
+
+
+class _ReuseBestCache(dict):
+ """Byte-copy of unsloth_zoo/compiler.py:4808 so the tests reproduce the real shape."""
+
+ def __contains__(self, key):
+ return len(self) > 0 or super().__contains__(key)
+
+ def __getitem__(self, key):
+ if not super().__contains__(key) and len(self) > 0:
+ return next(iter(self.values()))
+ return super().__getitem__(key)
+
+
+def _add_kernel(x_ptr, y_ptr, o_ptr, n, BLOCK: tl.constexpr):
+ pid = tl.program_id(0)
+ offs = pid * BLOCK + tl.arange(0, BLOCK)
+ mask = offs < n
+ tl.store(
+ o_ptr + offs, tl.load(x_ptr + offs, mask = mask) + tl.load(y_ptr + offs, mask = mask), mask = mask
+ )
+
+
+def _make(
+ configs,
+ key = ("n",),
+ pre_hook = None,
+):
+ from fla.ops.utils.cache import fla_cache_autotune
+ if pre_hook is not None:
+ configs = [
+ triton.Config(
+ c.kwargs, num_warps = c.num_warps, num_stages = c.num_stages, pre_hook = pre_hook
+ )
+ for c in configs
+ ]
+ return fla_cache_autotune(configs = configs, key = list(key))(_add_kernel)
+
+
+CFGS2 = [triton.Config({"BLOCK": 64}, num_warps = 2), triton.Config({"BLOCK": 128}, num_warps = 4)]
+CFGS1 = [triton.Config({"BLOCK": 128}, num_warps = 4)]
+
+
+def _grid(n):
+ # A meta-dependent grid, as every real fla kernel uses. A hardcoded grid would silently
+ # under-cover the tensor whenever the tuned BLOCK is smaller than the one assumed here,
+ # which makes the harness, not the patch, the source of any mismatch.
+ return lambda meta: (triton.cdiv(n, meta["BLOCK"]),)
+
+
+def _run(kern, n = 4096):
+ x = torch.randn(n, device = "cuda")
+ y = torch.randn(n, device = "cuda")
+ o = torch.empty_like(x)
+ kern[_grid(n)](x, y, o, n)
+ return x, y, o
+
+
+def _unpatch(cls, original):
+ cls.run = original
+
+
+def fresh():
+ """Reload the module so each test starts from an unpatched CachedAutotuner."""
+ importlib.reload(fla_cache)
+ orig = fla_cache.CachedAutotuner.run
+ yield fla_cache.CachedAutotuner
+ fla_cache.CachedAutotuner.run = orig
+
+
+# --------------------------------------------------------------------------- install
+@needs_patch
+def test_patch_installs_once_and_is_idempotent(fresh):
+ assert not getattr(fresh.run, "_unsloth_fast_path", False)
+ PATCHER()
+ first = fresh.run
+ assert getattr(first, "_unsloth_fast_path", False) is True
+ PATCHER()
+ assert fresh.run is first, "a second call must not stack another wrapper"
+
+
+@needs_patch
+def test_patch_declines_under_FLA_CACHE_MODE_always(fresh, monkeypatch):
+ class _Mode:
+ value = "always"
+
+ monkeypatch.setattr(fla_cache, "FLA_CACHE_MODE", _Mode(), raising = False)
+ PATCHER()
+ assert not getattr(
+ fresh.run, "_unsloth_fast_path", False
+ ), "the debug mode re-reads config files on every launch; the fast path must stand down"
+
+
+# --------------------------------------------------------------------------- behaviour
+@CUDA
+@needs_patch
+def test_multi_config_plain_dict_cache_falls_back(fresh):
+ kern = _make(CFGS2)
+ PATCHER()
+ _run(kern)
+ at = kern
+ while not hasattr(at, "configs"):
+ at = at.fn
+ assert type(at.cache) is not _ReuseBestCache
+ assert (
+ getattr(at, "_unsloth_fixed_config", None) is None
+ ), "without _ReuseBestCache the autotuner must keep triton's own lookup"
+
+
+@CUDA
+@needs_patch
+def test_reuse_best_cache_fast_path_matches_the_original_bit_for_bit(fresh):
+ kern_a = _make(CFGS2)
+ kern_b = _make(CFGS2)
+ # settle both the same way with the ORIGINAL run
+ _run(kern_a)
+ _run(kern_b)
+ at_a, at_b = kern_a, kern_b
+ while not hasattr(at_a, "configs"):
+ at_a = at_a.fn
+ while not hasattr(at_b, "configs"):
+ at_b = at_b.fn
+ at_a.cache = _ReuseBestCache(at_a.cache)
+ at_b.cache = _ReuseBestCache(at_b.cache)
+
+ n = 4096
+ x = torch.randn(n, device = "cuda")
+ y = torch.randn(n, device = "cuda")
+ o_ref = torch.empty_like(x)
+ o_fast = torch.empty_like(x)
+ kern_a[_grid(n)](x, y, o_ref, n) # unpatched
+
+ PATCHER()
+ kern_b[_grid(n)](x, y, o_fast, n) # patched
+
+ assert getattr(at_b, "_unsloth_fixed_config", None) is not None, "fast path did not latch"
+ assert at_b._unsloth_fixed_config.all_kwargs() == at_a.best_config.all_kwargs()
+ assert at_b.best_config is at_b._unsloth_fixed_config
+ assert torch.equal(o_ref, o_fast)
+ assert torch.equal(o_ref, x + y)
+
+
+@CUDA
+@needs_patch
+def test_single_config_fast_path_first_and_subsequent_launches(fresh):
+ kern = _make(CFGS1)
+ PATCHER()
+ x, y, o = _run(kern)
+ assert torch.equal(o, x + y)
+ at = kern
+ while not hasattr(at, "configs"):
+ at = at.fn
+ assert at._unsloth_fixed_config is at.configs[0]
+ x2, y2, o2 = _run(kern) # second launch uses the latch
+ assert torch.equal(o2, x2 + y2)
+
+
+@CUDA
+@needs_patch
+def test_per_config_pre_hook_falls_back_and_the_hook_still_fires(fresh):
+ fired = []
+ kern = _make(CFGS1, pre_hook = lambda nargs: fired.append(1))
+ PATCHER()
+ _run(kern)
+ _run(kern)
+ at = kern
+ while not hasattr(at, "configs"):
+ at = at.fn
+ assert getattr(at, "_unsloth_fixed_config", None) is None
+ assert len(fired) >= 2, "a per-Config pre_hook must run on every launch"
+
+
+# --------------------------------------------------------------------------- absence
+def test_patch_is_a_no_op_when_fla_is_missing(monkeypatch):
+ """fla < 0.5.1 has no fla.ops.utils.cache; the patch must not raise."""
+ if PATCHER is None:
+ pytest.skip("base tree")
+ import builtins
+
+ real_import = builtins.__import__
+
+ def _blocked(name, *a, **kw):
+ if name.startswith("fla"):
+ raise ImportError("no fla")
+ return real_import(name, *a, **kw)
+
+ monkeypatch.setattr(builtins, "__import__", _blocked)
+ PATCHER() # must not raise
+
+
+# --------------------------------------------------------------------------- cache probe
+CACHE_PROBE = getattr(U, "_unsloth_cache_reuses_one_config", None)
+needs_probe = pytest.mark.skipif(CACHE_PROBE is None, reason = "tree without the behaviour probe")
+
+
+@needs_probe
+def test_cache_probe_detects_the_reuse_cache_under_any_name():
+ """unsloth_zoo defines _ReuseBestCache inside a function, so there is no symbol to import.
+ The probe must key off behaviour, not the class name, or a rename silently turns the fast
+ path off."""
+
+ class RenamedTomorrow(_ReuseBestCache):
+ pass
+
+ for cls in (_ReuseBestCache, RenamedTomorrow):
+ c = cls()
+ assert CACHE_PROBE(c) is False, "an empty reuse cache reuses nothing yet"
+ c["k"] = object()
+ assert CACHE_PROBE(c) is True
+
+
+@needs_probe
+def test_cache_probe_rejects_a_plain_dict_and_anything_that_raises():
+ assert CACHE_PROBE({}) is False
+ assert CACHE_PROBE({"k": object()}) is False
+
+ class Hostile(dict):
+ def __contains__(self, key):
+ raise RuntimeError("no")
+
+ h = Hostile()
+ h["k"] = object()
+ assert CACHE_PROBE(h) is False, "a cache that raises must fall back, not propagate"
+
+
+@needs_probe
+def test_cache_probe_does_not_mutate_the_cache():
+ c = _ReuseBestCache()
+ c["k"] = object()
+ before = dict(c)
+ CACHE_PROBE(c)
+ assert dict(c) == before and len(c) == 1
diff --git a/tests/test_train_mode_asserted_once.py b/tests/test_train_mode_asserted_once.py
new file mode 100644
index 00000000000..8789a8a0569
--- /dev/null
+++ b/tests/test_train_mode_asserted_once.py
@@ -0,0 +1,182 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Behavioural difference between calling `model.train()` every micro-step and asserting
+train mode once, which is the only thing that can make that optimisation unsafe.
+"""
+
+import copy
+
+import pytest
+
+# These runners do not all ship torch; skip the module rather than erroring at collection.
+pytest.importorskip("torch")
+nn = pytest.importorskip("torch.nn")
+U = pytest.importorskip("unsloth.models._utils")
+
+HELPER = getattr(U, "_unsloth_train_if_needed", None)
+needs_helper = pytest.mark.skipif(HELPER is None, reason = "base tree: no _unsloth_train_if_needed")
+
+
+class _Tree(nn.Module):
+ """Root with a deep child, so a submodule can be flipped on its own."""
+
+ def __init__(self):
+ super().__init__()
+ self.block = nn.Sequential(nn.Linear(4, 4), nn.Dropout(0.5))
+
+ def forward(self, x):
+ return self.block(x)
+
+
+class _Wrapper(nn.Module):
+ """DDP shape: `training_step` gets this, other code holds `.module`."""
+
+ def __init__(self, inner):
+ super().__init__()
+ self.module = inner
+
+ def forward(self, x):
+ return self.module(x)
+
+
+def _modes(m):
+ return {n or "<root>": mod.training for n, mod in m.named_modules()}
+
+
+@needs_helper
+def test_first_call_asserts_train_mode_on_every_module():
+ m = _Tree()
+ m.eval()
+ assert not any(_modes(m).values())
+ HELPER(m)
+ assert all(_modes(m).values())
+ assert getattr(m, "_unsloth_train_mode_asserted", False) is True
+
+
+@needs_helper
+def test_root_eval_rearms_the_walk():
+ m = _Tree()
+ HELPER(m)
+ m.eval()
+ assert m.training is False
+ HELPER(m)
+ assert all(_modes(m).values()), "a root .eval() must be repaired by the next micro-step"
+
+
+@needs_helper
+def test_marker_is_not_in_state_dict_and_does_not_survive_a_fresh_module():
+ m = _Tree()
+ HELPER(m)
+ assert "_unsloth_train_mode_asserted" not in m.state_dict()
+ fresh = _Tree()
+ assert getattr(fresh, "_unsloth_train_mode_asserted", None) is None
+ clone = copy.deepcopy(m)
+ assert clone.training == m.training
+
+
+def test_reference_submodule_only_eval_is_repaired_by_stock_train():
+ """What the stock per-micro-step `model.train()` does. Passes on both arms."""
+ m = _Tree()
+ m.train()
+ m.block[1].eval()
+ assert m.block[1].training is False
+ m.train()
+ assert m.block[1].training is True
+
+
+@needs_helper
+def test_submodule_only_eval_is_NOT_repaired_by_the_helper():
+ """Documented semantic change: a submodule flipped to eval on its own keeps that mode."""
+ m = _Tree()
+ HELPER(m)
+ m.block[1].eval()
+ HELPER(m)
+ assert m.block[1].training is False, "if this repairs, the PR body's claim is wrong"
+
+
+def test_reference_wrapper_inner_eval_is_repaired_by_stock_train():
+ inner = _Tree()
+ w = _Wrapper(inner)
+ w.train()
+ inner.eval()
+ assert w.training is True and inner.training is False
+ w.train()
+ assert inner.training is True
+ assert all(_modes(w).values())
+
+
+@needs_helper
+ getattr(U, "_unsloth_wrappees_are_in_train_mode", None) is not None,
+ reason = "the wrappee check repairs this; kept to document what it repairs",
+)
+def test_wrapper_inner_eval_is_permanently_stale_under_the_helper():
+ """A wrapper's own `.training` never flips when the module inside it is eval'd, so a
+ root-flag check strands the whole model in eval for the rest of the run."""
+ inner = _Tree()
+ w = _Wrapper(inner)
+ HELPER(w)
+ inner.eval()
+ for _ in range(100):
+ HELPER(w)
+ assert w.training is True
+ assert inner.training is False
+ assert w.module.block[1].training is False
+
+
+@needs_helper
+def test_helper_survives_a_model_that_rejects_attribute_assignment():
+ class _Frozen(_Tree):
+ def __setattr__(self, k, v):
+ if k == "_unsloth_train_mode_asserted":
+ raise AttributeError("read-only")
+ super().__setattr__(k, v)
+
+ m = _Frozen()
+ HELPER(m)
+ assert m.training is True
+ HELPER(m)
+ assert m.training is True
+
+
+WRAPPEE_CHECK = getattr(U, "_unsloth_wrappees_are_in_train_mode", None)
+needs_fix = pytest.mark.skipif(WRAPPEE_CHECK is None, reason = "tree without the wrappee check")
+
+
+@needs_fix
+def test_wrapper_inner_eval_is_repaired_once_wrappees_are_checked():
+ """DDP + eval_strategy: evaluation_loop evals `self.model`, training_step gets
+ `self.model_wrapped`."""
+ inner = _Tree()
+ w = _Wrapper(inner)
+ HELPER(w)
+ inner.eval()
+ HELPER(w)
+ assert inner.training is True
+ assert all(_modes(w).values())
+
+
+@needs_fix
+def test_wrappee_check_is_cheap_and_stops_at_the_first_non_wrapper():
+ m = _Tree()
+ HELPER(m)
+ assert WRAPPEE_CHECK(m) is True
+ calls = []
+ orig = nn.Module.train
+ try:
+ nn.Module.train = lambda self, mode = True: calls.append(1) or orig(self, mode)
+ HELPER(m)
+ finally:
+ nn.Module.train = orig
+ assert calls == [], "an unwrapped model must still skip the walk"
+
+
+@needs_fix
+def test_wrappee_check_follows_a_nested_wrapper_chain():
+ inner = _Tree()
+ w = _Wrapper(_Wrapper(inner))
+ HELPER(w)
+ assert WRAPPEE_CHECK(w) is True
+ inner.eval()
+ assert WRAPPEE_CHECK(w) is False
diff --git a/tests/version_compat/test_transformers_pinned_symbols.py b/tests/version_compat/test_transformers_pinned_symbols.py
index 96d91aa2b62..b0188ccd414 100644
--- a/tests/version_compat/test_transformers_pinned_symbols.py
+++ b/tests/version_compat/test_transformers_pinned_symbols.py
@@ -332,3 +332,25 @@ def test_training_args_parallel_mode_importable(tag: str):
f"{tag}: transformers.training_args.ParallelMode missing; "
f"unsloth-zoo loss_utils.py:232 ImportError"
)
+
+
+def test_trainer_training_step_model_train_call_is_standalone(tag: str):
+ """unsloth#11238 rewrites the FIRST `model.train()` inside Trainer.training_step into
+ `_unsloth_train_if_needed(model)`. A release that wrote `self.model.train()` earlier in that
+ method would turn the same replace into `self._unsloth_train_if_needed(model)`, and every
+ training step would die with AttributeError. Nothing else in the rewrite guards that."""
+ candidates = ["src/transformers/trainer.py", "src/transformers/trainer/__init__.py"]
+ hit = first_match("huggingface/transformers", tag, candidates)
+ assert hit is not None
+ _, src = hit
+ m = re.search(r"\n def training_step\(.*?\n def ", src, re.DOTALL)
+ if m is None:
+ pytest.skip(f"{tag}: Trainer.training_step not found")
+ body = m.group(0)
+ idx = body.find("model.train()")
+ if idx < 0:
+ pytest.skip(f"{tag}: Trainer.training_step no longer calls model.train()")
+ assert body[max(0, idx - 11) : idx] != "self.model.", (
+ f"{tag}: the first `model.train()` in Trainer.training_step is `self.model.train()`; "
+ f"unsloth/models/_utils.py would rewrite it into `self._unsloth_train_if_needed(model)`"
+ )
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 9df944f352a..a0c479dfac1 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -3169,10 +3169,128 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):
return outputs
+# The attributes a training wrapper keeps its wrappee under. DDP, FSDP1 and DataParallel use
+# `module`; `torch.compile` uses `_orig_mod`; FSDP's own wrapper policy uses `_fsdp_wrapped_module`.
+_UNSLOTH_WRAPPED_MODULE_ATTRS = ("module", "_orig_mod", "_fsdp_wrapped_module")
+
+
+def _unsloth_wrappees_are_in_train_mode(model):
+ """Whether every module `model` merely wraps also reports training mode.
+
+ `training_step` is handed `self.model_wrapped`, but `Trainer.evaluation_loop` calls `.eval()`
+ on `self._wrap_model(self.model, training=False)`, which under DDP is the module INSIDE the
+ wrapper. That leaves the wrapper's own `.training` True while the whole model sits in eval,
+ so the root flag alone cannot tell the two apart and training would silently continue with
+ every dropout disabled. This is a handful of attribute reads, not the module walk being
+ skipped, and on an unwrapped model it stops at the first miss.
+ """
+ inner = model
+ for _ in range(4):
+ for attr in _UNSLOTH_WRAPPED_MODULE_ATTRS:
+ wrappee = getattr(inner, attr, None)
+ if isinstance(wrappee, torch.nn.Module):
+ break
+ else:
+ return True
+ if not wrappee.training:
+ return False
+ inner = wrappee
+ return True
+
+
+def _unsloth_train_if_needed(model):
+ """`Trainer.training_step` calls `model.train()` on every micro-step. On a PEFT-wrapped 9B
+ model that is a recursive walk over ~2k modules with a `__setattr__` each, several ms of
+ pure Python per micro-step while the GPU waits. The mode only has to be asserted once: skip
+ the walk when the root already reports training mode and we were the ones who set it. A root
+ `.eval()` (evaluation, `for_inference`) flips `model.training`, so the next call walks again.
+ """
+ if (
+ model.training
+ and getattr(model, "_unsloth_train_mode_asserted", False)
+ and _unsloth_wrappees_are_in_train_mode(model)
+ ):
+ return model
+ model.train()
+ try:
+ model._unsloth_train_mode_asserted = True
+ except Exception:
+ pass
+ return model
+
+
+def _unsloth_cache_reuses_one_config(cache):
+ """Whether this autotuner cache answers every key with its one stored config.
+
+ That is what unsloth_zoo's `compile_fla_no_autotune` installs, but its `_ReuseBestCache` is
+ defined inside that function, so there is no symbol to import and no class to isinstance
+ against. Probe the behaviour instead of the class name: `dict.keys()` membership always uses
+ dict's own lookup, so a key absent from the raw mapping yet reported present by `in` is
+ exactly the reuse behaviour, under any name a future rewrite gives it.
+ """
+ try:
+ if len(cache) == 0:
+ return False
+ probe = ("__unsloth_probe__",) * 2
+ return probe not in cache.keys() and probe in cache
+ except Exception:
+ return False
+
+
+def patch_fla_autotuner_fast_path():
+ """unsloth_zoo's `compile_fla_no_autotune` makes every fla Triton autotuner reuse its first
+ tuned config for every key (`_ReuseBestCache`). After that, fla's `CachedAutotuner.run` still
+ builds its own `AutotuneKey` (dict zips, dtype strings, JSON-able normalisation) and then
+ Triton's `Autotuner.run` builds a key a second time, on every launch, for a lookup whose answer
+ is always the same config. Linear-attention layers make thousands of such launches per optimizer
+ step. Once the config is settled, launch the kernel with it directly: same config, same kernel,
+ same numerics.
+ """
+ try:
+ import fla.ops.utils.cache as fla_cache
+ except Exception:
+ return
+ CachedAutotuner = getattr(fla_cache, "CachedAutotuner", None)
+ if CachedAutotuner is None or getattr(CachedAutotuner.run, "_unsloth_fast_path", False):
+ return
+ # FLA_CACHE_MODE=always re-reads the config files on every launch on purpose (a debug mode).
+ cache_mode = getattr(fla_cache, "FLA_CACHE_MODE", None)
+ if getattr(cache_mode, "value", None) == "always":
+ return
+ original_run = CachedAutotuner.run
+
+ @functools.wraps(original_run)
+ def run(self, *args, **kwargs):
+ cfg = getattr(self, "_unsloth_fixed_config", None)
+ if cfg is None:
+ cache = self.cache
+ if len(self.configs) == 1:
+ cfg = self.configs[0]
+ elif _unsloth_cache_reuses_one_config(cache):
+ cfg = next(iter(cache.values()))
+ else:
+ return original_run(self, *args, **kwargs)
+ if cfg.pre_hook is not None:
+ return original_run(self, *args, **kwargs)
+ self._unsloth_fixed_config = cfg
+ self._unsloth_fixed_kwargs = cfg.all_kwargs()
+ self.best_config = cfg
+ return self.fn.run(*args, **kwargs, **self._unsloth_fixed_kwargs)
+
+ run._unsloth_fast_path = True
+ CachedAutotuner.run = run
+
+
def patch_gradient_accumulation_fix(Trainer):
# Fixes "Output 0 of UnslothFusedLossBackward is a view and is being modified inplace" and gradient accumulation.
import inspect
+ # Before the early returns below: the fla patch is unrelated to gradient accumulation, and it
+ # can only install once fla has been imported. Loading a non-fla model first (llama.py's
+ # FastLlamaModel.from_pretrained never imports fla) and a gated-deltanet model second would
+ # otherwise leave it permanently uninstalled, since the second call returns here.
+ patch_fla_autotuner_fast_path()
+
if hasattr(Trainer, "get_batch_samples"):
if Trainer.get_batch_samples.__name__ == "_unsloth_get_batch_samples":
return
@@ -3254,6 +3372,9 @@ def patch_gradient_accumulation_fix(Trainer):
)
function = function.replace("def training_step", "def _unsloth_training_step", 1)
+ # Skip the per-micro-step recursive `model.train()` walk once train mode is set.
+ function = function.replace("model.train()", "_unsloth_train_if_needed(model)", 1)
+
# Fix 4.47.0 removing num_items_in_batch (huggingface/transformers#35121) and the case where it is nothing (huggingface/transformers#35207).
function = function.replace(
"if self.model_accepts_loss_kwargs:",