Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Flash impl: warn about an explicit fa3 / fa4 fallback on the Trainer path too (#15750)
Daniel Hane21cfc0d520d
diff --git a/tests/utils/test_flash_impl.py b/tests/utils/test_flash_impl.pyindex eb28ce06f01..9d7dd200b3a 100644--- a/tests/utils/test_flash_impl.py+++ b/tests/utils/test_flash_impl.py@@ -829,3 +829,42 @@ def test_auto_fallback_stays_quiet(fake, monkeypatch, caplog):with caplog.at_level(logging.WARNING, logger = fi.__name__):assert fi.maybe_activate_flash_attention_impl(_model()) is Noneassert not [r for r in caplog.records if r.levelno >= logging.WARNING]++[email protected]("value", ["fa3", "fa4", "auto"])[email protected]("case", ["device", "registry"])+def test_explicit_fallback_warns_once_through_trainer_setup(fake, monkeypatch, caplog, value, case):+ import logging++ from unsloth.utils import packing++ monkeypatch.setitem(fi.FLASH_IMPL_STATE, "logged", set())+ monkeypatch.setenv(fi._ENV, value)+ if case == "device":+ monkeypatch.setattr(fi, "_device_majors", lambda: {8})+ reason = "no supported impl for devices sm[8]"+ else:+ monkeypatch.setattr(fi, "_registry", lambda: None)+ reason = "torch has no flash attention impl registry (< 2.11)"+ calls = []++ class Trainer:+ def train(self):+ calls.append(fake.current_impl)+ return "done"++ collator = types.SimpleNamespace(padding_free = True, _unsloth_padding_free_lengths_wrapped = True)+ trainer = Trainer()+ trainer.model, trainer.data_collator = _model(), collator+ with caplog.at_level(logging.WARNING, logger = fi.__name__):+ packing.enable_padding_free_metadata(trainer.model, trainer)+ packing.enable_padding_free_metadata(trainer.model, trainer)+ assert trainer.train() == "done" and trainer.train() == "done"+ assert calls == [None, None]+ warnings = [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING]+ if value == "auto":+ assert warnings == []+ assert ("train" in vars(trainer)) == (case == "device")+ else:+ assert warnings == [f"Unsloth: {fi._ENV}={value} not used: {reason}"]+ assert fake.probes == [] and fake.current_impl is Nonediff --git a/unsloth/utils/flash_impl.py b/unsloth/utils/flash_impl.pyindex 891ddf15f8b..b59c2dadfc1 100644--- a/unsloth/utils/flash_impl.py+++ b/unsloth/utils/flash_impl.py@@ -553,7 +553,9 @@ def scope_flash_attention_impl_to_train("""Active only during trainer.train: FA4's launch cost makes decode attention ~6x slower."""if trainer is None or getattr(trainer, "_unsloth_flash_impl_wrapped", False):return- if _requested() is None or _registry() is None:+ requested = _requested()+ # Explicit fa3 / fa4 still wraps without a registry so train() reports the fallback.+ if requested is None or (requested == "auto" and _registry() is None):returnoriginal = getattr(trainer, "train", None)if not callable(original):