Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Flash impl: warn about an explicit fa3 / fa4 fallback on the Trainer path too (#15750)

Daniel Hane21cfc0d520d
diff --git a/tests/utils/test_flash_impl.py b/tests/utils/test_flash_impl.py
index eb28ce06f01..9d7dd200b3a 100644
--- a/tests/utils/test_flash_impl.py
+++ b/tests/utils/test_flash_impl.py
@@ -829,3 +829,42 @@ def test_auto_fallback_stays_quiet(fake, monkeypatch, caplog):
with caplog.at_level(logging.WARNING, logger = fi.__name__):
assert fi.maybe_activate_flash_attention_impl(_model()) is None
assert not [r for r in caplog.records if r.levelno >= logging.WARNING]
+
+
[email protected]("value", ["fa3", "fa4", "auto"])
[email protected]("case", ["device", "registry"])
+def test_explicit_fallback_warns_once_through_trainer_setup(fake, monkeypatch, caplog, value, case):
+ import logging
+
+ from unsloth.utils import packing
+
+ monkeypatch.setitem(fi.FLASH_IMPL_STATE, "logged", set())
+ monkeypatch.setenv(fi._ENV, value)
+ if case == "device":
+ monkeypatch.setattr(fi, "_device_majors", lambda: {8})
+ reason = "no supported impl for devices sm[8]"
+ else:
+ monkeypatch.setattr(fi, "_registry", lambda: None)
+ reason = "torch has no flash attention impl registry (< 2.11)"
+ calls = []
+
+ class Trainer:
+ def train(self):
+ calls.append(fake.current_impl)
+ return "done"
+
+ collator = types.SimpleNamespace(padding_free = True, _unsloth_padding_free_lengths_wrapped = True)
+ trainer = Trainer()
+ trainer.model, trainer.data_collator = _model(), collator
+ with caplog.at_level(logging.WARNING, logger = fi.__name__):
+ packing.enable_padding_free_metadata(trainer.model, trainer)
+ packing.enable_padding_free_metadata(trainer.model, trainer)
+ assert trainer.train() == "done" and trainer.train() == "done"
+ assert calls == [None, None]
+ warnings = [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING]
+ if value == "auto":
+ assert warnings == []
+ assert ("train" in vars(trainer)) == (case == "device")
+ else:
+ assert warnings == [f"Unsloth: {fi._ENV}={value} not used: {reason}"]
+ assert fake.probes == [] and fake.current_impl is None
diff --git a/unsloth/utils/flash_impl.py b/unsloth/utils/flash_impl.py
index 891ddf15f8b..b59c2dadfc1 100644
--- a/unsloth/utils/flash_impl.py
+++ b/unsloth/utils/flash_impl.py
@@ -553,7 +553,9 @@ def scope_flash_attention_impl_to_train(
"""Active only during trainer.train: FA4's launch cost makes decode attention ~6x slower."""
if trainer is None or getattr(trainer, "_unsloth_flash_impl_wrapped", False):
return
- if _requested() is None or _registry() is None:
+ requested = _requested()
+ # Explicit fa3 / fa4 still wraps without a registry so train() reports the fallback.
+ if requested is None or (requested == "auto" and _registry() is None):
return
original = getattr(trainer, "train", None)
if not callable(original):