Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep torch's bundled flash on Hopper for narrow models under UNSLOTH_FLASH_ATTENTION_IMPL=auto (#13544)

Daniel Hana96a4a799e14
diff --git a/tests/utils/test_flash_impl.py b/tests/utils/test_flash_impl.py
index 80888a51b3b..ec148884c29 100644
--- a/tests/utils/test_flash_impl.py
+++ b/tests/utils/test_flash_impl.py
@@ -131,6 +131,50 @@ def test_hopper_auto_prefers_fa3_then_fa4(fake, monkeypatch):
assert fi.maybe_activate_flash_attention_impl(_model()) is None
[email protected]("hidden_size", [896, 1024, fi._HOPPER_AUTO_MIN_HIDDEN - 8])
+def test_hopper_auto_keeps_bundled_flash_for_narrow_models(fake, monkeypatch, hidden_size):
+ # H100: FA3's extra host time per call outweighs its kernel win while a narrow model is launch bound.
+ monkeypatch.setattr(fi, "_device_majors", lambda: {9})
+ narrow = _model(hidden_size = hidden_size, head_dim = 128)
+ assert fi.maybe_activate_flash_attention_impl(narrow) is None
+ assert fake.probes == [] and fake.current_impl is None
+ assert str(hidden_size) in fi.FLASH_IMPL_STATE["reason"]
+ # No model: nothing to size, so auto stays off on sm90.
+ assert fi.maybe_activate_flash_attention_impl(None) is None
+ # fa3 / fa4 still opt in for any width.
+ for value, impl in (("fa3", "FA3"), ("fa4", "FA4")):
+ monkeypatch.setenv(fi._ENV, value)
+ assert fi.maybe_activate_flash_attention_impl(narrow) == impl
+ fi.deactivate_flash_attention_impl()
+ # sm100 is not gated on width.
+ monkeypatch.setenv(fi._ENV, "auto")
+ monkeypatch.setattr(fi, "_device_majors", lambda: {10})
+ assert fi.maybe_activate_flash_attention_impl(narrow) == "FA4"
+ fi.deactivate_flash_attention_impl()
+
+
+def test_hopper_auto_width_gate_reads_text_config_and_restores(fake, monkeypatch):
+ monkeypatch.setattr(fi, "_device_majors", lambda: {9})
+ wide = fi._HOPPER_AUTO_MIN_HIDDEN
+ multimodal = types.SimpleNamespace(
+ config = types.SimpleNamespace(
+ text_config = _config(hidden_size = wide),
+ vision_config = types.SimpleNamespace(
+ hidden_size = 1152, num_attention_heads = 16, head_dim = 72
+ ),
+ ),
+ dtype = torch.bfloat16,
+ )
+ assert fi.maybe_activate_flash_attention_impl(multimodal) == "FA3"
+ # A later narrow model restores torch's bundled flash.
+ assert fi.maybe_activate_flash_attention_impl(_model(hidden_size = 1024)) is None
+ assert fake.current_impl is None and fi.FLASH_IMPL_STATE["active"] is None
+ assert (
+ fi._hidden_size(types.SimpleNamespace(config = types.SimpleNamespace(hidden_size = True)))
+ is None
+ )
+
+
def test_hopper_auto_tries_fa4_when_fa3_fails_its_probe(fake, monkeypatch):
monkeypatch.setattr(fi, "_device_majors", lambda: {9})
tried = []
@@ -607,3 +651,21 @@ def test_hf_wrapper_skips_torch_varlen_for_shapes_the_impl_lacks(
assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before + 1
else:
assert out == ("orig", None) and not used
+
+
[email protected]("key", ["llm_config", "language_config"])
+def test_hidden_size_reads_a_decoder_exposed_only_as_llm_or_language_config(key):
+ # InternVL / Nemotron-style: a narrow vision tower, the wide decoder only under llm_config / language_config.
+ config = types.SimpleNamespace(vision_config = types.SimpleNamespace(hidden_size = 1024))
+ setattr(config, key, types.SimpleNamespace(hidden_size = 4096))
+ assert fi._hidden_size(types.SimpleNamespace(config = config)) == 4096
+
+
+def test_hidden_size_follows_get_text_config():
+ class Config:
+ vision_config = types.SimpleNamespace(hidden_size = 1152)
+
+ def get_text_config(self):
+ return types.SimpleNamespace(hidden_size = 3072)
+
+ assert fi._hidden_size(types.SimpleNamespace(config = Config())) == 3072
diff --git a/unsloth/utils/flash_impl.py b/unsloth/utils/flash_impl.py
index e365b3c1562..26f9d2bba60 100644
--- a/unsloth/utils/flash_impl.py
+++ b/unsloth/utils/flash_impl.py
@@ -5,7 +5,8 @@
torch (2.11+) swaps the flash impl process-wide, and FA4 raises (not falls back) on dropout and missing
head dims and costs ~0.2 ms per launch, so it is on only inside ``trainer.train()``, with no dropout,
supported head dims, and a probe matching torch's bundled flash.
-UNSLOTH_FLASH_ATTENTION_IMPL = auto (default: FA4 on sm100, FA3 then FA4 on sm90) | fa4 | fa3 | off.
+UNSLOTH_FLASH_ATTENTION_IMPL = auto (default: FA4 on sm100; on sm90 FA3 then FA4, only for models with
+hidden_size >= _HOPPER_AUTO_MIN_HIDDEN) | fa4 | fa3 | off.
"""
import contextlib
@@ -33,8 +34,20 @@ _DROPOUT_KEYS = (
"attention_probs_dropout_prob",
"attention_dropout_prob",
)
-_SUBCONFIG_KEYS = ("text_config", "vision_config", "audio_config", "encoder", "decoder")
+# llm_config / language_config: InternVL / Nemotron-style wrappers expose the decoder only there.
+_SUBCONFIG_KEYS = (
+ "text_config",
+ "vision_config",
+ "audio_config",
+ "encoder",
+ "decoder",
+ "llm_config",
+ "language_config",
+)
FLASH_IMPL_STATE = {"active": None, "reason": None, "verdicts": {}, "logged": set()}
+# H100: FA3 / FA4 kernels are faster but cost ~60 / ~110 us more host time per forward; at 8K tokens / step
+# hidden 1024 trains 2 to 7% slower on FA3, hidden 2048 to 4096 0.3 to 1.8% faster.
+_HOPPER_AUTO_MIN_HIDDEN = 2048
_logger = logging.getLogger(__name__)
@@ -95,7 +108,7 @@ def _device_majors():
def _candidates(requested, majors):
- """auto: FA4 on sm100; FA3 (the mature Hopper kernel) then FA4 on sm90."""
+ """auto: FA4 on sm100; FA3 (the mature Hopper kernel) then FA4 on sm90, for wide models only."""
if not majors or len(majors) != 1:
return []
major = next(iter(majors))
@@ -167,12 +180,22 @@ def _configs(model):
out.append(cfg)
for key in _SUBCONFIG_KEYS:
stack.append(getattr(cfg, key, None))
+ try:
+ stack.append(cfg.get_text_config())
+ except Exception:
+ pass
sub = getattr(cfg, "sub_configs", None)
if isinstance(sub, dict):
stack.extend(getattr(cfg, key, None) for key in sub)
return out
+def _hidden_size(model):
+ sizes = [getattr(cfg, "hidden_size", None) for cfg in _configs(model)]
+ sizes = [s for s in sizes if isinstance(s, int) and not isinstance(s, bool) and s > 0]
+ return max(sizes) if sizes else None
+
+
def _ceil8(value):
return -(-int(value) // 8) * 8
@@ -396,6 +419,15 @@ def maybe_activate_flash_attention_impl(model = None):
deactivate_flash_attention_impl()
return None
major = next(iter(majors))
+ if requested == "auto" and major == 9:
+ hidden = _hidden_size(model) if model is not None else None
+ if hidden is None or hidden < _HOPPER_AUTO_MIN_HIDDEN:
+ FLASH_IMPL_STATE["reason"] = (
+ f"auto keeps torch's bundled flash on sm90 below hidden_size {_HOPPER_AUTO_MIN_HIDDEN} "
+ f"(model: {hidden}); {_ENV}=fa3 or fa4 opts in"
+ )
+ deactivate_flash_attention_impl()
+ return None
facts = model_facts(model) if model is not None else (False, [], None, None, None, None)
cases = _probe_cases(facts)
# auto falls through to the next impl when one is missing, refuses this model or fails its probe.