Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep torch's bundled flash on Hopper for narrow models under UNSLOTH_FLASH_ATTENTION_IMPL=auto (#13544)
Daniel Hana96a4a799e14
diff --git a/tests/utils/test_flash_impl.py b/tests/utils/test_flash_impl.pyindex 80888a51b3b..ec148884c29 100644--- a/tests/utils/test_flash_impl.py+++ b/tests/utils/test_flash_impl.py@@ -131,6 +131,50 @@ def test_hopper_auto_prefers_fa3_then_fa4(fake, monkeypatch):assert fi.maybe_activate_flash_attention_impl(_model()) is None[email protected]("hidden_size", [896, 1024, fi._HOPPER_AUTO_MIN_HIDDEN - 8])+def test_hopper_auto_keeps_bundled_flash_for_narrow_models(fake, monkeypatch, hidden_size):+ # H100: FA3's extra host time per call outweighs its kernel win while a narrow model is launch bound.+ monkeypatch.setattr(fi, "_device_majors", lambda: {9})+ narrow = _model(hidden_size = hidden_size, head_dim = 128)+ assert fi.maybe_activate_flash_attention_impl(narrow) is None+ assert fake.probes == [] and fake.current_impl is None+ assert str(hidden_size) in fi.FLASH_IMPL_STATE["reason"]+ # No model: nothing to size, so auto stays off on sm90.+ assert fi.maybe_activate_flash_attention_impl(None) is None+ # fa3 / fa4 still opt in for any width.+ for value, impl in (("fa3", "FA3"), ("fa4", "FA4")):+ monkeypatch.setenv(fi._ENV, value)+ assert fi.maybe_activate_flash_attention_impl(narrow) == impl+ fi.deactivate_flash_attention_impl()+ # sm100 is not gated on width.+ monkeypatch.setenv(fi._ENV, "auto")+ monkeypatch.setattr(fi, "_device_majors", lambda: {10})+ assert fi.maybe_activate_flash_attention_impl(narrow) == "FA4"+ fi.deactivate_flash_attention_impl()+++def test_hopper_auto_width_gate_reads_text_config_and_restores(fake, monkeypatch):+ monkeypatch.setattr(fi, "_device_majors", lambda: {9})+ wide = fi._HOPPER_AUTO_MIN_HIDDEN+ multimodal = types.SimpleNamespace(+ config = types.SimpleNamespace(+ text_config = _config(hidden_size = wide),+ vision_config = types.SimpleNamespace(+ hidden_size = 1152, num_attention_heads = 16, head_dim = 72+ ),+ ),+ dtype = torch.bfloat16,+ )+ assert fi.maybe_activate_flash_attention_impl(multimodal) == "FA3"+ # A later narrow model restores torch's bundled flash.+ assert fi.maybe_activate_flash_attention_impl(_model(hidden_size = 1024)) is None+ assert fake.current_impl is None and fi.FLASH_IMPL_STATE["active"] is None+ assert (+ fi._hidden_size(types.SimpleNamespace(config = types.SimpleNamespace(hidden_size = True)))+ is None+ )++def test_hopper_auto_tries_fa4_when_fa3_fails_its_probe(fake, monkeypatch):monkeypatch.setattr(fi, "_device_majors", lambda: {9})tried = []@@ -607,3 +651,21 @@ def test_hf_wrapper_skips_torch_varlen_for_shapes_the_impl_lacks(assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before + 1else:assert out == ("orig", None) and not used++[email protected]("key", ["llm_config", "language_config"])+def test_hidden_size_reads_a_decoder_exposed_only_as_llm_or_language_config(key):+ # InternVL / Nemotron-style: a narrow vision tower, the wide decoder only under llm_config / language_config.+ config = types.SimpleNamespace(vision_config = types.SimpleNamespace(hidden_size = 1024))+ setattr(config, key, types.SimpleNamespace(hidden_size = 4096))+ assert fi._hidden_size(types.SimpleNamespace(config = config)) == 4096+++def test_hidden_size_follows_get_text_config():+ class Config:+ vision_config = types.SimpleNamespace(hidden_size = 1152)++ def get_text_config(self):+ return types.SimpleNamespace(hidden_size = 3072)++ assert fi._hidden_size(types.SimpleNamespace(config = Config())) == 3072diff --git a/unsloth/utils/flash_impl.py b/unsloth/utils/flash_impl.pyindex e365b3c1562..26f9d2bba60 100644--- a/unsloth/utils/flash_impl.py+++ b/unsloth/utils/flash_impl.py@@ -5,7 +5,8 @@torch (2.11+) swaps the flash impl process-wide, and FA4 raises (not falls back) on dropout and missinghead dims and costs ~0.2 ms per launch, so it is on only inside ``trainer.train()``, with no dropout,supported head dims, and a probe matching torch's bundled flash.-UNSLOTH_FLASH_ATTENTION_IMPL = auto (default: FA4 on sm100, FA3 then FA4 on sm90) | fa4 | fa3 | off.+UNSLOTH_FLASH_ATTENTION_IMPL = auto (default: FA4 on sm100; on sm90 FA3 then FA4, only for models with+hidden_size >= _HOPPER_AUTO_MIN_HIDDEN) | fa4 | fa3 | off."""import contextlib@@ -33,8 +34,20 @@ _DROPOUT_KEYS = ("attention_probs_dropout_prob","attention_dropout_prob",)-_SUBCONFIG_KEYS = ("text_config", "vision_config", "audio_config", "encoder", "decoder")+# llm_config / language_config: InternVL / Nemotron-style wrappers expose the decoder only there.+_SUBCONFIG_KEYS = (+ "text_config",+ "vision_config",+ "audio_config",+ "encoder",+ "decoder",+ "llm_config",+ "language_config",+)FLASH_IMPL_STATE = {"active": None, "reason": None, "verdicts": {}, "logged": set()}+# H100: FA3 / FA4 kernels are faster but cost ~60 / ~110 us more host time per forward; at 8K tokens / step+# hidden 1024 trains 2 to 7% slower on FA3, hidden 2048 to 4096 0.3 to 1.8% faster.+_HOPPER_AUTO_MIN_HIDDEN = 2048_logger = logging.getLogger(__name__)@@ -95,7 +108,7 @@ def _device_majors():def _candidates(requested, majors):- """auto: FA4 on sm100; FA3 (the mature Hopper kernel) then FA4 on sm90."""+ """auto: FA4 on sm100; FA3 (the mature Hopper kernel) then FA4 on sm90, for wide models only."""if not majors or len(majors) != 1:return []major = next(iter(majors))@@ -167,12 +180,22 @@ def _configs(model):out.append(cfg)for key in _SUBCONFIG_KEYS:stack.append(getattr(cfg, key, None))+ try:+ stack.append(cfg.get_text_config())+ except Exception:+ passsub = getattr(cfg, "sub_configs", None)if isinstance(sub, dict):stack.extend(getattr(cfg, key, None) for key in sub)return out+def _hidden_size(model):+ sizes = [getattr(cfg, "hidden_size", None) for cfg in _configs(model)]+ sizes = [s for s in sizes if isinstance(s, int) and not isinstance(s, bool) and s > 0]+ return max(sizes) if sizes else None++def _ceil8(value):return -(-int(value) // 8) * 8@@ -396,6 +419,15 @@ def maybe_activate_flash_attention_impl(model = None):deactivate_flash_attention_impl()return Nonemajor = next(iter(majors))+ if requested == "auto" and major == 9:+ hidden = _hidden_size(model) if model is not None else None+ if hidden is None or hidden < _HOPPER_AUTO_MIN_HIDDEN:+ FLASH_IMPL_STATE["reason"] = (+ f"auto keeps torch's bundled flash on sm90 below hidden_size {_HOPPER_AUTO_MIN_HIDDEN} "+ f"(model: {hidden}); {_ENV}=fa3 or fa4 opts in"+ )+ deactivate_flash_attention_impl()+ return Nonefacts = model_facts(model) if model is not None else (False, [], None, None, None, None)cases = _probe_cases(facts)# auto falls through to the next impl when one is missing, refuses this model or fails its probe.