Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Gemma 3: generate on the non-flex attention path instead of flex_attention's fallback (#16041)

Daniel Han3740f83358da
diff --git a/tests/test_gemma3_generate_off_flex.py b/tests/test_gemma3_generate_off_flex.py
new file mode 100644
index 00000000000..6f9c8ccab12
--- /dev/null
+++ b/tests/test_gemma3_generate_off_flex.py
@@ -0,0 +1,336 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Gemma 3 generate(): prefill stays on flex, one-token decode steps run the non-flex path."""
+
+import inspect
+import types
+
+import pytest
+import torch
+
+import unsloth.models.vision as vision
+from unsloth.models._utils import _EXPLICIT_FLEX_ATTR
+from unsloth.models.vision import _NonFlexGenerate, _flex_generate_configs
+
+
[email protected](autouse = True)
+def _no_opt_out(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_FLEX_ATTENTION_GENERATE", raising = False)
+
+
+def _text_model(model_type = "gemma3_text", attn = "flex_attention"):
+ from transformers import Gemma3TextConfig, LlamaConfig
+
+ cls = Gemma3TextConfig if model_type == "gemma3_text" else LlamaConfig
+ config = cls(
+ num_hidden_layers = 1,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 16,
+ vocab_size = 64,
+ )
+ config._attn_implementation = attn
+ return _with_prepare(types.SimpleNamespace(config = config))
+
+
+def _with_prepare(model):
+ model.seen = []
+
+ def prepare_inputs_for_generation(
+ input_ids,
+ cache_position = None,
+ **kwargs,
+ ):
+ text = model.config.get_text_config()
+ model.seen.append(text._attn_implementation)
+ if cache_position is not None:
+ input_ids = input_ids[:, -cache_position.shape[0] :]
+ return {"input_ids": input_ids, "cache_position": cache_position}
+
+ model.prepare_inputs_for_generation = prepare_inputs_for_generation
+ return model
+
+
+def _steps(
+ model,
+ prompt_len = 5,
+ new_tokens = 3,
+ cache_position = True,
+):
+ for step in range(new_tokens):
+ n = prompt_len + step
+ ids = torch.zeros(2, n, dtype = torch.long)
+ kw = {}
+ if cache_position:
+ kw["cache_position"] = torch.arange(n) if step == 0 else torch.tensor([n - 1])
+ elif step > 0:
+ ids = ids[:, -1:]
+ model.prepare_inputs_for_generation(ids, **kw)
+
+
+def _vlm_model():
+ from transformers import Gemma3Config
+
+ config = Gemma3Config()
+ config.text_config._attn_implementation = "flex_attention"
+ config.vision_config._attn_implementation = "sdpa"
+ return _with_prepare(types.SimpleNamespace(config = config))
+
+
+def test_prefill_on_flex_decode_off_flex_and_restores():
+ model = _text_model()
+ prepare = model.prepare_inputs_for_generation
+ with _NonFlexGenerate(model):
+ _steps(model)
+ assert model.seen == ["flex_attention", "eager", "eager"]
+ assert model.config._attn_implementation == "flex_attention"
+ assert model.prepare_inputs_for_generation is prepare
+ assert not [k for k in model.__dict__ if k.startswith("_unsloth_non_flex")]
+
+
+def test_step_kind_without_cache_position():
+ model = _text_model()
+ with _NonFlexGenerate(model):
+ _steps(model, cache_position = False)
+ model.prepare_inputs_for_generation(torch.zeros(1, 1, dtype = torch.long))
+ assert model.seen == ["flex_attention", "eager", "eager", "eager"]
+ assert model.config._attn_implementation == "flex_attention"
+
+
+def test_one_token_prompt_decodes_off_flex():
+ model = _text_model()
+ with _NonFlexGenerate(model):
+ _steps(model, prompt_len = 1)
+ assert model.seen == ["eager", "eager", "eager"]
+
+
+def test_second_generate_prefills_on_flex_again():
+ model = _text_model()
+ for _ in range(2):
+ with _NonFlexGenerate(model):
+ _steps(model)
+ assert model.seen == ["flex_attention", "eager", "eager"] * 2
+
+
+def test_instance_prepare_is_restored():
+ model = _text_model()
+ mine = model.prepare_inputs_for_generation
+ with _NonFlexGenerate(model):
+ assert model.prepare_inputs_for_generation is not mine
+ assert model.__dict__["prepare_inputs_for_generation"] is mine
+
+
+def test_overlapping_thread_waits_for_the_running_generate():
+ # B must wait for A to exit, not switch the shared config between A's mask build and forward.
+ import threading
+
+ model = _text_model()
+ outer = _NonFlexGenerate(model).__enter__()
+ model.prepare_inputs_for_generation(torch.zeros(1, 8, dtype = torch.long))
+ entered = threading.Event()
+
+ def other():
+ with _NonFlexGenerate(model):
+ entered.set()
+ _steps(model)
+
+ thread = threading.Thread(target = other)
+ thread.start()
+ assert not entered.wait(0.2)
+ assert model.config._attn_implementation == "flex_attention"
+ outer.__exit__(None, None, None)
+ thread.join(5)
+ assert entered.is_set() and not thread.is_alive()
+ assert model.seen == ["flex_attention", "flex_attention", "eager", "eager"]
+ assert model.config._attn_implementation == "flex_attention"
+ assert not [k for k in model.__dict__ if k.startswith("_unsloth_non_flex")]
+
+
+def test_model_stays_copyable_after_generate():
+ import copy, pickle
+
+ model = _text_model()
+ with _NonFlexGenerate(model):
+ _steps(model)
+ copy.deepcopy(model)
+ pickle.dumps(model.config)
+ assert not [k for k in model.__dict__ if "flex_generate" in k]
+
+
+def test_restored_when_generate_raises():
+ model = _text_model()
+ prepare = model.prepare_inputs_for_generation
+ with pytest.raises(RuntimeError):
+ with _NonFlexGenerate(model):
+ _steps(model)
+ raise RuntimeError("boom")
+ assert model.config._attn_implementation == "flex_attention"
+ assert model.prepare_inputs_for_generation is prepare
+
+
+def test_vlm_moves_only_the_decoder():
+ model = _vlm_model()
+ top = model.config._attn_implementation
+ with _NonFlexGenerate(model):
+ _steps(model)
+ assert model.config.text_config._attn_implementation == "eager"
+ assert model.config.vision_config._attn_implementation == "sdpa"
+ assert model.seen == ["flex_attention", "eager", "eager"]
+ assert model.config.text_config._attn_implementation == "flex_attention"
+ assert model.config.vision_config._attn_implementation == "sdpa"
+ assert model.config._attn_implementation == top
+
+
+def test_nested_calls_restore_once():
+ model = _text_model()
+ with _NonFlexGenerate(model):
+ with _NonFlexGenerate(model):
+ _steps(model)
+ assert model.config._attn_implementation == "eager"
+ assert model.config._attn_implementation == "eager"
+ _steps(model)
+ assert model.seen == ["flex_attention", "eager", "eager"] * 2
+ assert model.config._attn_implementation == "flex_attention"
+
+
+ "model",
+ [
+ _text_model(attn = "eager"),
+ _text_model(attn = "sdpa"),
+ _text_model(model_type = "llama"),
+ ],
+ ids = ["gemma3-eager", "gemma3-sdpa", "llama-flex"],
+)
+def test_untouched_unless_unsloth_chose_flex_for_gemma3(model):
+ before = model.config._attn_implementation
+ assert _flex_generate_configs(model) == []
+ with _NonFlexGenerate(model):
+ _steps(model)
+ assert model.config._attn_implementation == before
+ assert set(model.seen) == {before}
+ assert model.config._attn_implementation == before
+
+
+def test_explicit_flex_request_is_kept():
+ model = _text_model()
+ setattr(model, _EXPLICIT_FLEX_ATTR, True)
+ with _NonFlexGenerate(model):
+ _steps(model)
+ assert model.seen == ["flex_attention"] * 3
+
+
+def test_opt_out(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_FLEX_ATTENTION_GENERATE", "1")
+ model = _text_model()
+ with _NonFlexGenerate(model):
+ _steps(model)
+ assert model.seen == ["flex_attention"] * 3
+
+
+def test_generate_enters_the_scope():
+ source = inspect.getsource(vision.unsloth_base_fast_generate)
+ assert "_NonFlexGenerate(self)" in source
+
+
+def _tiny_gemma3():
+ from transformers import Gemma3ForCausalLM, Gemma3TextConfig
+
+ torch.manual_seed(0)
+ config = Gemma3TextConfig(
+ num_hidden_layers = 2,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 32,
+ vocab_size = 128,
+ sliding_window = 8,
+ max_position_embeddings = 256,
+ )
+ config._attn_implementation = "flex_attention"
+ return Gemma3ForCausalLM(config).eval()
+
+
+def _record_paths(model, layer):
+ seen = []
+
+ def hook(module, args, kwargs):
+ hidden = kwargs.get("hidden_states", args[0] if args else None)
+ mask = kwargs.get("attention_mask")
+ seen.append((model.config._attn_implementation, hidden.shape[1], type(mask).__name__))
+
+ layer.self_attn.register_forward_pre_hook(hook, with_kwargs = True)
+ return seen
+
+
+def _generate_kwargs(new_tokens = 5):
+ ids = torch.randint(0, 128, (2, 20))
+ mask = torch.ones_like(ids)
+ mask[0, :3] = 0
+ return dict(
+ input_ids = ids,
+ attention_mask = mask,
+ max_new_tokens = new_tokens,
+ do_sample = False,
+ cache_implementation = "static",
+ )
+
+
+def _skip_without_flex_prefill(model, kw):
+ # Only an unusable flex backend may skip: probe a plain flex prefill, then let generate() raise.
+ try:
+ with torch.no_grad():
+ model(input_ids = kw["input_ids"], attention_mask = kw["attention_mask"])
+ except Exception as e:
+ pytest.skip(f"flex_attention unavailable on this CPU build: {e!r}")
+
+
+def test_real_generate_prefills_on_flex_and_decodes_off_flex():
+ model = _tiny_gemma3()
+ kw = _generate_kwargs()
+ _skip_without_flex_prefill(model, kw)
+ seen = _record_paths(model, model.model.layers[0])
+ with _NonFlexGenerate(model):
+ out = model.generate(**kw)
+ assert seen[0] == ("flex_attention", 20, "BlockMask")
+ assert seen[1:] == [("eager", 1, "Tensor")] * 4
+ assert model.config._attn_implementation == "flex_attention"
+ model.config._attn_implementation = "sdpa"
+ assert torch.equal(out, model.generate(**kw))
+
+
[email protected]("base_patched", [False, True], ids = ["peft-only", "peft-and-base"])
+def test_lora_generate_routes_each_step(base_patched):
+ peft = pytest.importorskip("peft")
+ model = _tiny_gemma3()
+ kw = _generate_kwargs()
+ _skip_without_flex_prefill(model, kw)
+ seen = _record_paths(model, model.model.layers[0])
+ lora = peft.get_peft_model(model, peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))
+
+ def scoped(target):
+ original = target.generate
+
+ def generate(self, *args, **kwargs):
+ with _NonFlexGenerate(self):
+ return original(*args, **kwargs)
+
+ target.generate = types.MethodType(generate, target)
+
+ if base_patched:
+ scoped(model)
+ scoped(lora)
+ out = lora.generate(**kw)
+ assert seen[0] == ("flex_attention", 20, "BlockMask")
+ assert seen[1:] == [("eager", 1, "Tensor")] * 4
+ assert model.config._attn_implementation == "flex_attention"
+ for m in (lora, model, lora.base_model):
+ assert "prepare_inputs_for_generation" not in m.__dict__
+ assert not [k for k in m.__dict__ if k.startswith("_unsloth_non_flex")]
+ model.config._attn_implementation = "sdpa"
+ del lora.__dict__["generate"]
+ assert torch.equal(out, lora.generate(**kw))
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index f84a76b9377..1f2aacd0aa7 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -257,6 +257,10 @@ from ._utils import (
resolve_attention_implementation,
requests_flex_attention,
_EXPLICIT_FLEX_ATTR,
+ _config_prefers_flex_attention,
+ _config_set,
+ _iter_attention_configs,
+ _text_attention_configs,
_get_text_only_config,
_is_family_text_decoder,
_config_get,
@@ -393,6 +397,7 @@ import warnings
from typing import Optional, Tuple, List, Union
import re, inspect, sys
import contextlib
+import threading
try:
from huggingface_hub.utils import get_token
@@ -1532,6 +1537,210 @@ class _EagerDecodeSteps:
return False
+# One-token decode is slower on flex than SDPA, but prefill must stay on flex (its block mask skips tiles a
+# dense mask computes). The path is picked in prepare_inputs_for_generation, where the step's mask is built.
+_FLEX_GENERATE_ENV_VAR = "UNSLOTH_FLEX_ATTENTION_GENERATE"
+_FLEX_GENERATE_LOCK = threading.Lock()
+
+
+# Kept outside the model: an RLock in __dict__ breaks deepcopy / pickle.
+class _ModelMap:
+ def __init__(self):
+ self.weak, self.strong = weakref.WeakKeyDictionary(), {}
+
+ def _where(self, model):
+ try:
+ weakref.ref(model)
+ return self.weak, model
+ except TypeError:
+ return self.strong, id(model)
+
+ def get(
+ self,
+ model,
+ default = None,
+ ):
+ table, key = self._where(model)
+ return table.get(key, default)
+
+ def set(self, model, value):
+ table, key = self._where(model)
+ table[key] = value
+
+ def pop(self, model):
+ table, key = self._where(model)
+ table.pop(key, None)
+
+
+_FLEX_GENERATE_SERIAL = _ModelMap()
+_FLEX_GENERATE_ROUTING = _ModelMap()
+_NON_FLEX_DECODE = "eager"
+
+
+def _flex_generate_configs(model):
+ if os.environ.get(_FLEX_GENERATE_ENV_VAR, "0") == "1":
+ return []
+ config = getattr(model, "config", None)
+ if config is None or not _config_prefers_flex_attention(config):
+ return []
+ base = model.get_base_model() if hasattr(model, "get_base_model") else model
+ if getattr(base, _EXPLICIT_FLEX_ATTR, False) or getattr(model, _EXPLICIT_FLEX_ATTR, False):
+ return []
+ return [
+ c
+ for c in _text_attention_configs(config)
+ if _config_get(c, "_attn_implementation", None) == "flex_attention"
+ ]
+
+
+def _step_query_length(model_inputs):
+ if not isinstance(model_inputs, dict):
+ return None
+ for key in ("inputs_embeds", "input_ids", "decoder_input_ids"):
+ x = model_inputs.get(key)
+ if isinstance(x, torch.Tensor) and x.dim() >= 2:
+ return int(x.shape[1])
+ return None
+
+
+class _NonFlexGenerate:
+ """Threads are serialised per model: they share one config, switched between mask build and forward."""
+
+ def __init__(self, model):
+ self.model = model
+ self.active = False
+ self.serial = None
+ self.base = model.get_base_model() if hasattr(model, "get_base_model") else model
+
+ def _set(self, decode):
+ state = self.model.__dict__
+ if state.get("_unsloth_non_flex_generate_pinned", False):
+ decode = True
+ value = _NON_FLEX_DECODE if decode else "flex_attention"
+ for c in state.get("_unsloth_non_flex_generate_configs", ()):
+ if _config_get(c, "_attn_implementation", None) != value:
+ _config_set(c, "_attn_implementation", value)
+
+ def _wrap_prepare(self, prepare):
+ scope = self
+
+ @functools.wraps(prepare)
+ def prepare_inputs_for_generation(*args, **kwargs):
+ cache_position = kwargs.get("cache_position")
+ if isinstance(cache_position, torch.Tensor) and cache_position.dim() == 1:
+ decode = cache_position.shape[0] == 1
+ else:
+ decode = scope.steps > 0
+ scope.steps += 1
+ scope._set(decode)
+ model_inputs = prepare(*args, **kwargs)
+ q_len = _step_query_length(model_inputs)
+ if q_len is not None and (q_len == 1) != decode:
+ scope._set(q_len == 1)
+ model_inputs = prepare(*args, **kwargs)
+ return model_inputs
+
+ return prepare_inputs_for_generation
+
+ def __enter__(self):
+ state = self.model.__dict__
+ with _FLEX_GENERATE_LOCK:
+ routed = _FLEX_GENERATE_ROUTING.get(self.base, 0) > 0 or bool(
+ _flex_generate_configs(self.model)
+ )
+ serial = None
+ if routed:
+ serial = _FLEX_GENERATE_SERIAL.get(self.base)
+ if serial is None:
+ serial = threading.RLock()
+ _FLEX_GENERATE_SERIAL.set(self.base, serial)
+ if serial is not None:
+ serial.acquire()
+ self.serial = serial
+ with _FLEX_GENERATE_LOCK:
+ depth = state.get("_unsloth_non_flex_generate", 0)
+ if depth == 0:
+ configs = _flex_generate_configs(self.model)
+ if not configs:
+ self._release()
+ return self
+ # Parents first: the transformers setter also writes sub-configs.
+ saved = [
+ (c, field, _config_get(c, field, None))
+ for c in _iter_attention_configs(self.model.config)
+ for field in ("_attn_implementation", "attn_implementation")
+ if _config_get(c, field, None) is not None
+ ]
+ state["_unsloth_non_flex_generate_saved"] = saved
+ state["_unsloth_non_flex_generate_configs"] = configs
+ _FLEX_GENERATE_ROUTING.set(self.base, _FLEX_GENERATE_ROUTING.get(self.base, 0) + 1)
+ self.steps = 0
+ # PeftModelForCausalLM.generate calls the base model's prepare_inputs_for_generation: wrap both.
+ targets = [self.model]
+ base = (
+ self.model.get_base_model() if hasattr(self.model, "get_base_model") else None
+ )
+ if base is not None and base is not self.model:
+ targets.append(base)
+ wrapped = []
+ for target in targets:
+ prepare = getattr(target, "prepare_inputs_for_generation", None)
+ if prepare is None:
+ continue
+ wrapped.append(
+ (
+ target,
+ "prepare_inputs_for_generation" in target.__dict__,
+ target.__dict__.get("prepare_inputs_for_generation"),
+ )
+ )
+ target.prepare_inputs_for_generation = self._wrap_prepare(prepare)
+ if wrapped:
+ state["_unsloth_non_flex_generate_prepare"] = wrapped
+ else:
+ state["_unsloth_non_flex_generate_pinned"] = True
+ self._set(True)
+ state["_unsloth_non_flex_generate"] = depth + 1
+ self.active = True
+ return self
+
+ def _release(self):
+ if self.serial is not None:
+ self.serial.release()
+ self.serial = None
+
+ def __exit__(self, *exc):
+ if not self.active:
+ return False
+ try:
+ self._restore()
+ finally:
+ self._release()
+ return False
+
+ def _restore(self):
+ state = self.model.__dict__
+ with _FLEX_GENERATE_LOCK:
+ depth = state.get("_unsloth_non_flex_generate", 1) - 1
+ state["_unsloth_non_flex_generate"] = depth
+ if depth == 0:
+ state.pop("_unsloth_non_flex_generate", None)
+ if state.pop("_unsloth_non_flex_generate_configs", None) is not None:
+ left = _FLEX_GENERATE_ROUTING.get(self.base, 1) - 1
+ if left > 0:
+ _FLEX_GENERATE_ROUTING.set(self.base, left)
+ else:
+ _FLEX_GENERATE_ROUTING.pop(self.base)
+ state.pop("_unsloth_non_flex_generate_pinned", None)
+ for target, had, previous in state.pop("_unsloth_non_flex_generate_prepare", ()):
+ if had:
+ target.__dict__["prepare_inputs_for_generation"] = previous
+ else:
+ target.__dict__.pop("prepare_inputs_for_generation", None)
+ for c, field, value in state.pop("_unsloth_non_flex_generate_saved", ()):
+ _config_set(c, field, value)
+
+
def _compiles_decode(model):
if unsloth_decode_compile is None:
return False
@@ -2179,7 +2388,13 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
decode_scope = _CompileDecodeOnRepeat(self) if compile_decode else contextlib.nullcontext()
eager_scope = _EagerDecodeSteps(self) if _eager_decodes(self) else contextlib.nullcontext()
try:
- with decode_scope, eager_scope, _get_inference_mode_context_manager(self), autocaster:
+ with (
+ _NonFlexGenerate(self),
+ decode_scope,
+ eager_scope,
+ _get_inference_mode_context_manager(self),
+ autocaster,
+ ):
output = self._old_generate(*args, **kwargs)
finally:
_clear_generation_caches(self)