Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Gemma 3: generate on the non-flex attention path instead of flex_attention's fallback (#16041)
Daniel Han3740f83358da
diff --git a/tests/test_gemma3_generate_off_flex.py b/tests/test_gemma3_generate_off_flex.pynew file mode 100644index 00000000000..6f9c8ccab12--- /dev/null+++ b/tests/test_gemma3_generate_off_flex.py@@ -0,0 +1,336 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Gemma 3 generate(): prefill stays on flex, one-token decode steps run the non-flex path."""++import inspect+import types++import pytest+import torch++import unsloth.models.vision as vision+from unsloth.models._utils import _EXPLICIT_FLEX_ATTR+from unsloth.models.vision import _NonFlexGenerate, _flex_generate_configs++[email protected](autouse = True)+def _no_opt_out(monkeypatch):+ monkeypatch.delenv("UNSLOTH_FLEX_ATTENTION_GENERATE", raising = False)+++def _text_model(model_type = "gemma3_text", attn = "flex_attention"):+ from transformers import Gemma3TextConfig, LlamaConfig++ cls = Gemma3TextConfig if model_type == "gemma3_text" else LlamaConfig+ config = cls(+ num_hidden_layers = 1,+ hidden_size = 32,+ intermediate_size = 64,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 16,+ vocab_size = 64,+ )+ config._attn_implementation = attn+ return _with_prepare(types.SimpleNamespace(config = config))+++def _with_prepare(model):+ model.seen = []++ def prepare_inputs_for_generation(+ input_ids,+ cache_position = None,+ **kwargs,+ ):+ text = model.config.get_text_config()+ model.seen.append(text._attn_implementation)+ if cache_position is not None:+ input_ids = input_ids[:, -cache_position.shape[0] :]+ return {"input_ids": input_ids, "cache_position": cache_position}++ model.prepare_inputs_for_generation = prepare_inputs_for_generation+ return model+++def _steps(+ model,+ prompt_len = 5,+ new_tokens = 3,+ cache_position = True,+):+ for step in range(new_tokens):+ n = prompt_len + step+ ids = torch.zeros(2, n, dtype = torch.long)+ kw = {}+ if cache_position:+ kw["cache_position"] = torch.arange(n) if step == 0 else torch.tensor([n - 1])+ elif step > 0:+ ids = ids[:, -1:]+ model.prepare_inputs_for_generation(ids, **kw)+++def _vlm_model():+ from transformers import Gemma3Config++ config = Gemma3Config()+ config.text_config._attn_implementation = "flex_attention"+ config.vision_config._attn_implementation = "sdpa"+ return _with_prepare(types.SimpleNamespace(config = config))+++def test_prefill_on_flex_decode_off_flex_and_restores():+ model = _text_model()+ prepare = model.prepare_inputs_for_generation+ with _NonFlexGenerate(model):+ _steps(model)+ assert model.seen == ["flex_attention", "eager", "eager"]+ assert model.config._attn_implementation == "flex_attention"+ assert model.prepare_inputs_for_generation is prepare+ assert not [k for k in model.__dict__ if k.startswith("_unsloth_non_flex")]+++def test_step_kind_without_cache_position():+ model = _text_model()+ with _NonFlexGenerate(model):+ _steps(model, cache_position = False)+ model.prepare_inputs_for_generation(torch.zeros(1, 1, dtype = torch.long))+ assert model.seen == ["flex_attention", "eager", "eager", "eager"]+ assert model.config._attn_implementation == "flex_attention"+++def test_one_token_prompt_decodes_off_flex():+ model = _text_model()+ with _NonFlexGenerate(model):+ _steps(model, prompt_len = 1)+ assert model.seen == ["eager", "eager", "eager"]+++def test_second_generate_prefills_on_flex_again():+ model = _text_model()+ for _ in range(2):+ with _NonFlexGenerate(model):+ _steps(model)+ assert model.seen == ["flex_attention", "eager", "eager"] * 2+++def test_instance_prepare_is_restored():+ model = _text_model()+ mine = model.prepare_inputs_for_generation+ with _NonFlexGenerate(model):+ assert model.prepare_inputs_for_generation is not mine+ assert model.__dict__["prepare_inputs_for_generation"] is mine+++def test_overlapping_thread_waits_for_the_running_generate():+ # B must wait for A to exit, not switch the shared config between A's mask build and forward.+ import threading++ model = _text_model()+ outer = _NonFlexGenerate(model).__enter__()+ model.prepare_inputs_for_generation(torch.zeros(1, 8, dtype = torch.long))+ entered = threading.Event()++ def other():+ with _NonFlexGenerate(model):+ entered.set()+ _steps(model)++ thread = threading.Thread(target = other)+ thread.start()+ assert not entered.wait(0.2)+ assert model.config._attn_implementation == "flex_attention"+ outer.__exit__(None, None, None)+ thread.join(5)+ assert entered.is_set() and not thread.is_alive()+ assert model.seen == ["flex_attention", "flex_attention", "eager", "eager"]+ assert model.config._attn_implementation == "flex_attention"+ assert not [k for k in model.__dict__ if k.startswith("_unsloth_non_flex")]+++def test_model_stays_copyable_after_generate():+ import copy, pickle++ model = _text_model()+ with _NonFlexGenerate(model):+ _steps(model)+ copy.deepcopy(model)+ pickle.dumps(model.config)+ assert not [k for k in model.__dict__ if "flex_generate" in k]+++def test_restored_when_generate_raises():+ model = _text_model()+ prepare = model.prepare_inputs_for_generation+ with pytest.raises(RuntimeError):+ with _NonFlexGenerate(model):+ _steps(model)+ raise RuntimeError("boom")+ assert model.config._attn_implementation == "flex_attention"+ assert model.prepare_inputs_for_generation is prepare+++def test_vlm_moves_only_the_decoder():+ model = _vlm_model()+ top = model.config._attn_implementation+ with _NonFlexGenerate(model):+ _steps(model)+ assert model.config.text_config._attn_implementation == "eager"+ assert model.config.vision_config._attn_implementation == "sdpa"+ assert model.seen == ["flex_attention", "eager", "eager"]+ assert model.config.text_config._attn_implementation == "flex_attention"+ assert model.config.vision_config._attn_implementation == "sdpa"+ assert model.config._attn_implementation == top+++def test_nested_calls_restore_once():+ model = _text_model()+ with _NonFlexGenerate(model):+ with _NonFlexGenerate(model):+ _steps(model)+ assert model.config._attn_implementation == "eager"+ assert model.config._attn_implementation == "eager"+ _steps(model)+ assert model.seen == ["flex_attention", "eager", "eager"] * 2+ assert model.config._attn_implementation == "flex_attention"+++ "model",+ [+ _text_model(attn = "eager"),+ _text_model(attn = "sdpa"),+ _text_model(model_type = "llama"),+ ],+ ids = ["gemma3-eager", "gemma3-sdpa", "llama-flex"],+)+def test_untouched_unless_unsloth_chose_flex_for_gemma3(model):+ before = model.config._attn_implementation+ assert _flex_generate_configs(model) == []+ with _NonFlexGenerate(model):+ _steps(model)+ assert model.config._attn_implementation == before+ assert set(model.seen) == {before}+ assert model.config._attn_implementation == before+++def test_explicit_flex_request_is_kept():+ model = _text_model()+ setattr(model, _EXPLICIT_FLEX_ATTR, True)+ with _NonFlexGenerate(model):+ _steps(model)+ assert model.seen == ["flex_attention"] * 3+++def test_opt_out(monkeypatch):+ monkeypatch.setenv("UNSLOTH_FLEX_ATTENTION_GENERATE", "1")+ model = _text_model()+ with _NonFlexGenerate(model):+ _steps(model)+ assert model.seen == ["flex_attention"] * 3+++def test_generate_enters_the_scope():+ source = inspect.getsource(vision.unsloth_base_fast_generate)+ assert "_NonFlexGenerate(self)" in source+++def _tiny_gemma3():+ from transformers import Gemma3ForCausalLM, Gemma3TextConfig++ torch.manual_seed(0)+ config = Gemma3TextConfig(+ num_hidden_layers = 2,+ hidden_size = 64,+ intermediate_size = 128,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 32,+ vocab_size = 128,+ sliding_window = 8,+ max_position_embeddings = 256,+ )+ config._attn_implementation = "flex_attention"+ return Gemma3ForCausalLM(config).eval()+++def _record_paths(model, layer):+ seen = []++ def hook(module, args, kwargs):+ hidden = kwargs.get("hidden_states", args[0] if args else None)+ mask = kwargs.get("attention_mask")+ seen.append((model.config._attn_implementation, hidden.shape[1], type(mask).__name__))++ layer.self_attn.register_forward_pre_hook(hook, with_kwargs = True)+ return seen+++def _generate_kwargs(new_tokens = 5):+ ids = torch.randint(0, 128, (2, 20))+ mask = torch.ones_like(ids)+ mask[0, :3] = 0+ return dict(+ input_ids = ids,+ attention_mask = mask,+ max_new_tokens = new_tokens,+ do_sample = False,+ cache_implementation = "static",+ )+++def _skip_without_flex_prefill(model, kw):+ # Only an unusable flex backend may skip: probe a plain flex prefill, then let generate() raise.+ try:+ with torch.no_grad():+ model(input_ids = kw["input_ids"], attention_mask = kw["attention_mask"])+ except Exception as e:+ pytest.skip(f"flex_attention unavailable on this CPU build: {e!r}")+++def test_real_generate_prefills_on_flex_and_decodes_off_flex():+ model = _tiny_gemma3()+ kw = _generate_kwargs()+ _skip_without_flex_prefill(model, kw)+ seen = _record_paths(model, model.model.layers[0])+ with _NonFlexGenerate(model):+ out = model.generate(**kw)+ assert seen[0] == ("flex_attention", 20, "BlockMask")+ assert seen[1:] == [("eager", 1, "Tensor")] * 4+ assert model.config._attn_implementation == "flex_attention"+ model.config._attn_implementation = "sdpa"+ assert torch.equal(out, model.generate(**kw))++[email protected]("base_patched", [False, True], ids = ["peft-only", "peft-and-base"])+def test_lora_generate_routes_each_step(base_patched):+ peft = pytest.importorskip("peft")+ model = _tiny_gemma3()+ kw = _generate_kwargs()+ _skip_without_flex_prefill(model, kw)+ seen = _record_paths(model, model.model.layers[0])+ lora = peft.get_peft_model(model, peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))++ def scoped(target):+ original = target.generate++ def generate(self, *args, **kwargs):+ with _NonFlexGenerate(self):+ return original(*args, **kwargs)++ target.generate = types.MethodType(generate, target)++ if base_patched:+ scoped(model)+ scoped(lora)+ out = lora.generate(**kw)+ assert seen[0] == ("flex_attention", 20, "BlockMask")+ assert seen[1:] == [("eager", 1, "Tensor")] * 4+ assert model.config._attn_implementation == "flex_attention"+ for m in (lora, model, lora.base_model):+ assert "prepare_inputs_for_generation" not in m.__dict__+ assert not [k for k in m.__dict__ if k.startswith("_unsloth_non_flex")]+ model.config._attn_implementation = "sdpa"+ del lora.__dict__["generate"]+ assert torch.equal(out, lora.generate(**kw))diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex f84a76b9377..1f2aacd0aa7 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -257,6 +257,10 @@ from ._utils import (resolve_attention_implementation,requests_flex_attention,_EXPLICIT_FLEX_ATTR,+ _config_prefers_flex_attention,+ _config_set,+ _iter_attention_configs,+ _text_attention_configs,_get_text_only_config,_is_family_text_decoder,_config_get,@@ -393,6 +397,7 @@ import warningsfrom typing import Optional, Tuple, List, Unionimport re, inspect, sysimport contextlib+import threadingtry:from huggingface_hub.utils import get_token@@ -1532,6 +1537,210 @@ class _EagerDecodeSteps:return False+# One-token decode is slower on flex than SDPA, but prefill must stay on flex (its block mask skips tiles a+# dense mask computes). The path is picked in prepare_inputs_for_generation, where the step's mask is built.+_FLEX_GENERATE_ENV_VAR = "UNSLOTH_FLEX_ATTENTION_GENERATE"+_FLEX_GENERATE_LOCK = threading.Lock()+++# Kept outside the model: an RLock in __dict__ breaks deepcopy / pickle.+class _ModelMap:+ def __init__(self):+ self.weak, self.strong = weakref.WeakKeyDictionary(), {}++ def _where(self, model):+ try:+ weakref.ref(model)+ return self.weak, model+ except TypeError:+ return self.strong, id(model)++ def get(+ self,+ model,+ default = None,+ ):+ table, key = self._where(model)+ return table.get(key, default)++ def set(self, model, value):+ table, key = self._where(model)+ table[key] = value++ def pop(self, model):+ table, key = self._where(model)+ table.pop(key, None)+++_FLEX_GENERATE_SERIAL = _ModelMap()+_FLEX_GENERATE_ROUTING = _ModelMap()+_NON_FLEX_DECODE = "eager"+++def _flex_generate_configs(model):+ if os.environ.get(_FLEX_GENERATE_ENV_VAR, "0") == "1":+ return []+ config = getattr(model, "config", None)+ if config is None or not _config_prefers_flex_attention(config):+ return []+ base = model.get_base_model() if hasattr(model, "get_base_model") else model+ if getattr(base, _EXPLICIT_FLEX_ATTR, False) or getattr(model, _EXPLICIT_FLEX_ATTR, False):+ return []+ return [+ c+ for c in _text_attention_configs(config)+ if _config_get(c, "_attn_implementation", None) == "flex_attention"+ ]+++def _step_query_length(model_inputs):+ if not isinstance(model_inputs, dict):+ return None+ for key in ("inputs_embeds", "input_ids", "decoder_input_ids"):+ x = model_inputs.get(key)+ if isinstance(x, torch.Tensor) and x.dim() >= 2:+ return int(x.shape[1])+ return None+++class _NonFlexGenerate:+ """Threads are serialised per model: they share one config, switched between mask build and forward."""++ def __init__(self, model):+ self.model = model+ self.active = False+ self.serial = None+ self.base = model.get_base_model() if hasattr(model, "get_base_model") else model++ def _set(self, decode):+ state = self.model.__dict__+ if state.get("_unsloth_non_flex_generate_pinned", False):+ decode = True+ value = _NON_FLEX_DECODE if decode else "flex_attention"+ for c in state.get("_unsloth_non_flex_generate_configs", ()):+ if _config_get(c, "_attn_implementation", None) != value:+ _config_set(c, "_attn_implementation", value)++ def _wrap_prepare(self, prepare):+ scope = self++ @functools.wraps(prepare)+ def prepare_inputs_for_generation(*args, **kwargs):+ cache_position = kwargs.get("cache_position")+ if isinstance(cache_position, torch.Tensor) and cache_position.dim() == 1:+ decode = cache_position.shape[0] == 1+ else:+ decode = scope.steps > 0+ scope.steps += 1+ scope._set(decode)+ model_inputs = prepare(*args, **kwargs)+ q_len = _step_query_length(model_inputs)+ if q_len is not None and (q_len == 1) != decode:+ scope._set(q_len == 1)+ model_inputs = prepare(*args, **kwargs)+ return model_inputs++ return prepare_inputs_for_generation++ def __enter__(self):+ state = self.model.__dict__+ with _FLEX_GENERATE_LOCK:+ routed = _FLEX_GENERATE_ROUTING.get(self.base, 0) > 0 or bool(+ _flex_generate_configs(self.model)+ )+ serial = None+ if routed:+ serial = _FLEX_GENERATE_SERIAL.get(self.base)+ if serial is None:+ serial = threading.RLock()+ _FLEX_GENERATE_SERIAL.set(self.base, serial)+ if serial is not None:+ serial.acquire()+ self.serial = serial+ with _FLEX_GENERATE_LOCK:+ depth = state.get("_unsloth_non_flex_generate", 0)+ if depth == 0:+ configs = _flex_generate_configs(self.model)+ if not configs:+ self._release()+ return self+ # Parents first: the transformers setter also writes sub-configs.+ saved = [+ (c, field, _config_get(c, field, None))+ for c in _iter_attention_configs(self.model.config)+ for field in ("_attn_implementation", "attn_implementation")+ if _config_get(c, field, None) is not None+ ]+ state["_unsloth_non_flex_generate_saved"] = saved+ state["_unsloth_non_flex_generate_configs"] = configs+ _FLEX_GENERATE_ROUTING.set(self.base, _FLEX_GENERATE_ROUTING.get(self.base, 0) + 1)+ self.steps = 0+ # PeftModelForCausalLM.generate calls the base model's prepare_inputs_for_generation: wrap both.+ targets = [self.model]+ base = (+ self.model.get_base_model() if hasattr(self.model, "get_base_model") else None+ )+ if base is not None and base is not self.model:+ targets.append(base)+ wrapped = []+ for target in targets:+ prepare = getattr(target, "prepare_inputs_for_generation", None)+ if prepare is None:+ continue+ wrapped.append(+ (+ target,+ "prepare_inputs_for_generation" in target.__dict__,+ target.__dict__.get("prepare_inputs_for_generation"),+ )+ )+ target.prepare_inputs_for_generation = self._wrap_prepare(prepare)+ if wrapped:+ state["_unsloth_non_flex_generate_prepare"] = wrapped+ else:+ state["_unsloth_non_flex_generate_pinned"] = True+ self._set(True)+ state["_unsloth_non_flex_generate"] = depth + 1+ self.active = True+ return self++ def _release(self):+ if self.serial is not None:+ self.serial.release()+ self.serial = None++ def __exit__(self, *exc):+ if not self.active:+ return False+ try:+ self._restore()+ finally:+ self._release()+ return False++ def _restore(self):+ state = self.model.__dict__+ with _FLEX_GENERATE_LOCK:+ depth = state.get("_unsloth_non_flex_generate", 1) - 1+ state["_unsloth_non_flex_generate"] = depth+ if depth == 0:+ state.pop("_unsloth_non_flex_generate", None)+ if state.pop("_unsloth_non_flex_generate_configs", None) is not None:+ left = _FLEX_GENERATE_ROUTING.get(self.base, 1) - 1+ if left > 0:+ _FLEX_GENERATE_ROUTING.set(self.base, left)+ else:+ _FLEX_GENERATE_ROUTING.pop(self.base)+ state.pop("_unsloth_non_flex_generate_pinned", None)+ for target, had, previous in state.pop("_unsloth_non_flex_generate_prepare", ()):+ if had:+ target.__dict__["prepare_inputs_for_generation"] = previous+ else:+ target.__dict__.pop("prepare_inputs_for_generation", None)+ for c, field, value in state.pop("_unsloth_non_flex_generate_saved", ()):+ _config_set(c, field, value)++def _compiles_decode(model):if unsloth_decode_compile is None:return False@@ -2179,7 +2388,13 @@ def unsloth_base_fast_generate(self, *args, **kwargs):decode_scope = _CompileDecodeOnRepeat(self) if compile_decode else contextlib.nullcontext()eager_scope = _EagerDecodeSteps(self) if _eager_decodes(self) else contextlib.nullcontext()try:- with decode_scope, eager_scope, _get_inference_mode_context_manager(self), autocaster:+ with (+ _NonFlexGenerate(self),+ decode_scope,+ eager_scope,+ _get_inference_mode_context_manager(self),+ autocaster,+ ):output = self._old_generate(*args, **kwargs)finally:_clear_generation_caches(self)