Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

12.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

"""Gemma 3 generate(): prefill stays on flex, one-token decode steps run the non-flex path."""

import ast
import inspect
import textwrap
import types

import pytest
import torch

import unsloth.models.vision as vision
from unsloth.models._utils import _EXPLICIT_FLEX_ATTR
from unsloth.models.vision import _NonFlexGenerate, _flex_generate_configs


@pytest.fixture(autouse = True)
def _no_opt_out(monkeypatch):
    monkeypatch.delenv("UNSLOTH_FLEX_ATTENTION_GENERATE", raising = False)


def _text_model(model_type = "gemma3_text", attn = "flex_attention"):
    from transformers import Gemma3TextConfig, LlamaConfig

    cls = Gemma3TextConfig if model_type == "gemma3_text" else LlamaConfig
    config = cls(
        num_hidden_layers = 1,
        hidden_size = 32,
        intermediate_size = 64,
        num_attention_heads = 2,
        num_key_value_heads = 1,
        head_dim = 16,
        vocab_size = 64,
    )
    config._attn_implementation = attn
    return _with_prepare(types.SimpleNamespace(config = config))


def _with_prepare(model):
    model.seen = []

    def prepare_inputs_for_generation(
        input_ids,
        cache_position = None,
        **kwargs,
    ):
        text = model.config.get_text_config()
        model.seen.append(text._attn_implementation)
        if cache_position is not None:
            input_ids = input_ids[:, -cache_position.shape[0] :]
        return {"input_ids": input_ids, "cache_position": cache_position}

    model.prepare_inputs_for_generation = prepare_inputs_for_generation
    return model


def _steps(
    model,
    prompt_len = 5,
    new_tokens = 3,
    cache_position = True,
):
    for step in range(new_tokens):
        n = prompt_len + step
        ids = torch.zeros(2, n, dtype = torch.long)
        kw = {}
        if cache_position:
            kw["cache_position"] = torch.arange(n) if step == 0 else torch.tensor([n - 1])
        elif step > 0:
            ids = ids[:, -1:]
        model.prepare_inputs_for_generation(ids, **kw)


def _vlm_model():
    from transformers import Gemma3Config

    config = Gemma3Config()
    config.text_config._attn_implementation = "flex_attention"
    config.vision_config._attn_implementation = "sdpa"
    return _with_prepare(types.SimpleNamespace(config = config))


def test_prefill_on_flex_decode_off_flex_and_restores():
    model = _text_model()
    prepare = model.prepare_inputs_for_generation
    with _NonFlexGenerate(model):
        _steps(model)
    assert model.seen == ["flex_attention", "eager", "eager"]
    assert model.config._attn_implementation == "flex_attention"
    assert model.prepare_inputs_for_generation is prepare
    assert not [k for k in model.__dict__ if k.startswith("_unsloth_non_flex")]


def test_step_kind_without_cache_position():
    model = _text_model()
    with _NonFlexGenerate(model):
        _steps(model, cache_position = False)
        model.prepare_inputs_for_generation(torch.zeros(1, 1, dtype = torch.long))
    assert model.seen == ["flex_attention", "eager", "eager", "eager"]
    assert model.config._attn_implementation == "flex_attention"


def test_one_token_prompt_decodes_off_flex():
    model = _text_model()
    with _NonFlexGenerate(model):
        _steps(model, prompt_len = 1)
    assert model.seen == ["eager", "eager", "eager"]


def test_second_generate_prefills_on_flex_again():
    model = _text_model()
    for _ in range(2):
        with _NonFlexGenerate(model):
            _steps(model)
    assert model.seen == ["flex_attention", "eager", "eager"] * 2


def test_instance_prepare_is_restored():
    model = _text_model()
    mine = model.prepare_inputs_for_generation
    with _NonFlexGenerate(model):
        assert model.prepare_inputs_for_generation is not mine
    assert model.__dict__["prepare_inputs_for_generation"] is mine


def test_overlapping_thread_waits_for_the_running_generate():
    # B must wait for A to exit, not switch the shared config between A's mask build and forward.
    import threading

    model = _text_model()
    outer = _NonFlexGenerate(model).__enter__()
    model.prepare_inputs_for_generation(torch.zeros(1, 8, dtype = torch.long))
    entered = threading.Event()

    def other():
        with _NonFlexGenerate(model):
            entered.set()
            _steps(model)

    thread = threading.Thread(target = other)
    thread.start()
    assert not entered.wait(0.2)
    assert model.config._attn_implementation == "flex_attention"
    outer.__exit__(None, None, None)
    thread.join(5)
    assert entered.is_set() and not thread.is_alive()
    assert model.seen == ["flex_attention", "flex_attention", "eager", "eager"]
    assert model.config._attn_implementation == "flex_attention"
    assert not [k for k in model.__dict__ if k.startswith("_unsloth_non_flex")]


def test_model_stays_copyable_after_generate():
    import copy, pickle

    model = _text_model()
    with _NonFlexGenerate(model):
        _steps(model)
    copy.deepcopy(model)
    pickle.dumps(model.config)
    assert not [k for k in model.__dict__ if "flex_generate" in k]


def test_restored_when_generate_raises():
    model = _text_model()
    prepare = model.prepare_inputs_for_generation
    with pytest.raises(RuntimeError):
        with _NonFlexGenerate(model):
            _steps(model)
            raise RuntimeError("boom")
    assert model.config._attn_implementation == "flex_attention"
    assert model.prepare_inputs_for_generation is prepare


def test_vlm_moves_only_the_decoder():
    model = _vlm_model()
    top = model.config._attn_implementation
    with _NonFlexGenerate(model):
        _steps(model)
        assert model.config.text_config._attn_implementation == "eager"
        assert model.config.vision_config._attn_implementation == "sdpa"
    assert model.seen == ["flex_attention", "eager", "eager"]
    assert model.config.text_config._attn_implementation == "flex_attention"
    assert model.config.vision_config._attn_implementation == "sdpa"
    assert model.config._attn_implementation == top


def test_nested_calls_restore_once():
    model = _text_model()
    with _NonFlexGenerate(model):
        with _NonFlexGenerate(model):
            _steps(model)
            assert model.config._attn_implementation == "eager"
        assert model.config._attn_implementation == "eager"
        _steps(model)
    assert model.seen == ["flex_attention", "eager", "eager"] * 2
    assert model.config._attn_implementation == "flex_attention"


@pytest.mark.parametrize(
    "model",
    [
        _text_model(attn = "eager"),
        _text_model(attn = "sdpa"),
        _text_model(model_type = "llama"),
    ],
    ids = ["gemma3-eager", "gemma3-sdpa", "llama-flex"],
)
def test_untouched_unless_unsloth_chose_flex_for_gemma3(model):
    before = model.config._attn_implementation
    assert _flex_generate_configs(model) == []
    with _NonFlexGenerate(model):
        _steps(model)
        assert model.config._attn_implementation == before
    assert set(model.seen) == {before}
    assert model.config._attn_implementation == before


def test_explicit_flex_request_is_kept():
    model = _text_model()
    setattr(model, _EXPLICIT_FLEX_ATTR, True)
    with _NonFlexGenerate(model):
        _steps(model)
    assert model.seen == ["flex_attention"] * 3


def test_opt_out(monkeypatch):
    monkeypatch.setenv("UNSLOTH_FLEX_ATTENTION_GENERATE", "1")
    model = _text_model()
    with _NonFlexGenerate(model):
        _steps(model)
    assert model.seen == ["flex_attention"] * 3


def _scope_entries(fn) -> list:
    """`_NonFlexGenerate(...)` calls in `fn` whose first argument is `self`, by AST so extra keyword
    arguments (flex_decode = ...) or a reflow do not hide the scope."""
    tree = ast.parse(textwrap.dedent(inspect.getsource(fn)))
    return [
        node
        for node in ast.walk(tree)
        if isinstance(node, ast.Call)
        and isinstance(node.func, ast.Name)
        and node.func.id == "_NonFlexGenerate"
        and node.args
        and isinstance(node.args[0], ast.Name)
        and node.args[0].id == "self"
    ]


def test_generate_enters_the_scope():
    assert _scope_entries(vision.unsloth_base_fast_generate)


def test_the_scope_check_reads_calls_not_text():
    def with_kwargs(self):
        with _NonFlexGenerate(self, flex_decode = True):
            pass

    def bare(self):
        with _NonFlexGenerate(self):
            pass

    def mentions_only(self):
        # _NonFlexGenerate(self)
        return "_NonFlexGenerate(self)"

    def other_model(self, model):
        with _NonFlexGenerate(model):
            pass

    assert _scope_entries(with_kwargs) and _scope_entries(bare)
    assert not _scope_entries(mentions_only) and not _scope_entries(other_model)


def _tiny_gemma3():
    from transformers import Gemma3ForCausalLM, Gemma3TextConfig

    torch.manual_seed(0)
    config = Gemma3TextConfig(
        num_hidden_layers = 2,
        hidden_size = 64,
        intermediate_size = 128,
        num_attention_heads = 2,
        num_key_value_heads = 1,
        head_dim = 32,
        vocab_size = 128,
        sliding_window = 8,
        max_position_embeddings = 256,
    )
    config._attn_implementation = "flex_attention"
    return Gemma3ForCausalLM(config).eval()


def _record_paths(model, layer):
    seen = []

    def hook(module, args, kwargs):
        hidden = kwargs.get("hidden_states", args[0] if args else None)
        mask = kwargs.get("attention_mask")
        seen.append((model.config._attn_implementation, hidden.shape[1], type(mask).__name__))

    layer.self_attn.register_forward_pre_hook(hook, with_kwargs = True)
    return seen


def _generate_kwargs(new_tokens = 5):
    ids = torch.randint(0, 128, (2, 20))
    mask = torch.ones_like(ids)
    mask[0, :3] = 0
    return dict(
        input_ids = ids,
        attention_mask = mask,
        max_new_tokens = new_tokens,
        do_sample = False,
        cache_implementation = "static",
    )


def _skip_without_flex_prefill(model, kw):
    # Only an unusable flex backend may skip: probe a plain flex prefill, then let generate() raise.
    try:
        with torch.no_grad():
            model(input_ids = kw["input_ids"], attention_mask = kw["attention_mask"])
    except Exception as e:
        pytest.skip(f"flex_attention unavailable on this CPU build: {e!r}")


def test_real_generate_prefills_on_flex_and_decodes_off_flex():
    model = _tiny_gemma3()
    kw = _generate_kwargs()
    _skip_without_flex_prefill(model, kw)
    seen = _record_paths(model, model.model.layers[0])
    with _NonFlexGenerate(model):
        out = model.generate(**kw)
    assert seen[0] == ("flex_attention", 20, "BlockMask")
    assert seen[1:] == [("eager", 1, "Tensor")] * 4
    assert model.config._attn_implementation == "flex_attention"
    model.config._attn_implementation = "sdpa"
    assert torch.equal(out, model.generate(**kw))


@pytest.mark.parametrize("base_patched", [False, True], ids = ["peft-only", "peft-and-base"])
def test_lora_generate_routes_each_step(base_patched):
    peft = pytest.importorskip("peft")
    model = _tiny_gemma3()
    kw = _generate_kwargs()
    _skip_without_flex_prefill(model, kw)
    seen = _record_paths(model, model.model.layers[0])
    lora = peft.get_peft_model(model, peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"]))

    def scoped(target):
        original = target.generate

        def generate(self, *args, **kwargs):
            with _NonFlexGenerate(self):
                return original(*args, **kwargs)

        target.generate = types.MethodType(generate, target)

    if base_patched:
        scoped(model)
    scoped(lora)
    out = lora.generate(**kw)
    assert seen[0] == ("flex_attention", 20, "BlockMask")
    assert seen[1:] == [("eager", 1, "Tensor")] * 4
    assert model.config._attn_implementation == "flex_attention"
    for m in (lora, model, lora.base_model):
        assert "prepare_inputs_for_generation" not in m.__dict__
        assert not [k for k in m.__dict__ if k.startswith("_unsloth_non_flex")]
    model.config._attn_implementation = "sdpa"
    del lora.__dict__["generate"]
    assert torch.equal(out, lora.generate(**kw))