Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Use the dynamic cache for Gemma multimodal generation (#10903)

Daniel Hanb58ea42a0a64
diff --git a/tests/test_gemma_multimodal_cache_selection.py b/tests/test_gemma_multimodal_cache_selection.py
new file mode 100644
index 00000000000..e81f7ac0c11
--- /dev/null
+++ b/tests/test_gemma_multimodal_cache_selection.py
@@ -0,0 +1,426 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Cache selection for multimodal generation (unslothai/unsloth#6028).
+
+A static cache makes transformers skip mask materialisation at prefill and rely
+on `is_causal`, which drops the bidirectional image/audio block overlay the Gemma
+multimodal families build. Image tokens then attend causally and spatial
+grounding degrades. `generate` must fall back to the dynamic cache for those
+requests, and only those.
+"""
+
+import sys
+import types
+
+import torch
+
+import pytest
+
+from unsloth.models.vision import (
+ _BIDIRECTIONAL_MASK_BUILDERS,
+ _MEDIA_TOKEN_TYPES,
+ _STATIC_CACHE_IMPLEMENTATIONS,
+ _dynamic_cache_choice,
+ _needs_bidirectional_multimodal_mask,
+)
+
+# Both helpers landed in transformers 5.10; earlier versions expose neither.
+_FIRST_VERSION_WITH_BLOCK_MASK = "5.10.0"
+
+
+def _make_model(
+ module_name,
+ has_helper,
+ helper = None,
+):
+ module = types.ModuleType(module_name)
+ if has_helper:
+ setattr(module, helper or _BIDIRECTIONAL_MASK_BUILDERS[0], lambda *args, **kwargs: None)
+ sys.modules[module_name] = module
+
+ model_cls = type("Model", (), {})
+ model_cls.__module__ = module_name
+ return model_cls()
+
+
+def gemma_like(request):
+ name = f"_fake_gemma_{request.node.name}"
+ yield _make_model(name, True)
+ sys.modules.pop(name, None)
+
+
+def causal_vlm(request):
+ name = f"_fake_qwen_{request.node.name}"
+ yield _make_model(name, False)
+ sys.modules.pop(name, None)
+
+
[email protected]("media_kwarg", ["pixel_values", "pixel_values_videos"])
+def test_media_request_on_gemma_uses_dynamic_cache(gemma_like, media_kwarg):
+ assert _needs_bidirectional_multimodal_mask(gemma_like, {media_kwarg: object()})
+
+
+def test_audio_only_keeps_the_static_cache(gemma_like):
+ """get_block_sequence_ids_for_mask blocks token types 1 and 2 only, so audio
+ stays causal and must not lose the compiled path."""
+ assert not _needs_bidirectional_multimodal_mask(gemma_like, {"input_features": object()})
+
+
+def test_text_only_keeps_static_cache(gemma_like):
+ assert not _needs_bidirectional_multimodal_mask(gemma_like, {"input_ids": object()})
+
+
+def test_none_media_kwarg_keeps_static_cache(gemma_like):
+ assert not _needs_bidirectional_multimodal_mask(gemma_like, {"pixel_values": None})
+
+
+def test_causal_vlm_keeps_static_cache(causal_vlm):
+ """Qwen2-VL, Llava and PaliGemma have no bidirectional block overlay, so they
+ must not lose the static-cache path."""
+ assert not _needs_bidirectional_multimodal_mask(causal_vlm, {"pixel_values": object()})
+
+
+def test_unknown_module_does_not_raise():
+ orphan_cls = type("Orphan", (), {})
+ orphan_cls.__module__ = "_module_that_does_not_exist"
+ assert not _needs_bidirectional_multimodal_mask(orphan_cls(), {"pixel_values": object()})
+
+
[email protected]("helper", _BIDIRECTIONAL_MASK_BUILDERS)
+def test_either_mask_builder_gates_the_guard(request, helper):
+ """The two names span transformers 5.10 to current, so either one alone has
+ to be enough."""
+ name = f"_fake_helper_{request.node.name}"
+ model = _make_model(name, True, helper = helper)
+ try:
+ assert _needs_bidirectional_multimodal_mask(model, {"pixel_values": object()})
+ finally:
+ sys.modules.pop(name, None)
+
+
+def test_real_gemma_modules_expose_the_helper():
+ """Pin the upstream symbols the guard keys on, so a rename is caught here.
+
+ These helpers landed in transformers 5.10. Before that neither name exists
+ (measured on 4.57.6 and 5.5.0), the guard is inert by design, and Gemma keeps
+ the static path it always had, so there is nothing to pin.
+ """
+ transformers = pytest.importorskip("transformers")
+ from packaging.version import InvalidVersion, Version
+
+ try:
+ new_enough = Version(transformers.__version__) >= Version(_FIRST_VERSION_WITH_BLOCK_MASK)
+ except InvalidVersion:
+ pytest.skip(f"cannot order version {transformers.__version__!r}")
+ if not new_enough:
+ pytest.skip(f"no block mask helper before transformers {_FIRST_VERSION_WITH_BLOCK_MASK}")
+
+ seen = False
+ for name in ("gemma3", "gemma4", "gemma4_unified"):
+ try:
+ module = __import__(f"transformers.models.{name}.modeling_{name}", fromlist = ["x"])
+ except Exception:
+ continue
+ seen = True
+ assert any(hasattr(module, h) for h in _BIDIRECTIONAL_MASK_BUILDERS), name
+ if not seen:
+ pytest.skip("no Gemma multimodal module in this transformers version")
+
+
+def test_causal_vlms_never_expose_a_mask_builder():
+ """The other half of the gate: these must stay on the static path on every
+ version, so neither name may appear on them."""
+ pytest.importorskip("transformers")
+ for name in ("qwen2_vl", "llava", "paligemma"):
+ try:
+ module = __import__(f"transformers.models.{name}.modeling_{name}", fromlist = ["x"])
+ except Exception:
+ continue
+ assert not any(hasattr(module, h) for h in _BIDIRECTIONAL_MASK_BUILDERS), name
+
+
+class _Cfg:
+ def __init__(self, bidir):
+ self.use_bidirectional_attention = bidir
+
+ def get_text_config(self):
+ return self
+
+
+def _gemma4_shaped(module, bidir):
+ """A model whose create_masks_for_generate takes mm_token_type_ids, which is
+ the Gemma 4 form and the one upstream gates on the config value."""
+
+ class Model:
+ config = _Cfg(bidir)
+
+ @staticmethod
+ def create_masks_for_generate(
+ config,
+ inputs_embeds,
+ attention_mask,
+ past_key_values,
+ position_ids,
+ mm_token_type_ids = None,
+ **kwargs,
+ ):
+ return None
+
+ Model.__module__ = module.__name__
+ return Model()
+
+
+def _gemma3_shaped(module):
+ """Gemma 3 takes token_type_ids and overlays regardless of the config flag,
+ which it sets to False."""
+
+ class Model:
+ config = _Cfg(False)
+
+ @staticmethod
+ def create_masks_for_generate(
+ config,
+ inputs_embeds,
+ attention_mask,
+ past_key_values,
+ position_ids,
+ token_type_ids = None,
+ **kwargs,
+ ):
+ return None
+
+ Model.__module__ = module.__name__
+ return Model()
+
+
+def test_causal_gemma4_variant_keeps_the_static_cache(gemma_like):
+ """E2B / E4B leave use_bidirectional_attention None, so upstream builds no
+ overlay and they must not lose the compiled path."""
+ module = sys.modules[type(gemma_like).__module__]
+ assert not _needs_bidirectional_multimodal_mask(
+ _gemma4_shaped(module, None), {"pixel_values": object()}
+ )
+
+
+def test_vision_gemma4_variant_is_gated(gemma_like):
+ module = sys.modules[type(gemma_like).__module__]
+ assert _needs_bidirectional_multimodal_mask(
+ _gemma4_shaped(module, "vision"), {"pixel_values": object()}
+ )
+
+
+def test_gemma3_is_gated_despite_the_flag_being_false(gemma_like):
+ module = sys.modules[type(gemma_like).__module__]
+ assert _needs_bidirectional_multimodal_mask(_gemma3_shaped(module), {"pixel_values": object()})
+
+
+def test_precomputed_embeds_with_media_tokens_are_gated(gemma_like):
+ """inputs_embeds carries no media kwarg; the token types are the signal."""
+ module = sys.modules[type(gemma_like).__module__]
+ model = _gemma4_shaped(module, "vision")
+ ids = torch.tensor([[0, 0, 1, 1, 0]])
+ assert _needs_bidirectional_multimodal_mask(model, {"mm_token_type_ids": ids})
+
+
+def test_text_only_token_types_keep_the_static_cache(gemma_like):
+ """The processor emits mm_token_type_ids for text-only prompts too, all
+ zeros, so presence alone must not cost the static path."""
+ module = sys.modules[type(gemma_like).__module__]
+ model = _gemma4_shaped(module, "vision")
+ ids = torch.zeros(1, 5, dtype = torch.long)
+ assert not _needs_bidirectional_multimodal_mask(model, {"mm_token_type_ids": ids})
+
+
+def test_gemma3_token_type_ids_are_read(gemma_like):
+ module = sys.modules[type(gemma_like).__module__]
+ model = _gemma3_shaped(module)
+ assert _needs_bidirectional_multimodal_mask(
+ model, {"token_type_ids": torch.tensor([[0, 1, 0]])}
+ )
+ assert not _needs_bidirectional_multimodal_mask(
+ model, {"token_type_ids": torch.zeros(1, 3, dtype = torch.long)}
+ )
+
+
+def test_non_tensor_token_types_do_not_raise(gemma_like):
+ module = sys.modules[type(gemma_like).__module__]
+ model = _gemma4_shaped(module, "vision")
+ assert not _needs_bidirectional_multimodal_mask(model, {"mm_token_type_ids": "not a tensor"})
+
+
+def _resolve_cache_choice(force_dynamic, cache_implementation, kwargs):
+ """The assignment block from unsloth_base_fast_generate, isolated."""
+ forced = _dynamic_cache_choice(kwargs) if force_dynamic else None
+ if "generation_config" in kwargs:
+ kwargs["generation_config"].cache_implementation = (
+ forced if force_dynamic else cache_implementation
+ )
+ return kwargs["generation_config"].cache_implementation
+ kwargs["cache_implementation"] = forced if force_dynamic else cache_implementation
+ return kwargs["cache_implementation"]
+
+
+class _GenCfg:
+ cache_implementation = "static"
+
+
+def test_media_request_pins_the_literal_dynamic():
+ """None is refilled from the model default by _prepare_generation_config, so
+ the guard has to name the cache it wants."""
+ assert _resolve_cache_choice(True, None, {}) == "dynamic"
+ assert _resolve_cache_choice(True, None, {"generation_config": _GenCfg()}) == "dynamic"
+
+
+def test_caller_generation_config_is_overridden():
+ """The reported hole: a caller-supplied config whose field says static."""
+ cfg = _GenCfg()
+ assert cfg.cache_implementation == "static"
+ _resolve_cache_choice(True, None, {"generation_config": cfg})
+ assert cfg.cache_implementation == "dynamic"
+
+
+def test_text_only_still_gets_the_static_cache():
+ assert _resolve_cache_choice(False, "static", {}) == "static"
+ cfg = _GenCfg()
+ _resolve_cache_choice(False, "static", {"generation_config": cfg})
+ assert cfg.cache_implementation == "static"
+
+
+def test_none_is_preserved_when_not_forcing():
+ """bfloat16 mixed precision clears the cache by setting None; that path must
+ keep its own meaning."""
+ assert _resolve_cache_choice(False, None, {}) is None
+
+
+def test_dynamic_is_a_cache_implementation_transformers_accepts():
+ """Pin the literal against upstream so a rename does not silently no-op."""
+ import inspect
+
+ generation_utils = pytest.importorskip("transformers.generation.utils")
+ prepare = getattr(generation_utils.GenerationMixin, "_prepare_cache_for_generation", None)
+ if prepare is None:
+ pytest.skip("no _prepare_cache_for_generation in this transformers")
+ source = inspect.getsource(prepare)
+ assert '"dynamic"' in source or "'dynamic'" in source
+ static = getattr(generation_utils, "ALL_STATIC_CACHE_IMPLEMENTATIONS", ())
+ assert "dynamic" not in static
+
+
+def test_audio_token_type_alone_keeps_the_static_cache(gemma_like):
+ """Audio marks a different token type than the blocked 1 and 2."""
+ module = sys.modules[type(gemma_like).__module__]
+ model = _gemma4_shaped(module, "vision")
+ audio_only = torch.tensor([[0, 3, 3, 0]])
+ assert not _needs_bidirectional_multimodal_mask(model, {"mm_token_type_ids": audio_only})
+
+
+def test_image_token_type_beside_audio_is_gated(gemma_like):
+ module = sys.modules[type(gemma_like).__module__]
+ model = _gemma4_shaped(module, "vision")
+ mixed = torch.tensor([[0, 3, 1, 0]])
+ assert _needs_bidirectional_multimodal_mask(model, {"mm_token_type_ids": mixed})
+
+
+def test_video_token_type_is_gated(gemma_like):
+ module = sys.modules[type(gemma_like).__module__]
+ model = _gemma4_shaped(module, "vision")
+ assert _needs_bidirectional_multimodal_mask(
+ model, {"mm_token_type_ids": torch.tensor([[0, 2, 0]])}
+ )
+
+
+def test_explicit_static_kwarg_is_overridden_alongside_the_config():
+ """Generation kwargs are applied after the config merge, so an explicit
+ cache_implementation would otherwise outlive the config assignment."""
+ cfg = _GenCfg()
+ kwargs = {"generation_config": cfg, "cache_implementation": "static"}
+ _resolve_cache_choice(True, None, kwargs)
+ if kwargs.get("cache_implementation") is not None:
+ kwargs["cache_implementation"] = "dynamic"
+ assert cfg.cache_implementation == "dynamic"
+ assert kwargs["cache_implementation"] == "dynamic"
+
+
+def test_blocked_token_types_match_upstream():
+ """Pin the values upstream actually blocks, so a change there is caught."""
+ import inspect
+
+ modeling = pytest.importorskip("transformers.models.gemma4_unified.modeling_gemma4_unified")
+ builder = getattr(modeling, "get_block_sequence_ids_for_mask", None)
+ if builder is None:
+ pytest.skip("no block mask builder in this transformers version")
+ source = inspect.getsource(builder)
+ for value in _MEDIA_TOKEN_TYPES:
+ assert f"== {value}" in source
+ assert "== 3" not in source # audio is not blocked
+
+
+def _force_gate(local_cache_implementation, kwargs, is_media):
+ """The force decision from unsloth_base_fast_generate, isolated.
+
+ Deliberately independent of the local default: clearing it does not make the
+ effective cache dynamic, since kwargs and the caller's generation_config are
+ applied afterwards.
+ """
+ return kwargs.get("past_key_values") is None and is_media
+
+
+def test_mixed_precision_still_forces_dynamic_for_media():
+ """UNSLOTH_BFLOAT16_MIXED_PRECISION clears the local default to None. An
+ explicit static kwarg would otherwise survive and bring the bug back."""
+ kwargs = {"generation_config": _GenCfg(), "cache_implementation": "static"}
+ assert _force_gate(None, kwargs, is_media = True)
+ _resolve_cache_choice(True, None, kwargs)
+ kwargs["cache_implementation"] = "dynamic"
+ assert kwargs["generation_config"].cache_implementation == "dynamic"
+ assert kwargs["cache_implementation"] == "dynamic"
+
+
+def test_a_caller_supplied_cache_is_never_overridden():
+ kwargs = {"past_key_values": object()}
+ assert not _force_gate("static", kwargs, is_media = True)
+
+
+def test_text_only_never_forces_even_with_a_cleared_default():
+ assert not _force_gate(None, {}, is_media = False)
+
+
[email protected]("requested", ["offloaded", "quantized"])
+def test_a_growing_cache_the_caller_asked_for_survives(requested):
+ """offloaded and quantized grow per step, so they keep the media mask.
+ Downgrading them to plain dynamic would throw away the memory they buy."""
+ assert _dynamic_cache_choice({"cache_implementation": requested}) == requested
+ cfg = _GenCfg()
+ cfg.cache_implementation = requested
+ assert _dynamic_cache_choice({"generation_config": cfg}) == requested
+ assert _resolve_cache_choice(True, None, {"generation_config": cfg}) == requested
+
+
[email protected]("requested", ["static", "offloaded_static"])
+def test_a_static_cache_the_caller_asked_for_is_replaced(requested):
+ assert _dynamic_cache_choice({"cache_implementation": requested}) == "dynamic"
+ cfg = _GenCfg()
+ cfg.cache_implementation = requested
+ assert _resolve_cache_choice(True, None, {"generation_config": cfg}) == "dynamic"
+
+
+def test_the_kwarg_outranks_the_config_when_both_are_given():
+ cfg = _GenCfg()
+ cfg.cache_implementation = "offloaded"
+ assert (
+ _dynamic_cache_choice({"generation_config": cfg, "cache_implementation": "static"})
+ == "dynamic"
+ )
+
+
+def test_static_implementations_match_upstream():
+ """A name upstream treats as static must be replaced, not preserved."""
+ configuration_utils = pytest.importorskip("transformers.generation.configuration_utils")
+ upstream = getattr(configuration_utils, "ALL_STATIC_CACHE_IMPLEMENTATIONS", None)
+ if upstream is None:
+ pytest.skip("no ALL_STATIC_CACHE_IMPLEMENTATIONS in this transformers")
+ assert set(upstream) <= set(_STATIC_CACHE_IMPLEMENTATIONS)
+ assert "dynamic" not in _STATIC_CACHE_IMPLEMENTATIONS
diff --git a/tests/test_high_precision_layernorm_coverage.py b/tests/test_high_precision_layernorm_coverage.py
new file mode 100644
index 00000000000..95a30030108
--- /dev/null
+++ b/tests/test_high_precision_layernorm_coverage.py
@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_HIGH_PRECISION_LAYERNORM must upcast every norm in a block, not some.
+
+The selector used to be name-only, so a block whose norms are not all named
+"*norm" got a float32 norm feeding a bfloat16 one on the same chain. Gemma 4's
+`embed_vision` is the live case: `pos_norm` matched, its siblings `patch_ln1`
+and `patch_ln2` did not.
+"""
+
+import torch
+import torch.nn as nn
+
+from unsloth.models.vision import _NORM_MODULE_TYPES
+
+
+def _selected(model):
+ """Mirror of the loader's selector in unsloth/models/vision.py."""
+ out = []
+ for name, module in model.named_modules():
+ if (
+ name.endswith(("norm", "norm1", "norm2", "norm3", "norm4"))
+ or "layernorm" in name
+ or "layer_norm" in name
+ or isinstance(module, _NORM_MODULE_TYPES)
+ ) and hasattr(module, "weight"):
+ out.append(name)
+ return set(out)
+
+
+class _VisionEmbedderLike(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.patch_ln1 = nn.LayerNorm(8)
+ self.patch_ln2 = nn.LayerNorm(8)
+ self.pos_norm = nn.LayerNorm(8)
+ self.proj = nn.Linear(8, 8)
+
+
+def test_sibling_norms_are_all_selected():
+ got = _selected(_VisionEmbedderLike())
+ assert got == {"patch_ln1", "patch_ln2", "pos_norm"}
+
+
+def test_non_norm_modules_are_not_selected():
+ assert "proj" not in _selected(_VisionEmbedderLike())
+
+
+def test_custom_rmsnorm_still_matched_by_name():
+ """Model-specific RMSNorm classes are not torch.nn types, so the name rule
+ still has to carry them."""
+
+ class RMSNorm(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(8))
+
+ class Block(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.input_layernorm = RMSNorm()
+ self.q_norm = RMSNorm()
+
+ assert _selected(Block()) == {"input_layernorm", "q_norm"}
+
+
+def test_norm_types_is_non_empty_and_typed():
+ assert _NORM_MODULE_TYPES
+ assert nn.LayerNorm in _NORM_MODULE_TYPES
+ assert all(isinstance(t, type) for t in _NORM_MODULE_TYPES)
+
+
+def test_weightless_norm_is_skipped():
+ """`.to(float32)` on a norm with no weight is a no-op, and named_modules
+ exposes such norms, so the hasattr guard must stay."""
+
+ class Weightless(nn.LayerNorm):
+ def __init__(self):
+ super().__init__(8, elementwise_affine = False)
+
+ class Block(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.v_norm = Weightless()
+
+ m = Block()
+ assert m.v_norm.weight is None
+ assert "v_norm" in _selected(m) # selected, but .to() is harmless
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 17429f7283e..6b8c46d11c5 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -113,6 +113,13 @@ from ..device_type import (
ALLOW_PREQUANTIZED_MODELS,
)
+# torch.nn.RMSNorm only exists on torch >= 2.4.
+_NORM_MODULE_TYPES = tuple(
+ t
+ for t in (getattr(torch.nn, "LayerNorm", None), getattr(torch.nn, "RMSNorm", None))
+ if t is not None
+)
+
__all__ = [
"FastBaseModel",
]
@@ -583,6 +590,95 @@ except:
torch_compiler_set_stance = None
+# Image and video only: get_block_sequence_ids_for_mask blocks token types 1 and 2.
+_MEDIA_GENERATE_KWARGS = ("pixel_values", "pixel_values_videos")
+_MEDIA_TOKEN_TYPES = (1, 2)
+# Either name marks the overlay: the first from transformers 5.10, the second 5.17.
+_BIDIRECTIONAL_MASK_BUILDERS = (
+ "get_block_sequence_ids_for_mask",
+ "create_masks_for_vision_model",
+)
+# Gemma 3 names it token_type_ids, Gemma 4 mm_token_type_ids.
+_TOKEN_TYPE_KWARGS = ("mm_token_type_ids", "token_type_ids")
+# Only these preallocate; offloaded and quantized caches still grow per step.
+try:
+ from transformers.generation.configuration_utils import (
+ ALL_STATIC_CACHE_IMPLEMENTATIONS as _STATIC_CACHE_IMPLEMENTATIONS,
+ )
+except ImportError:
+ _STATIC_CACHE_IMPLEMENTATIONS = (
+ "static",
+ "offloaded_static",
+ "sliding_window",
+ "hybrid",
+ "hybrid_chunked",
+ "offloaded_hybrid",
+ "offloaded_hybrid_chunked",
+ )
+
+
+def _overlay_is_configured(model):
+ """Mirror upstream: Gemma 4 overlays only when the text config says "vision"
+ (E2B and E4B leave it None), while Gemma 3 sets it False and overlays anyway.
+ The token-type argument of create_masks_for_generate tells them apart.
+ """
+ builder = getattr(type(model), "create_masks_for_generate", None)
+ if builder is None:
+ return True
+ try:
+ params = inspect.signature(builder).parameters
+ except (TypeError, ValueError):
+ return True
+ if "mm_token_type_ids" not in params:
+ return True
+ config = getattr(model, "config", None)
+ text_config = config.get_text_config() if hasattr(config, "get_text_config") else config
+ return getattr(text_config, "use_bidirectional_attention", None) == "vision"
+
+
+def _has_media_token_types(kwargs):
+ """Media markers in the token-type ids: the signal upstream keys on, and the
+ only one an `inputs_embeds` request carries. Text-only prompts emit the ids
+ too, so the values decide, not the presence."""
+ for name in _TOKEN_TYPE_KWARGS:
+ ids = kwargs.get(name)
+ if ids is None:
+ continue
+ try:
+ if bool(sum((ids == v).any() for v in _MEDIA_TOKEN_TYPES)):
+ return True
+ except (AttributeError, TypeError, RuntimeError):
+ continue
+ return False
+
+
+def _needs_bidirectional_multimodal_mask(model, kwargs):
+ """True when the request carries media and the model overlays a bidirectional
+ block on the causal mask (Gemma 3 / 4), which a static cache drops. Qwen2-VL,
+ Llava and PaliGemma have no overlay and stay on the static path."""
+ module = sys.modules.get(type(model).__module__, None)
+ if module is None:
+ return False
+ if not any(hasattr(module, name) for name in _BIDIRECTIONAL_MASK_BUILDERS):
+ return False
+ if not _overlay_is_configured(model):
+ return False
+ if any(kwargs.get(name) is not None for name in _MEDIA_GENERATE_KWARGS):
+ return True
+ return _has_media_token_types(kwargs)
+
+
+def _dynamic_cache_choice(kwargs):
+ """Cache to force for a media request. Only a static one drops the mask, so a
+ growing cache the caller named, like offloaded, is kept along with its budget."""
+ requested = kwargs.get("cache_implementation")
+ if requested is None and "generation_config" in kwargs:
+ requested = getattr(kwargs["generation_config"], "cache_implementation", None)
+ if requested is None or requested in _STATIC_CACHE_IMPLEMENTATIONS:
+ return "dynamic"
+ return requested
+
+
def _uses_flash_attention_for_generation(config):
language_config_names = (
"text_config",
@@ -846,13 +942,28 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
cache_implementation = "static"
if do_bfloat16_mixed_precision:
cache_implementation = None
+ # A static cache drops the media block mask at prefill (#6028). Name the cache
+ # rather than None, which _prepare_generation_config refills from the default.
+ force_dynamic_cache = kwargs.get(
+ "past_key_values"
+ ) is None and _needs_bidirectional_multimodal_mask(self, kwargs)
+ if force_dynamic_cache:
+ cache_implementation = None
+ dynamic_implementation = _dynamic_cache_choice(kwargs)
if "generation_config" in kwargs:
- kwargs["generation_config"].cache_implementation = cache_implementation
+ kwargs["generation_config"].cache_implementation = (
+ dynamic_implementation if force_dynamic_cache else cache_implementation
+ )
+ # kwargs are applied after the config merge, so an explicit value survives.
+ if force_dynamic_cache:
+ kwargs["cache_implementation"] = dynamic_implementation
if cache_implementation is not None:
kwargs["generation_config"].compile_config = _compile_config
else:
- kwargs["cache_implementation"] = cache_implementation
+ kwargs["cache_implementation"] = (
+ dynamic_implementation if force_dynamic_cache else cache_implementation
+ )
if cache_implementation is not None:
kwargs["compile_config"] = _compile_config
@@ -1701,6 +1812,8 @@ class FastBaseModel:
name.endswith(("norm", "norm1", "norm2", "norm3", "norm4"))
or "layernorm" in name
or "layer_norm" in name
+ # Name alone splits a block: pos_norm matches, patch_ln1 does not.
+ or isinstance(module, _NORM_MODULE_TYPES)
) and hasattr(module, "weight"):
module._pre_set_compute_dtype = torch.float32
if custom_datatype is not None: