Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Use the dynamic cache for Gemma multimodal generation (#10903)
Daniel Hanb58ea42a0a64
diff --git a/tests/test_gemma_multimodal_cache_selection.py b/tests/test_gemma_multimodal_cache_selection.pynew file mode 100644index 00000000000..e81f7ac0c11--- /dev/null+++ b/tests/test_gemma_multimodal_cache_selection.py@@ -0,0 +1,426 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Cache selection for multimodal generation (unslothai/unsloth#6028).++A static cache makes transformers skip mask materialisation at prefill and rely+on `is_causal`, which drops the bidirectional image/audio block overlay the Gemma+multimodal families build. Image tokens then attend causally and spatial+grounding degrades. `generate` must fall back to the dynamic cache for those+requests, and only those.+"""++import sys+import types++import torch++import pytest++from unsloth.models.vision import (+ _BIDIRECTIONAL_MASK_BUILDERS,+ _MEDIA_TOKEN_TYPES,+ _STATIC_CACHE_IMPLEMENTATIONS,+ _dynamic_cache_choice,+ _needs_bidirectional_multimodal_mask,+)++# Both helpers landed in transformers 5.10; earlier versions expose neither.+_FIRST_VERSION_WITH_BLOCK_MASK = "5.10.0"+++def _make_model(+ module_name,+ has_helper,+ helper = None,+):+ module = types.ModuleType(module_name)+ if has_helper:+ setattr(module, helper or _BIDIRECTIONAL_MASK_BUILDERS[0], lambda *args, **kwargs: None)+ sys.modules[module_name] = module++ model_cls = type("Model", (), {})+ model_cls.__module__ = module_name+ return model_cls()+++def gemma_like(request):+ name = f"_fake_gemma_{request.node.name}"+ yield _make_model(name, True)+ sys.modules.pop(name, None)+++def causal_vlm(request):+ name = f"_fake_qwen_{request.node.name}"+ yield _make_model(name, False)+ sys.modules.pop(name, None)++[email protected]("media_kwarg", ["pixel_values", "pixel_values_videos"])+def test_media_request_on_gemma_uses_dynamic_cache(gemma_like, media_kwarg):+ assert _needs_bidirectional_multimodal_mask(gemma_like, {media_kwarg: object()})+++def test_audio_only_keeps_the_static_cache(gemma_like):+ """get_block_sequence_ids_for_mask blocks token types 1 and 2 only, so audio+ stays causal and must not lose the compiled path."""+ assert not _needs_bidirectional_multimodal_mask(gemma_like, {"input_features": object()})+++def test_text_only_keeps_static_cache(gemma_like):+ assert not _needs_bidirectional_multimodal_mask(gemma_like, {"input_ids": object()})+++def test_none_media_kwarg_keeps_static_cache(gemma_like):+ assert not _needs_bidirectional_multimodal_mask(gemma_like, {"pixel_values": None})+++def test_causal_vlm_keeps_static_cache(causal_vlm):+ """Qwen2-VL, Llava and PaliGemma have no bidirectional block overlay, so they+ must not lose the static-cache path."""+ assert not _needs_bidirectional_multimodal_mask(causal_vlm, {"pixel_values": object()})+++def test_unknown_module_does_not_raise():+ orphan_cls = type("Orphan", (), {})+ orphan_cls.__module__ = "_module_that_does_not_exist"+ assert not _needs_bidirectional_multimodal_mask(orphan_cls(), {"pixel_values": object()})++[email protected]("helper", _BIDIRECTIONAL_MASK_BUILDERS)+def test_either_mask_builder_gates_the_guard(request, helper):+ """The two names span transformers 5.10 to current, so either one alone has+ to be enough."""+ name = f"_fake_helper_{request.node.name}"+ model = _make_model(name, True, helper = helper)+ try:+ assert _needs_bidirectional_multimodal_mask(model, {"pixel_values": object()})+ finally:+ sys.modules.pop(name, None)+++def test_real_gemma_modules_expose_the_helper():+ """Pin the upstream symbols the guard keys on, so a rename is caught here.++ These helpers landed in transformers 5.10. Before that neither name exists+ (measured on 4.57.6 and 5.5.0), the guard is inert by design, and Gemma keeps+ the static path it always had, so there is nothing to pin.+ """+ transformers = pytest.importorskip("transformers")+ from packaging.version import InvalidVersion, Version++ try:+ new_enough = Version(transformers.__version__) >= Version(_FIRST_VERSION_WITH_BLOCK_MASK)+ except InvalidVersion:+ pytest.skip(f"cannot order version {transformers.__version__!r}")+ if not new_enough:+ pytest.skip(f"no block mask helper before transformers {_FIRST_VERSION_WITH_BLOCK_MASK}")++ seen = False+ for name in ("gemma3", "gemma4", "gemma4_unified"):+ try:+ module = __import__(f"transformers.models.{name}.modeling_{name}", fromlist = ["x"])+ except Exception:+ continue+ seen = True+ assert any(hasattr(module, h) for h in _BIDIRECTIONAL_MASK_BUILDERS), name+ if not seen:+ pytest.skip("no Gemma multimodal module in this transformers version")+++def test_causal_vlms_never_expose_a_mask_builder():+ """The other half of the gate: these must stay on the static path on every+ version, so neither name may appear on them."""+ pytest.importorskip("transformers")+ for name in ("qwen2_vl", "llava", "paligemma"):+ try:+ module = __import__(f"transformers.models.{name}.modeling_{name}", fromlist = ["x"])+ except Exception:+ continue+ assert not any(hasattr(module, h) for h in _BIDIRECTIONAL_MASK_BUILDERS), name+++class _Cfg:+ def __init__(self, bidir):+ self.use_bidirectional_attention = bidir++ def get_text_config(self):+ return self+++def _gemma4_shaped(module, bidir):+ """A model whose create_masks_for_generate takes mm_token_type_ids, which is+ the Gemma 4 form and the one upstream gates on the config value."""++ class Model:+ config = _Cfg(bidir)++ @staticmethod+ def create_masks_for_generate(+ config,+ inputs_embeds,+ attention_mask,+ past_key_values,+ position_ids,+ mm_token_type_ids = None,+ **kwargs,+ ):+ return None++ Model.__module__ = module.__name__+ return Model()+++def _gemma3_shaped(module):+ """Gemma 3 takes token_type_ids and overlays regardless of the config flag,+ which it sets to False."""++ class Model:+ config = _Cfg(False)++ @staticmethod+ def create_masks_for_generate(+ config,+ inputs_embeds,+ attention_mask,+ past_key_values,+ position_ids,+ token_type_ids = None,+ **kwargs,+ ):+ return None++ Model.__module__ = module.__name__+ return Model()+++def test_causal_gemma4_variant_keeps_the_static_cache(gemma_like):+ """E2B / E4B leave use_bidirectional_attention None, so upstream builds no+ overlay and they must not lose the compiled path."""+ module = sys.modules[type(gemma_like).__module__]+ assert not _needs_bidirectional_multimodal_mask(+ _gemma4_shaped(module, None), {"pixel_values": object()}+ )+++def test_vision_gemma4_variant_is_gated(gemma_like):+ module = sys.modules[type(gemma_like).__module__]+ assert _needs_bidirectional_multimodal_mask(+ _gemma4_shaped(module, "vision"), {"pixel_values": object()}+ )+++def test_gemma3_is_gated_despite_the_flag_being_false(gemma_like):+ module = sys.modules[type(gemma_like).__module__]+ assert _needs_bidirectional_multimodal_mask(_gemma3_shaped(module), {"pixel_values": object()})+++def test_precomputed_embeds_with_media_tokens_are_gated(gemma_like):+ """inputs_embeds carries no media kwarg; the token types are the signal."""+ module = sys.modules[type(gemma_like).__module__]+ model = _gemma4_shaped(module, "vision")+ ids = torch.tensor([[0, 0, 1, 1, 0]])+ assert _needs_bidirectional_multimodal_mask(model, {"mm_token_type_ids": ids})+++def test_text_only_token_types_keep_the_static_cache(gemma_like):+ """The processor emits mm_token_type_ids for text-only prompts too, all+ zeros, so presence alone must not cost the static path."""+ module = sys.modules[type(gemma_like).__module__]+ model = _gemma4_shaped(module, "vision")+ ids = torch.zeros(1, 5, dtype = torch.long)+ assert not _needs_bidirectional_multimodal_mask(model, {"mm_token_type_ids": ids})+++def test_gemma3_token_type_ids_are_read(gemma_like):+ module = sys.modules[type(gemma_like).__module__]+ model = _gemma3_shaped(module)+ assert _needs_bidirectional_multimodal_mask(+ model, {"token_type_ids": torch.tensor([[0, 1, 0]])}+ )+ assert not _needs_bidirectional_multimodal_mask(+ model, {"token_type_ids": torch.zeros(1, 3, dtype = torch.long)}+ )+++def test_non_tensor_token_types_do_not_raise(gemma_like):+ module = sys.modules[type(gemma_like).__module__]+ model = _gemma4_shaped(module, "vision")+ assert not _needs_bidirectional_multimodal_mask(model, {"mm_token_type_ids": "not a tensor"})+++def _resolve_cache_choice(force_dynamic, cache_implementation, kwargs):+ """The assignment block from unsloth_base_fast_generate, isolated."""+ forced = _dynamic_cache_choice(kwargs) if force_dynamic else None+ if "generation_config" in kwargs:+ kwargs["generation_config"].cache_implementation = (+ forced if force_dynamic else cache_implementation+ )+ return kwargs["generation_config"].cache_implementation+ kwargs["cache_implementation"] = forced if force_dynamic else cache_implementation+ return kwargs["cache_implementation"]+++class _GenCfg:+ cache_implementation = "static"+++def test_media_request_pins_the_literal_dynamic():+ """None is refilled from the model default by _prepare_generation_config, so+ the guard has to name the cache it wants."""+ assert _resolve_cache_choice(True, None, {}) == "dynamic"+ assert _resolve_cache_choice(True, None, {"generation_config": _GenCfg()}) == "dynamic"+++def test_caller_generation_config_is_overridden():+ """The reported hole: a caller-supplied config whose field says static."""+ cfg = _GenCfg()+ assert cfg.cache_implementation == "static"+ _resolve_cache_choice(True, None, {"generation_config": cfg})+ assert cfg.cache_implementation == "dynamic"+++def test_text_only_still_gets_the_static_cache():+ assert _resolve_cache_choice(False, "static", {}) == "static"+ cfg = _GenCfg()+ _resolve_cache_choice(False, "static", {"generation_config": cfg})+ assert cfg.cache_implementation == "static"+++def test_none_is_preserved_when_not_forcing():+ """bfloat16 mixed precision clears the cache by setting None; that path must+ keep its own meaning."""+ assert _resolve_cache_choice(False, None, {}) is None+++def test_dynamic_is_a_cache_implementation_transformers_accepts():+ """Pin the literal against upstream so a rename does not silently no-op."""+ import inspect++ generation_utils = pytest.importorskip("transformers.generation.utils")+ prepare = getattr(generation_utils.GenerationMixin, "_prepare_cache_for_generation", None)+ if prepare is None:+ pytest.skip("no _prepare_cache_for_generation in this transformers")+ source = inspect.getsource(prepare)+ assert '"dynamic"' in source or "'dynamic'" in source+ static = getattr(generation_utils, "ALL_STATIC_CACHE_IMPLEMENTATIONS", ())+ assert "dynamic" not in static+++def test_audio_token_type_alone_keeps_the_static_cache(gemma_like):+ """Audio marks a different token type than the blocked 1 and 2."""+ module = sys.modules[type(gemma_like).__module__]+ model = _gemma4_shaped(module, "vision")+ audio_only = torch.tensor([[0, 3, 3, 0]])+ assert not _needs_bidirectional_multimodal_mask(model, {"mm_token_type_ids": audio_only})+++def test_image_token_type_beside_audio_is_gated(gemma_like):+ module = sys.modules[type(gemma_like).__module__]+ model = _gemma4_shaped(module, "vision")+ mixed = torch.tensor([[0, 3, 1, 0]])+ assert _needs_bidirectional_multimodal_mask(model, {"mm_token_type_ids": mixed})+++def test_video_token_type_is_gated(gemma_like):+ module = sys.modules[type(gemma_like).__module__]+ model = _gemma4_shaped(module, "vision")+ assert _needs_bidirectional_multimodal_mask(+ model, {"mm_token_type_ids": torch.tensor([[0, 2, 0]])}+ )+++def test_explicit_static_kwarg_is_overridden_alongside_the_config():+ """Generation kwargs are applied after the config merge, so an explicit+ cache_implementation would otherwise outlive the config assignment."""+ cfg = _GenCfg()+ kwargs = {"generation_config": cfg, "cache_implementation": "static"}+ _resolve_cache_choice(True, None, kwargs)+ if kwargs.get("cache_implementation") is not None:+ kwargs["cache_implementation"] = "dynamic"+ assert cfg.cache_implementation == "dynamic"+ assert kwargs["cache_implementation"] == "dynamic"+++def test_blocked_token_types_match_upstream():+ """Pin the values upstream actually blocks, so a change there is caught."""+ import inspect++ modeling = pytest.importorskip("transformers.models.gemma4_unified.modeling_gemma4_unified")+ builder = getattr(modeling, "get_block_sequence_ids_for_mask", None)+ if builder is None:+ pytest.skip("no block mask builder in this transformers version")+ source = inspect.getsource(builder)+ for value in _MEDIA_TOKEN_TYPES:+ assert f"== {value}" in source+ assert "== 3" not in source # audio is not blocked+++def _force_gate(local_cache_implementation, kwargs, is_media):+ """The force decision from unsloth_base_fast_generate, isolated.++ Deliberately independent of the local default: clearing it does not make the+ effective cache dynamic, since kwargs and the caller's generation_config are+ applied afterwards.+ """+ return kwargs.get("past_key_values") is None and is_media+++def test_mixed_precision_still_forces_dynamic_for_media():+ """UNSLOTH_BFLOAT16_MIXED_PRECISION clears the local default to None. An+ explicit static kwarg would otherwise survive and bring the bug back."""+ kwargs = {"generation_config": _GenCfg(), "cache_implementation": "static"}+ assert _force_gate(None, kwargs, is_media = True)+ _resolve_cache_choice(True, None, kwargs)+ kwargs["cache_implementation"] = "dynamic"+ assert kwargs["generation_config"].cache_implementation == "dynamic"+ assert kwargs["cache_implementation"] == "dynamic"+++def test_a_caller_supplied_cache_is_never_overridden():+ kwargs = {"past_key_values": object()}+ assert not _force_gate("static", kwargs, is_media = True)+++def test_text_only_never_forces_even_with_a_cleared_default():+ assert not _force_gate(None, {}, is_media = False)++[email protected]("requested", ["offloaded", "quantized"])+def test_a_growing_cache_the_caller_asked_for_survives(requested):+ """offloaded and quantized grow per step, so they keep the media mask.+ Downgrading them to plain dynamic would throw away the memory they buy."""+ assert _dynamic_cache_choice({"cache_implementation": requested}) == requested+ cfg = _GenCfg()+ cfg.cache_implementation = requested+ assert _dynamic_cache_choice({"generation_config": cfg}) == requested+ assert _resolve_cache_choice(True, None, {"generation_config": cfg}) == requested++[email protected]("requested", ["static", "offloaded_static"])+def test_a_static_cache_the_caller_asked_for_is_replaced(requested):+ assert _dynamic_cache_choice({"cache_implementation": requested}) == "dynamic"+ cfg = _GenCfg()+ cfg.cache_implementation = requested+ assert _resolve_cache_choice(True, None, {"generation_config": cfg}) == "dynamic"+++def test_the_kwarg_outranks_the_config_when_both_are_given():+ cfg = _GenCfg()+ cfg.cache_implementation = "offloaded"+ assert (+ _dynamic_cache_choice({"generation_config": cfg, "cache_implementation": "static"})+ == "dynamic"+ )+++def test_static_implementations_match_upstream():+ """A name upstream treats as static must be replaced, not preserved."""+ configuration_utils = pytest.importorskip("transformers.generation.configuration_utils")+ upstream = getattr(configuration_utils, "ALL_STATIC_CACHE_IMPLEMENTATIONS", None)+ if upstream is None:+ pytest.skip("no ALL_STATIC_CACHE_IMPLEMENTATIONS in this transformers")+ assert set(upstream) <= set(_STATIC_CACHE_IMPLEMENTATIONS)+ assert "dynamic" not in _STATIC_CACHE_IMPLEMENTATIONSdiff --git a/tests/test_high_precision_layernorm_coverage.py b/tests/test_high_precision_layernorm_coverage.pynew file mode 100644index 00000000000..95a30030108--- /dev/null+++ b/tests/test_high_precision_layernorm_coverage.py@@ -0,0 +1,89 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_HIGH_PRECISION_LAYERNORM must upcast every norm in a block, not some.++The selector used to be name-only, so a block whose norms are not all named+"*norm" got a float32 norm feeding a bfloat16 one on the same chain. Gemma 4's+`embed_vision` is the live case: `pos_norm` matched, its siblings `patch_ln1`+and `patch_ln2` did not.+"""++import torch+import torch.nn as nn++from unsloth.models.vision import _NORM_MODULE_TYPES+++def _selected(model):+ """Mirror of the loader's selector in unsloth/models/vision.py."""+ out = []+ for name, module in model.named_modules():+ if (+ name.endswith(("norm", "norm1", "norm2", "norm3", "norm4"))+ or "layernorm" in name+ or "layer_norm" in name+ or isinstance(module, _NORM_MODULE_TYPES)+ ) and hasattr(module, "weight"):+ out.append(name)+ return set(out)+++class _VisionEmbedderLike(nn.Module):+ def __init__(self):+ super().__init__()+ self.patch_ln1 = nn.LayerNorm(8)+ self.patch_ln2 = nn.LayerNorm(8)+ self.pos_norm = nn.LayerNorm(8)+ self.proj = nn.Linear(8, 8)+++def test_sibling_norms_are_all_selected():+ got = _selected(_VisionEmbedderLike())+ assert got == {"patch_ln1", "patch_ln2", "pos_norm"}+++def test_non_norm_modules_are_not_selected():+ assert "proj" not in _selected(_VisionEmbedderLike())+++def test_custom_rmsnorm_still_matched_by_name():+ """Model-specific RMSNorm classes are not torch.nn types, so the name rule+ still has to carry them."""++ class RMSNorm(nn.Module):+ def __init__(self):+ super().__init__()+ self.weight = nn.Parameter(torch.ones(8))++ class Block(nn.Module):+ def __init__(self):+ super().__init__()+ self.input_layernorm = RMSNorm()+ self.q_norm = RMSNorm()++ assert _selected(Block()) == {"input_layernorm", "q_norm"}+++def test_norm_types_is_non_empty_and_typed():+ assert _NORM_MODULE_TYPES+ assert nn.LayerNorm in _NORM_MODULE_TYPES+ assert all(isinstance(t, type) for t in _NORM_MODULE_TYPES)+++def test_weightless_norm_is_skipped():+ """`.to(float32)` on a norm with no weight is a no-op, and named_modules+ exposes such norms, so the hasattr guard must stay."""++ class Weightless(nn.LayerNorm):+ def __init__(self):+ super().__init__(8, elementwise_affine = False)++ class Block(nn.Module):+ def __init__(self):+ super().__init__()+ self.v_norm = Weightless()++ m = Block()+ assert m.v_norm.weight is None+ assert "v_norm" in _selected(m) # selected, but .to() is harmlessdiff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 17429f7283e..6b8c46d11c5 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -113,6 +113,13 @@ from ..device_type import (ALLOW_PREQUANTIZED_MODELS,)+# torch.nn.RMSNorm only exists on torch >= 2.4.+_NORM_MODULE_TYPES = tuple(+ t+ for t in (getattr(torch.nn, "LayerNorm", None), getattr(torch.nn, "RMSNorm", None))+ if t is not None+)+__all__ = ["FastBaseModel",]@@ -583,6 +590,95 @@ except:torch_compiler_set_stance = None+# Image and video only: get_block_sequence_ids_for_mask blocks token types 1 and 2.+_MEDIA_GENERATE_KWARGS = ("pixel_values", "pixel_values_videos")+_MEDIA_TOKEN_TYPES = (1, 2)+# Either name marks the overlay: the first from transformers 5.10, the second 5.17.+_BIDIRECTIONAL_MASK_BUILDERS = (+ "get_block_sequence_ids_for_mask",+ "create_masks_for_vision_model",+)+# Gemma 3 names it token_type_ids, Gemma 4 mm_token_type_ids.+_TOKEN_TYPE_KWARGS = ("mm_token_type_ids", "token_type_ids")+# Only these preallocate; offloaded and quantized caches still grow per step.+try:+ from transformers.generation.configuration_utils import (+ ALL_STATIC_CACHE_IMPLEMENTATIONS as _STATIC_CACHE_IMPLEMENTATIONS,+ )+except ImportError:+ _STATIC_CACHE_IMPLEMENTATIONS = (+ "static",+ "offloaded_static",+ "sliding_window",+ "hybrid",+ "hybrid_chunked",+ "offloaded_hybrid",+ "offloaded_hybrid_chunked",+ )+++def _overlay_is_configured(model):+ """Mirror upstream: Gemma 4 overlays only when the text config says "vision"+ (E2B and E4B leave it None), while Gemma 3 sets it False and overlays anyway.+ The token-type argument of create_masks_for_generate tells them apart.+ """+ builder = getattr(type(model), "create_masks_for_generate", None)+ if builder is None:+ return True+ try:+ params = inspect.signature(builder).parameters+ except (TypeError, ValueError):+ return True+ if "mm_token_type_ids" not in params:+ return True+ config = getattr(model, "config", None)+ text_config = config.get_text_config() if hasattr(config, "get_text_config") else config+ return getattr(text_config, "use_bidirectional_attention", None) == "vision"+++def _has_media_token_types(kwargs):+ """Media markers in the token-type ids: the signal upstream keys on, and the+ only one an `inputs_embeds` request carries. Text-only prompts emit the ids+ too, so the values decide, not the presence."""+ for name in _TOKEN_TYPE_KWARGS:+ ids = kwargs.get(name)+ if ids is None:+ continue+ try:+ if bool(sum((ids == v).any() for v in _MEDIA_TOKEN_TYPES)):+ return True+ except (AttributeError, TypeError, RuntimeError):+ continue+ return False+++def _needs_bidirectional_multimodal_mask(model, kwargs):+ """True when the request carries media and the model overlays a bidirectional+ block on the causal mask (Gemma 3 / 4), which a static cache drops. Qwen2-VL,+ Llava and PaliGemma have no overlay and stay on the static path."""+ module = sys.modules.get(type(model).__module__, None)+ if module is None:+ return False+ if not any(hasattr(module, name) for name in _BIDIRECTIONAL_MASK_BUILDERS):+ return False+ if not _overlay_is_configured(model):+ return False+ if any(kwargs.get(name) is not None for name in _MEDIA_GENERATE_KWARGS):+ return True+ return _has_media_token_types(kwargs)+++def _dynamic_cache_choice(kwargs):+ """Cache to force for a media request. Only a static one drops the mask, so a+ growing cache the caller named, like offloaded, is kept along with its budget."""+ requested = kwargs.get("cache_implementation")+ if requested is None and "generation_config" in kwargs:+ requested = getattr(kwargs["generation_config"], "cache_implementation", None)+ if requested is None or requested in _STATIC_CACHE_IMPLEMENTATIONS:+ return "dynamic"+ return requested++def _uses_flash_attention_for_generation(config):language_config_names = ("text_config",@@ -846,13 +942,28 @@ def unsloth_base_fast_generate(self, *args, **kwargs):cache_implementation = "static"if do_bfloat16_mixed_precision:cache_implementation = None+ # A static cache drops the media block mask at prefill (#6028). Name the cache+ # rather than None, which _prepare_generation_config refills from the default.+ force_dynamic_cache = kwargs.get(+ "past_key_values"+ ) is None and _needs_bidirectional_multimodal_mask(self, kwargs)+ if force_dynamic_cache:+ cache_implementation = None+ dynamic_implementation = _dynamic_cache_choice(kwargs)if "generation_config" in kwargs:- kwargs["generation_config"].cache_implementation = cache_implementation+ kwargs["generation_config"].cache_implementation = (+ dynamic_implementation if force_dynamic_cache else cache_implementation+ )+ # kwargs are applied after the config merge, so an explicit value survives.+ if force_dynamic_cache:+ kwargs["cache_implementation"] = dynamic_implementationif cache_implementation is not None:kwargs["generation_config"].compile_config = _compile_configelse:- kwargs["cache_implementation"] = cache_implementation+ kwargs["cache_implementation"] = (+ dynamic_implementation if force_dynamic_cache else cache_implementation+ )if cache_implementation is not None:kwargs["compile_config"] = _compile_config@@ -1701,6 +1812,8 @@ class FastBaseModel:name.endswith(("norm", "norm1", "norm2", "norm3", "norm4"))or "layernorm" in nameor "layer_norm" in name+ # Name alone splits a block: pos_norm matches, patch_ln1 does not.+ or isinstance(module, _NORM_MODULE_TYPES)) and hasattr(module, "weight"):module._pre_set_compute_dtype = torch.float32if custom_datatype is not None: