Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Phi-4-reasoning-vision: give remote multimodal prep an indexable cache view on transformers 5 (#12123)
Daniel Han3040eca30216
diff --git a/tests/test_remote_multimodal_cache_view.py b/tests/test_remote_multimodal_cache_view.pynew file mode 100644index 00000000000..2ce5c8e3175--- /dev/null+++ b/tests/test_remote_multimodal_cache_view.py@@ -0,0 +1,127 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Phi-4-reasoning-vision generate() on transformers 5: "'DynamicCache' object is not subscriptable"."""++import pytest+import torch+from transformers import PretrainedConfig, PreTrainedModel+from transformers.cache_utils import DynamicCache++from unsloth.models.remote_code_shims import apply_remote_code_shims+++class _Config(PretrainedConfig):+ model_type = "phi4-siglip-stub"+++def _remote_class():+ class Phi4ForCausalLMVStub(PreTrainedModel):+ config_class = _Config++ def __init__(self, config):+ super().__init__(config)+ self.embed_tokens = torch.nn.Embedding(8, 4)++ def get_input_embeddings(self):+ return self.embed_tokens++ def forward(+ self,+ input_ids = None,+ labels = None,+ ):+ return None++ def prepare_inputs_labels_for_multimodal(+ self, input_ids, position_ids, attention_mask, past_key_values, labels, images+ ):+ if past_key_values is not None and images is not None and input_ids.shape[1] == 1:+ target_shape = past_key_values[-1][-1].shape[-2] + 1+ attention_mask = torch.cat(+ (+ attention_mask,+ torch.ones(+ (attention_mask.shape[0], target_shape - attention_mask.shape[1]),+ dtype = attention_mask.dtype,+ device = attention_mask.device,+ ),+ ),+ dim = 1,+ )+ position_ids = torch.sum(attention_mask, dim = 1).unsqueeze(-1) - 1+ return input_ids, position_ids, attention_mask, past_key_values, None, labels++ Phi4ForCausalLMVStub.__module__ = "transformers_modules.microsoft.phi4v.modeling_phi4_visionr"+ return Phi4ForCausalLMVStub+++def _cache(past_len = 7):+ cache = DynamicCache()+ for layer in range(2):+ cache.update(torch.zeros(1, 2, past_len, 4), torch.zeros(1, 2, past_len, 4), layer)+ return cache+++def _decode_step(model, cache, positional):+ args = (torch.ones(1, 1, dtype = torch.long), None, torch.ones(1, 3, dtype = torch.long))+ if positional:+ return model.prepare_inputs_labels_for_multimodal(*args, cache, None, torch.zeros(1))+ return model.prepare_inputs_labels_for_multimodal(+ *args,+ past_key_values = cache,+ labels = None,+ images = torch.zeros(1),+ )+++ hasattr(DynamicCache, "__getitem__"), reason = "transformers 4.x Cache is subscriptable"+)+def test_unrepaired_remote_code_fails_on_transformers_5():+ model = _remote_class()(_Config())+ with pytest.raises(TypeError, match = "not subscriptable"):+ _decode_step(model, _cache(), positional = True)++[email protected]("positional", [True, False])+def test_repaired_decode_step_extends_mask_to_cache_length(positional):+ cls = _remote_class()+ model = cls(_Config())+ repaired = apply_remote_code_shims(model)+ assert f"{cls.__name__}.prepare_inputs_labels_for_multimodal" in repaired+ assert model._supports_static_cache is False+ cache = _cache(past_len = 7)+ _, position_ids, attention_mask, returned, _, _ = _decode_step(model, cache, positional)+ assert returned is cache+ assert attention_mask.shape == (1, 8)+ assert position_ids.tolist() == [[7]]+ assert f"{cls.__name__}.prepare_inputs_labels_for_multimodal" not in apply_remote_code_shims(+ model+ )+++def test_non_remote_class_is_untouched():+ cls = _remote_class()+ cls.__module__ = "my_package.models"+ model = cls(_Config())+ assert not any(+ "prepare_inputs_labels_for_multimodal" in r for r in apply_remote_code_shims(model)+ )+++def test_cache_api_remote_prep_keeps_real_cache():+ cls = _remote_class()++ def prepare_inputs_labels_for_multimodal(+ self, input_ids, position_ids, attention_mask, past_key_values, labels, images+ ):+ past = past_key_values.get_seq_length() if past_key_values is not None else 0+ return input_ids, position_ids, attention_mask, past_key_values, past, labels++ cls.prepare_inputs_labels_for_multimodal = prepare_inputs_labels_for_multimodal+ model = cls(_Config())+ assert not any(+ "prepare_inputs_labels_for_multimodal" in r for r in apply_remote_code_shims(model)+ )+ cache = _cache(past_len = 5)+ output = model.prepare_inputs_labels_for_multimodal(None, None, None, cache, None, None)+ assert output[3] is cache and output[4] == 5diff --git a/unsloth/models/remote_code_shims.py b/unsloth/models/remote_code_shims.pyindex 123ae32f5dc..57b683f0853 100644--- a/unsloth/models/remote_code_shims.py+++ b/unsloth/models/remote_code_shims.py@@ -5,6 +5,7 @@import functoolsimport inspect+import reimport torch@@ -260,6 +261,88 @@ def _fill_missing_loss(cls):return True+class _LegacyCacheView:+ """Read-only `view[i] == (keys, values)` of a Cache; transformers 5 removed Cache.__getitem__."""++ __slots__ = ("cache",)++ def __init__(self, cache):+ self.cache = cache++ def _layers(self):+ return getattr(self.cache, "layers", None) or []++ def __len__(self):+ return len(self._layers())++ def __getitem__(self, index):+ layer = self._layers()[index]+ return (layer.keys, layer.values)++ def __iter__(self):+ for i in range(len(self)):+ yield self[i]++ def __getattr__(self, name):+ return getattr(self.cache, name)+++def _needs_legacy_view(cache):+ if cache is None or isinstance(cache, (tuple, list)):+ return False+ return not hasattr(type(cache), "__getitem__") and hasattr(cache, "layers")+++def _repair_multimodal_cache_indexing(cls):+ """Phi-4-reasoning-vision indexes `past_key_values[-1][-1]` here per decode step: pass a tuple view, return the real cache."""+ name = "prepare_inputs_labels_for_multimodal"+ original = None+ for klass in cls.__mro__:+ if name in klass.__dict__:+ original = klass.__dict__[name]+ break+ if original is None or getattr(original, "_unsloth_cache_view", False):+ return False+ try:+ parameters = list(inspect.signature(original).parameters)+ except (TypeError, ValueError):+ return False+ if "past_key_values" not in parameters:+ return False+ try:+ source = inspect.getsource(original)+ except (OSError, TypeError):+ return False+ # Code already on the Cache API (get_seq_length, isinstance Cache) must keep the real cache.+ if re.search(r"past_key_values\s*\[", source) is None:+ return False+ position = parameters.index("past_key_values") - 1++ @functools.wraps(original)+ def prepare_inputs_labels_for_multimodal(self, *args, **kwargs):+ args = list(args)+ if "past_key_values" in kwargs:+ cache = kwargs["past_key_values"]+ if _needs_legacy_view(cache):+ kwargs["past_key_values"] = _LegacyCacheView(cache)+ elif len(args) > position:+ cache = args[position]+ if _needs_legacy_view(cache):+ args[position] = _LegacyCacheView(cache)+ else:+ cache = None+ output = original(self, *args, **kwargs)+ if isinstance(output, tuple):+ output = tuple(cache if isinstance(x, _LegacyCacheView) else x for x in output)+ return output++ prepare_inputs_labels_for_multimodal._unsloth_cache_view = True+ setattr(cls, name, prepare_inputs_labels_for_multimodal)+ # Static cache: prefill mask arrives as 4D / BlockMask, not the 2D padding mask this prep compacts by.+ cls._supports_static_cache = False+ return True++def _rebind_accelerate_hook(model):# device_map loading keeps the bound original as `_old_forward`, bypassing class repairs.if getattr(type(model), "_unsloth_original_forward", None) is None:@@ -290,6 +373,8 @@ def apply_remote_code_shims(model):repaired.append(f"{cls.__name__}.get_output_embeddings")if _is_remote_code(cls) and _fill_missing_loss(cls):repaired.append(f"{cls.__name__}.forward")+ if _is_remote_code(cls) and _repair_multimodal_cache_indexing(cls):+ repaired.append(f"{cls.__name__}.prepare_inputs_labels_for_multimodal")_rebind_accelerate_hook(model)if repaired:print("Unsloth: Repaired remote modeling code so it trains: " + ", ".join(repaired) + ".")