Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Phi-4-reasoning-vision: give remote multimodal prep an indexable cache view on transformers 5 (#12123)

Daniel Han3040eca30216
diff --git a/tests/test_remote_multimodal_cache_view.py b/tests/test_remote_multimodal_cache_view.py
new file mode 100644
index 00000000000..2ce5c8e3175
--- /dev/null
+++ b/tests/test_remote_multimodal_cache_view.py
@@ -0,0 +1,127 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Phi-4-reasoning-vision generate() on transformers 5: "'DynamicCache' object is not subscriptable"."""
+
+import pytest
+import torch
+from transformers import PretrainedConfig, PreTrainedModel
+from transformers.cache_utils import DynamicCache
+
+from unsloth.models.remote_code_shims import apply_remote_code_shims
+
+
+class _Config(PretrainedConfig):
+ model_type = "phi4-siglip-stub"
+
+
+def _remote_class():
+ class Phi4ForCausalLMVStub(PreTrainedModel):
+ config_class = _Config
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.embed_tokens = torch.nn.Embedding(8, 4)
+
+ def get_input_embeddings(self):
+ return self.embed_tokens
+
+ def forward(
+ self,
+ input_ids = None,
+ labels = None,
+ ):
+ return None
+
+ def prepare_inputs_labels_for_multimodal(
+ self, input_ids, position_ids, attention_mask, past_key_values, labels, images
+ ):
+ if past_key_values is not None and images is not None and input_ids.shape[1] == 1:
+ target_shape = past_key_values[-1][-1].shape[-2] + 1
+ attention_mask = torch.cat(
+ (
+ attention_mask,
+ torch.ones(
+ (attention_mask.shape[0], target_shape - attention_mask.shape[1]),
+ dtype = attention_mask.dtype,
+ device = attention_mask.device,
+ ),
+ ),
+ dim = 1,
+ )
+ position_ids = torch.sum(attention_mask, dim = 1).unsqueeze(-1) - 1
+ return input_ids, position_ids, attention_mask, past_key_values, None, labels
+
+ Phi4ForCausalLMVStub.__module__ = "transformers_modules.microsoft.phi4v.modeling_phi4_visionr"
+ return Phi4ForCausalLMVStub
+
+
+def _cache(past_len = 7):
+ cache = DynamicCache()
+ for layer in range(2):
+ cache.update(torch.zeros(1, 2, past_len, 4), torch.zeros(1, 2, past_len, 4), layer)
+ return cache
+
+
+def _decode_step(model, cache, positional):
+ args = (torch.ones(1, 1, dtype = torch.long), None, torch.ones(1, 3, dtype = torch.long))
+ if positional:
+ return model.prepare_inputs_labels_for_multimodal(*args, cache, None, torch.zeros(1))
+ return model.prepare_inputs_labels_for_multimodal(
+ *args,
+ past_key_values = cache,
+ labels = None,
+ images = torch.zeros(1),
+ )
+
+
+ hasattr(DynamicCache, "__getitem__"), reason = "transformers 4.x Cache is subscriptable"
+)
+def test_unrepaired_remote_code_fails_on_transformers_5():
+ model = _remote_class()(_Config())
+ with pytest.raises(TypeError, match = "not subscriptable"):
+ _decode_step(model, _cache(), positional = True)
+
+
[email protected]("positional", [True, False])
+def test_repaired_decode_step_extends_mask_to_cache_length(positional):
+ cls = _remote_class()
+ model = cls(_Config())
+ repaired = apply_remote_code_shims(model)
+ assert f"{cls.__name__}.prepare_inputs_labels_for_multimodal" in repaired
+ assert model._supports_static_cache is False
+ cache = _cache(past_len = 7)
+ _, position_ids, attention_mask, returned, _, _ = _decode_step(model, cache, positional)
+ assert returned is cache
+ assert attention_mask.shape == (1, 8)
+ assert position_ids.tolist() == [[7]]
+ assert f"{cls.__name__}.prepare_inputs_labels_for_multimodal" not in apply_remote_code_shims(
+ model
+ )
+
+
+def test_non_remote_class_is_untouched():
+ cls = _remote_class()
+ cls.__module__ = "my_package.models"
+ model = cls(_Config())
+ assert not any(
+ "prepare_inputs_labels_for_multimodal" in r for r in apply_remote_code_shims(model)
+ )
+
+
+def test_cache_api_remote_prep_keeps_real_cache():
+ cls = _remote_class()
+
+ def prepare_inputs_labels_for_multimodal(
+ self, input_ids, position_ids, attention_mask, past_key_values, labels, images
+ ):
+ past = past_key_values.get_seq_length() if past_key_values is not None else 0
+ return input_ids, position_ids, attention_mask, past_key_values, past, labels
+
+ cls.prepare_inputs_labels_for_multimodal = prepare_inputs_labels_for_multimodal
+ model = cls(_Config())
+ assert not any(
+ "prepare_inputs_labels_for_multimodal" in r for r in apply_remote_code_shims(model)
+ )
+ cache = _cache(past_len = 5)
+ output = model.prepare_inputs_labels_for_multimodal(None, None, None, cache, None, None)
+ assert output[3] is cache and output[4] == 5
diff --git a/unsloth/models/remote_code_shims.py b/unsloth/models/remote_code_shims.py
index 123ae32f5dc..57b683f0853 100644
--- a/unsloth/models/remote_code_shims.py
+++ b/unsloth/models/remote_code_shims.py
@@ -5,6 +5,7 @@
import functools
import inspect
+import re
import torch
@@ -260,6 +261,88 @@ def _fill_missing_loss(cls):
return True
+class _LegacyCacheView:
+ """Read-only `view[i] == (keys, values)` of a Cache; transformers 5 removed Cache.__getitem__."""
+
+ __slots__ = ("cache",)
+
+ def __init__(self, cache):
+ self.cache = cache
+
+ def _layers(self):
+ return getattr(self.cache, "layers", None) or []
+
+ def __len__(self):
+ return len(self._layers())
+
+ def __getitem__(self, index):
+ layer = self._layers()[index]
+ return (layer.keys, layer.values)
+
+ def __iter__(self):
+ for i in range(len(self)):
+ yield self[i]
+
+ def __getattr__(self, name):
+ return getattr(self.cache, name)
+
+
+def _needs_legacy_view(cache):
+ if cache is None or isinstance(cache, (tuple, list)):
+ return False
+ return not hasattr(type(cache), "__getitem__") and hasattr(cache, "layers")
+
+
+def _repair_multimodal_cache_indexing(cls):
+ """Phi-4-reasoning-vision indexes `past_key_values[-1][-1]` here per decode step: pass a tuple view, return the real cache."""
+ name = "prepare_inputs_labels_for_multimodal"
+ original = None
+ for klass in cls.__mro__:
+ if name in klass.__dict__:
+ original = klass.__dict__[name]
+ break
+ if original is None or getattr(original, "_unsloth_cache_view", False):
+ return False
+ try:
+ parameters = list(inspect.signature(original).parameters)
+ except (TypeError, ValueError):
+ return False
+ if "past_key_values" not in parameters:
+ return False
+ try:
+ source = inspect.getsource(original)
+ except (OSError, TypeError):
+ return False
+ # Code already on the Cache API (get_seq_length, isinstance Cache) must keep the real cache.
+ if re.search(r"past_key_values\s*\[", source) is None:
+ return False
+ position = parameters.index("past_key_values") - 1
+
+ @functools.wraps(original)
+ def prepare_inputs_labels_for_multimodal(self, *args, **kwargs):
+ args = list(args)
+ if "past_key_values" in kwargs:
+ cache = kwargs["past_key_values"]
+ if _needs_legacy_view(cache):
+ kwargs["past_key_values"] = _LegacyCacheView(cache)
+ elif len(args) > position:
+ cache = args[position]
+ if _needs_legacy_view(cache):
+ args[position] = _LegacyCacheView(cache)
+ else:
+ cache = None
+ output = original(self, *args, **kwargs)
+ if isinstance(output, tuple):
+ output = tuple(cache if isinstance(x, _LegacyCacheView) else x for x in output)
+ return output
+
+ prepare_inputs_labels_for_multimodal._unsloth_cache_view = True
+ setattr(cls, name, prepare_inputs_labels_for_multimodal)
+ # Static cache: prefill mask arrives as 4D / BlockMask, not the 2D padding mask this prep compacts by.
+ cls._supports_static_cache = False
+ return True
+
+
def _rebind_accelerate_hook(model):
# device_map loading keeps the bound original as `_old_forward`, bypassing class repairs.
if getattr(type(model), "_unsloth_original_forward", None) is None:
@@ -290,6 +373,8 @@ def apply_remote_code_shims(model):
repaired.append(f"{cls.__name__}.get_output_embeddings")
if _is_remote_code(cls) and _fill_missing_loss(cls):
repaired.append(f"{cls.__name__}.forward")
+ if _is_remote_code(cls) and _repair_multimodal_cache_indexing(cls):
+ repaired.append(f"{cls.__name__}.prepare_inputs_labels_for_multimodal")
_rebind_accelerate_hook(model)
if repaired:
print("Unsloth: Repaired remote modeling code so it trains: " + ", ".join(repaired) + ".")