Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: load LFM3 custom-code models (route by saving transformers version, fix generate, chat and export) (#16096)

Daniel Han87a4d574389d
diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py
index 60894b33c10..ac20044ba22 100644
--- a/studio/backend/core/export/export.py
+++ b/studio/backend/core/export/export.py
@@ -713,6 +713,33 @@ def _checkpoint_load_refusal(checkpoint_path, pinned_base) -> Optional[str]:
return None
+def _with_base_processor(tokenizer, base_model: str, **load_kw):
+ """A text-only VLM adapter saves a bare tokenizer; the export still needs the base's processor (mmproj)."""
+ if tokenizer is None or hasattr(tokenizer, "tokenizer"):
+ return tokenizer
+ try:
+ from transformers import AutoProcessor
+ processor = AutoProcessor.from_pretrained(base_model, **load_kw)
+ except Exception as e:
+ logger.info(f"No processor from base model {base_model}: {e}")
+ return tokenizer
+ if not hasattr(processor, "tokenizer"):
+ return tokenizer
+ processor.tokenizer = tokenizer
+ if getattr(tokenizer, "chat_template", None):
+ processor.chat_template = tokenizer.chat_template
+ return processor
+
+
+def _restore_use_cache(model) -> None:
+ """The load prepares the model for training (use_cache=False on every config); export saves it."""
+ try:
+ from unsloth_zoo.training_utils import restore_use_cache
+ except ImportError:
+ return
+ restore_use_cache(model)
+
+
def _cached_hub_adapter_commit(checkpoint_path: str) -> Optional[str]:
"""The commit of a Hub adapter's cached snapshot (the one its load just used), else None."""
if not checkpoint_path or os.path.isdir(checkpoint_path):
@@ -1032,6 +1059,15 @@ class ExportBackend:
)
if _IS_MLX and adapter_config.exists():
processor = load_mlx_adapter_processor(processor, checkpoint_path)
+ elif base_model:
+ with _offline_window_if(local_files_only):
+ processor = _with_base_processor(
+ processor,
+ base_model,
+ trust_remote_code = trust_remote_code,
+ token = token,
+ local_files_only = local_files_only,
+ )
tokenizer = processor
else:
@@ -1061,6 +1097,7 @@ class ExportBackend:
self.is_peft = adapter_config.exists()
else:
self.is_peft = isinstance(model, (PeftModel, PeftModelForCausalLM))
+ _restore_use_cache(model)
restored_repo_id = restore_hf_cache_repo_identity(model, base_model)
if restored_repo_id:
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 1a2c3b80043..c6fb36adf88 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -11,6 +11,7 @@ from transformers import TextIteratorStreamer, TextStreamer
from peft import PeftModel, PeftModelForCausalLM
import contextlib
+import inspect
import json
import re
import sys
@@ -694,6 +695,32 @@ class _StopSequenceStreamer:
abort()
+_CUSTOM_GENERATE_REJECTED_KWARGS = ("logits_processor", "stopping_criteria")
+
+
+def _supported_generate_kwargs(model, generation_kwargs: dict) -> dict:
+ """Drops kwargs a custom-code generate rejects (no presence penalty or early cancel there)."""
+ if not isinstance(getattr(model, "_generation_fields_unsupported", None), dict):
+ return generation_kwargs
+ return {k: v for k, v in generation_kwargs.items() if k not in _CUSTOM_GENERATE_REJECTED_KWARGS}
+
+
+def _processor_takes_text_first(processor) -> bool:
+ """LFM3-Omni's processor is (text, audio, ..., images); the usual VLM order is (images, text)."""
+ # unsloth_zoo wraps __call__ as (*args, **kwargs); read the first class that names its arguments.
+ for cls in type(processor).__mro__:
+ call = cls.__dict__.get("__call__")
+ if call is None:
+ continue
+ try:
+ params = list(inspect.signature(call).parameters.values())[1:]
+ except (TypeError, ValueError):
+ return False
+ if params and params[0].kind is not inspect.Parameter.VAR_POSITIONAL:
+ return params[0].name == "text"
+ return False
+
+
# LlamaModel_fast_forward cuts input_ids at max_seq_length, so decoding past it crashes; FastModel's stock forward does not.
def _truncates_past_window(model) -> bool:
return any(
@@ -2172,12 +2199,21 @@ class InferenceBackend:
input_text = _render_plain_vision(vision_messages)
else:
raise
- inputs = processor(
- attached[0] if len(attached) == 1 else attached,
- input_text,
- add_special_tokens = False,
- return_tensors = "pt",
- ).to(model.device)
+ _images = attached[0] if len(attached) == 1 else attached
+ if _processor_takes_text_first(processor):
+ inputs = processor(
+ text = input_text,
+ images = _images,
+ add_special_tokens = False,
+ return_tensors = "pt",
+ ).to(model.device)
+ else:
+ inputs = processor(
+ _images,
+ input_text,
+ add_special_tokens = False,
+ return_tensors = "pt",
+ ).to(model.device)
prompt_text = input_text
else:
formatted_prompt = self.format_chat_prompt(
@@ -2281,7 +2317,9 @@ class InferenceBackend:
# Started inside the lock so a queued request's wait is not billed as prefill.
timer.start()
# Only the returned sequences carry an exact generated-token count; see generate_stream.
- gen_outputs["sequences"] = model.generate(**generation_kwargs)
+ gen_outputs["sequences"] = model.generate(
+ **_supported_generate_kwargs(model, generation_kwargs)
+ )
except Exception as e:
err["msg"] = str(e)
stop_streamer.abort()
@@ -2478,7 +2516,9 @@ class InferenceBackend:
# Started after the adapter swap so only model.generate() is timed.
timer.start()
# Only the returned sequences carry an exact generated-token count; see generate_stream.
- gen_outputs["sequences"] = model.generate(**generation_kwargs)
+ gen_outputs["sequences"] = model.generate(
+ **_supported_generate_kwargs(model, generation_kwargs)
+ )
except Exception as e:
err["msg"] = str(e)
logger.error(f"Audio input generation error in thread: {e}")
@@ -2870,7 +2910,9 @@ class InferenceBackend:
# Started after the adapter swap so only model.generate() is timed.
timer.start()
# Only the returned sequences carry an exact token count.
- gen_outputs["sequences"] = model.generate(**generation_kwargs)
+ gen_outputs["sequences"] = model.generate(
+ **_supported_generate_kwargs(model, generation_kwargs)
+ )
except Exception as e:
err["msg"] = str(e)
stop_streamer.abort()
diff --git a/studio/backend/tests/test_custom_generate_kwargs.py b/studio/backend/tests/test_custom_generate_kwargs.py
new file mode 100644
index 00000000000..a14af9a52fb
--- /dev/null
+++ b/studio/backend/tests/test_custom_generate_kwargs.py
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""LFM3-Omni's custom generate raises on logits_processor and stopping_criteria (#16054)."""
+
+import ast
+import inspect
+import types
+from pathlib import Path
+
+# Loaded from source: importing core.inference.inference needs unsloth, which the backend matrix lacks.
+_SRC = Path(__file__).resolve().parents[1] / "core" / "inference" / "inference.py"
+_WANTED = {
+ "_CUSTOM_GENERATE_REJECTED_KWARGS",
+ "_supported_generate_kwargs",
+ "_processor_takes_text_first",
+}
+_NS = {"inspect": inspect}
+exec(
+ compile(
+ ast.Module(
+ [
+ n
+ for n in ast.parse(_SRC.read_text(encoding = "utf-8")).body
+ if getattr(n, "name", None) in _WANTED
+ or any(getattr(t, "id", None) in _WANTED for t in getattr(n, "targets", ()))
+ ],
+ [],
+ ),
+ str(_SRC),
+ "exec",
+ ),
+ _NS,
+)
+_supported_generate_kwargs = _NS["_supported_generate_kwargs"]
+_processor_takes_text_first = _NS["_processor_takes_text_first"]
+
+_KWARGS = {"input_ids": 1, "logits_processor": [], "stopping_criteria": [], "streamer": object()}
+
+
+def test_custom_generate_drops_rejected_kwargs():
+ model = types.SimpleNamespace(_generation_fields_unsupported = {"cache_implementation": None})
+ out = _supported_generate_kwargs(model, _KWARGS)
+ assert "logits_processor" not in out and "stopping_criteria" not in out
+ assert out["input_ids"] == 1 and "streamer" in out
+
+
+def test_regular_model_keeps_all_kwargs():
+ assert _supported_generate_kwargs(types.SimpleNamespace(), _KWARGS) is _KWARGS
+
+
+class _TextFirst:
+ def __call__(
+ self,
+ text = None,
+ audio = None,
+ images = None,
+ **kwargs,
+ ):
+ return text, images
+
+
+class _ImagesFirst:
+ def __call__(
+ self,
+ images = None,
+ text = None,
+ **kwargs,
+ ):
+ return images, text
+
+
+def test_text_first_processor_detected():
+ assert _processor_takes_text_first(_TextFirst()) is True
+ assert _processor_takes_text_first(_ImagesFirst()) is False
+
+
+def test_zoo_wrapped_call_still_reads_original_signature():
+ class _Patched(_TextFirst):
+ def __call__(self, *args, **kwargs):
+ return super().__call__(*args, **kwargs)
+
+ assert _processor_takes_text_first(_Patched()) is True
diff --git a/studio/backend/tests/test_export_vlm_processor_use_cache.py b/studio/backend/tests/test_export_vlm_processor_use_cache.py
new file mode 100644
index 00000000000..133a5f9484b
--- /dev/null
+++ b/studio/backend/tests/test_export_vlm_processor_use_cache.py
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A text-only LoRA on a VLM exports with the base's processor and without training's use_cache=False."""
+
+from __future__ import annotations
+
+import sys
+import types
+from pathlib import Path
+
+_BACKEND_DIR = Path(__file__).resolve().parent.parent
+if str(_BACKEND_DIR) not in sys.path:
+ sys.path.insert(0, str(_BACKEND_DIR))
+_TESTS_DIR = Path(__file__).resolve().parent
+if str(_TESTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_TESTS_DIR))
+
+from test_export_absolute_paths import ( # noqa: E402
+ _install_export_backend_stubs,
+ _load_module,
+)
+
+
+def _export_mod(monkeypatch):
+ _install_export_backend_stubs(monkeypatch)
+ return _load_module("test_core_export_vlm_processor", "core/export/export.py", monkeypatch)
+
+
+class _Tokenizer:
+ chat_template = "{{ trained }}"
+
+
+class _Processor:
+ def __init__(self):
+ self.tokenizer = object()
+ self.chat_template = "{{ base }}"
+
+
+def _stub_auto_processor(monkeypatch, load):
+ calls = []
+
+ class AutoProcessor:
+ @staticmethod
+ def from_pretrained(name, **kw):
+ calls.append((name, kw))
+ return load()
+
+ monkeypatch.setitem(
+ sys.modules, "transformers", types.SimpleNamespace(AutoProcessor = AutoProcessor)
+ )
+ return calls
+
+
+def test_bare_tokenizer_is_wrapped_in_the_base_processor(monkeypatch):
+ mod = _export_mod(monkeypatch)
+ calls = _stub_auto_processor(monkeypatch, _Processor)
+ tok = _Tokenizer()
+ out = mod._with_base_processor(tok, "org/vlm", trust_remote_code = True, token = None)
+ assert isinstance(out, _Processor)
+ assert out.tokenizer is tok
+ assert out.chat_template == "{{ trained }}"
+ assert calls == [("org/vlm", {"trust_remote_code": True, "token": None})]
+
+
+def test_a_loaded_processor_is_kept(monkeypatch):
+ mod = _export_mod(monkeypatch)
+ calls = _stub_auto_processor(monkeypatch, _Processor)
+ proc = _Processor()
+ assert mod._with_base_processor(proc, "org/vlm") is proc
+ assert calls == []
+
+
+def test_base_without_processor_keeps_the_tokenizer(monkeypatch):
+ mod = _export_mod(monkeypatch)
+ _stub_auto_processor(monkeypatch, _Tokenizer)
+ tok = _Tokenizer()
+ assert mod._with_base_processor(tok, "org/text") is tok
+
+
+def test_processor_load_failure_keeps_the_tokenizer(monkeypatch):
+ mod = _export_mod(monkeypatch)
+
+ def _boom():
+ raise OSError("no processor_config.json")
+
+ _stub_auto_processor(monkeypatch, _boom)
+ tok = _Tokenizer()
+ assert mod._with_base_processor(tok, "org/vlm") is tok
+
+
+def test_export_load_restores_use_cache(monkeypatch):
+ mod = _export_mod(monkeypatch)
+ restored = []
+ zoo = types.ModuleType("unsloth_zoo")
+ training_utils = types.ModuleType("unsloth_zoo.training_utils")
+ training_utils.restore_use_cache = restored.append
+ zoo.training_utils = training_utils
+ monkeypatch.setitem(sys.modules, "unsloth_zoo", zoo)
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.training_utils", training_utils)
+ model = object()
+ mod._restore_use_cache(model)
+ assert restored == [model]
+
+
+def test_restore_use_cache_tolerates_an_old_zoo(monkeypatch):
+ mod = _export_mod(monkeypatch)
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.training_utils", types.ModuleType("x"))
+ mod._restore_use_cache(object())
diff --git a/studio/backend/tests/test_transformers_latest.py b/studio/backend/tests/test_transformers_latest.py
index 264382a3cd8..89ca0668402 100644
--- a/studio/backend/tests/test_transformers_latest.py
+++ b/studio/backend/tests/test_transformers_latest.py
@@ -1614,3 +1614,95 @@ def test_fetch_main_version_requires_dev_string(monkeypatch):
assert tl._fetch_main_version() is None
monkeypatch.setattr(tl, "_fetch_text", lambda url: '__version__ = "5.19.0.dev0"\n')
assert tl._fetch_main_version() == "5.19.0.dev0"
+
+
+def _custom_code_model(tmp_path: Path, saved_by: str) -> str:
+ """LFM3-shaped checkpoint: model_type no transformers ships, config class in repo code."""
+ d = tmp_path / f"custom_{saved_by}"
+ d.mkdir()
+ cfg = {
+ "model_type": "lfm3_omni",
+ "architectures": ["Lfm3OmniForConditionalGeneration"],
+ "auto_map": {"AutoConfig": "configuration_lfm3_omni.Lfm3OmniConfig"},
+ "transformers_version": saved_by,
+ }
+ (d / "config.json").write_text(json.dumps(cfg))
+ return str(d)
+
+
+class TestCustomCodeRouting:
+ """The probe parses with trust_remote_code=False, so custom code fell to 5.3.0 (#16054)."""
+
+ @pytest.fixture(autouse = True)
+ def _no_probe(self, monkeypatch):
+ _fake_overlays(monkeypatch)
+ monkeypatch.setattr(tv, "_probe_tier", lambda *a, floor = "530", **k: floor)
+ monkeypatch.setattr(tv, "latest_venv_pinned_version", lambda: None)
+ monkeypatch.setattr(tv, "TRANSFORMERS_DEFAULT_VERSION", "5.5.0")
+
+ def test_newer_than_every_tier_routes_to_newest_fixed_tier(self, tmp_path: Path):
+ assert get_transformers_tier(_custom_code_model(tmp_path, "5.19.0.dev0")) == "510"
+
+ def test_routes_to_pinned_latest_when_new_enough(self, tmp_path: Path, monkeypatch):
+ monkeypatch.setattr(tv, "latest_venv_pinned_version", lambda: "5.22.0")
+ assert get_transformers_tier(_custom_code_model(tmp_path, "5.19.0.dev0")) == "latest"
+
+ def test_oldest_sidecar_at_or_above_saving_version(self, tmp_path: Path):
+ assert get_transformers_tier(_custom_code_model(tmp_path, "5.8.0")) == "510"
+
+ def test_default_kept_when_new_enough(self, tmp_path: Path):
+ assert get_transformers_tier(_custom_code_model(tmp_path, "5.3.0")) == "default"
+
+ def test_patch_release_fits_its_minor(self, tmp_path: Path):
+ assert get_transformers_tier(_custom_code_model(tmp_path, "5.5.4")) == "default"
+
+ def test_v5_tokenizer_saved_at_530_keeps_530(self, tmp_path: Path):
+ d = Path(_custom_code_model(tmp_path, "5.3.0"))
+ (d / "tokenizer_config.json").write_text(
+ json.dumps({"tokenizer_class": "TokenizersBackend"})
+ )
+ assert get_transformers_tier(str(d)) == "530"
+
+ def test_upgrade_offered_when_pypi_is_new_enough(self, tmp_path: Path, monkeypatch):
+ monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen_factory({}))
+ result = check_upgrade_for_model(_custom_code_model(tmp_path, "5.12.0.dev0"))
+ assert result is not None
+ assert result["model_type"] == "lfm3_omni" and result["pypi_version"] == "5.13.0"
+ assert result["supported_in_pypi"] is True
+
+ def test_no_offer_when_pypi_too_old(self, tmp_path: Path, monkeypatch):
+ monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen_factory({}))
+ assert check_upgrade_for_model(_custom_code_model(tmp_path, "5.19.0.dev0")) is None
+
+ def test_no_offer_when_installed_tier_fits(self, tmp_path: Path, monkeypatch):
+ calls = _no_network(monkeypatch)
+ assert check_upgrade_for_model(_custom_code_model(tmp_path, "5.8.0")) is None
+ assert calls["n"] == 0
+
+ def test_native_tier_is_raised_to_saving_version(self, tmp_path: Path):
+ d = tmp_path / "custom_native"
+ d.mkdir()
+ cfg = {
+ "model_type": "qwen3_moe",
+ "architectures": ["Qwen3MoeForCausalLM"],
+ "auto_map": {"AutoConfig": "configuration_x.XConfig"},
+ "transformers_version": "5.8.0",
+ }
+ (d / "config.json").write_text(json.dumps(cfg))
+ assert get_transformers_tier(str(d)) == "510"
+
+ def test_main_offered_when_pypi_is_behind(self, tmp_path: Path, monkeypatch):
+ monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen_factory({}))
+ result = check_upgrade_for_model(_custom_code_model(tmp_path, "5.14.0.dev0"))
+ assert result is not None
+ assert result["supported_in_pypi"] is False and result["supported_in_main"] is True
+ assert result["main_version"] == "5.14.0.dev0"
+
+ def test_hub_name_match_is_raised_to_saving_version(self, monkeypatch):
+ cfg = {
+ "model_type": "qwen3_5",
+ "auto_map": {"AutoConfig": "configuration_x.XConfig"},
+ "transformers_version": "5.8.0",
+ }
+ monkeypatch.setattr(tv, "_load_config_json", lambda name, token = None: cfg)
+ assert get_transformers_tier("someorg/Qwen3.5-Custom") == "510"
diff --git a/studio/backend/utils/transformers_latest.py b/studio/backend/utils/transformers_latest.py
index 114315946f9..83500598012 100644
--- a/studio/backend/utils/transformers_latest.py
+++ b/studio/backend/utils/transformers_latest.py
@@ -23,6 +23,9 @@ from utils.transformers_version import (
_tier_from_config_mapping,
_config_model_types,
_NESTED_CONFIG_KEYS,
+ _release_tuple,
+ _remote_code_min_version,
+ _remote_code_tier,
_TIER_RANK,
_model_types_from_config,
_TRANSFORMERS_510_MODEL_TYPES,
@@ -389,6 +392,38 @@ def _hardcoded_tier_ships(model_type: str) -> bool:
return True
+def _remote_code_upgrade(model_name: str, cfg: dict) -> dict | None:
+ """Offer latest transformers when it meets a custom-code config's saving version and no installed tier does."""
+ required = _remote_code_min_version(cfg)
+ if required is None or _remote_code_tier(cfg) is not None:
+ return None
+ snapshot = _get_snapshot()
+ if snapshot is None:
+ return None
+ pypi = _release_tuple(snapshot["pypi_version"])
+ main = _release_tuple(snapshot.get("main_version"))
+ in_pypi = pypi is not None and pypi >= required
+ # Main ships as the next release's .dev0, so a dev build of the required release counts.
+ offerable = in_pypi or (main is not None and main >= required)
+ if not offerable:
+ return None
+ logger.info(
+ "Model %s has custom code saved by transformers %s, newer than every installed tier "
+ "(latest PyPI %s)",
+ model_name,
+ cfg.get("transformers_version"),
+ snapshot["pypi_version"],
+ )
+ types = _model_types_from_config(cfg)
+ return {
+ "model_type": types[0] if types else str(cfg.get("model_type") or ""),
+ "pypi_version": snapshot["pypi_version"],
+ "supported_in_pypi": in_pypi,
+ "supported_in_main": True,
+ "main_version": snapshot.get("main_version"),
+ }
+
+
def check_upgrade_for_model(model_name: str, hf_token: str | None = None) -> dict | None:
"""Fires only when model_type is in no installed overlay or tier table, i.e. the load would fail."""
try:
@@ -399,6 +434,9 @@ def check_upgrade_for_model(model_name: str, hf_token: str | None = None) -> dic
return None
if _architecture_cannot_come_from_transformers(model_name, cfg):
return None
+ # Custom code loads from the repo, so only the saving version matters, not the mapping.
+ if _remote_code_min_version(cfg) is not None:
+ return _remote_code_upgrade(model_name, cfg)
candidates = _model_types_from_config(cfg)
if not candidates:
return None
diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py
index 2a7eff3917f..6aabb1ae7f0 100644
--- a/studio/backend/utils/transformers_version.py
+++ b/studio/backend/utils/transformers_version.py
@@ -1104,6 +1104,66 @@ def _config_saved_by_transformers_5(cfg: dict | None) -> bool:
return False
+def _release_tuple(ver) -> tuple[int, int, int] | None:
+ """``"5.19.0.dev0"`` -> ``(5, 19, 0)``; a dev build counts as its release."""
+ m = re.match(r"\s*(\d+)\.(\d+)(?:\.(\d+))?", ver) if isinstance(ver, str) else None
+ return (int(m.group(1)), int(m.group(2)), int(m.group(3) or 0)) if m else None
+
+
+def _remote_code_min_version(cfg: dict | None) -> tuple[int, int, int] | None:
+ """Saving transformers 5.x version of a custom-code (``auto_map`` AutoConfig) config, else None."""
+ if not isinstance(cfg, dict) or not _config_saved_by_transformers_5(cfg):
+ return None
+ auto_map = cfg.get("auto_map")
+ if not isinstance(auto_map, dict) or "AutoConfig" not in auto_map:
+ return None
+ v = _release_tuple(cfg.get("transformers_version"))
+ # Patch releases keep remote-code APIs: 5.5.4 still fits the 5.5.0 default.
+ return (v[0], v[1], 0) if v else None
+
+
+def _remote_code_tier(cfg: dict | None) -> str | None:
+ """Default if new enough, else the oldest sidecar at or above the saving version; None if none is."""
+ required = _remote_code_min_version(cfg)
+ if required is None:
+ return None
+ tiers = [
+ ("default", TRANSFORMERS_DEFAULT_VERSION),
+ ("530", TRANSFORMERS_530_VERSION),
+ ("550", TRANSFORMERS_550_VERSION),
+ ("510", TRANSFORMERS_510_VERSION),
+ ]
+ if not _latest_tier_disabled():
+ tiers.append(("latest", latest_venv_pinned_version()))
+ fits = [
+ (tier != "default", v, _TIER_RANK[tier], tier)
+ for tier, ver in tiers
+ if (v := _release_tuple(ver)) is not None and v >= required
+ ]
+ return min(fits)[3] if fits else None
+
+
+def _route_remote_code(
+ model_name: str,
+ cfg: dict | None,
+ above: str | None = None,
+) -> str | None:
+ """Tier for a custom-code config; None falls through, also when ``above`` already satisfies it."""
+ required = _remote_code_min_version(cfg)
+ if required is None or (above is not None and required <= _release_tuple(above)):
+ return None
+ tier = _remote_code_tier(cfg)
+ if tier is None:
+ tier = "latest" if (not _latest_tier_disabled() and latest_venv_pinned_version()) else "510"
+ logger.info(
+ "Transformers tier %s selected for %s (custom code saved by transformers %s)",
+ tier,
+ model_name,
+ cfg.get("transformers_version"),
+ )
+ return tier
+
+
def _cached_config_json(model_name: str, hf_token: str | None) -> dict | None:
"""Already-fetched config.json from the in-process cache (no new fetch); the tier checks
above populate it, and a miss just skips the version-field probe."""
@@ -1618,6 +1678,30 @@ def get_transformers_tier(
probe: bool = True,
) -> str:
"""Returns 510, 550, 530 or default; local config.json is checked before name heuristics (dir-name false positives); probe=False skips sidecar subprocesses."""
+ tier = _native_transformers_tier(model_name, hf_token, probe)
+ # Raise only; the name fast path fetched no config, so a real load (probe=True) reads it here.
+ if probe and tier != "default":
+ cfg = _load_config_json(model_name, hf_token)
+ else:
+ cfg = _cached_config_json(model_name, hf_token)
+ required = _remote_code_min_version(cfg)
+ current = _release_tuple(_tier_version(tier))
+ if required is not None and current is not None and current < required:
+ return _route_remote_code(model_name, cfg) or tier
+ return tier
+
+
+def _tier_version(tier: str) -> str | None:
+ return {
+ "default": TRANSFORMERS_DEFAULT_VERSION,
+ "457": TRANSFORMERS_457_VERSION,
+ "530": TRANSFORMERS_530_VERSION,
+ "550": TRANSFORMERS_550_VERSION,
+ "510": TRANSFORMERS_510_VERSION,
+ }.get(tier) or (latest_venv_pinned_version() if tier == "latest" else None)
+
+
+def _native_transformers_tier(model_name: str, hf_token: str | None, probe: bool) -> str:
local_cfg = Path(model_name) / "config.json"
if _safe_is_file(local_cfg):
cfg = _load_config_json(model_name, hf_token)
@@ -1700,6 +1784,10 @@ def get_transformers_tier(
model_name,
)
return static
+ # The fall-through's 530 floor fits these; routing here would lower them to default.
+ remote_code = _route_remote_code(model_name, cfg, above = TRANSFORMERS_530_VERSION)
+ if remote_code is not None:
+ return remote_code
local_tc = Path(model_name) / "tokenizer_config.json"
if _safe_is_file(local_tc) and _check_tokenizer_config_needs_v5(model_name, hf_token):
if not probe:
@@ -1780,6 +1868,9 @@ def get_transformers_tier(
model_name,
)
return static
+ remote_code = _route_remote_code(model_name, remote_cfg, above = TRANSFORMERS_530_VERSION)
+ if remote_code is not None:
+ return remote_code
if _check_tokenizer_config_needs_v5(model_name, hf_token):
if not probe:
return "530"
diff --git a/tests/test_fa2_fast_generate_bypass.py b/tests/test_fa2_fast_generate_bypass.py
index 96565b8d29c..e82c15fcdef 100644
--- a/tests/test_fa2_fast_generate_bypass.py
+++ b/tests/test_fa2_fast_generate_bypass.py
@@ -345,6 +345,13 @@ def test_flash_attention_fallback_pins_a_dynamic_cache():
assert "cache_implementation" not in captured
assert captured["past_key_values"] is cache
+ # LFM3's custom generate lists cache_implementation as unsupported and raises on any value (#16054).
+ class CustomGenerateModel(Model):
+ _generation_fields_unsupported = {"cache_implementation": None}
+
+ fast_generate(CustomGenerateModel(), input_ids = input_ids)
+ assert "cache_implementation" not in captured
+
if __name__ == "__main__":
tests = [
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index f264f8fa027..bd2f228b7a6 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -2064,9 +2064,15 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
# FlashAttention breaks on the forced static cache below (unfilled slots stay unmasked while decoding), so delegate after normalization but before it.
_clear_generation_caches(self)
+ # Custom-code generate (LFM3) raises on any cache field it lists as unsupported; leave them unset.
+ # Inline, not a helper: tests exec this function's source with a hand-built namespace.
+ unsupported_fields = getattr(self, "_generation_fields_unsupported", None)
+ rejects_cache_fields = (
+ isinstance(unsupported_fields, dict) and "cache_implementation" in unsupported_fields
+ )
if _uses_flash_attention_for_generation(self.config):
# Pin the literal "dynamic": None is merged back to the model default, and a static cache still arrives via kwargs or the caller's generation_config (TRL). The kwarg wins, since update runs last; skip it when the caller passed a cache.
- if kwargs.get("past_key_values") is None:
+ if kwargs.get("past_key_values") is None and not rejects_cache_fields:
kwargs["cache_implementation"] = "dynamic"
try:
with _get_inference_mode_context_manager(self), autocaster:
@@ -2118,6 +2124,9 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
if force_dynamic_cache:
cache_implementation = None
dynamic_implementation = _dynamic_cache_choice(kwargs)
+ if rejects_cache_fields:
+ cache_implementation = None
+ force_dynamic_cache = False
generation_config = kwargs.get("generation_config") or getattr(self, "generation_config", None)
compile_decode = (