Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix text_only 4-bit load and generate for Gemma-4 and other VLM text configs (#11684)

Daniel Haned18d9c8a630
diff --git a/tests/python/test_fast_language_model_text_only.py b/tests/python/test_fast_language_model_text_only.py
index 046f9197224..1f842792acb 100644
--- a/tests/python/test_fast_language_model_text_only.py
+++ b/tests/python/test_fast_language_model_text_only.py
@@ -307,6 +307,46 @@ def test_text_only_helper_preserves_quantization_config():
assert getattr(config.get_text_config(), "quantization_config", None) is None
+class _ReadOnlyTextConfigProxy:
+ # Mirrors unsloth_zoo's _Gemma4KVSharedSafeProxy.
+ __slots__ = ("_real",)
+
+ def __init__(self, real):
+ object.__setattr__(self, "_real", real)
+
+ def __getattr__(self, name):
+ return getattr(object.__getattribute__(self, "_real"), name)
+
+
+def test_text_only_helper_copies_config_behind_read_only_proxy():
+ transformers = pytest.importorskip("transformers")
+ helper = _load_text_only_helper()
+ config = transformers.Gemma3Config()
+ real_text = config.get_text_config()
+ proxy = _ReadOnlyTextConfigProxy(real_text)
+ config.get_text_config = lambda *a, **k: proxy
+ sentinel = object()
+ config.quantization_config = sentinel
+ text_config = helper(config, "google/gemma-4-31B-it")
+ assert getattr(text_config, "quantization_config", None) is sentinel
+ assert type(text_config) is type(real_text)
+ assert text_config is not real_text
+ assert getattr(real_text, "quantization_config", None) is None
+
+
+def test_text_only_helper_unwraps_read_only_proxy_without_quantization():
+ transformers = pytest.importorskip("transformers")
+ ns = _load_text_only_namespace()
+ helper, resolve = ns["_get_text_only_config"], ns["resolve_model_class"]
+ config = transformers.Gemma3Config()
+ real_text = config.get_text_config()
+ config.get_text_config = lambda *a, **k: _ReadOnlyTextConfigProxy(real_text)
+ text_config = helper(config, "google/gemma-4-31B-it")
+ assert type(text_config) is type(real_text)
+ model_class = resolve(transformers.AutoModelForCausalLM, text_config)
+ assert model_class is not None and model_class.__name__ == "Gemma3ForCausalLM"
+
+
def test_text_only_key_mapping_targets_published_prefixes():
# Remap the published VLM decoder prefixes, applying only on transformers >=5
# (on 4.x base_model_prefix handles it and a mapping hurts).
@@ -416,3 +456,28 @@ def test_gemma3_text_only_loads_real_language_weights_from_vlm_checkpoint(tmp_pa
assert not any(
"vision_tower" in n for n, _ in model.named_modules()
), "vision tower should be skipped on the text-only path"
+
+
+def _module_function(tree, name):
+ for node in tree.body:
+ if isinstance(node, ast.FunctionDef) and node.name == name:
+ return node
+ raise AssertionError(f"{name} not found")
+
+
+def test_base_fast_generate_tolerates_missing_architectures():
+ method = _module_function(ast.parse(_source(VISION_PATH)), "unsloth_base_fast_generate")
+
+ def is_raw_architectures(node):
+ return (
+ isinstance(node, ast.Attribute)
+ and node.attr == "architectures"
+ and isinstance(node.value, ast.Attribute)
+ and node.value.attr == "config"
+ )
+
+ for node in ast.walk(method):
+ if isinstance(node, ast.comprehension):
+ assert not is_raw_architectures(node.iter)
+ if isinstance(node, ast.Subscript):
+ assert not is_raw_architectures(node.value)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index c14b96bd86f..8c7351e92cd 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -1127,6 +1127,11 @@ def _get_text_only_config(model_config, model_name):
text_config = getattr(model_config, "text_config", None)
if text_config is None:
raise ValueError(f"Cannot load {model_name} as text-only; use FastVisionModel")
+ # unsloth_zoo's Gemma-4 proxy (__slots__) refuses new attrs and resolve_model_class cannot map it; unwrap.
+ try:
+ text_config = object.__getattribute__(text_config, "_real")
+ except AttributeError:
+ pass
# Carry over quantization_config; copy first, since get_text_config() shares the parent's object.
qc = getattr(model_config, "quantization_config", None)
if qc is not None and getattr(text_config, "quantization_config", None) is None:
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index d888c9cecde..ee8146ee50f 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -882,12 +882,13 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
if do_bfloat16_mixed_precision:
dtype = torch.bfloat16
+ # text_only configs have architectures=None.
+ architectures = getattr(self.config, "architectures", None) or [type(self).__name__]
is_vlm = any(
- x.endswith(("ForConditionalGeneration", "ForVisionText2Text"))
- for x in self.config.architectures
+ x.endswith(("ForConditionalGeneration", "ForVisionText2Text")) for x in architectures
)
is_vlm = is_vlm or hasattr(self.config, "vision_config")
- arch = self.config.architectures[0]
+ arch = architectures[0]
# Removing token_type_ids is WRONG for Gemma 3, which uses bidirectional attention.
if hasattr(self, "generate") and hasattr(self, "forward"):