Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix text_only 4-bit load and generate for Gemma-4 and other VLM text configs (#11684)
Daniel Haned18d9c8a630
diff --git a/tests/python/test_fast_language_model_text_only.py b/tests/python/test_fast_language_model_text_only.pyindex 046f9197224..1f842792acb 100644--- a/tests/python/test_fast_language_model_text_only.py+++ b/tests/python/test_fast_language_model_text_only.py@@ -307,6 +307,46 @@ def test_text_only_helper_preserves_quantization_config():assert getattr(config.get_text_config(), "quantization_config", None) is None+class _ReadOnlyTextConfigProxy:+ # Mirrors unsloth_zoo's _Gemma4KVSharedSafeProxy.+ __slots__ = ("_real",)++ def __init__(self, real):+ object.__setattr__(self, "_real", real)++ def __getattr__(self, name):+ return getattr(object.__getattribute__(self, "_real"), name)+++def test_text_only_helper_copies_config_behind_read_only_proxy():+ transformers = pytest.importorskip("transformers")+ helper = _load_text_only_helper()+ config = transformers.Gemma3Config()+ real_text = config.get_text_config()+ proxy = _ReadOnlyTextConfigProxy(real_text)+ config.get_text_config = lambda *a, **k: proxy+ sentinel = object()+ config.quantization_config = sentinel+ text_config = helper(config, "google/gemma-4-31B-it")+ assert getattr(text_config, "quantization_config", None) is sentinel+ assert type(text_config) is type(real_text)+ assert text_config is not real_text+ assert getattr(real_text, "quantization_config", None) is None+++def test_text_only_helper_unwraps_read_only_proxy_without_quantization():+ transformers = pytest.importorskip("transformers")+ ns = _load_text_only_namespace()+ helper, resolve = ns["_get_text_only_config"], ns["resolve_model_class"]+ config = transformers.Gemma3Config()+ real_text = config.get_text_config()+ config.get_text_config = lambda *a, **k: _ReadOnlyTextConfigProxy(real_text)+ text_config = helper(config, "google/gemma-4-31B-it")+ assert type(text_config) is type(real_text)+ model_class = resolve(transformers.AutoModelForCausalLM, text_config)+ assert model_class is not None and model_class.__name__ == "Gemma3ForCausalLM"++def test_text_only_key_mapping_targets_published_prefixes():# Remap the published VLM decoder prefixes, applying only on transformers >=5# (on 4.x base_model_prefix handles it and a mapping hurts).@@ -416,3 +456,28 @@ def test_gemma3_text_only_loads_real_language_weights_from_vlm_checkpoint(tmp_paassert not any("vision_tower" in n for n, _ in model.named_modules()), "vision tower should be skipped on the text-only path"+++def _module_function(tree, name):+ for node in tree.body:+ if isinstance(node, ast.FunctionDef) and node.name == name:+ return node+ raise AssertionError(f"{name} not found")+++def test_base_fast_generate_tolerates_missing_architectures():+ method = _module_function(ast.parse(_source(VISION_PATH)), "unsloth_base_fast_generate")++ def is_raw_architectures(node):+ return (+ isinstance(node, ast.Attribute)+ and node.attr == "architectures"+ and isinstance(node.value, ast.Attribute)+ and node.value.attr == "config"+ )++ for node in ast.walk(method):+ if isinstance(node, ast.comprehension):+ assert not is_raw_architectures(node.iter)+ if isinstance(node, ast.Subscript):+ assert not is_raw_architectures(node.value)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex c14b96bd86f..8c7351e92cd 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -1127,6 +1127,11 @@ def _get_text_only_config(model_config, model_name):text_config = getattr(model_config, "text_config", None)if text_config is None:raise ValueError(f"Cannot load {model_name} as text-only; use FastVisionModel")+ # unsloth_zoo's Gemma-4 proxy (__slots__) refuses new attrs and resolve_model_class cannot map it; unwrap.+ try:+ text_config = object.__getattribute__(text_config, "_real")+ except AttributeError:+ pass# Carry over quantization_config; copy first, since get_text_config() shares the parent's object.qc = getattr(model_config, "quantization_config", None)if qc is not None and getattr(text_config, "quantization_config", None) is None:diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex d888c9cecde..ee8146ee50f 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -882,12 +882,13 @@ def unsloth_base_fast_generate(self, *args, **kwargs):if do_bfloat16_mixed_precision:dtype = torch.bfloat16+ # text_only configs have architectures=None.+ architectures = getattr(self.config, "architectures", None) or [type(self).__name__]is_vlm = any(- x.endswith(("ForConditionalGeneration", "ForVisionText2Text"))- for x in self.config.architectures+ x.endswith(("ForConditionalGeneration", "ForVisionText2Text")) for x in architectures)is_vlm = is_vlm or hasattr(self.config, "vision_config")- arch = self.config.architectures[0]+ arch = architectures[0]# Removing token_type_ids is WRONG for Gemma 3, which uses bidirectional attention.if hasattr(self, "generate") and hasattr(self, "forward"):