Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fast_inference: switch Gemma 4 to Unsloth inference on GPUs below sm80 (#13864)
Daniel Hanf6c34ddbc495
diff --git a/tests/test_fast_inference_old_gpu_fallback.py b/tests/test_fast_inference_old_gpu_fallback.pynew file mode 100644index 00000000000..80758e87f1a--- /dev/null+++ b/tests/test_fast_inference_old_gpu_fallback.py@@ -0,0 +1,85 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import ast+import sys+import types+from pathlib import Path++import pytest++LOADER = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "loader.py"+GEMMA4_TYPES = ["gemma4", "gemma4_text", "gemma4_unified", "gemma4_unified_text"]+++def _helper(monkeypatch, device_type, capability):+ fake_torch = types.ModuleType("torch")+ fake_torch.cuda = types.SimpleNamespace(get_device_capability = capability)+ monkeypatch.setitem(sys.modules, "torch", fake_torch)+ tree = ast.parse(LOADER.read_text(encoding = "utf-8"))+ wanted = []+ for node in tree.body:+ if isinstance(node, ast.FunctionDef) and node.name == "_vllm_needs_newer_gpu":+ wanted.append(node)+ elif isinstance(node, ast.Assign) and any(+ getattr(t, "id", None) == "VLLM_NEEDS_SM80_MODEL_TYPES" for t in node.targets+ ):+ wanted.append(node)+ assert len(wanted) == 2+ namespace = {"DEVICE_TYPE": device_type}+ exec(compile(ast.Module(body = wanted, type_ignores = []), "loader.py", "exec"), namespace)+ return namespace["_vllm_needs_newer_gpu"]++[email protected]("model_type", GEMMA4_TYPES)+def test_gemma4_falls_back_below_sm80(monkeypatch, model_type):+ f = _helper(monkeypatch, "cuda", lambda: (7, 5))+ assert f([model_type]) == model_type++[email protected]("capability", [(8, 0), (8, 9), (9, 0), (12, 0)])+def test_gemma4_keeps_vllm_on_sm80_and_newer(monkeypatch, capability):+ f = _helper(monkeypatch, "cuda", lambda: capability)+ assert f(GEMMA4_TYPES) is None+++def test_other_models_keep_vllm_on_t4(monkeypatch):+ f = _helper(monkeypatch, "cuda", lambda: (7, 5))+ for model_types in (["qwen3"], ["llama"], ["gemma3", "gemma3_text"], ["qwen3_5"]):+ assert f(model_types) is None+++def test_non_cuda_devices_untouched(monkeypatch):+ def boom():+ raise AssertionError("capability must not be probed off CUDA")++ for device_type in ("hip", "xpu", "mps", "cpu"):+ f = _helper(monkeypatch, device_type, boom)+ assert f(["gemma4"]) is None+++def test_failed_probe_keeps_vllm(monkeypatch):+ def broken():+ raise RuntimeError("no device")++ f = _helper(monkeypatch, "cuda", broken)+ assert f(["gemma4"]) is None+++def test_fallback_drops_max_num_batched_tokens(capsys):+ # load_vllm takes it; transformers' from_pretrained rejects it once vLLM is switched off.+ tree = ast.parse(LOADER.read_text(encoding = "utf-8"))+ wanted = [+ node+ for node in tree.body+ if (isinstance(node, ast.FunctionDef) and node.name == "_drop_vllm_engine_kwargs")+ or (+ isinstance(node, ast.Assign)+ and any(getattr(t, "id", None) == "_VLLM_ENGINE_KWARGS" for t in node.targets)+ )+ ]+ namespace = {}+ exec(compile(ast.Module(body = wanted, type_ignores = []), "loader.py", "exec"), namespace)+ kwargs = {"max_num_batched_tokens": 4096, "token": "x"}+ namespace["_drop_vllm_engine_kwargs"](kwargs)+ assert kwargs == {"token": "x"}diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 671a13cb25d..33302ae8cd4 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -105,6 +105,9 @@ SUPPORTS_GEMMA4 = transformers_version >= Version("5.5.0")VLLM_UNSUPPORTED_MODEL_TYPES = ("gpt_oss",)# model_type -> unsloth_zoo.empty_model symbol that rebuilds it from vLLM; an older zoo falls back._VLLM_NEEDS_ZOO_SYMBOL = {"nemotron_h": "extract_nemotron_h_mixer"}+# vLLM 0.31 has no attention backend for these below sm80: TRITON_ATTN needs 96 KB shared memory+# for Gemma 4's 512-wide global heads (T4 has 64 KB), FlexAttention rejects its KV sharing.+VLLM_NEEDS_SM80_MODEL_TYPES = ("gemma4", "gemma4_text", "gemma4_unified", "gemma4_unified_text")def _vllm_unsupported_model_types(model_types):@@ -117,6 +120,23 @@ def _vllm_unsupported_model_types(model_types):return unsupported+def _vllm_needs_newer_gpu(model_types):+ """First model type vLLM cannot run on this CUDA GPU (compute capability < 8), else None."""+ if DEVICE_TYPE != "cuda":+ return None+ model_type = next((t for t in model_types if t in VLLM_NEEDS_SM80_MODEL_TYPES), None)+ if model_type is None:+ return None+ import torch++ try:+ if torch.cuda.get_device_capability()[0] >= 8:+ return None+ except Exception:+ return None+ return model_type++# Transformers v5 meta-device loading corrupts non-persistent buffers (inv_freq); see _fix_rope_inv_freq() below._NEEDS_ROPE_FIX = transformers_version >= Version("5.0.0")if SUPPORTS_GEMMA:@@ -1042,6 +1062,7 @@ _VLLM_ENGINE_KWARGS = ("conservativeness","max_logprobs","max_num_seqs",+ "max_num_batched_tokens","return_args","training",)@@ -2440,6 +2461,14 @@ class FastModel(FastBaseModel):)fast_inference = False_drop_vllm_engine_kwargs(kwargs)+ _vllm_old_gpu = _vllm_needs_newer_gpu(model_types) if fast_inference else None+ if _vllm_old_gpu is not None:+ print(+ f"Unsloth: fast_inference (vLLM) cannot run {_vllm_old_gpu} on GPUs older than compute capability 8.0 "+ "(such as T4) - will switch to Unsloth inference!"+ )+ fast_inference = False+ _drop_vllm_engine_kwargs(kwargs)# Text-diffusion models (DiffusionGemma) take a transformers-only slow path: a custom block-diffusion generate over a novel backbone, so Unsloth's autoregressive kernel/compile patching is skipped and the unmodified HF model is loaded, keeping 4bit/8bit and PEFT LoRA.# Remote-code checkpoints can report their backbone's type (Nemotron-Labs-Diffusion says "nemotron"), so also match the raw config.