Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fast_inference: switch Gemma 4 to Unsloth inference on GPUs below sm80 (#13864)

Daniel Hanf6c34ddbc495
diff --git a/tests/test_fast_inference_old_gpu_fallback.py b/tests/test_fast_inference_old_gpu_fallback.py
new file mode 100644
index 00000000000..80758e87f1a
--- /dev/null
+++ b/tests/test_fast_inference_old_gpu_fallback.py
@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import ast
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+LOADER = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "loader.py"
+GEMMA4_TYPES = ["gemma4", "gemma4_text", "gemma4_unified", "gemma4_unified_text"]
+
+
+def _helper(monkeypatch, device_type, capability):
+ fake_torch = types.ModuleType("torch")
+ fake_torch.cuda = types.SimpleNamespace(get_device_capability = capability)
+ monkeypatch.setitem(sys.modules, "torch", fake_torch)
+ tree = ast.parse(LOADER.read_text(encoding = "utf-8"))
+ wanted = []
+ for node in tree.body:
+ if isinstance(node, ast.FunctionDef) and node.name == "_vllm_needs_newer_gpu":
+ wanted.append(node)
+ elif isinstance(node, ast.Assign) and any(
+ getattr(t, "id", None) == "VLLM_NEEDS_SM80_MODEL_TYPES" for t in node.targets
+ ):
+ wanted.append(node)
+ assert len(wanted) == 2
+ namespace = {"DEVICE_TYPE": device_type}
+ exec(compile(ast.Module(body = wanted, type_ignores = []), "loader.py", "exec"), namespace)
+ return namespace["_vllm_needs_newer_gpu"]
+
+
[email protected]("model_type", GEMMA4_TYPES)
+def test_gemma4_falls_back_below_sm80(monkeypatch, model_type):
+ f = _helper(monkeypatch, "cuda", lambda: (7, 5))
+ assert f([model_type]) == model_type
+
+
[email protected]("capability", [(8, 0), (8, 9), (9, 0), (12, 0)])
+def test_gemma4_keeps_vllm_on_sm80_and_newer(monkeypatch, capability):
+ f = _helper(monkeypatch, "cuda", lambda: capability)
+ assert f(GEMMA4_TYPES) is None
+
+
+def test_other_models_keep_vllm_on_t4(monkeypatch):
+ f = _helper(monkeypatch, "cuda", lambda: (7, 5))
+ for model_types in (["qwen3"], ["llama"], ["gemma3", "gemma3_text"], ["qwen3_5"]):
+ assert f(model_types) is None
+
+
+def test_non_cuda_devices_untouched(monkeypatch):
+ def boom():
+ raise AssertionError("capability must not be probed off CUDA")
+
+ for device_type in ("hip", "xpu", "mps", "cpu"):
+ f = _helper(monkeypatch, device_type, boom)
+ assert f(["gemma4"]) is None
+
+
+def test_failed_probe_keeps_vllm(monkeypatch):
+ def broken():
+ raise RuntimeError("no device")
+
+ f = _helper(monkeypatch, "cuda", broken)
+ assert f(["gemma4"]) is None
+
+
+def test_fallback_drops_max_num_batched_tokens(capsys):
+ # load_vllm takes it; transformers' from_pretrained rejects it once vLLM is switched off.
+ tree = ast.parse(LOADER.read_text(encoding = "utf-8"))
+ wanted = [
+ node
+ for node in tree.body
+ if (isinstance(node, ast.FunctionDef) and node.name == "_drop_vllm_engine_kwargs")
+ or (
+ isinstance(node, ast.Assign)
+ and any(getattr(t, "id", None) == "_VLLM_ENGINE_KWARGS" for t in node.targets)
+ )
+ ]
+ namespace = {}
+ exec(compile(ast.Module(body = wanted, type_ignores = []), "loader.py", "exec"), namespace)
+ kwargs = {"max_num_batched_tokens": 4096, "token": "x"}
+ namespace["_drop_vllm_engine_kwargs"](kwargs)
+ assert kwargs == {"token": "x"}
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 671a13cb25d..33302ae8cd4 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -105,6 +105,9 @@ SUPPORTS_GEMMA4 = transformers_version >= Version("5.5.0")
VLLM_UNSUPPORTED_MODEL_TYPES = ("gpt_oss",)
# model_type -> unsloth_zoo.empty_model symbol that rebuilds it from vLLM; an older zoo falls back.
_VLLM_NEEDS_ZOO_SYMBOL = {"nemotron_h": "extract_nemotron_h_mixer"}
+# vLLM 0.31 has no attention backend for these below sm80: TRITON_ATTN needs 96 KB shared memory
+# for Gemma 4's 512-wide global heads (T4 has 64 KB), FlexAttention rejects its KV sharing.
+VLLM_NEEDS_SM80_MODEL_TYPES = ("gemma4", "gemma4_text", "gemma4_unified", "gemma4_unified_text")
def _vllm_unsupported_model_types(model_types):
@@ -117,6 +120,23 @@ def _vllm_unsupported_model_types(model_types):
return unsupported
+def _vllm_needs_newer_gpu(model_types):
+ """First model type vLLM cannot run on this CUDA GPU (compute capability < 8), else None."""
+ if DEVICE_TYPE != "cuda":
+ return None
+ model_type = next((t for t in model_types if t in VLLM_NEEDS_SM80_MODEL_TYPES), None)
+ if model_type is None:
+ return None
+ import torch
+
+ try:
+ if torch.cuda.get_device_capability()[0] >= 8:
+ return None
+ except Exception:
+ return None
+ return model_type
+
+
# Transformers v5 meta-device loading corrupts non-persistent buffers (inv_freq); see _fix_rope_inv_freq() below.
_NEEDS_ROPE_FIX = transformers_version >= Version("5.0.0")
if SUPPORTS_GEMMA:
@@ -1042,6 +1062,7 @@ _VLLM_ENGINE_KWARGS = (
"conservativeness",
"max_logprobs",
"max_num_seqs",
+ "max_num_batched_tokens",
"return_args",
"training",
)
@@ -2440,6 +2461,14 @@ class FastModel(FastBaseModel):
)
fast_inference = False
_drop_vllm_engine_kwargs(kwargs)
+ _vllm_old_gpu = _vllm_needs_newer_gpu(model_types) if fast_inference else None
+ if _vllm_old_gpu is not None:
+ print(
+ f"Unsloth: fast_inference (vLLM) cannot run {_vllm_old_gpu} on GPUs older than compute capability 8.0 "
+ "(such as T4) - will switch to Unsloth inference!"
+ )
+ fast_inference = False
+ _drop_vllm_engine_kwargs(kwargs)
# Text-diffusion models (DiffusionGemma) take a transformers-only slow path: a custom block-diffusion generate over a novel backbone, so Unsloth's autoregressive kernel/compile patching is skipped and the unmodified HF model is loaded, keeping 4bit/8bit and PEFT LoRA.
# Remote-code checkpoints can report their backbone's type (Nemotron-Labs-Diffusion says "nemotron"), so also match the raw config.