Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FastModel: fall back to Unsloth inference on GPUs older than Volta (#12959)
Daniel Hane9c8662c61e5
diff --git a/tests/test_fastmodel_prevolta_fast_inference.py b/tests/test_fastmodel_prevolta_fast_inference.pynew file mode 100644index 00000000000..92dd60f7996--- /dev/null+++ b/tests/test_fastmodel_prevolta_fast_inference.py@@ -0,0 +1,97 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FastModel pre-Volta fast_inference fallback; sliced with `ast` because importing the loader needs a GPU."""++import ast+import types+from pathlib import Path++import pytest++LOADER_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "loader.py"+++def _fast_inference_blocks():+ tree = ast.parse(LOADER_PATH.read_text(encoding = "utf-8"))+ cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "FastModel")+ fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")+ for parent in ast.walk(fn):+ body = getattr(parent, "body", None)+ if not isinstance(body, list):+ continue+ for i, node in enumerate(body):+ if (+ isinstance(node, ast.If)+ and isinstance(node.test, ast.Name)+ and node.test.id == "fast_inference"+ and "GB10" in ast.unparse(node)+ ):+ gate = body[i - 1]+ assert isinstance(gate, ast.If) and "get_device_capability" in ast.unparse(+ gate.test+ )+ return [gate, node]+ raise AssertionError("FastModel.from_pretrained has no pre-Volta gate before the GB10 block")+++def _run(+ device_type,+ capability,+ name = "Tesla P100-PCIE-16GB",+ has_vllm = True,+):+ device_type_torch = {"cuda": "cuda", "hip": "cuda", "xpu": "xpu"}[device_type]+ cuda = types.SimpleNamespace(+ get_device_name = lambda i = 0: name,+ get_device_capability = lambda *a: capability,+ )+ ns = {+ "fast_inference": True,+ "importlib": types.SimpleNamespace(+ util = types.SimpleNamespace(find_spec = lambda m: object() if has_vllm else None)+ ),+ "_vllm_unavailable_error": lambda: ImportError("no vllm"),+ "DEVICE_TYPE": device_type,+ "DEVICE_TYPE_TORCH": device_type_torch,+ "DEVICE_COUNT": 1,+ "torch": types.SimpleNamespace(cuda = cuda),+ }+ exec(compile(ast.Module(_fast_inference_blocks(), []), str(LOADER_PATH), "exec"), ns)+ return ns["fast_inference"]++[email protected]("capability", [(6, 1), (6, 0), (5, 2)])+def test_pre_volta_cuda_falls_back(capability, capsys):+ assert _run("cuda", capability) is False+ assert "vLLM does not work on older GPUs" in capsys.readouterr().out+++ "device_type,capability",+ [+ ("cuda", (7, 0)),+ ("cuda", (7, 5)),+ ("cuda", (9, 0)),+ ("hip", (9, 4)),+ ("hip", (6, 0)), # gfx arch major, not a CUDA capability: left to vLLM+ ("xpu", (0, 0)),+ ],+)+def test_other_devices_keep_fast_inference(device_type, capability, capsys):+ assert _run(device_type, capability) is True+ assert "older GPUs" not in capsys.readouterr().out+++def test_gb10_still_falls_back():+ assert _run("cuda", (12, 1), name = "NVIDIA GB10") is False+++def test_pre_volta_falls_back_without_requiring_vllm(capsys):+ assert _run("cuda", (6, 1), has_vllm = False) is False+ assert "vLLM does not work on older GPUs" in capsys.readouterr().out+++def test_volta_still_requires_vllm():+ with pytest.raises(ImportError):+ _run("cuda", (7, 0), has_vllm = False)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex ef402a8f82b..9c65fc97f06 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -1884,6 +1884,10 @@ class FastModel(FastBaseModel):# One whole model per rank; sharding one across the ranks' GPUs too would have every rank fighting for the same cards.device_map = distributed_device_map+ # Same fallback as FastLanguageModel, before requiring vLLM: zoo's vLLM loader raises on compute capability < 7.+ if fast_inference and DEVICE_TYPE == "cuda" and torch.cuda.get_device_capability()[0] < 7:+ print("Unsloth: vLLM does not work on older GPUs - will switch to Unsloth inference!")+ fast_inference = Falseif fast_inference:if importlib.util.find_spec("vllm") is None:raise _vllm_unavailable_error()