Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Refuse load_in_8bit with fast_inference instead of silently loading vLLM in 16-bit (#13019)
Daniel Hanc2f9ce1944b7
diff --git a/tests/test_fast_inference_8bit_refused.py b/tests/test_fast_inference_8bit_refused.pynew file mode 100644index 00000000000..ea1cfb10001--- /dev/null+++ b/tests/test_fast_inference_8bit_refused.py@@ -0,0 +1,139 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""fast_inference + bnb 8-bit must refuse before vLLM starts (it silently ran 16-bit)."""++import ast+from pathlib import Path+from types import SimpleNamespace++import pytest++from unsloth.models.loader_utils import refuse_fast_inference_load_in_8bit++ROOT = Path(__file__).resolve().parents[1]+BNB_8BIT_CHECKPOINT = {"quant_method": "bitsandbytes", "load_in_8bit": True}+BNB_4BIT_CHECKPOINT = {+ "quant_method": "bitsandbytes",+ "load_in_4bit": True,+ "bnb_4bit_quant_type": "nf4",+}+++def _config(quantization_config = None):+ if quantization_config is None:+ return SimpleNamespace(model_type = "qwen3")+ return SimpleNamespace(model_type = "qwen3", quantization_config = quantization_config)+++def test_load_in_8bit_flag_is_refused():+ with pytest.raises(NotImplementedError, match = "load_in_8bit") as err:+ refuse_fast_inference_load_in_8bit(True, _config())+ message = str(err.value)+ assert "load_in_4bit = True" in message+ assert "fast_inference = False" in message+++def test_caller_bitsandbytes_8bit_config_is_refused():+ from transformers import BitsAndBytesConfig+ with pytest.raises(NotImplementedError):+ refuse_fast_inference_load_in_8bit(False, _config(), BitsAndBytesConfig(load_in_8bit = True))+ with pytest.raises(NotImplementedError):+ refuse_fast_inference_load_in_8bit(False, _config(), {"load_in_8bit": True})+++def test_prequantized_8bit_checkpoint_is_refused():+ # Flag off; the checkpoint's own config makes it 8-bit.+ with pytest.raises(NotImplementedError):+ refuse_fast_inference_load_in_8bit(False, _config(BNB_8BIT_CHECKPOINT))+++ "checkpoint",+ [+ None,+ BNB_4BIT_CHECKPOINT,+ {"quant_method": "fp8", "weight_block_size": [128, 128]},+ {"quant_method": "fbgemm_fp8"},+ {"quant_method": "compressed-tensors"},+ {"quant_method": "gptq", "bits": 8},+ ],+)+def test_4bit_16bit_fp8_and_other_formats_pass(checkpoint):+ refuse_fast_inference_load_in_8bit(False, _config(checkpoint))+ refuse_fast_inference_load_in_8bit(False, _config(checkpoint), None)+++def test_caller_4bit_config_passes():+ from transformers import BitsAndBytesConfig+ refuse_fast_inference_load_in_8bit(False, _config(), BitsAndBytesConfig(load_in_4bit = True))+++def test_missing_config_does_not_crash():+ refuse_fast_inference_load_in_8bit(False, None, None)+ with pytest.raises(NotImplementedError):+ refuse_fast_inference_load_in_8bit(True, None, None)+++def _calls(function, name):+ return [+ node+ for node in ast.walk(function)+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == name+ ]+++def _from_pretrained(path, class_name):+ tree = ast.parse((ROOT / path).read_text(encoding = "utf-8"))+ for node in ast.walk(tree):+ if isinstance(node, ast.ClassDef) and node.name == class_name:+ for item in node.body:+ if isinstance(item, ast.FunctionDef) and item.name == "from_pretrained":+ return item+ raise AssertionError(f"{class_name}.from_pretrained not found in {path}")+++ "path, class_name",+ [+ ("unsloth/models/llama.py", "FastLlamaModel"),+ ("unsloth/models/vision.py", "FastBaseModel"),+ ],+)+def test_every_vllm_load_is_gated_before_the_engine(path, class_name):+ function = _from_pretrained(path, class_name)+ loads = _calls(function, "load_vllm")+ guards = _calls(function, "refuse_fast_inference_load_in_8bit")+ assert len(loads) == 1, f"{path}: expected one load_vllm call"+ assert len(guards) == 1, f"{path}: missing the 8-bit fast_inference refusal"+ guard = guards[0]+ assert guard.lineno < loads[0].lineno+ args = [ast.unparse(arg) for arg in guard.args]+ assert args[0] == "load_in_8bit"+ assert args[1] == "model_config"+ assert "quantization_config" in args[2]+ # Same If branch as load_vllm.+ parents = {}+ for node in ast.walk(function):+ for child in ast.iter_child_nodes(node):+ parents[child] = node++ def chain(node):+ out = []+ while node in parents:+ node = parents[node]+ out.append(node)+ return out++ assert next(n for n in chain(guard) if isinstance(n, ast.If)) is next(+ n for n in chain(loads[0]) if isinstance(n, ast.If)+ )+++def test_only_load_vllm_callers_are_the_two_loaders():+ # A new load_vllm call site in the model loaders needs the same refusal.+ callers = []+ for path in (ROOT / "unsloth" / "models").rglob("*.py"):+ if "load_vllm(" in path.read_text(encoding = "utf-8"):+ callers.append(path.relative_to(ROOT).as_posix())+ assert sorted(callers) == ["unsloth/models/llama.py", "unsloth/models/vision.py"]diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex aba2f68328e..87422c0e37c 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -3927,6 +3927,14 @@ class FastLlamaModel:fast_inference,)+ from .loader_utils import refuse_fast_inference_load_in_8bit++ refuse_fast_inference_load_in_8bit(+ load_in_8bit,+ model_config,+ kwargs.get("quantization_config", None),+ )+allowed_args = inspect.getfullargspec(load_vllm).args_vllm_model_name = resolve_vllm_model_path(model_name,diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 97c82b85cbb..c6b267da4d0 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -3557,6 +3557,28 @@ def gptq_trainable_quantization_config(model_config, user_quantization_config):return GPTQConfig(bits = qc["bits"], backend = "auto_trainable")+def refuse_fast_inference_load_in_8bit(+ load_in_8bit,+ model_config = None,+ quantization_config = None,+):+ """Refuse bnb 8-bit (flag, caller config or checkpoint config) under fast_inference: no 8-bit vLLM path."""+ checkpoint_config = getattr(model_config, "quantization_config", None)+ if not (+ load_in_8bit+ or _bnb_bits_requested(quantization_config) == 8+ or _bnb_bits_requested(checkpoint_config) == 8+ ):+ return+ raise NotImplementedError(+ "Unsloth: fast_inference = True does not support bitsandbytes 8-bit "+ "(load_in_8bit = True or an 8-bit bitsandbytes checkpoint): vLLM would "+ "not run in 8-bit. Use load_in_4bit = True or 16-bit "+ "(load_in_4bit = False, load_in_8bit = False) with fast_inference, or set "+ "fast_inference = False to keep load_in_8bit."+ )++def warn_if_bitsandbytes_quantized_nothing(model,quantization_config,diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 5112ab62505..73e515ef359 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -4062,6 +4062,13 @@ class FastBaseModel:)from unsloth_zoo.utils import get_quant_type+ from .loader_utils import refuse_fast_inference_load_in_8bit++ refuse_fast_inference_load_in_8bit(+ load_in_8bit,+ model_config,+ kwargs.get("quantization_config", None),+ )# Mirrors load_vllm's bnb loader test, so prequantized bnb-4bit is refused too.# Every sparse MoE, text ones too: vLLM's MoE LoRA layer rejects the bnb MoE