Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Refuse load_in_8bit with fast_inference instead of silently loading vLLM in 16-bit (#13019)

Daniel Hanc2f9ce1944b7
diff --git a/tests/test_fast_inference_8bit_refused.py b/tests/test_fast_inference_8bit_refused.py
new file mode 100644
index 00000000000..ea1cfb10001
--- /dev/null
+++ b/tests/test_fast_inference_8bit_refused.py
@@ -0,0 +1,139 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""fast_inference + bnb 8-bit must refuse before vLLM starts (it silently ran 16-bit)."""
+
+import ast
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from unsloth.models.loader_utils import refuse_fast_inference_load_in_8bit
+
+ROOT = Path(__file__).resolve().parents[1]
+BNB_8BIT_CHECKPOINT = {"quant_method": "bitsandbytes", "load_in_8bit": True}
+BNB_4BIT_CHECKPOINT = {
+ "quant_method": "bitsandbytes",
+ "load_in_4bit": True,
+ "bnb_4bit_quant_type": "nf4",
+}
+
+
+def _config(quantization_config = None):
+ if quantization_config is None:
+ return SimpleNamespace(model_type = "qwen3")
+ return SimpleNamespace(model_type = "qwen3", quantization_config = quantization_config)
+
+
+def test_load_in_8bit_flag_is_refused():
+ with pytest.raises(NotImplementedError, match = "load_in_8bit") as err:
+ refuse_fast_inference_load_in_8bit(True, _config())
+ message = str(err.value)
+ assert "load_in_4bit = True" in message
+ assert "fast_inference = False" in message
+
+
+def test_caller_bitsandbytes_8bit_config_is_refused():
+ from transformers import BitsAndBytesConfig
+ with pytest.raises(NotImplementedError):
+ refuse_fast_inference_load_in_8bit(False, _config(), BitsAndBytesConfig(load_in_8bit = True))
+ with pytest.raises(NotImplementedError):
+ refuse_fast_inference_load_in_8bit(False, _config(), {"load_in_8bit": True})
+
+
+def test_prequantized_8bit_checkpoint_is_refused():
+ # Flag off; the checkpoint's own config makes it 8-bit.
+ with pytest.raises(NotImplementedError):
+ refuse_fast_inference_load_in_8bit(False, _config(BNB_8BIT_CHECKPOINT))
+
+
+ "checkpoint",
+ [
+ None,
+ BNB_4BIT_CHECKPOINT,
+ {"quant_method": "fp8", "weight_block_size": [128, 128]},
+ {"quant_method": "fbgemm_fp8"},
+ {"quant_method": "compressed-tensors"},
+ {"quant_method": "gptq", "bits": 8},
+ ],
+)
+def test_4bit_16bit_fp8_and_other_formats_pass(checkpoint):
+ refuse_fast_inference_load_in_8bit(False, _config(checkpoint))
+ refuse_fast_inference_load_in_8bit(False, _config(checkpoint), None)
+
+
+def test_caller_4bit_config_passes():
+ from transformers import BitsAndBytesConfig
+ refuse_fast_inference_load_in_8bit(False, _config(), BitsAndBytesConfig(load_in_4bit = True))
+
+
+def test_missing_config_does_not_crash():
+ refuse_fast_inference_load_in_8bit(False, None, None)
+ with pytest.raises(NotImplementedError):
+ refuse_fast_inference_load_in_8bit(True, None, None)
+
+
+def _calls(function, name):
+ return [
+ node
+ for node in ast.walk(function)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == name
+ ]
+
+
+def _from_pretrained(path, class_name):
+ tree = ast.parse((ROOT / path).read_text(encoding = "utf-8"))
+ for node in ast.walk(tree):
+ if isinstance(node, ast.ClassDef) and node.name == class_name:
+ for item in node.body:
+ if isinstance(item, ast.FunctionDef) and item.name == "from_pretrained":
+ return item
+ raise AssertionError(f"{class_name}.from_pretrained not found in {path}")
+
+
+ "path, class_name",
+ [
+ ("unsloth/models/llama.py", "FastLlamaModel"),
+ ("unsloth/models/vision.py", "FastBaseModel"),
+ ],
+)
+def test_every_vllm_load_is_gated_before_the_engine(path, class_name):
+ function = _from_pretrained(path, class_name)
+ loads = _calls(function, "load_vllm")
+ guards = _calls(function, "refuse_fast_inference_load_in_8bit")
+ assert len(loads) == 1, f"{path}: expected one load_vllm call"
+ assert len(guards) == 1, f"{path}: missing the 8-bit fast_inference refusal"
+ guard = guards[0]
+ assert guard.lineno < loads[0].lineno
+ args = [ast.unparse(arg) for arg in guard.args]
+ assert args[0] == "load_in_8bit"
+ assert args[1] == "model_config"
+ assert "quantization_config" in args[2]
+ # Same If branch as load_vllm.
+ parents = {}
+ for node in ast.walk(function):
+ for child in ast.iter_child_nodes(node):
+ parents[child] = node
+
+ def chain(node):
+ out = []
+ while node in parents:
+ node = parents[node]
+ out.append(node)
+ return out
+
+ assert next(n for n in chain(guard) if isinstance(n, ast.If)) is next(
+ n for n in chain(loads[0]) if isinstance(n, ast.If)
+ )
+
+
+def test_only_load_vllm_callers_are_the_two_loaders():
+ # A new load_vllm call site in the model loaders needs the same refusal.
+ callers = []
+ for path in (ROOT / "unsloth" / "models").rglob("*.py"):
+ if "load_vllm(" in path.read_text(encoding = "utf-8"):
+ callers.append(path.relative_to(ROOT).as_posix())
+ assert sorted(callers) == ["unsloth/models/llama.py", "unsloth/models/vision.py"]
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index aba2f68328e..87422c0e37c 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -3927,6 +3927,14 @@ class FastLlamaModel:
fast_inference,
)
+ from .loader_utils import refuse_fast_inference_load_in_8bit
+
+ refuse_fast_inference_load_in_8bit(
+ load_in_8bit,
+ model_config,
+ kwargs.get("quantization_config", None),
+ )
+
allowed_args = inspect.getfullargspec(load_vllm).args
_vllm_model_name = resolve_vllm_model_path(
model_name,
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 97c82b85cbb..c6b267da4d0 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -3557,6 +3557,28 @@ def gptq_trainable_quantization_config(model_config, user_quantization_config):
return GPTQConfig(bits = qc["bits"], backend = "auto_trainable")
+def refuse_fast_inference_load_in_8bit(
+ load_in_8bit,
+ model_config = None,
+ quantization_config = None,
+):
+ """Refuse bnb 8-bit (flag, caller config or checkpoint config) under fast_inference: no 8-bit vLLM path."""
+ checkpoint_config = getattr(model_config, "quantization_config", None)
+ if not (
+ load_in_8bit
+ or _bnb_bits_requested(quantization_config) == 8
+ or _bnb_bits_requested(checkpoint_config) == 8
+ ):
+ return
+ raise NotImplementedError(
+ "Unsloth: fast_inference = True does not support bitsandbytes 8-bit "
+ "(load_in_8bit = True or an 8-bit bitsandbytes checkpoint): vLLM would "
+ "not run in 8-bit. Use load_in_4bit = True or 16-bit "
+ "(load_in_4bit = False, load_in_8bit = False) with fast_inference, or set "
+ "fast_inference = False to keep load_in_8bit."
+ )
+
+
def warn_if_bitsandbytes_quantized_nothing(
model,
quantization_config,
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 5112ab62505..73e515ef359 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -4062,6 +4062,13 @@ class FastBaseModel:
)
from unsloth_zoo.utils import get_quant_type
+ from .loader_utils import refuse_fast_inference_load_in_8bit
+
+ refuse_fast_inference_load_in_8bit(
+ load_in_8bit,
+ model_config,
+ kwargs.get("quantization_config", None),
+ )
# Mirrors load_vllm's bnb loader test, so prequantized bnb-4bit is refused too.
# Every sparse MoE, text ones too: vLLM's MoE LoRA layer rejects the bnb MoE