Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Load FP8 checkpoints dequantized to 16-bit on GPUs below sm89 (#14531)

Daniel Hanb7e66d7fd141
diff --git a/tests/test_fp8_device_gate.py b/tests/test_fp8_device_gate.py
index 7f88d9a7ea4..9291398c7ef 100644
--- a/tests/test_fp8_device_gate.py
+++ b/tests/test_fp8_device_gate.py
@@ -44,6 +44,7 @@ def test_non_fp8_checkpoints_load_anywhere(monkeypatch, device_type):
],
)
def test_cuda_capability_rules_are_unchanged(monkeypatch, capability, quant_method, refused):
+ monkeypatch.setattr(_utils, "_fp8_checkpoint_dequantizes_below_sm89", lambda: False)
monkeypatch.setattr(_utils, "DEVICE_TYPE", "cuda")
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: capability)
monkeypatch.setattr(torch.cuda, "get_device_name", lambda *a: "spoofed")
@@ -52,3 +53,84 @@ def test_cuda_capability_rules_are_unchanged(monkeypatch, capability, quant_meth
_utils.verify_fp8_support_if_applicable(_config(quant_method))
else:
_utils.verify_fp8_support_if_applicable(_config(quant_method))
+
+
+def _spoof_cuda(monkeypatch, capability):
+ monkeypatch.setattr(_utils, "DEVICE_TYPE", "cuda")
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: capability)
+ monkeypatch.setattr(torch.cuda, "get_device_name", lambda *a: "spoofed")
+
+
[email protected]("capability", [(7, 5), (8, 0), (8, 6)])
+def test_fp8_checkpoint_loads_dequantized_below_sm89(monkeypatch, capability):
+ _spoof_cuda(monkeypatch, capability)
+ monkeypatch.setattr(_utils, "_fp8_checkpoint_dequantizes_below_sm89", lambda: True)
+ _utils.verify_fp8_support_if_applicable(_config("fp8"))
+ _utils.verify_fp8_support_if_applicable(_config("fp8"), fast_inference = False)
+
+
[email protected]("capability", [(7, 5), (8, 0)])
+def test_vllm_load_of_fp8_checkpoint_still_needs_sm89(monkeypatch, capability):
+ _spoof_cuda(monkeypatch, capability)
+ monkeypatch.setattr(_utils, "_fp8_checkpoint_dequantizes_below_sm89", lambda: True)
+ with pytest.raises(ValueError, match = "compute capability 8.9"):
+ _utils.verify_fp8_support_if_applicable(_config("fp8"), fast_inference = True)
+
+
[email protected]("capability", [(7, 5), (8, 0), (8, 9)])
+def test_fbgemm_fp8_rule_is_unchanged_by_the_fallback(monkeypatch, capability):
+ _spoof_cuda(monkeypatch, capability)
+ monkeypatch.setattr(_utils, "_fp8_checkpoint_dequantizes_below_sm89", lambda: True)
+ with pytest.raises(ValueError, match = "FBGEMM"):
+ _utils.verify_fp8_support_if_applicable(_config("fbgemm_fp8"))
+
+
[email protected]("device_type", ["hip", "mps", "cpu"])
+def test_fallback_does_not_widen_other_devices(monkeypatch, device_type):
+ monkeypatch.setattr(_utils, "DEVICE_TYPE", device_type)
+ monkeypatch.setattr(_utils, "_fp8_checkpoint_dequantizes_below_sm89", lambda: True)
+ with pytest.raises(ValueError, match = "only supported on CUDA and XPU"):
+ _utils.verify_fp8_support_if_applicable(_config("fp8"))
+
+
+def test_fallback_detection_follows_the_installed_transformers():
+ import inspect
+
+ from transformers.utils.quantization_config import FineGrainedFP8Config
+
+ has_flag = "dequantize" in inspect.signature(FineGrainedFP8Config.__init__).parameters
+ assert _utils._fp8_checkpoint_dequantizes_below_sm89() is has_flag
+
+
+def test_real_check_admits_t4_on_this_transformers(monkeypatch):
+ from transformers.utils.quantization_config import FineGrainedFP8Config
+ import inspect
+
+ _spoof_cuda(monkeypatch, (7, 5))
+ if "dequantize" in inspect.signature(FineGrainedFP8Config.__init__).parameters:
+ _utils.verify_fp8_support_if_applicable(_config("fp8"))
+ else:
+ with pytest.raises(ValueError, match = "compute capability 8.9"):
+ _utils.verify_fp8_support_if_applicable(_config("fp8"))
+
+
+def test_fast_language_model_gates_on_the_effective_weight_loader():
+ # fast_inference with num_labels loads through transformers, so the 16-bit fallback applies.
+ import ast
+ import inspect
+ import textwrap
+
+ from unsloth.models import llama
+
+ assert llama._vllm_will_load_weights(True, num_labels = 2) is False
+ tree = ast.parse(textwrap.dedent(inspect.getsource(llama.FastLlamaModel.from_pretrained)))
+ calls = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", None) == "verify_fp8_support_if_applicable"
+ ]
+ assert len(calls) == 2
+ for call in calls:
+ (kw,) = [k for k in call.keywords if k.arg == "fast_inference"]
+ assert isinstance(kw.value, ast.Call) and kw.value.func.id == "_vllm_will_load_weights"
diff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.py
index 5a692c3bbaa..a7ec3e7f6cd 100644
--- a/tests/test_modelopt_fp8_loading.py
+++ b/tests/test_modelopt_fp8_loading.py
@@ -532,7 +532,7 @@ from unsloth import FastLanguageModel
from unsloth.models import llama
from unsloth.models._utils import get_quant_type
seen = []
-llama.verify_fp8_support_if_applicable = lambda config: seen.append(get_quant_type(config))
+llama.verify_fp8_support_if_applicable = lambda config, **kw: seen.append(get_quant_type(config))
try:
FastLanguageModel.from_pretrained({str(tmp_path)!r}, load_in_4bit = False, max_seq_length = 64)
except Exception:
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 2a061a81b70..04e8e58fefa 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -8311,7 +8311,19 @@ def patch_hf_quantizer():
patch_hf_quantizer()
-def verify_fp8_support_if_applicable(model_config):
+def _fp8_checkpoint_dequantizes_below_sm89():
+ # transformers 5.x dequantizes FP8 below sm89 (same release added `dequantize`); 4.57.x raises.
+ try:
+ from transformers.utils.quantization_config import FineGrainedFP8Config
+ except Exception:
+ return False
+ try:
+ return "dequantize" in inspect.signature(FineGrainedFP8Config.__init__).parameters
+ except (TypeError, ValueError):
+ return False
+
+
+def verify_fp8_support_if_applicable(model_config, fast_inference = False):
quant_method = get_quant_type(model_config)
if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE not in ("cuda", "xpu"):
# In case of block quantized, we allow L4 because we fall back to torchao kernels.
@@ -8327,6 +8339,12 @@ def verify_fp8_support_if_applicable(model_config):
f"Unsloth: FBGEMM FP8 quantization is only supported on H100 and higher GPUs. L4 is not supported. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details."
)
if quant_method == "fp8" and major_version * 10 + minor_version < 89:
+ # Only vLLM keeps FP8 weights; transformers dequantizes them.
+ if not fast_inference and _fp8_checkpoint_dequantizes_below_sm89():
+ logger.warning_once(
+ f"Unsloth: {torch.cuda.get_device_name()} has no FP8 compute (compute capability 8.9 or higher), so this FP8 checkpoint is dequantized to 16-bit on load."
+ )
+ return
raise ValueError(
f"Unsloth: FP8 quantization is only supported on L4 and higher GPUs with compute capability 8.9 or higher. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details."
)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 46a606b778e..7ced84fb87b 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -3406,7 +3406,9 @@ class FastLlamaModel:
model_config.model_name = model_name
model_max_seq_length = model_config.max_position_embeddings
- verify_fp8_support_if_applicable(model_config)
+ verify_fp8_support_if_applicable(
+ model_config, fast_inference = _vllm_will_load_weights(fast_inference, num_labels)
+ )
# Check if RoPE Scaling is even allowed
model_function = MODEL_FOR_CAUSAL_LM_MAPPING[model_config.__class__]
@@ -3624,7 +3626,9 @@ class FastLlamaModel:
_fp8_to_nf4 = fp8_to_nf4_armed(model_config)
_modelopt_rewritten = modelopt_rewritten(model_config)
if _modelopt_rewritten:
- verify_fp8_support_if_applicable(model_config)
+ verify_fp8_support_if_applicable(
+ model_config, fast_inference = _vllm_will_load_weights(fast_inference, num_labels)
+ )
if _modelopt_rewritten and num_labels is not None:
keep_task_heads_unquantized(model_config, AutoModelForSequenceClassification)
pop_modelopt_key_mapping(model_config, kwargs)
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 7d22eba694f..4f32ba7f2f6 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -3892,7 +3892,9 @@ class FastBaseModel:
_set_attn_impl(auto_config, config_attn_impl)
model_config = auto_config
- verify_fp8_support_if_applicable(model_config)
+ verify_fp8_support_if_applicable(
+ model_config, fast_inference = bool(fast_inference and is_vLLM_available())
+ )
raise_handler = RaiseUninitialized()
try: