Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Load FP8 checkpoints dequantized to 16-bit on GPUs below sm89 (#14531)
Daniel Hanb7e66d7fd141
diff --git a/tests/test_fp8_device_gate.py b/tests/test_fp8_device_gate.pyindex 7f88d9a7ea4..9291398c7ef 100644--- a/tests/test_fp8_device_gate.py+++ b/tests/test_fp8_device_gate.py@@ -44,6 +44,7 @@ def test_non_fp8_checkpoints_load_anywhere(monkeypatch, device_type):],)def test_cuda_capability_rules_are_unchanged(monkeypatch, capability, quant_method, refused):+ monkeypatch.setattr(_utils, "_fp8_checkpoint_dequantizes_below_sm89", lambda: False)monkeypatch.setattr(_utils, "DEVICE_TYPE", "cuda")monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: capability)monkeypatch.setattr(torch.cuda, "get_device_name", lambda *a: "spoofed")@@ -52,3 +53,84 @@ def test_cuda_capability_rules_are_unchanged(monkeypatch, capability, quant_meth_utils.verify_fp8_support_if_applicable(_config(quant_method))else:_utils.verify_fp8_support_if_applicable(_config(quant_method))+++def _spoof_cuda(monkeypatch, capability):+ monkeypatch.setattr(_utils, "DEVICE_TYPE", "cuda")+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: capability)+ monkeypatch.setattr(torch.cuda, "get_device_name", lambda *a: "spoofed")++[email protected]("capability", [(7, 5), (8, 0), (8, 6)])+def test_fp8_checkpoint_loads_dequantized_below_sm89(monkeypatch, capability):+ _spoof_cuda(monkeypatch, capability)+ monkeypatch.setattr(_utils, "_fp8_checkpoint_dequantizes_below_sm89", lambda: True)+ _utils.verify_fp8_support_if_applicable(_config("fp8"))+ _utils.verify_fp8_support_if_applicable(_config("fp8"), fast_inference = False)++[email protected]("capability", [(7, 5), (8, 0)])+def test_vllm_load_of_fp8_checkpoint_still_needs_sm89(monkeypatch, capability):+ _spoof_cuda(monkeypatch, capability)+ monkeypatch.setattr(_utils, "_fp8_checkpoint_dequantizes_below_sm89", lambda: True)+ with pytest.raises(ValueError, match = "compute capability 8.9"):+ _utils.verify_fp8_support_if_applicable(_config("fp8"), fast_inference = True)++[email protected]("capability", [(7, 5), (8, 0), (8, 9)])+def test_fbgemm_fp8_rule_is_unchanged_by_the_fallback(monkeypatch, capability):+ _spoof_cuda(monkeypatch, capability)+ monkeypatch.setattr(_utils, "_fp8_checkpoint_dequantizes_below_sm89", lambda: True)+ with pytest.raises(ValueError, match = "FBGEMM"):+ _utils.verify_fp8_support_if_applicable(_config("fbgemm_fp8"))++[email protected]("device_type", ["hip", "mps", "cpu"])+def test_fallback_does_not_widen_other_devices(monkeypatch, device_type):+ monkeypatch.setattr(_utils, "DEVICE_TYPE", device_type)+ monkeypatch.setattr(_utils, "_fp8_checkpoint_dequantizes_below_sm89", lambda: True)+ with pytest.raises(ValueError, match = "only supported on CUDA and XPU"):+ _utils.verify_fp8_support_if_applicable(_config("fp8"))+++def test_fallback_detection_follows_the_installed_transformers():+ import inspect++ from transformers.utils.quantization_config import FineGrainedFP8Config++ has_flag = "dequantize" in inspect.signature(FineGrainedFP8Config.__init__).parameters+ assert _utils._fp8_checkpoint_dequantizes_below_sm89() is has_flag+++def test_real_check_admits_t4_on_this_transformers(monkeypatch):+ from transformers.utils.quantization_config import FineGrainedFP8Config+ import inspect++ _spoof_cuda(monkeypatch, (7, 5))+ if "dequantize" in inspect.signature(FineGrainedFP8Config.__init__).parameters:+ _utils.verify_fp8_support_if_applicable(_config("fp8"))+ else:+ with pytest.raises(ValueError, match = "compute capability 8.9"):+ _utils.verify_fp8_support_if_applicable(_config("fp8"))+++def test_fast_language_model_gates_on_the_effective_weight_loader():+ # fast_inference with num_labels loads through transformers, so the 16-bit fallback applies.+ import ast+ import inspect+ import textwrap++ from unsloth.models import llama++ assert llama._vllm_will_load_weights(True, num_labels = 2) is False+ tree = ast.parse(textwrap.dedent(inspect.getsource(llama.FastLlamaModel.from_pretrained)))+ calls = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and getattr(node.func, "id", None) == "verify_fp8_support_if_applicable"+ ]+ assert len(calls) == 2+ for call in calls:+ (kw,) = [k for k in call.keywords if k.arg == "fast_inference"]+ assert isinstance(kw.value, ast.Call) and kw.value.func.id == "_vllm_will_load_weights"diff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.pyindex 5a692c3bbaa..a7ec3e7f6cd 100644--- a/tests/test_modelopt_fp8_loading.py+++ b/tests/test_modelopt_fp8_loading.py@@ -532,7 +532,7 @@ from unsloth import FastLanguageModelfrom unsloth.models import llamafrom unsloth.models._utils import get_quant_typeseen = []-llama.verify_fp8_support_if_applicable = lambda config: seen.append(get_quant_type(config))+llama.verify_fp8_support_if_applicable = lambda config, **kw: seen.append(get_quant_type(config))try:FastLanguageModel.from_pretrained({str(tmp_path)!r}, load_in_4bit = False, max_seq_length = 64)except Exception:diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 2a061a81b70..04e8e58fefa 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -8311,7 +8311,19 @@ def patch_hf_quantizer():patch_hf_quantizer()-def verify_fp8_support_if_applicable(model_config):+def _fp8_checkpoint_dequantizes_below_sm89():+ # transformers 5.x dequantizes FP8 below sm89 (same release added `dequantize`); 4.57.x raises.+ try:+ from transformers.utils.quantization_config import FineGrainedFP8Config+ except Exception:+ return False+ try:+ return "dequantize" in inspect.signature(FineGrainedFP8Config.__init__).parameters+ except (TypeError, ValueError):+ return False+++def verify_fp8_support_if_applicable(model_config, fast_inference = False):quant_method = get_quant_type(model_config)if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE not in ("cuda", "xpu"):# In case of block quantized, we allow L4 because we fall back to torchao kernels.@@ -8327,6 +8339,12 @@ def verify_fp8_support_if_applicable(model_config):f"Unsloth: FBGEMM FP8 quantization is only supported on H100 and higher GPUs. L4 is not supported. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details.")if quant_method == "fp8" and major_version * 10 + minor_version < 89:+ # Only vLLM keeps FP8 weights; transformers dequantizes them.+ if not fast_inference and _fp8_checkpoint_dequantizes_below_sm89():+ logger.warning_once(+ f"Unsloth: {torch.cuda.get_device_name()} has no FP8 compute (compute capability 8.9 or higher), so this FP8 checkpoint is dequantized to 16-bit on load."+ )+ returnraise ValueError(f"Unsloth: FP8 quantization is only supported on L4 and higher GPUs with compute capability 8.9 or higher. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details.")diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 46a606b778e..7ced84fb87b 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -3406,7 +3406,9 @@ class FastLlamaModel:model_config.model_name = model_namemodel_max_seq_length = model_config.max_position_embeddings- verify_fp8_support_if_applicable(model_config)+ verify_fp8_support_if_applicable(+ model_config, fast_inference = _vllm_will_load_weights(fast_inference, num_labels)+ )# Check if RoPE Scaling is even allowedmodel_function = MODEL_FOR_CAUSAL_LM_MAPPING[model_config.__class__]@@ -3624,7 +3626,9 @@ class FastLlamaModel:_fp8_to_nf4 = fp8_to_nf4_armed(model_config)_modelopt_rewritten = modelopt_rewritten(model_config)if _modelopt_rewritten:- verify_fp8_support_if_applicable(model_config)+ verify_fp8_support_if_applicable(+ model_config, fast_inference = _vllm_will_load_weights(fast_inference, num_labels)+ )if _modelopt_rewritten and num_labels is not None:keep_task_heads_unquantized(model_config, AutoModelForSequenceClassification)pop_modelopt_key_mapping(model_config, kwargs)diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 7d22eba694f..4f32ba7f2f6 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -3892,7 +3892,9 @@ class FastBaseModel:_set_attn_impl(auto_config, config_attn_impl)model_config = auto_config- verify_fp8_support_if_applicable(model_config)+ verify_fp8_support_if_applicable(+ model_config, fast_inference = bool(fast_inference and is_vLLM_available())+ )raise_handler = RaiseUninitialized()try: